aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/arm64/booting.txt45
-rw-r--r--Documentation/arm64/memory.txt29
-rw-r--r--arch/arm64/Kconfig17
-rw-r--r--arch/arm64/Makefile6
-rw-r--r--arch/arm64/configs/defconfig2
-rw-r--r--arch/arm64/include/asm/assembler.h31
-rw-r--r--arch/arm64/include/asm/cmpxchg.h2
-rw-r--r--arch/arm64/include/asm/compat.h14
-rw-r--r--arch/arm64/include/asm/cpu_ops.h59
-rw-r--r--arch/arm64/include/asm/elf.h18
-rw-r--r--arch/arm64/include/asm/io.h2
-rw-r--r--arch/arm64/include/asm/irq.h1
-rw-r--r--arch/arm64/include/asm/memory.h11
-rw-r--r--arch/arm64/include/asm/pgtable-2level-hwdef.h4
-rw-r--r--arch/arm64/include/asm/pgtable.h2
-rw-r--r--arch/arm64/include/asm/processor.h5
-rw-r--r--arch/arm64/include/asm/psci.h19
-rw-r--r--arch/arm64/include/asm/ptrace.h1
-rw-r--r--arch/arm64/include/asm/smp.h15
-rw-r--r--arch/arm64/include/asm/spinlock.h83
-rw-r--r--arch/arm64/include/asm/spinlock_types.h15
-rw-r--r--arch/arm64/include/asm/syscall.h6
-rw-r--r--arch/arm64/include/asm/virt.h3
-rw-r--r--arch/arm64/include/uapi/asm/byteorder.h4
-rw-r--r--arch/arm64/kernel/Makefile4
-rw-r--r--arch/arm64/kernel/arm64ksyms.c1
-rw-r--r--arch/arm64/kernel/cpu_ops.c87
-rw-r--r--arch/arm64/kernel/cputable.c2
-rw-r--r--arch/arm64/kernel/entry.S22
-rw-r--r--arch/arm64/kernel/head.S61
-rw-r--r--arch/arm64/kernel/irq.c61
-rw-r--r--arch/arm64/kernel/kuser32.S57
-rw-r--r--arch/arm64/kernel/module.c5
-rw-r--r--arch/arm64/kernel/perf_event.c7
-rw-r--r--arch/arm64/kernel/process.c7
-rw-r--r--arch/arm64/kernel/psci.c87
-rw-r--r--arch/arm64/kernel/setup.c9
-rw-r--r--arch/arm64/kernel/signal32.c37
-rw-r--r--arch/arm64/kernel/smp.c214
-rw-r--r--arch/arm64/kernel/smp_psci.c53
-rw-r--r--arch/arm64/kernel/smp_spin_table.c97
-rw-r--r--arch/arm64/kernel/sys32.S22
-rw-r--r--arch/arm64/kernel/vdso.c5
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S32
-rw-r--r--arch/arm64/kvm/hyp-init.S5
-rw-r--r--arch/arm64/kvm/hyp.S13
-rw-r--r--arch/arm64/mm/ioremap.c20
-rw-r--r--arch/arm64/mm/proc.S4
48 files changed, 936 insertions, 370 deletions
diff --git a/Documentation/arm64/booting.txt b/Documentation/arm64/booting.txt
index 98df4a03807e..a9691cc48fe3 100644
--- a/Documentation/arm64/booting.txt
+++ b/Documentation/arm64/booting.txt
@@ -115,9 +115,10 @@ Before jumping into the kernel, the following conditions must be met:
115 External caches (if present) must be configured and disabled. 115 External caches (if present) must be configured and disabled.
116 116
117- Architected timers 117- Architected timers
118 CNTFRQ must be programmed with the timer frequency. 118 CNTFRQ must be programmed with the timer frequency and CNTVOFF must
119 If entering the kernel at EL1, CNTHCTL_EL2 must have EL1PCTEN (bit 0) 119 be programmed with a consistent value on all CPUs. If entering the
120 set where available. 120 kernel at EL1, CNTHCTL_EL2 must have EL1PCTEN (bit 0) set where
121 available.
121 122
122- Coherency 123- Coherency
123 All CPUs to be booted by the kernel must be part of the same coherency 124 All CPUs to be booted by the kernel must be part of the same coherency
@@ -130,30 +131,46 @@ Before jumping into the kernel, the following conditions must be met:
130 the kernel image will be entered must be initialised by software at a 131 the kernel image will be entered must be initialised by software at a
131 higher exception level to prevent execution in an UNKNOWN state. 132 higher exception level to prevent execution in an UNKNOWN state.
132 133
134The requirements described above for CPU mode, caches, MMUs, architected
135timers, coherency and system registers apply to all CPUs. All CPUs must
136enter the kernel in the same exception level.
137
133The boot loader is expected to enter the kernel on each CPU in the 138The boot loader is expected to enter the kernel on each CPU in the
134following manner: 139following manner:
135 140
136- The primary CPU must jump directly to the first instruction of the 141- The primary CPU must jump directly to the first instruction of the
137 kernel image. The device tree blob passed by this CPU must contain 142 kernel image. The device tree blob passed by this CPU must contain
138 for each CPU node: 143 an 'enable-method' property for each cpu node. The supported
139 144 enable-methods are described below.
140 1. An 'enable-method' property. Currently, the only supported value
141 for this field is the string "spin-table".
142
143 2. A 'cpu-release-addr' property identifying a 64-bit,
144 zero-initialised memory location.
145 145
146 It is expected that the bootloader will generate these device tree 146 It is expected that the bootloader will generate these device tree
147 properties and insert them into the blob prior to kernel entry. 147 properties and insert them into the blob prior to kernel entry.
148 148
149- Any secondary CPUs must spin outside of the kernel in a reserved area 149- CPUs with a "spin-table" enable-method must have a 'cpu-release-addr'
150 of memory (communicated to the kernel by a /memreserve/ region in the 150 property in their cpu node. This property identifies a
151 naturally-aligned 64-bit zero-initalised memory location.
152
153 These CPUs should spin outside of the kernel in a reserved area of
154 memory (communicated to the kernel by a /memreserve/ region in the
151 device tree) polling their cpu-release-addr location, which must be 155 device tree) polling their cpu-release-addr location, which must be
152 contained in the reserved region. A wfe instruction may be inserted 156 contained in the reserved region. A wfe instruction may be inserted
153 to reduce the overhead of the busy-loop and a sev will be issued by 157 to reduce the overhead of the busy-loop and a sev will be issued by
154 the primary CPU. When a read of the location pointed to by the 158 the primary CPU. When a read of the location pointed to by the
155 cpu-release-addr returns a non-zero value, the CPU must jump directly 159 cpu-release-addr returns a non-zero value, the CPU must jump to this
156 to this value. 160 value. The value will be written as a single 64-bit little-endian
161 value, so CPUs must convert the read value to their native endianness
162 before jumping to it.
163
164- CPUs with a "psci" enable method should remain outside of
165 the kernel (i.e. outside of the regions of memory described to the
166 kernel in the memory node, or in a reserved area of memory described
167 to the kernel by a /memreserve/ region in the device tree). The
168 kernel will issue CPU_ON calls as described in ARM document number ARM
169 DEN 0022A ("Power State Coordination Interface System Software on ARM
170 processors") to bring CPUs into the kernel.
171
172 The device tree should contain a 'psci' node, as described in
173 Documentation/devicetree/bindings/arm/psci.txt.
157 174
158- Secondary CPU general-purpose register settings 175- Secondary CPU general-purpose register settings
159 x0 = 0 (reserved for future use) 176 x0 = 0 (reserved for future use)
diff --git a/Documentation/arm64/memory.txt b/Documentation/arm64/memory.txt
index 78a377124ef0..5e054bfe4dde 100644
--- a/Documentation/arm64/memory.txt
+++ b/Documentation/arm64/memory.txt
@@ -21,7 +21,7 @@ The swapper_pgd_dir address is written to TTBR1 and never written to
21TTBR0. 21TTBR0.
22 22
23 23
24AArch64 Linux memory layout: 24AArch64 Linux memory layout with 4KB pages:
25 25
26Start End Size Use 26Start End Size Use
27----------------------------------------------------------------------- 27-----------------------------------------------------------------------
@@ -39,13 +39,38 @@ ffffffbffbc00000 ffffffbffbdfffff 2MB earlyprintk device
39 39
40ffffffbffbe00000 ffffffbffbe0ffff 64KB PCI I/O space 40ffffffbffbe00000 ffffffbffbe0ffff 64KB PCI I/O space
41 41
42ffffffbbffff0000 ffffffbcffffffff ~2MB [guard] 42ffffffbffbe10000 ffffffbcffffffff ~2MB [guard]
43 43
44ffffffbffc000000 ffffffbfffffffff 64MB modules 44ffffffbffc000000 ffffffbfffffffff 64MB modules
45 45
46ffffffc000000000 ffffffffffffffff 256GB kernel logical memory map 46ffffffc000000000 ffffffffffffffff 256GB kernel logical memory map
47 47
48 48
49AArch64 Linux memory layout with 64KB pages:
50
51Start End Size Use
52-----------------------------------------------------------------------
530000000000000000 000003ffffffffff 4TB user
54
55fffffc0000000000 fffffdfbfffeffff ~2TB vmalloc
56
57fffffdfbffff0000 fffffdfbffffffff 64KB [guard page]
58
59fffffdfc00000000 fffffdfdffffffff 8GB vmemmap
60
61fffffdfe00000000 fffffdfffbbfffff ~8GB [guard, future vmmemap]
62
63fffffdfffbc00000 fffffdfffbdfffff 2MB earlyprintk device
64
65fffffdfffbe00000 fffffdfffbe0ffff 64KB PCI I/O space
66
67fffffdfffbe10000 fffffdfffbffffff ~2MB [guard]
68
69fffffdfffc000000 fffffdffffffffff 64MB modules
70
71fffffe0000000000 ffffffffffffffff 2TB kernel logical memory map
72
73
49Translation table lookup with 4KB pages: 74Translation table lookup with 4KB pages:
50 75
51+--------+--------+--------+--------+--------+--------+--------+--------+ 76+--------+--------+--------+--------+--------+--------+--------+--------+
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index c04454876bcb..ce6ae9497492 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1,6 +1,7 @@
1config ARM64 1config ARM64
2 def_bool y 2 def_bool y
3 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE 3 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
4 select ARCH_USE_CMPXCHG_LOCKREF
4 select ARCH_WANT_OPTIONAL_GPIOLIB 5 select ARCH_WANT_OPTIONAL_GPIOLIB
5 select ARCH_WANT_COMPAT_IPC_PARSE_VERSION 6 select ARCH_WANT_COMPAT_IPC_PARSE_VERSION
6 select ARCH_WANT_FRAME_POINTERS 7 select ARCH_WANT_FRAME_POINTERS
@@ -61,10 +62,6 @@ config LOCKDEP_SUPPORT
61config TRACE_IRQFLAGS_SUPPORT 62config TRACE_IRQFLAGS_SUPPORT
62 def_bool y 63 def_bool y
63 64
64config GENERIC_LOCKBREAK
65 def_bool y
66 depends on SMP && PREEMPT
67
68config RWSEM_GENERIC_SPINLOCK 65config RWSEM_GENERIC_SPINLOCK
69 def_bool y 66 def_bool y
70 67
@@ -138,6 +135,11 @@ config ARM64_64K_PAGES
138 look-up. AArch32 emulation is not available when this feature 135 look-up. AArch32 emulation is not available when this feature
139 is enabled. 136 is enabled.
140 137
138config CPU_BIG_ENDIAN
139 bool "Build big-endian kernel"
140 help
141 Say Y if you plan on running a kernel in big-endian mode.
142
141config SMP 143config SMP
142 bool "Symmetric Multi-Processing" 144 bool "Symmetric Multi-Processing"
143 select USE_GENERIC_SMP_HELPERS 145 select USE_GENERIC_SMP_HELPERS
@@ -160,6 +162,13 @@ config NR_CPUS
160 default "8" if ARCH_XGENE 162 default "8" if ARCH_XGENE
161 default "4" 163 default "4"
162 164
165config HOTPLUG_CPU
166 bool "Support for hot-pluggable CPUs"
167 depends on SMP
168 help
169 Say Y here to experiment with turning CPUs off and on. CPUs
170 can be controlled through /sys/devices/system/cpu.
171
163source kernel/Kconfig.preempt 172source kernel/Kconfig.preempt
164 173
165config HZ 174config HZ
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index d90cf79f233a..2fceb71ac3b7 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -20,9 +20,15 @@ LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
20KBUILD_DEFCONFIG := defconfig 20KBUILD_DEFCONFIG := defconfig
21 21
22KBUILD_CFLAGS += -mgeneral-regs-only 22KBUILD_CFLAGS += -mgeneral-regs-only
23ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
24KBUILD_CPPFLAGS += -mbig-endian
25AS += -EB
26LD += -EB
27else
23KBUILD_CPPFLAGS += -mlittle-endian 28KBUILD_CPPFLAGS += -mlittle-endian
24AS += -EL 29AS += -EL
25LD += -EL 30LD += -EL
31endif
26 32
27comma = , 33comma = ,
28 34
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 31c81e9b792e..84139be62ae6 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -26,7 +26,7 @@ CONFIG_MODULE_UNLOAD=y
26CONFIG_ARCH_VEXPRESS=y 26CONFIG_ARCH_VEXPRESS=y
27CONFIG_ARCH_XGENE=y 27CONFIG_ARCH_XGENE=y
28CONFIG_SMP=y 28CONFIG_SMP=y
29CONFIG_PREEMPT_VOLUNTARY=y 29CONFIG_PREEMPT=y
30CONFIG_CMDLINE="console=ttyAMA0" 30CONFIG_CMDLINE="console=ttyAMA0"
31# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set 31# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
32CONFIG_COMPAT=y 32CONFIG_COMPAT=y
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 5aceb83b3f5c..fd3e3924041b 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -115,3 +115,34 @@ lr .req x30 // link register
115 .align 7 115 .align 7
116 b \label 116 b \label
117 .endm 117 .endm
118
119/*
120 * Select code when configured for BE.
121 */
122#ifdef CONFIG_CPU_BIG_ENDIAN
123#define CPU_BE(code...) code
124#else
125#define CPU_BE(code...)
126#endif
127
128/*
129 * Select code when configured for LE.
130 */
131#ifdef CONFIG_CPU_BIG_ENDIAN
132#define CPU_LE(code...)
133#else
134#define CPU_LE(code...) code
135#endif
136
137/*
138 * Define a macro that constructs a 64-bit value by concatenating two
139 * 32-bit registers. Note that on big endian systems the order of the
140 * registers is swapped.
141 */
142#ifndef CONFIG_CPU_BIG_ENDIAN
143 .macro regs_to_64, rd, lbits, hbits
144#else
145 .macro regs_to_64, rd, hbits, lbits
146#endif
147 orr \rd, \lbits, \hbits, lsl #32
148 .endm
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 8a8ce0e73a38..3914c0dcd09c 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -173,4 +173,6 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
173#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) 173#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
174#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 174#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
175 175
176#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
177
176#endif /* __ASM_CMPXCHG_H */ 178#endif /* __ASM_CMPXCHG_H */
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index 899af807ef0f..fda2704b3f9f 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -26,7 +26,11 @@
26#include <linux/ptrace.h> 26#include <linux/ptrace.h>
27 27
28#define COMPAT_USER_HZ 100 28#define COMPAT_USER_HZ 100
29#ifdef __AARCH64EB__
30#define COMPAT_UTS_MACHINE "armv8b\0\0"
31#else
29#define COMPAT_UTS_MACHINE "armv8l\0\0" 32#define COMPAT_UTS_MACHINE "armv8l\0\0"
33#endif
30 34
31typedef u32 compat_size_t; 35typedef u32 compat_size_t;
32typedef s32 compat_ssize_t; 36typedef s32 compat_ssize_t;
@@ -73,13 +77,23 @@ struct compat_timeval {
73}; 77};
74 78
75struct compat_stat { 79struct compat_stat {
80#ifdef __AARCH64EB__
81 short st_dev;
82 short __pad1;
83#else
76 compat_dev_t st_dev; 84 compat_dev_t st_dev;
85#endif
77 compat_ino_t st_ino; 86 compat_ino_t st_ino;
78 compat_mode_t st_mode; 87 compat_mode_t st_mode;
79 compat_ushort_t st_nlink; 88 compat_ushort_t st_nlink;
80 __compat_uid16_t st_uid; 89 __compat_uid16_t st_uid;
81 __compat_gid16_t st_gid; 90 __compat_gid16_t st_gid;
91#ifdef __AARCH64EB__
92 short st_rdev;
93 short __pad2;
94#else
82 compat_dev_t st_rdev; 95 compat_dev_t st_rdev;
96#endif
83 compat_off_t st_size; 97 compat_off_t st_size;
84 compat_off_t st_blksize; 98 compat_off_t st_blksize;
85 compat_off_t st_blocks; 99 compat_off_t st_blocks;
diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h
new file mode 100644
index 000000000000..c4cdb5e5b73d
--- /dev/null
+++ b/arch/arm64/include/asm/cpu_ops.h
@@ -0,0 +1,59 @@
1/*
2 * Copyright (C) 2013 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_CPU_OPS_H
17#define __ASM_CPU_OPS_H
18
19#include <linux/init.h>
20#include <linux/threads.h>
21
22struct device_node;
23
24/**
25 * struct cpu_operations - Callback operations for hotplugging CPUs.
26 *
27 * @name: Name of the property as appears in a devicetree cpu node's
28 * enable-method property.
29 * @cpu_init: Reads any data necessary for a specific enable-method from the
30 * devicetree, for a given cpu node and proposed logical id.
31 * @cpu_prepare: Early one-time preparation step for a cpu. If there is a
32 * mechanism for doing so, tests whether it is possible to boot
33 * the given CPU.
34 * @cpu_boot: Boots a cpu into the kernel.
35 * @cpu_postboot: Optionally, perform any post-boot cleanup or necesary
36 * synchronisation. Called from the cpu being booted.
37 * @cpu_disable: Prepares a cpu to die. May fail for some mechanism-specific
38 * reason, which will cause the hot unplug to be aborted. Called
39 * from the cpu to be killed.
40 * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
41 * cpu being killed.
42 */
43struct cpu_operations {
44 const char *name;
45 int (*cpu_init)(struct device_node *, unsigned int);
46 int (*cpu_prepare)(unsigned int);
47 int (*cpu_boot)(unsigned int);
48 void (*cpu_postboot)(void);
49#ifdef CONFIG_HOTPLUG_CPU
50 int (*cpu_disable)(unsigned int cpu);
51 void (*cpu_die)(unsigned int cpu);
52#endif
53};
54
55extern const struct cpu_operations *cpu_ops[NR_CPUS];
56extern int __init cpu_read_ops(struct device_node *dn, int cpu);
57extern void __init cpu_read_bootcpu_ops(void);
58
59#endif /* ifndef __ASM_CPU_OPS_H */
diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h
index e7fa87f9201b..01d3aab64b79 100644
--- a/arch/arm64/include/asm/elf.h
+++ b/arch/arm64/include/asm/elf.h
@@ -90,11 +90,24 @@ typedef struct user_fpsimd_state elf_fpregset_t;
90 * These are used to set parameters in the core dumps. 90 * These are used to set parameters in the core dumps.
91 */ 91 */
92#define ELF_CLASS ELFCLASS64 92#define ELF_CLASS ELFCLASS64
93#ifdef __AARCH64EB__
94#define ELF_DATA ELFDATA2MSB
95#else
93#define ELF_DATA ELFDATA2LSB 96#define ELF_DATA ELFDATA2LSB
97#endif
94#define ELF_ARCH EM_AARCH64 98#define ELF_ARCH EM_AARCH64
95 99
100/*
101 * This yields a string that ld.so will use to load implementation
102 * specific libraries for optimization. This is more specific in
103 * intent than poking at uname or /proc/cpuinfo.
104 */
96#define ELF_PLATFORM_SIZE 16 105#define ELF_PLATFORM_SIZE 16
106#ifdef __AARCH64EB__
107#define ELF_PLATFORM ("aarch64_be")
108#else
97#define ELF_PLATFORM ("aarch64") 109#define ELF_PLATFORM ("aarch64")
110#endif
98 111
99/* 112/*
100 * This is used to ensure we don't load something for the wrong architecture. 113 * This is used to ensure we don't load something for the wrong architecture.
@@ -149,7 +162,12 @@ extern unsigned long arch_randomize_brk(struct mm_struct *mm);
149#define arch_randomize_brk arch_randomize_brk 162#define arch_randomize_brk arch_randomize_brk
150 163
151#ifdef CONFIG_COMPAT 164#ifdef CONFIG_COMPAT
165
166#ifdef __AARCH64EB__
167#define COMPAT_ELF_PLATFORM ("v8b")
168#else
152#define COMPAT_ELF_PLATFORM ("v8l") 169#define COMPAT_ELF_PLATFORM ("v8l")
170#endif
153 171
154#define COMPAT_ELF_ET_DYN_BASE (randomize_et_dyn(2 * TASK_SIZE_32 / 3)) 172#define COMPAT_ELF_ET_DYN_BASE (randomize_et_dyn(2 * TASK_SIZE_32 / 3))
155 173
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 1d12f89140ba..b56e5b5df881 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -224,6 +224,7 @@ extern void __memset_io(volatile void __iomem *, int, size_t);
224 */ 224 */
225extern void __iomem *__ioremap(phys_addr_t phys_addr, size_t size, pgprot_t prot); 225extern void __iomem *__ioremap(phys_addr_t phys_addr, size_t size, pgprot_t prot);
226extern void __iounmap(volatile void __iomem *addr); 226extern void __iounmap(volatile void __iomem *addr);
227extern void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size);
227 228
228#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_DIRTY) 229#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_DIRTY)
229#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE)) 230#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE))
@@ -233,7 +234,6 @@ extern void __iounmap(volatile void __iomem *addr);
233#define ioremap(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) 234#define ioremap(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE))
234#define ioremap_nocache(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) 235#define ioremap_nocache(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE))
235#define ioremap_wc(addr, size) __ioremap((addr), (size), __pgprot(PROT_NORMAL_NC)) 236#define ioremap_wc(addr, size) __ioremap((addr), (size), __pgprot(PROT_NORMAL_NC))
236#define ioremap_cached(addr, size) __ioremap((addr), (size), __pgprot(PROT_NORMAL))
237#define iounmap __iounmap 237#define iounmap __iounmap
238 238
239#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF) 239#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF)
diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h
index 0332fc077f6e..e1f7ecdde11f 100644
--- a/arch/arm64/include/asm/irq.h
+++ b/arch/arm64/include/asm/irq.h
@@ -4,6 +4,7 @@
4#include <asm-generic/irq.h> 4#include <asm-generic/irq.h>
5 5
6extern void (*handle_arch_irq)(struct pt_regs *); 6extern void (*handle_arch_irq)(struct pt_regs *);
7extern void migrate_irqs(void);
7extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); 8extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
8 9
9#endif 10#endif
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 20925bcf4e2a..37762175896f 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -33,18 +33,23 @@
33#define UL(x) _AC(x, UL) 33#define UL(x) _AC(x, UL)
34 34
35/* 35/*
36 * PAGE_OFFSET - the virtual address of the start of the kernel image. 36 * PAGE_OFFSET - the virtual address of the start of the kernel image (top
37 * (VA_BITS - 1))
37 * VA_BITS - the maximum number of bits for virtual addresses. 38 * VA_BITS - the maximum number of bits for virtual addresses.
38 * TASK_SIZE - the maximum size of a user space task. 39 * TASK_SIZE - the maximum size of a user space task.
39 * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area. 40 * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
40 * The module space lives between the addresses given by TASK_SIZE 41 * The module space lives between the addresses given by TASK_SIZE
41 * and PAGE_OFFSET - it must be within 128MB of the kernel text. 42 * and PAGE_OFFSET - it must be within 128MB of the kernel text.
42 */ 43 */
43#define PAGE_OFFSET UL(0xffffffc000000000) 44#ifdef CONFIG_ARM64_64K_PAGES
45#define VA_BITS (42)
46#else
47#define VA_BITS (39)
48#endif
49#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
44#define MODULES_END (PAGE_OFFSET) 50#define MODULES_END (PAGE_OFFSET)
45#define MODULES_VADDR (MODULES_END - SZ_64M) 51#define MODULES_VADDR (MODULES_END - SZ_64M)
46#define EARLYCON_IOBASE (MODULES_VADDR - SZ_4M) 52#define EARLYCON_IOBASE (MODULES_VADDR - SZ_4M)
47#define VA_BITS (39)
48#define TASK_SIZE_64 (UL(1) << VA_BITS) 53#define TASK_SIZE_64 (UL(1) << VA_BITS)
49 54
50#ifdef CONFIG_COMPAT 55#ifdef CONFIG_COMPAT
diff --git a/arch/arm64/include/asm/pgtable-2level-hwdef.h b/arch/arm64/include/asm/pgtable-2level-hwdef.h
index 0a8ed3f94e93..2593b490c56a 100644
--- a/arch/arm64/include/asm/pgtable-2level-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-2level-hwdef.h
@@ -21,10 +21,10 @@
21 * 8192 entries of 8 bytes each, occupying a 64KB page. Levels 0 and 1 are not 21 * 8192 entries of 8 bytes each, occupying a 64KB page. Levels 0 and 1 are not
22 * used. The 2nd level table (PGD for Linux) can cover a range of 4TB, each 22 * used. The 2nd level table (PGD for Linux) can cover a range of 4TB, each
23 * entry representing 512MB. The user and kernel address spaces are limited to 23 * entry representing 512MB. The user and kernel address spaces are limited to
24 * 512GB and therefore we only use 1024 entries in the PGD. 24 * 4TB in the 64KB page configuration.
25 */ 25 */
26#define PTRS_PER_PTE 8192 26#define PTRS_PER_PTE 8192
27#define PTRS_PER_PGD 1024 27#define PTRS_PER_PGD 8192
28 28
29/* 29/*
30 * PGDIR_SHIFT determines the size a top-level page table entry can map. 30 * PGDIR_SHIFT determines the size a top-level page table entry can map.
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index f0bebc5e22cd..17bd3af0a117 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -33,7 +33,7 @@
33/* 33/*
34 * VMALLOC and SPARSEMEM_VMEMMAP ranges. 34 * VMALLOC and SPARSEMEM_VMEMMAP ranges.
35 */ 35 */
36#define VMALLOC_START UL(0xffffff8000000000) 36#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS)
37#define VMALLOC_END (PAGE_OFFSET - UL(0x400000000) - SZ_64K) 37#define VMALLOC_END (PAGE_OFFSET - UL(0x400000000) - SZ_64K)
38 38
39#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K)) 39#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index ab239b2c456f..45b20cd6cbca 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -107,6 +107,11 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
107 regs->pstate = COMPAT_PSR_MODE_USR; 107 regs->pstate = COMPAT_PSR_MODE_USR;
108 if (pc & 1) 108 if (pc & 1)
109 regs->pstate |= COMPAT_PSR_T_BIT; 109 regs->pstate |= COMPAT_PSR_T_BIT;
110
111#ifdef __AARCH64EB__
112 regs->pstate |= COMPAT_PSR_E_BIT;
113#endif
114
110 regs->compat_sp = sp; 115 regs->compat_sp = sp;
111} 116}
112#endif 117#endif
diff --git a/arch/arm64/include/asm/psci.h b/arch/arm64/include/asm/psci.h
index 0604237ecd99..e5312ea0ec1a 100644
--- a/arch/arm64/include/asm/psci.h
+++ b/arch/arm64/include/asm/psci.h
@@ -14,25 +14,6 @@
14#ifndef __ASM_PSCI_H 14#ifndef __ASM_PSCI_H
15#define __ASM_PSCI_H 15#define __ASM_PSCI_H
16 16
17#define PSCI_POWER_STATE_TYPE_STANDBY 0
18#define PSCI_POWER_STATE_TYPE_POWER_DOWN 1
19
20struct psci_power_state {
21 u16 id;
22 u8 type;
23 u8 affinity_level;
24};
25
26struct psci_operations {
27 int (*cpu_suspend)(struct psci_power_state state,
28 unsigned long entry_point);
29 int (*cpu_off)(struct psci_power_state state);
30 int (*cpu_on)(unsigned long cpuid, unsigned long entry_point);
31 int (*migrate)(unsigned long cpuid);
32};
33
34extern struct psci_operations psci_ops;
35
36int psci_init(void); 17int psci_init(void);
37 18
38#endif /* __ASM_PSCI_H */ 19#endif /* __ASM_PSCI_H */
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index 0dacbbf9458b..0e7fa4963735 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -42,6 +42,7 @@
42#define COMPAT_PSR_MODE_UND 0x0000001b 42#define COMPAT_PSR_MODE_UND 0x0000001b
43#define COMPAT_PSR_MODE_SYS 0x0000001f 43#define COMPAT_PSR_MODE_SYS 0x0000001f
44#define COMPAT_PSR_T_BIT 0x00000020 44#define COMPAT_PSR_T_BIT 0x00000020
45#define COMPAT_PSR_E_BIT 0x00000200
45#define COMPAT_PSR_F_BIT 0x00000040 46#define COMPAT_PSR_F_BIT 0x00000040
46#define COMPAT_PSR_I_BIT 0x00000080 47#define COMPAT_PSR_I_BIT 0x00000080
47#define COMPAT_PSR_A_BIT 0x00000100 48#define COMPAT_PSR_A_BIT 0x00000100
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index 4b8023c5d146..a498f2cd2c2a 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -60,21 +60,14 @@ struct secondary_data {
60 void *stack; 60 void *stack;
61}; 61};
62extern struct secondary_data secondary_data; 62extern struct secondary_data secondary_data;
63extern void secondary_holding_pen(void); 63extern void secondary_entry(void);
64extern volatile unsigned long secondary_holding_pen_release;
65 64
66extern void arch_send_call_function_single_ipi(int cpu); 65extern void arch_send_call_function_single_ipi(int cpu);
67extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); 66extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
68 67
69struct device_node; 68extern int __cpu_disable(void);
70 69
71struct smp_enable_ops { 70extern void __cpu_die(unsigned int cpu);
72 const char *name; 71extern void cpu_die(void);
73 int (*init_cpu)(struct device_node *, int);
74 int (*prepare_cpu)(int);
75};
76
77extern const struct smp_enable_ops smp_spin_table_ops;
78extern const struct smp_enable_ops smp_psci_ops;
79 72
80#endif /* ifndef __ASM_SMP_H */ 73#endif /* ifndef __ASM_SMP_H */
diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h
index 0defa0728a9b..3d5cf064d7a1 100644
--- a/arch/arm64/include/asm/spinlock.h
+++ b/arch/arm64/include/asm/spinlock.h
@@ -22,17 +22,10 @@
22/* 22/*
23 * Spinlock implementation. 23 * Spinlock implementation.
24 * 24 *
25 * The old value is read exclusively and the new one, if unlocked, is written
26 * exclusively. In case of failure, the loop is restarted.
27 *
28 * The memory barriers are implicit with the load-acquire and store-release 25 * The memory barriers are implicit with the load-acquire and store-release
29 * instructions. 26 * instructions.
30 *
31 * Unlocked value: 0
32 * Locked value: 1
33 */ 27 */
34 28
35#define arch_spin_is_locked(x) ((x)->lock != 0)
36#define arch_spin_unlock_wait(lock) \ 29#define arch_spin_unlock_wait(lock) \
37 do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) 30 do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
38 31
@@ -41,32 +34,51 @@
41static inline void arch_spin_lock(arch_spinlock_t *lock) 34static inline void arch_spin_lock(arch_spinlock_t *lock)
42{ 35{
43 unsigned int tmp; 36 unsigned int tmp;
37 arch_spinlock_t lockval, newval;
44 38
45 asm volatile( 39 asm volatile(
46 " sevl\n" 40 /* Atomically increment the next ticket. */
47 "1: wfe\n" 41" prfm pstl1strm, %3\n"
48 "2: ldaxr %w0, %1\n" 42"1: ldaxr %w0, %3\n"
49 " cbnz %w0, 1b\n" 43" add %w1, %w0, %w5\n"
50 " stxr %w0, %w2, %1\n" 44" stxr %w2, %w1, %3\n"
51 " cbnz %w0, 2b\n" 45" cbnz %w2, 1b\n"
52 : "=&r" (tmp), "+Q" (lock->lock) 46 /* Did we get the lock? */
53 : "r" (1) 47" eor %w1, %w0, %w0, ror #16\n"
54 : "cc", "memory"); 48" cbz %w1, 3f\n"
49 /*
50 * No: spin on the owner. Send a local event to avoid missing an
51 * unlock before the exclusive load.
52 */
53" sevl\n"
54"2: wfe\n"
55" ldaxrh %w2, %4\n"
56" eor %w1, %w2, %w0, lsr #16\n"
57" cbnz %w1, 2b\n"
58 /* We got the lock. Critical section starts here. */
59"3:"
60 : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
61 : "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
62 : "memory");
55} 63}
56 64
57static inline int arch_spin_trylock(arch_spinlock_t *lock) 65static inline int arch_spin_trylock(arch_spinlock_t *lock)
58{ 66{
59 unsigned int tmp; 67 unsigned int tmp;
68 arch_spinlock_t lockval;
60 69
61 asm volatile( 70 asm volatile(
62 "2: ldaxr %w0, %1\n" 71" prfm pstl1strm, %2\n"
63 " cbnz %w0, 1f\n" 72"1: ldaxr %w0, %2\n"
64 " stxr %w0, %w2, %1\n" 73" eor %w1, %w0, %w0, ror #16\n"
65 " cbnz %w0, 2b\n" 74" cbnz %w1, 2f\n"
66 "1:\n" 75" add %w0, %w0, %3\n"
67 : "=&r" (tmp), "+Q" (lock->lock) 76" stxr %w1, %w0, %2\n"
68 : "r" (1) 77" cbnz %w1, 1b\n"
69 : "cc", "memory"); 78"2:"
79 : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
80 : "I" (1 << TICKET_SHIFT)
81 : "memory");
70 82
71 return !tmp; 83 return !tmp;
72} 84}
@@ -74,9 +86,28 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)
74static inline void arch_spin_unlock(arch_spinlock_t *lock) 86static inline void arch_spin_unlock(arch_spinlock_t *lock)
75{ 87{
76 asm volatile( 88 asm volatile(
77 " stlr %w1, %0\n" 89" stlrh %w1, %0\n"
78 : "=Q" (lock->lock) : "r" (0) : "memory"); 90 : "=Q" (lock->owner)
91 : "r" (lock->owner + 1)
92 : "memory");
93}
94
95static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
96{
97 return lock.owner == lock.next;
98}
99
100static inline int arch_spin_is_locked(arch_spinlock_t *lock)
101{
102 return !arch_spin_value_unlocked(ACCESS_ONCE(*lock));
103}
104
105static inline int arch_spin_is_contended(arch_spinlock_t *lock)
106{
107 arch_spinlock_t lockval = ACCESS_ONCE(*lock);
108 return (lockval.next - lockval.owner) > 1;
79} 109}
110#define arch_spin_is_contended arch_spin_is_contended
80 111
81/* 112/*
82 * Write lock implementation. 113 * Write lock implementation.
diff --git a/arch/arm64/include/asm/spinlock_types.h b/arch/arm64/include/asm/spinlock_types.h
index 9a494346efed..b8d383665f56 100644
--- a/arch/arm64/include/asm/spinlock_types.h
+++ b/arch/arm64/include/asm/spinlock_types.h
@@ -20,14 +20,19 @@
20# error "please don't include this file directly" 20# error "please don't include this file directly"
21#endif 21#endif
22 22
23/* We only require natural alignment for exclusive accesses. */ 23#define TICKET_SHIFT 16
24#define __lock_aligned
25 24
26typedef struct { 25typedef struct {
27 volatile unsigned int lock; 26#ifdef __AARCH64EB__
28} arch_spinlock_t; 27 u16 next;
28 u16 owner;
29#else
30 u16 owner;
31 u16 next;
32#endif
33} __aligned(4) arch_spinlock_t;
29 34
30#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } 35#define __ARCH_SPIN_LOCK_UNLOCKED { 0 , 0 }
31 36
32typedef struct { 37typedef struct {
33 volatile unsigned int lock; 38 volatile unsigned int lock;
diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
index 89c047f9a971..70ba9d4ee978 100644
--- a/arch/arm64/include/asm/syscall.h
+++ b/arch/arm64/include/asm/syscall.h
@@ -59,6 +59,9 @@ static inline void syscall_get_arguments(struct task_struct *task,
59 unsigned int i, unsigned int n, 59 unsigned int i, unsigned int n,
60 unsigned long *args) 60 unsigned long *args)
61{ 61{
62 if (n == 0)
63 return;
64
62 if (i + n > SYSCALL_MAX_ARGS) { 65 if (i + n > SYSCALL_MAX_ARGS) {
63 unsigned long *args_bad = args + SYSCALL_MAX_ARGS - i; 66 unsigned long *args_bad = args + SYSCALL_MAX_ARGS - i;
64 unsigned int n_bad = n + i - SYSCALL_MAX_ARGS; 67 unsigned int n_bad = n + i - SYSCALL_MAX_ARGS;
@@ -82,6 +85,9 @@ static inline void syscall_set_arguments(struct task_struct *task,
82 unsigned int i, unsigned int n, 85 unsigned int i, unsigned int n,
83 const unsigned long *args) 86 const unsigned long *args)
84{ 87{
88 if (n == 0)
89 return;
90
85 if (i + n > SYSCALL_MAX_ARGS) { 91 if (i + n > SYSCALL_MAX_ARGS) {
86 pr_warning("%s called with max args %d, handling only %d\n", 92 pr_warning("%s called with max args %d, handling only %d\n",
87 __func__, i + n, SYSCALL_MAX_ARGS); 93 __func__, i + n, SYSCALL_MAX_ARGS);
diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h
index 26e310c54344..130e2be952cf 100644
--- a/arch/arm64/include/asm/virt.h
+++ b/arch/arm64/include/asm/virt.h
@@ -18,7 +18,8 @@
18#ifndef __ASM__VIRT_H 18#ifndef __ASM__VIRT_H
19#define __ASM__VIRT_H 19#define __ASM__VIRT_H
20 20
21#define BOOT_CPU_MODE_EL2 (0x0e12b007) 21#define BOOT_CPU_MODE_EL1 (0xe11)
22#define BOOT_CPU_MODE_EL2 (0xe12)
22 23
23#ifndef __ASSEMBLY__ 24#ifndef __ASSEMBLY__
24#include <asm/cacheflush.h> 25#include <asm/cacheflush.h>
diff --git a/arch/arm64/include/uapi/asm/byteorder.h b/arch/arm64/include/uapi/asm/byteorder.h
index 2b92046aafc5..dc19e9537f0d 100644
--- a/arch/arm64/include/uapi/asm/byteorder.h
+++ b/arch/arm64/include/uapi/asm/byteorder.h
@@ -16,6 +16,10 @@
16#ifndef __ASM_BYTEORDER_H 16#ifndef __ASM_BYTEORDER_H
17#define __ASM_BYTEORDER_H 17#define __ASM_BYTEORDER_H
18 18
19#ifdef __AARCH64EB__
20#include <linux/byteorder/big_endian.h>
21#else
19#include <linux/byteorder/little_endian.h> 22#include <linux/byteorder/little_endian.h>
23#endif
20 24
21#endif /* __ASM_BYTEORDER_H */ 25#endif /* __ASM_BYTEORDER_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 7b4b564961d4..5ba2fd43a75b 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -9,12 +9,12 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
9arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ 9arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
10 entry-fpsimd.o process.o ptrace.o setup.o signal.o \ 10 entry-fpsimd.o process.o ptrace.o setup.o signal.o \
11 sys.o stacktrace.o time.o traps.o io.o vdso.o \ 11 sys.o stacktrace.o time.o traps.o io.o vdso.o \
12 hyp-stub.o psci.o 12 hyp-stub.o psci.o cpu_ops.o
13 13
14arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ 14arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
15 sys_compat.o 15 sys_compat.o
16arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o 16arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
17arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o smp_psci.o 17arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o
18arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o 18arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
19arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o 19arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
20arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o 20arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index 41b4f626d554..e7ee770c0697 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -39,6 +39,7 @@ EXPORT_SYMBOL(clear_page);
39EXPORT_SYMBOL(__copy_from_user); 39EXPORT_SYMBOL(__copy_from_user);
40EXPORT_SYMBOL(__copy_to_user); 40EXPORT_SYMBOL(__copy_to_user);
41EXPORT_SYMBOL(__clear_user); 41EXPORT_SYMBOL(__clear_user);
42EXPORT_SYMBOL(__copy_in_user);
42 43
43 /* physical memory */ 44 /* physical memory */
44EXPORT_SYMBOL(memstart_addr); 45EXPORT_SYMBOL(memstart_addr);
diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c
new file mode 100644
index 000000000000..d62d12fb36c8
--- /dev/null
+++ b/arch/arm64/kernel/cpu_ops.c
@@ -0,0 +1,87 @@
1/*
2 * CPU kernel entry/exit control
3 *
4 * Copyright (C) 2013 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <asm/cpu_ops.h>
20#include <asm/smp_plat.h>
21#include <linux/errno.h>
22#include <linux/of.h>
23#include <linux/string.h>
24
25extern const struct cpu_operations smp_spin_table_ops;
26extern const struct cpu_operations cpu_psci_ops;
27
28const struct cpu_operations *cpu_ops[NR_CPUS];
29
30static const struct cpu_operations *supported_cpu_ops[] __initconst = {
31#ifdef CONFIG_SMP
32 &smp_spin_table_ops,
33 &cpu_psci_ops,
34#endif
35 NULL,
36};
37
38static const struct cpu_operations * __init cpu_get_ops(const char *name)
39{
40 const struct cpu_operations **ops = supported_cpu_ops;
41
42 while (*ops) {
43 if (!strcmp(name, (*ops)->name))
44 return *ops;
45
46 ops++;
47 }
48
49 return NULL;
50}
51
52/*
53 * Read a cpu's enable method from the device tree and record it in cpu_ops.
54 */
55int __init cpu_read_ops(struct device_node *dn, int cpu)
56{
57 const char *enable_method = of_get_property(dn, "enable-method", NULL);
58 if (!enable_method) {
59 /*
60 * The boot CPU may not have an enable method (e.g. when
61 * spin-table is used for secondaries). Don't warn spuriously.
62 */
63 if (cpu != 0)
64 pr_err("%s: missing enable-method property\n",
65 dn->full_name);
66 return -ENOENT;
67 }
68
69 cpu_ops[cpu] = cpu_get_ops(enable_method);
70 if (!cpu_ops[cpu]) {
71 pr_warn("%s: unsupported enable-method property: %s\n",
72 dn->full_name, enable_method);
73 return -EOPNOTSUPP;
74 }
75
76 return 0;
77}
78
79void __init cpu_read_bootcpu_ops(void)
80{
81 struct device_node *dn = of_get_cpu_node(0, NULL);
82 if (!dn) {
83 pr_err("Failed to find device node for boot cpu\n");
84 return;
85 }
86 cpu_read_ops(dn, 0);
87}
diff --git a/arch/arm64/kernel/cputable.c b/arch/arm64/kernel/cputable.c
index 63cfc4a43f4e..fd3993cb060f 100644
--- a/arch/arm64/kernel/cputable.c
+++ b/arch/arm64/kernel/cputable.c
@@ -22,7 +22,7 @@
22 22
23extern unsigned long __cpu_setup(void); 23extern unsigned long __cpu_setup(void);
24 24
25struct cpu_info __initdata cpu_table[] = { 25struct cpu_info cpu_table[] = {
26 { 26 {
27 .cpu_id_val = 0x000f0000, 27 .cpu_id_val = 0x000f0000,
28 .cpu_id_mask = 0x000f0000, 28 .cpu_id_mask = 0x000f0000,
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 3881fd115ebb..e1166145ca29 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -311,14 +311,14 @@ el1_irq:
311#endif 311#endif
312#ifdef CONFIG_PREEMPT 312#ifdef CONFIG_PREEMPT
313 get_thread_info tsk 313 get_thread_info tsk
314 ldr x24, [tsk, #TI_PREEMPT] // get preempt count 314 ldr w24, [tsk, #TI_PREEMPT] // get preempt count
315 add x0, x24, #1 // increment it 315 add w0, w24, #1 // increment it
316 str x0, [tsk, #TI_PREEMPT] 316 str w0, [tsk, #TI_PREEMPT]
317#endif 317#endif
318 irq_handler 318 irq_handler
319#ifdef CONFIG_PREEMPT 319#ifdef CONFIG_PREEMPT
320 str x24, [tsk, #TI_PREEMPT] // restore preempt count 320 str w24, [tsk, #TI_PREEMPT] // restore preempt count
321 cbnz x24, 1f // preempt count != 0 321 cbnz w24, 1f // preempt count != 0
322 ldr x0, [tsk, #TI_FLAGS] // get flags 322 ldr x0, [tsk, #TI_FLAGS] // get flags
323 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? 323 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
324 bl el1_preempt 324 bl el1_preempt
@@ -509,15 +509,15 @@ el0_irq_naked:
509#endif 509#endif
510 get_thread_info tsk 510 get_thread_info tsk
511#ifdef CONFIG_PREEMPT 511#ifdef CONFIG_PREEMPT
512 ldr x24, [tsk, #TI_PREEMPT] // get preempt count 512 ldr w24, [tsk, #TI_PREEMPT] // get preempt count
513 add x23, x24, #1 // increment it 513 add w23, w24, #1 // increment it
514 str x23, [tsk, #TI_PREEMPT] 514 str w23, [tsk, #TI_PREEMPT]
515#endif 515#endif
516 irq_handler 516 irq_handler
517#ifdef CONFIG_PREEMPT 517#ifdef CONFIG_PREEMPT
518 ldr x0, [tsk, #TI_PREEMPT] 518 ldr w0, [tsk, #TI_PREEMPT]
519 str x24, [tsk, #TI_PREEMPT] 519 str w24, [tsk, #TI_PREEMPT]
520 cmp x0, x23 520 cmp w0, w23
521 b.eq 1f 521 b.eq 1f
522 mov x1, #0 522 mov x1, #0
523 str x1, [x1] // BUG 523 str x1, [x1] // BUG
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 7090c126797c..7009387348b7 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -123,8 +123,9 @@
123 123
124ENTRY(stext) 124ENTRY(stext)
125 mov x21, x0 // x21=FDT 125 mov x21, x0 // x21=FDT
126 bl el2_setup // Drop to EL1, w20=cpu_boot_mode
126 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 127 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
127 bl el2_setup // Drop to EL1 128 bl set_cpu_boot_mode_flag
128 mrs x22, midr_el1 // x22=cpuid 129 mrs x22, midr_el1 // x22=cpuid
129 mov x0, x22 130 mov x0, x22
130 bl lookup_processor_type 131 bl lookup_processor_type
@@ -150,21 +151,30 @@ ENDPROC(stext)
150/* 151/*
151 * If we're fortunate enough to boot at EL2, ensure that the world is 152 * If we're fortunate enough to boot at EL2, ensure that the world is
152 * sane before dropping to EL1. 153 * sane before dropping to EL1.
154 *
155 * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if
156 * booted in EL1 or EL2 respectively.
153 */ 157 */
154ENTRY(el2_setup) 158ENTRY(el2_setup)
155 mrs x0, CurrentEL 159 mrs x0, CurrentEL
156 cmp x0, #PSR_MODE_EL2t 160 cmp x0, #PSR_MODE_EL2t
157 ccmp x0, #PSR_MODE_EL2h, #0x4, ne 161 ccmp x0, #PSR_MODE_EL2h, #0x4, ne
158 ldr x0, =__boot_cpu_mode // Compute __boot_cpu_mode 162 b.ne 1f
159 add x0, x0, x28 163 mrs x0, sctlr_el2
160 b.eq 1f 164CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2
161 str wzr, [x0] // Remember we don't have EL2... 165CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2
166 msr sctlr_el2, x0
167 b 2f
1681: mrs x0, sctlr_el1
169CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1
170CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1
171 msr sctlr_el1, x0
172 mov w20, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1
173 isb
162 ret 174 ret
163 175
164 /* Hyp configuration. */ 176 /* Hyp configuration. */
1651: ldr w1, =BOOT_CPU_MODE_EL2 1772: mov x0, #(1 << 31) // 64-bit EL1
166 str w1, [x0, #4] // This CPU has EL2
167 mov x0, #(1 << 31) // 64-bit EL1
168 msr hcr_el2, x0 178 msr hcr_el2, x0
169 179
170 /* Generic timers. */ 180 /* Generic timers. */
@@ -181,7 +191,8 @@ ENTRY(el2_setup)
181 191
182 /* sctlr_el1 */ 192 /* sctlr_el1 */
183 mov x0, #0x0800 // Set/clear RES{1,0} bits 193 mov x0, #0x0800 // Set/clear RES{1,0} bits
184 movk x0, #0x30d0, lsl #16 194CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems
195CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems
185 msr sctlr_el1, x0 196 msr sctlr_el1, x0
186 197
187 /* Coprocessor traps. */ 198 /* Coprocessor traps. */
@@ -204,10 +215,25 @@ ENTRY(el2_setup)
204 PSR_MODE_EL1h) 215 PSR_MODE_EL1h)
205 msr spsr_el2, x0 216 msr spsr_el2, x0
206 msr elr_el2, lr 217 msr elr_el2, lr
218 mov w20, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2
207 eret 219 eret
208ENDPROC(el2_setup) 220ENDPROC(el2_setup)
209 221
210/* 222/*
223 * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
224 * in x20. See arch/arm64/include/asm/virt.h for more info.
225 */
226ENTRY(set_cpu_boot_mode_flag)
227 ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode
228 add x1, x1, x28
229 cmp w20, #BOOT_CPU_MODE_EL2
230 b.ne 1f
231 add x1, x1, #4
2321: str w20, [x1] // This CPU has booted in EL1
233 ret
234ENDPROC(set_cpu_boot_mode_flag)
235
236/*
211 * We need to find out the CPU boot mode long after boot, so we need to 237 * We need to find out the CPU boot mode long after boot, so we need to
212 * store it in a writable variable. 238 * store it in a writable variable.
213 * 239 *
@@ -225,7 +251,6 @@ ENTRY(__boot_cpu_mode)
225 .quad PAGE_OFFSET 251 .quad PAGE_OFFSET
226 252
227#ifdef CONFIG_SMP 253#ifdef CONFIG_SMP
228 .pushsection .smp.pen.text, "ax"
229 .align 3 254 .align 3
2301: .quad . 2551: .quad .
231 .quad secondary_holding_pen_release 256 .quad secondary_holding_pen_release
@@ -235,8 +260,9 @@ ENTRY(__boot_cpu_mode)
235 * cores are held until we're ready for them to initialise. 260 * cores are held until we're ready for them to initialise.
236 */ 261 */
237ENTRY(secondary_holding_pen) 262ENTRY(secondary_holding_pen)
238 bl __calc_phys_offset // x24=phys offset 263 bl el2_setup // Drop to EL1, w20=cpu_boot_mode
239 bl el2_setup // Drop to EL1 264 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
265 bl set_cpu_boot_mode_flag
240 mrs x0, mpidr_el1 266 mrs x0, mpidr_el1
241 ldr x1, =MPIDR_HWID_BITMASK 267 ldr x1, =MPIDR_HWID_BITMASK
242 and x0, x0, x1 268 and x0, x0, x1
@@ -250,7 +276,16 @@ pen: ldr x4, [x3]
250 wfe 276 wfe
251 b pen 277 b pen
252ENDPROC(secondary_holding_pen) 278ENDPROC(secondary_holding_pen)
253 .popsection 279
280 /*
281 * Secondary entry point that jumps straight into the kernel. Only to
282 * be used where CPUs are brought online dynamically by the kernel.
283 */
284ENTRY(secondary_entry)
285 bl __calc_phys_offset // x2=phys offset
286 bl el2_setup // Drop to EL1
287 b secondary_startup
288ENDPROC(secondary_entry)
254 289
255ENTRY(secondary_startup) 290ENTRY(secondary_startup)
256 /* 291 /*
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index ecb3354292ed..473e5dbf8f39 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -81,3 +81,64 @@ void __init init_IRQ(void)
81 if (!handle_arch_irq) 81 if (!handle_arch_irq)
82 panic("No interrupt controller found."); 82 panic("No interrupt controller found.");
83} 83}
84
85#ifdef CONFIG_HOTPLUG_CPU
86static bool migrate_one_irq(struct irq_desc *desc)
87{
88 struct irq_data *d = irq_desc_get_irq_data(desc);
89 const struct cpumask *affinity = d->affinity;
90 struct irq_chip *c;
91 bool ret = false;
92
93 /*
94 * If this is a per-CPU interrupt, or the affinity does not
95 * include this CPU, then we have nothing to do.
96 */
97 if (irqd_is_per_cpu(d) || !cpumask_test_cpu(smp_processor_id(), affinity))
98 return false;
99
100 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
101 affinity = cpu_online_mask;
102 ret = true;
103 }
104
105 c = irq_data_get_irq_chip(d);
106 if (!c->irq_set_affinity)
107 pr_debug("IRQ%u: unable to set affinity\n", d->irq);
108 else if (c->irq_set_affinity(d, affinity, true) == IRQ_SET_MASK_OK && ret)
109 cpumask_copy(d->affinity, affinity);
110
111 return ret;
112}
113
114/*
115 * The current CPU has been marked offline. Migrate IRQs off this CPU.
116 * If the affinity settings do not allow other CPUs, force them onto any
117 * available CPU.
118 *
119 * Note: we must iterate over all IRQs, whether they have an attached
120 * action structure or not, as we need to get chained interrupts too.
121 */
122void migrate_irqs(void)
123{
124 unsigned int i;
125 struct irq_desc *desc;
126 unsigned long flags;
127
128 local_irq_save(flags);
129
130 for_each_irq_desc(i, desc) {
131 bool affinity_broken;
132
133 raw_spin_lock(&desc->lock);
134 affinity_broken = migrate_one_irq(desc);
135 raw_spin_unlock(&desc->lock);
136
137 if (affinity_broken)
138 pr_warn_ratelimited("IRQ%u no longer affine to CPU%u\n",
139 i, smp_processor_id());
140 }
141
142 local_irq_restore(flags);
143}
144#endif /* CONFIG_HOTPLUG_CPU */
diff --git a/arch/arm64/kernel/kuser32.S b/arch/arm64/kernel/kuser32.S
index 8b69ecb1d8bc..63c48ffdf230 100644
--- a/arch/arm64/kernel/kuser32.S
+++ b/arch/arm64/kernel/kuser32.S
@@ -27,6 +27,9 @@
27 * 27 *
28 * See Documentation/arm/kernel_user_helpers.txt for formal definitions. 28 * See Documentation/arm/kernel_user_helpers.txt for formal definitions.
29 */ 29 */
30
31#include <asm/unistd32.h>
32
30 .align 5 33 .align 5
31 .globl __kuser_helper_start 34 .globl __kuser_helper_start
32__kuser_helper_start: 35__kuser_helper_start:
@@ -35,33 +38,30 @@ __kuser_cmpxchg64: // 0xffff0f60
35 .inst 0xe92d00f0 // push {r4, r5, r6, r7} 38 .inst 0xe92d00f0 // push {r4, r5, r6, r7}
36 .inst 0xe1c040d0 // ldrd r4, r5, [r0] 39 .inst 0xe1c040d0 // ldrd r4, r5, [r0]
37 .inst 0xe1c160d0 // ldrd r6, r7, [r1] 40 .inst 0xe1c160d0 // ldrd r6, r7, [r1]
38 .inst 0xf57ff05f // dmb sy 41 .inst 0xe1b20e9f // 1: ldaexd r0, r1, [r2]
39 .inst 0xe1b20f9f // 1: ldrexd r0, r1, [r2]
40 .inst 0xe0303004 // eors r3, r0, r4 42 .inst 0xe0303004 // eors r3, r0, r4
41 .inst 0x00313005 // eoreqs r3, r1, r5 43 .inst 0x00313005 // eoreqs r3, r1, r5
42 .inst 0x01a23f96 // strexdeq r3, r6, [r2] 44 .inst 0x01a23e96 // stlexdeq r3, r6, [r2]
43 .inst 0x03330001 // teqeq r3, #1 45 .inst 0x03330001 // teqeq r3, #1
44 .inst 0x0afffff9 // beq 1b 46 .inst 0x0afffff9 // beq 1b
45 .inst 0xf57ff05f // dmb sy
46 .inst 0xe2730000 // rsbs r0, r3, #0 47 .inst 0xe2730000 // rsbs r0, r3, #0
47 .inst 0xe8bd00f0 // pop {r4, r5, r6, r7} 48 .inst 0xe8bd00f0 // pop {r4, r5, r6, r7}
48 .inst 0xe12fff1e // bx lr 49 .inst 0xe12fff1e // bx lr
49 50
50 .align 5 51 .align 5
51__kuser_memory_barrier: // 0xffff0fa0 52__kuser_memory_barrier: // 0xffff0fa0
52 .inst 0xf57ff05f // dmb sy 53 .inst 0xf57ff05b // dmb ish
53 .inst 0xe12fff1e // bx lr 54 .inst 0xe12fff1e // bx lr
54 55
55 .align 5 56 .align 5
56__kuser_cmpxchg: // 0xffff0fc0 57__kuser_cmpxchg: // 0xffff0fc0
57 .inst 0xf57ff05f // dmb sy 58 .inst 0xe1923e9f // 1: ldaex r3, [r2]
58 .inst 0xe1923f9f // 1: ldrex r3, [r2]
59 .inst 0xe0533000 // subs r3, r3, r0 59 .inst 0xe0533000 // subs r3, r3, r0
60 .inst 0x01823f91 // strexeq r3, r1, [r2] 60 .inst 0x01823e91 // stlexeq r3, r1, [r2]
61 .inst 0x03330001 // teqeq r3, #1 61 .inst 0x03330001 // teqeq r3, #1
62 .inst 0x0afffffa // beq 1b 62 .inst 0x0afffffa // beq 1b
63 .inst 0xe2730000 // rsbs r0, r3, #0 63 .inst 0xe2730000 // rsbs r0, r3, #0
64 .inst 0xeaffffef // b <__kuser_memory_barrier> 64 .inst 0xe12fff1e // bx lr
65 65
66 .align 5 66 .align 5
67__kuser_get_tls: // 0xffff0fe0 67__kuser_get_tls: // 0xffff0fe0
@@ -75,3 +75,42 @@ __kuser_helper_version: // 0xffff0ffc
75 .word ((__kuser_helper_end - __kuser_helper_start) >> 5) 75 .word ((__kuser_helper_end - __kuser_helper_start) >> 5)
76 .globl __kuser_helper_end 76 .globl __kuser_helper_end
77__kuser_helper_end: 77__kuser_helper_end:
78
79/*
80 * AArch32 sigreturn code
81 *
82 * For ARM syscalls, the syscall number has to be loaded into r7.
83 * We do not support an OABI userspace.
84 *
85 * For Thumb syscalls, we also pass the syscall number via r7. We therefore
86 * need two 16-bit instructions.
87 */
88 .globl __aarch32_sigret_code_start
89__aarch32_sigret_code_start:
90
91 /*
92 * ARM Code
93 */
94 .byte __NR_compat_sigreturn, 0x70, 0xa0, 0xe3 // mov r7, #__NR_compat_sigreturn
95 .byte __NR_compat_sigreturn, 0x00, 0x00, 0xef // svc #__NR_compat_sigreturn
96
97 /*
98 * Thumb code
99 */
100 .byte __NR_compat_sigreturn, 0x27 // svc #__NR_compat_sigreturn
101 .byte __NR_compat_sigreturn, 0xdf // mov r7, #__NR_compat_sigreturn
102
103 /*
104 * ARM code
105 */
106 .byte __NR_compat_rt_sigreturn, 0x70, 0xa0, 0xe3 // mov r7, #__NR_compat_rt_sigreturn
107 .byte __NR_compat_rt_sigreturn, 0x00, 0x00, 0xef // svc #__NR_compat_rt_sigreturn
108
109 /*
110 * Thumb code
111 */
112 .byte __NR_compat_rt_sigreturn, 0x27 // svc #__NR_compat_rt_sigreturn
113 .byte __NR_compat_rt_sigreturn, 0xdf // mov r7, #__NR_compat_rt_sigreturn
114
115 .globl __aarch32_sigret_code_end
116__aarch32_sigret_code_end:
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index ca0e3d55da99..2c28a6cf93e6 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -111,6 +111,9 @@ static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm)
111 u32 immlo, immhi, lomask, himask, mask; 111 u32 immlo, immhi, lomask, himask, mask;
112 int shift; 112 int shift;
113 113
114 /* The instruction stream is always little endian. */
115 insn = le32_to_cpu(insn);
116
114 switch (type) { 117 switch (type) {
115 case INSN_IMM_MOVNZ: 118 case INSN_IMM_MOVNZ:
116 /* 119 /*
@@ -179,7 +182,7 @@ static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm)
179 insn &= ~(mask << shift); 182 insn &= ~(mask << shift);
180 insn |= (imm & mask) << shift; 183 insn |= (imm & mask) << shift;
181 184
182 return insn; 185 return cpu_to_le32(insn);
183} 186}
184 187
185static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val, 188static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index cea1594ff933..5d14470452ac 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -784,8 +784,8 @@ static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
784/* 784/*
785 * PMXEVTYPER: Event selection reg 785 * PMXEVTYPER: Event selection reg
786 */ 786 */
787#define ARMV8_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */ 787#define ARMV8_EVTYPE_MASK 0xc80003ff /* Mask for writable bits */
788#define ARMV8_EVTYPE_EVENT 0xff /* Mask for EVENT bits */ 788#define ARMV8_EVTYPE_EVENT 0x3ff /* Mask for EVENT bits */
789 789
790/* 790/*
791 * Event filters for PMUv3 791 * Event filters for PMUv3
@@ -1175,7 +1175,8 @@ static void armv8pmu_reset(void *info)
1175static int armv8_pmuv3_map_event(struct perf_event *event) 1175static int armv8_pmuv3_map_event(struct perf_event *event)
1176{ 1176{
1177 return map_cpu_event(event, &armv8_pmuv3_perf_map, 1177 return map_cpu_event(event, &armv8_pmuv3_perf_map,
1178 &armv8_pmuv3_perf_cache_map, 0xFF); 1178 &armv8_pmuv3_perf_cache_map,
1179 ARMV8_EVTYPE_EVENT);
1179} 1180}
1180 1181
1181static struct arm_pmu armv8pmu = { 1182static struct arm_pmu armv8pmu = {
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 7ae8a1f00c3c..de17c89985db 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -102,6 +102,13 @@ void arch_cpu_idle(void)
102 local_irq_enable(); 102 local_irq_enable();
103} 103}
104 104
105#ifdef CONFIG_HOTPLUG_CPU
106void arch_cpu_idle_dead(void)
107{
108 cpu_die();
109}
110#endif
111
105void machine_shutdown(void) 112void machine_shutdown(void)
106{ 113{
107#ifdef CONFIG_SMP 114#ifdef CONFIG_SMP
diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c
index 14f73c445ff5..4f97db3d7363 100644
--- a/arch/arm64/kernel/psci.c
+++ b/arch/arm64/kernel/psci.c
@@ -17,12 +17,32 @@
17 17
18#include <linux/init.h> 18#include <linux/init.h>
19#include <linux/of.h> 19#include <linux/of.h>
20#include <linux/smp.h>
20 21
21#include <asm/compiler.h> 22#include <asm/compiler.h>
23#include <asm/cpu_ops.h>
22#include <asm/errno.h> 24#include <asm/errno.h>
23#include <asm/psci.h> 25#include <asm/psci.h>
26#include <asm/smp_plat.h>
24 27
25struct psci_operations psci_ops; 28#define PSCI_POWER_STATE_TYPE_STANDBY 0
29#define PSCI_POWER_STATE_TYPE_POWER_DOWN 1
30
31struct psci_power_state {
32 u16 id;
33 u8 type;
34 u8 affinity_level;
35};
36
37struct psci_operations {
38 int (*cpu_suspend)(struct psci_power_state state,
39 unsigned long entry_point);
40 int (*cpu_off)(struct psci_power_state state);
41 int (*cpu_on)(unsigned long cpuid, unsigned long entry_point);
42 int (*migrate)(unsigned long cpuid);
43};
44
45static struct psci_operations psci_ops;
26 46
27static int (*invoke_psci_fn)(u64, u64, u64, u64); 47static int (*invoke_psci_fn)(u64, u64, u64, u64);
28 48
@@ -209,3 +229,68 @@ out_put_node:
209 of_node_put(np); 229 of_node_put(np);
210 return err; 230 return err;
211} 231}
232
233#ifdef CONFIG_SMP
234
235static int __init cpu_psci_cpu_init(struct device_node *dn, unsigned int cpu)
236{
237 return 0;
238}
239
240static int __init cpu_psci_cpu_prepare(unsigned int cpu)
241{
242 if (!psci_ops.cpu_on) {
243 pr_err("no cpu_on method, not booting CPU%d\n", cpu);
244 return -ENODEV;
245 }
246
247 return 0;
248}
249
250static int cpu_psci_cpu_boot(unsigned int cpu)
251{
252 int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_entry));
253 if (err)
254 pr_err("psci: failed to boot CPU%d (%d)\n", cpu, err);
255
256 return err;
257}
258
259#ifdef CONFIG_HOTPLUG_CPU
260static int cpu_psci_cpu_disable(unsigned int cpu)
261{
262 /* Fail early if we don't have CPU_OFF support */
263 if (!psci_ops.cpu_off)
264 return -EOPNOTSUPP;
265 return 0;
266}
267
268static void cpu_psci_cpu_die(unsigned int cpu)
269{
270 int ret;
271 /*
272 * There are no known implementations of PSCI actually using the
273 * power state field, pass a sensible default for now.
274 */
275 struct psci_power_state state = {
276 .type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
277 };
278
279 ret = psci_ops.cpu_off(state);
280
281 pr_crit("psci: unable to power off CPU%u (%d)\n", cpu, ret);
282}
283#endif
284
285const struct cpu_operations cpu_psci_ops = {
286 .name = "psci",
287 .cpu_init = cpu_psci_cpu_init,
288 .cpu_prepare = cpu_psci_cpu_prepare,
289 .cpu_boot = cpu_psci_cpu_boot,
290#ifdef CONFIG_HOTPLUG_CPU
291 .cpu_disable = cpu_psci_cpu_disable,
292 .cpu_die = cpu_psci_cpu_die,
293#endif
294};
295
296#endif
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 055cfb80e05c..9cf30f49610d 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -45,6 +45,7 @@
45#include <asm/cputype.h> 45#include <asm/cputype.h>
46#include <asm/elf.h> 46#include <asm/elf.h>
47#include <asm/cputable.h> 47#include <asm/cputable.h>
48#include <asm/cpu_ops.h>
48#include <asm/sections.h> 49#include <asm/sections.h>
49#include <asm/setup.h> 50#include <asm/setup.h>
50#include <asm/smp_plat.h> 51#include <asm/smp_plat.h>
@@ -97,6 +98,11 @@ void __init early_print(const char *str, ...)
97 printk("%s", buf); 98 printk("%s", buf);
98} 99}
99 100
101bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
102{
103 return phys_id == cpu_logical_map(cpu);
104}
105
100static void __init setup_processor(void) 106static void __init setup_processor(void)
101{ 107{
102 struct cpu_info *cpu_info; 108 struct cpu_info *cpu_info;
@@ -118,7 +124,7 @@ static void __init setup_processor(void)
118 printk("CPU: %s [%08x] revision %d\n", 124 printk("CPU: %s [%08x] revision %d\n",
119 cpu_name, read_cpuid_id(), read_cpuid_id() & 15); 125 cpu_name, read_cpuid_id(), read_cpuid_id() & 15);
120 126
121 sprintf(init_utsname()->machine, "aarch64"); 127 sprintf(init_utsname()->machine, ELF_PLATFORM);
122 elf_hwcap = 0; 128 elf_hwcap = 0;
123} 129}
124 130
@@ -264,6 +270,7 @@ void __init setup_arch(char **cmdline_p)
264 psci_init(); 270 psci_init();
265 271
266 cpu_logical_map(0) = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; 272 cpu_logical_map(0) = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
273 cpu_read_bootcpu_ops();
267#ifdef CONFIG_SMP 274#ifdef CONFIG_SMP
268 smp_init_cpus(); 275 smp_init_cpus();
269#endif 276#endif
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index e393174fe859..e51bbe79f5b5 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -100,34 +100,6 @@ struct compat_rt_sigframe {
100 100
101#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) 101#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP)))
102 102
103/*
104 * For ARM syscalls, the syscall number has to be loaded into r7.
105 * We do not support an OABI userspace.
106 */
107#define MOV_R7_NR_SIGRETURN (0xe3a07000 | __NR_compat_sigreturn)
108#define SVC_SYS_SIGRETURN (0xef000000 | __NR_compat_sigreturn)
109#define MOV_R7_NR_RT_SIGRETURN (0xe3a07000 | __NR_compat_rt_sigreturn)
110#define SVC_SYS_RT_SIGRETURN (0xef000000 | __NR_compat_rt_sigreturn)
111
112/*
113 * For Thumb syscalls, we also pass the syscall number via r7. We therefore
114 * need two 16-bit instructions.
115 */
116#define SVC_THUMB_SIGRETURN (((0xdf00 | __NR_compat_sigreturn) << 16) | \
117 0x2700 | __NR_compat_sigreturn)
118#define SVC_THUMB_RT_SIGRETURN (((0xdf00 | __NR_compat_rt_sigreturn) << 16) | \
119 0x2700 | __NR_compat_rt_sigreturn)
120
121const compat_ulong_t aarch32_sigret_code[6] = {
122 /*
123 * AArch32 sigreturn code.
124 * We don't construct an OABI SWI - instead we just set the imm24 field
125 * to the EABI syscall number so that we create a sane disassembly.
126 */
127 MOV_R7_NR_SIGRETURN, SVC_SYS_SIGRETURN, SVC_THUMB_SIGRETURN,
128 MOV_R7_NR_RT_SIGRETURN, SVC_SYS_RT_SIGRETURN, SVC_THUMB_RT_SIGRETURN,
129};
130
131static inline int put_sigset_t(compat_sigset_t __user *uset, sigset_t *set) 103static inline int put_sigset_t(compat_sigset_t __user *uset, sigset_t *set)
132{ 104{
133 compat_sigset_t cset; 105 compat_sigset_t cset;
@@ -474,12 +446,13 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
474 /* Check if the handler is written for ARM or Thumb */ 446 /* Check if the handler is written for ARM or Thumb */
475 thumb = handler & 1; 447 thumb = handler & 1;
476 448
477 if (thumb) { 449 if (thumb)
478 spsr |= COMPAT_PSR_T_BIT; 450 spsr |= COMPAT_PSR_T_BIT;
479 spsr &= ~COMPAT_PSR_IT_MASK; 451 else
480 } else {
481 spsr &= ~COMPAT_PSR_T_BIT; 452 spsr &= ~COMPAT_PSR_T_BIT;
482 } 453
454 /* The IT state must be cleared for both ARM and Thumb-2 */
455 spsr &= ~COMPAT_PSR_IT_MASK;
483 456
484 if (ka->sa.sa_flags & SA_RESTORER) { 457 if (ka->sa.sa_flags & SA_RESTORER) {
485 retcode = ptr_to_compat(ka->sa.sa_restorer); 458 retcode = ptr_to_compat(ka->sa.sa_restorer);
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 78db90dcc910..a5aeefab03c3 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -39,6 +39,7 @@
39#include <asm/atomic.h> 39#include <asm/atomic.h>
40#include <asm/cacheflush.h> 40#include <asm/cacheflush.h>
41#include <asm/cputype.h> 41#include <asm/cputype.h>
42#include <asm/cpu_ops.h>
42#include <asm/mmu_context.h> 43#include <asm/mmu_context.h>
43#include <asm/pgtable.h> 44#include <asm/pgtable.h>
44#include <asm/pgalloc.h> 45#include <asm/pgalloc.h>
@@ -54,7 +55,6 @@
54 * where to place its SVC stack 55 * where to place its SVC stack
55 */ 56 */
56struct secondary_data secondary_data; 57struct secondary_data secondary_data;
57volatile unsigned long secondary_holding_pen_release = INVALID_HWID;
58 58
59enum ipi_msg_type { 59enum ipi_msg_type {
60 IPI_RESCHEDULE, 60 IPI_RESCHEDULE,
@@ -63,61 +63,16 @@ enum ipi_msg_type {
63 IPI_CPU_STOP, 63 IPI_CPU_STOP,
64}; 64};
65 65
66static DEFINE_RAW_SPINLOCK(boot_lock);
67
68/*
69 * Write secondary_holding_pen_release in a way that is guaranteed to be
70 * visible to all observers, irrespective of whether they're taking part
71 * in coherency or not. This is necessary for the hotplug code to work
72 * reliably.
73 */
74static void write_pen_release(u64 val)
75{
76 void *start = (void *)&secondary_holding_pen_release;
77 unsigned long size = sizeof(secondary_holding_pen_release);
78
79 secondary_holding_pen_release = val;
80 __flush_dcache_area(start, size);
81}
82
83/* 66/*
84 * Boot a secondary CPU, and assign it the specified idle task. 67 * Boot a secondary CPU, and assign it the specified idle task.
85 * This also gives us the initial stack to use for this CPU. 68 * This also gives us the initial stack to use for this CPU.
86 */ 69 */
87static int boot_secondary(unsigned int cpu, struct task_struct *idle) 70static int boot_secondary(unsigned int cpu, struct task_struct *idle)
88{ 71{
89 unsigned long timeout; 72 if (cpu_ops[cpu]->cpu_boot)
90 73 return cpu_ops[cpu]->cpu_boot(cpu);
91 /*
92 * Set synchronisation state between this boot processor
93 * and the secondary one
94 */
95 raw_spin_lock(&boot_lock);
96
97 /*
98 * Update the pen release flag.
99 */
100 write_pen_release(cpu_logical_map(cpu));
101
102 /*
103 * Send an event, causing the secondaries to read pen_release.
104 */
105 sev();
106
107 timeout = jiffies + (1 * HZ);
108 while (time_before(jiffies, timeout)) {
109 if (secondary_holding_pen_release == INVALID_HWID)
110 break;
111 udelay(10);
112 }
113
114 /*
115 * Now the secondary core is starting up let it run its
116 * calibrations, then wait for it to finish
117 */
118 raw_spin_unlock(&boot_lock);
119 74
120 return secondary_holding_pen_release != INVALID_HWID ? -ENOSYS : 0; 75 return -EOPNOTSUPP;
121} 76}
122 77
123static DECLARE_COMPLETION(cpu_running); 78static DECLARE_COMPLETION(cpu_running);
@@ -187,17 +142,13 @@ asmlinkage void secondary_start_kernel(void)
187 preempt_disable(); 142 preempt_disable();
188 trace_hardirqs_off(); 143 trace_hardirqs_off();
189 144
190 /* 145 if (cpu_ops[cpu]->cpu_postboot)
191 * Let the primary processor know we're out of the 146 cpu_ops[cpu]->cpu_postboot();
192 * pen, then head off into the C entry point
193 */
194 write_pen_release(INVALID_HWID);
195 147
196 /* 148 /*
197 * Synchronise with the boot thread. 149 * Enable GIC and timers.
198 */ 150 */
199 raw_spin_lock(&boot_lock); 151 notify_cpu_starting(cpu);
200 raw_spin_unlock(&boot_lock);
201 152
202 /* 153 /*
203 * OK, now it's safe to let the boot CPU continue. Wait for 154 * OK, now it's safe to let the boot CPU continue. Wait for
@@ -207,11 +158,6 @@ asmlinkage void secondary_start_kernel(void)
207 set_cpu_online(cpu, true); 158 set_cpu_online(cpu, true);
208 complete(&cpu_running); 159 complete(&cpu_running);
209 160
210 /*
211 * Enable GIC and timers.
212 */
213 notify_cpu_starting(cpu);
214
215 local_irq_enable(); 161 local_irq_enable();
216 local_fiq_enable(); 162 local_fiq_enable();
217 163
@@ -221,39 +167,113 @@ asmlinkage void secondary_start_kernel(void)
221 cpu_startup_entry(CPUHP_ONLINE); 167 cpu_startup_entry(CPUHP_ONLINE);
222} 168}
223 169
224void __init smp_cpus_done(unsigned int max_cpus) 170#ifdef CONFIG_HOTPLUG_CPU
171static int op_cpu_disable(unsigned int cpu)
225{ 172{
226 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); 173 /*
174 * If we don't have a cpu_die method, abort before we reach the point
175 * of no return. CPU0 may not have an cpu_ops, so test for it.
176 */
177 if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_die)
178 return -EOPNOTSUPP;
179
180 /*
181 * We may need to abort a hot unplug for some other mechanism-specific
182 * reason.
183 */
184 if (cpu_ops[cpu]->cpu_disable)
185 return cpu_ops[cpu]->cpu_disable(cpu);
186
187 return 0;
227} 188}
228 189
229void __init smp_prepare_boot_cpu(void) 190/*
191 * __cpu_disable runs on the processor to be shutdown.
192 */
193int __cpu_disable(void)
230{ 194{
231} 195 unsigned int cpu = smp_processor_id();
196 int ret;
232 197
233static void (*smp_cross_call)(const struct cpumask *, unsigned int); 198 ret = op_cpu_disable(cpu);
199 if (ret)
200 return ret;
234 201
235static const struct smp_enable_ops *enable_ops[] __initconst = { 202 /*
236 &smp_spin_table_ops, 203 * Take this CPU offline. Once we clear this, we can't return,
237 &smp_psci_ops, 204 * and we must not schedule until we're ready to give up the cpu.
238 NULL, 205 */
239}; 206 set_cpu_online(cpu, false);
207
208 /*
209 * OK - migrate IRQs away from this CPU
210 */
211 migrate_irqs();
240 212
241static const struct smp_enable_ops *smp_enable_ops[NR_CPUS]; 213 /*
214 * Remove this CPU from the vm mask set of all processes.
215 */
216 clear_tasks_mm_cpumask(cpu);
242 217
243static const struct smp_enable_ops * __init smp_get_enable_ops(const char *name) 218 return 0;
244{ 219}
245 const struct smp_enable_ops **ops = enable_ops;
246 220
247 while (*ops) { 221static DECLARE_COMPLETION(cpu_died);
248 if (!strcmp(name, (*ops)->name))
249 return *ops;
250 222
251 ops++; 223/*
224 * called on the thread which is asking for a CPU to be shutdown -
225 * waits until shutdown has completed, or it is timed out.
226 */
227void __cpu_die(unsigned int cpu)
228{
229 if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) {
230 pr_crit("CPU%u: cpu didn't die\n", cpu);
231 return;
252 } 232 }
233 pr_notice("CPU%u: shutdown\n", cpu);
234}
235
236/*
237 * Called from the idle thread for the CPU which has been shutdown.
238 *
239 * Note that we disable IRQs here, but do not re-enable them
240 * before returning to the caller. This is also the behaviour
241 * of the other hotplug-cpu capable cores, so presumably coming
242 * out of idle fixes this.
243 */
244void cpu_die(void)
245{
246 unsigned int cpu = smp_processor_id();
247
248 idle_task_exit();
253 249
254 return NULL; 250 local_irq_disable();
251
252 /* Tell __cpu_die() that this CPU is now safe to dispose of */
253 complete(&cpu_died);
254
255 /*
256 * Actually shutdown the CPU. This must never fail. The specific hotplug
257 * mechanism must perform all required cache maintenance to ensure that
258 * no dirty lines are lost in the process of shutting down the CPU.
259 */
260 cpu_ops[cpu]->cpu_die(cpu);
261
262 BUG();
263}
264#endif
265
266void __init smp_cpus_done(unsigned int max_cpus)
267{
268 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
255} 269}
256 270
271void __init smp_prepare_boot_cpu(void)
272{
273}
274
275static void (*smp_cross_call)(const struct cpumask *, unsigned int);
276
257/* 277/*
258 * Enumerate the possible CPU set from the device tree and build the 278 * Enumerate the possible CPU set from the device tree and build the
259 * cpu logical map array containing MPIDR values related to logical 279 * cpu logical map array containing MPIDR values related to logical
@@ -261,9 +281,8 @@ static const struct smp_enable_ops * __init smp_get_enable_ops(const char *name)
261 */ 281 */
262void __init smp_init_cpus(void) 282void __init smp_init_cpus(void)
263{ 283{
264 const char *enable_method;
265 struct device_node *dn = NULL; 284 struct device_node *dn = NULL;
266 int i, cpu = 1; 285 unsigned int i, cpu = 1;
267 bool bootcpu_valid = false; 286 bool bootcpu_valid = false;
268 287
269 while ((dn = of_find_node_by_type(dn, "cpu"))) { 288 while ((dn = of_find_node_by_type(dn, "cpu"))) {
@@ -332,25 +351,10 @@ void __init smp_init_cpus(void)
332 if (cpu >= NR_CPUS) 351 if (cpu >= NR_CPUS)
333 goto next; 352 goto next;
334 353
335 /* 354 if (cpu_read_ops(dn, cpu) != 0)
336 * We currently support only the "spin-table" enable-method.
337 */
338 enable_method = of_get_property(dn, "enable-method", NULL);
339 if (!enable_method) {
340 pr_err("%s: missing enable-method property\n",
341 dn->full_name);
342 goto next; 355 goto next;
343 }
344
345 smp_enable_ops[cpu] = smp_get_enable_ops(enable_method);
346
347 if (!smp_enable_ops[cpu]) {
348 pr_err("%s: invalid enable-method property: %s\n",
349 dn->full_name, enable_method);
350 goto next;
351 }
352 356
353 if (smp_enable_ops[cpu]->init_cpu(dn, cpu)) 357 if (cpu_ops[cpu]->cpu_init(dn, cpu))
354 goto next; 358 goto next;
355 359
356 pr_debug("cpu logical map 0x%llx\n", hwid); 360 pr_debug("cpu logical map 0x%llx\n", hwid);
@@ -380,8 +384,8 @@ next:
380 384
381void __init smp_prepare_cpus(unsigned int max_cpus) 385void __init smp_prepare_cpus(unsigned int max_cpus)
382{ 386{
383 int cpu, err; 387 int err;
384 unsigned int ncores = num_possible_cpus(); 388 unsigned int cpu, ncores = num_possible_cpus();
385 389
386 /* 390 /*
387 * are we trying to boot more cores than exist? 391 * are we trying to boot more cores than exist?
@@ -408,10 +412,10 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
408 if (cpu == smp_processor_id()) 412 if (cpu == smp_processor_id())
409 continue; 413 continue;
410 414
411 if (!smp_enable_ops[cpu]) 415 if (!cpu_ops[cpu])
412 continue; 416 continue;
413 417
414 err = smp_enable_ops[cpu]->prepare_cpu(cpu); 418 err = cpu_ops[cpu]->cpu_prepare(cpu);
415 if (err) 419 if (err)
416 continue; 420 continue;
417 421
@@ -451,7 +455,7 @@ void show_ipi_list(struct seq_file *p, int prec)
451 for (i = 0; i < NR_IPI; i++) { 455 for (i = 0; i < NR_IPI; i++) {
452 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i + IPI_RESCHEDULE, 456 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i + IPI_RESCHEDULE,
453 prec >= 4 ? " " : ""); 457 prec >= 4 ? " " : "");
454 for_each_present_cpu(cpu) 458 for_each_online_cpu(cpu)
455 seq_printf(p, "%10u ", 459 seq_printf(p, "%10u ",
456 __get_irq_stat(cpu, ipi_irqs[i])); 460 __get_irq_stat(cpu, ipi_irqs[i]));
457 seq_printf(p, " %s\n", ipi_types[i]); 461 seq_printf(p, " %s\n", ipi_types[i]);
diff --git a/arch/arm64/kernel/smp_psci.c b/arch/arm64/kernel/smp_psci.c
deleted file mode 100644
index 0c533301be77..000000000000
--- a/arch/arm64/kernel/smp_psci.c
+++ /dev/null
@@ -1,53 +0,0 @@
1/*
2 * PSCI SMP initialisation
3 *
4 * Copyright (C) 2013 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/init.h>
20#include <linux/of.h>
21#include <linux/smp.h>
22
23#include <asm/psci.h>
24#include <asm/smp_plat.h>
25
26static int __init smp_psci_init_cpu(struct device_node *dn, int cpu)
27{
28 return 0;
29}
30
31static int __init smp_psci_prepare_cpu(int cpu)
32{
33 int err;
34
35 if (!psci_ops.cpu_on) {
36 pr_err("psci: no cpu_on method, not booting CPU%d\n", cpu);
37 return -ENODEV;
38 }
39
40 err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_holding_pen));
41 if (err) {
42 pr_err("psci: failed to boot CPU%d (%d)\n", cpu, err);
43 return err;
44 }
45
46 return 0;
47}
48
49const struct smp_enable_ops smp_psci_ops __initconst = {
50 .name = "psci",
51 .init_cpu = smp_psci_init_cpu,
52 .prepare_cpu = smp_psci_prepare_cpu,
53};
diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c
index 7c35fa682f76..44c22805d2e2 100644
--- a/arch/arm64/kernel/smp_spin_table.c
+++ b/arch/arm64/kernel/smp_spin_table.c
@@ -16,15 +16,39 @@
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */ 17 */
18 18
19#include <linux/delay.h>
19#include <linux/init.h> 20#include <linux/init.h>
20#include <linux/of.h> 21#include <linux/of.h>
21#include <linux/smp.h> 22#include <linux/smp.h>
22 23
23#include <asm/cacheflush.h> 24#include <asm/cacheflush.h>
25#include <asm/cpu_ops.h>
26#include <asm/cputype.h>
27#include <asm/smp_plat.h>
28
29extern void secondary_holding_pen(void);
30volatile unsigned long secondary_holding_pen_release = INVALID_HWID;
24 31
25static phys_addr_t cpu_release_addr[NR_CPUS]; 32static phys_addr_t cpu_release_addr[NR_CPUS];
33static DEFINE_RAW_SPINLOCK(boot_lock);
34
35/*
36 * Write secondary_holding_pen_release in a way that is guaranteed to be
37 * visible to all observers, irrespective of whether they're taking part
38 * in coherency or not. This is necessary for the hotplug code to work
39 * reliably.
40 */
41static void write_pen_release(u64 val)
42{
43 void *start = (void *)&secondary_holding_pen_release;
44 unsigned long size = sizeof(secondary_holding_pen_release);
26 45
27static int __init smp_spin_table_init_cpu(struct device_node *dn, int cpu) 46 secondary_holding_pen_release = val;
47 __flush_dcache_area(start, size);
48}
49
50
51static int smp_spin_table_cpu_init(struct device_node *dn, unsigned int cpu)
28{ 52{
29 /* 53 /*
30 * Determine the address from which the CPU is polling. 54 * Determine the address from which the CPU is polling.
@@ -40,7 +64,7 @@ static int __init smp_spin_table_init_cpu(struct device_node *dn, int cpu)
40 return 0; 64 return 0;
41} 65}
42 66
43static int __init smp_spin_table_prepare_cpu(int cpu) 67static int smp_spin_table_cpu_prepare(unsigned int cpu)
44{ 68{
45 void **release_addr; 69 void **release_addr;
46 70
@@ -48,7 +72,16 @@ static int __init smp_spin_table_prepare_cpu(int cpu)
48 return -ENODEV; 72 return -ENODEV;
49 73
50 release_addr = __va(cpu_release_addr[cpu]); 74 release_addr = __va(cpu_release_addr[cpu]);
51 release_addr[0] = (void *)__pa(secondary_holding_pen); 75
76 /*
77 * We write the release address as LE regardless of the native
78 * endianess of the kernel. Therefore, any boot-loaders that
79 * read this address need to convert this address to the
80 * boot-loader's endianess before jumping. This is mandated by
81 * the boot protocol.
82 */
83 release_addr[0] = (void *) cpu_to_le64(__pa(secondary_holding_pen));
84
52 __flush_dcache_area(release_addr, sizeof(release_addr[0])); 85 __flush_dcache_area(release_addr, sizeof(release_addr[0]));
53 86
54 /* 87 /*
@@ -59,8 +92,60 @@ static int __init smp_spin_table_prepare_cpu(int cpu)
59 return 0; 92 return 0;
60} 93}
61 94
62const struct smp_enable_ops smp_spin_table_ops __initconst = { 95static int smp_spin_table_cpu_boot(unsigned int cpu)
96{
97 unsigned long timeout;
98
99 /*
100 * Set synchronisation state between this boot processor
101 * and the secondary one
102 */
103 raw_spin_lock(&boot_lock);
104
105 /*
106 * Update the pen release flag.
107 */
108 write_pen_release(cpu_logical_map(cpu));
109
110 /*
111 * Send an event, causing the secondaries to read pen_release.
112 */
113 sev();
114
115 timeout = jiffies + (1 * HZ);
116 while (time_before(jiffies, timeout)) {
117 if (secondary_holding_pen_release == INVALID_HWID)
118 break;
119 udelay(10);
120 }
121
122 /*
123 * Now the secondary core is starting up let it run its
124 * calibrations, then wait for it to finish
125 */
126 raw_spin_unlock(&boot_lock);
127
128 return secondary_holding_pen_release != INVALID_HWID ? -ENOSYS : 0;
129}
130
131void smp_spin_table_cpu_postboot(void)
132{
133 /*
134 * Let the primary processor know we're out of the pen.
135 */
136 write_pen_release(INVALID_HWID);
137
138 /*
139 * Synchronise with the boot thread.
140 */
141 raw_spin_lock(&boot_lock);
142 raw_spin_unlock(&boot_lock);
143}
144
145const struct cpu_operations smp_spin_table_ops = {
63 .name = "spin-table", 146 .name = "spin-table",
64 .init_cpu = smp_spin_table_init_cpu, 147 .cpu_init = smp_spin_table_cpu_init,
65 .prepare_cpu = smp_spin_table_prepare_cpu, 148 .cpu_prepare = smp_spin_table_cpu_prepare,
149 .cpu_boot = smp_spin_table_cpu_boot,
150 .cpu_postboot = smp_spin_table_cpu_postboot,
66}; 151};
diff --git a/arch/arm64/kernel/sys32.S b/arch/arm64/kernel/sys32.S
index a1b19ed7467c..423a5b3fc2be 100644
--- a/arch/arm64/kernel/sys32.S
+++ b/arch/arm64/kernel/sys32.S
@@ -59,48 +59,48 @@ ENDPROC(compat_sys_fstatfs64_wrapper)
59 * extension. 59 * extension.
60 */ 60 */
61compat_sys_pread64_wrapper: 61compat_sys_pread64_wrapper:
62 orr x3, x4, x5, lsl #32 62 regs_to_64 x3, x4, x5
63 b sys_pread64 63 b sys_pread64
64ENDPROC(compat_sys_pread64_wrapper) 64ENDPROC(compat_sys_pread64_wrapper)
65 65
66compat_sys_pwrite64_wrapper: 66compat_sys_pwrite64_wrapper:
67 orr x3, x4, x5, lsl #32 67 regs_to_64 x3, x4, x5
68 b sys_pwrite64 68 b sys_pwrite64
69ENDPROC(compat_sys_pwrite64_wrapper) 69ENDPROC(compat_sys_pwrite64_wrapper)
70 70
71compat_sys_truncate64_wrapper: 71compat_sys_truncate64_wrapper:
72 orr x1, x2, x3, lsl #32 72 regs_to_64 x1, x2, x3
73 b sys_truncate 73 b sys_truncate
74ENDPROC(compat_sys_truncate64_wrapper) 74ENDPROC(compat_sys_truncate64_wrapper)
75 75
76compat_sys_ftruncate64_wrapper: 76compat_sys_ftruncate64_wrapper:
77 orr x1, x2, x3, lsl #32 77 regs_to_64 x1, x2, x3
78 b sys_ftruncate 78 b sys_ftruncate
79ENDPROC(compat_sys_ftruncate64_wrapper) 79ENDPROC(compat_sys_ftruncate64_wrapper)
80 80
81compat_sys_readahead_wrapper: 81compat_sys_readahead_wrapper:
82 orr x1, x2, x3, lsl #32 82 regs_to_64 x1, x2, x3
83 mov w2, w4 83 mov w2, w4
84 b sys_readahead 84 b sys_readahead
85ENDPROC(compat_sys_readahead_wrapper) 85ENDPROC(compat_sys_readahead_wrapper)
86 86
87compat_sys_fadvise64_64_wrapper: 87compat_sys_fadvise64_64_wrapper:
88 mov w6, w1 88 mov w6, w1
89 orr x1, x2, x3, lsl #32 89 regs_to_64 x1, x2, x3
90 orr x2, x4, x5, lsl #32 90 regs_to_64 x2, x4, x5
91 mov w3, w6 91 mov w3, w6
92 b sys_fadvise64_64 92 b sys_fadvise64_64
93ENDPROC(compat_sys_fadvise64_64_wrapper) 93ENDPROC(compat_sys_fadvise64_64_wrapper)
94 94
95compat_sys_sync_file_range2_wrapper: 95compat_sys_sync_file_range2_wrapper:
96 orr x2, x2, x3, lsl #32 96 regs_to_64 x2, x2, x3
97 orr x3, x4, x5, lsl #32 97 regs_to_64 x3, x4, x5
98 b sys_sync_file_range2 98 b sys_sync_file_range2
99ENDPROC(compat_sys_sync_file_range2_wrapper) 99ENDPROC(compat_sys_sync_file_range2_wrapper)
100 100
101compat_sys_fallocate_wrapper: 101compat_sys_fallocate_wrapper:
102 orr x2, x2, x3, lsl #32 102 regs_to_64 x2, x2, x3
103 orr x3, x4, x5, lsl #32 103 regs_to_64 x3, x4, x5
104 b sys_fallocate 104 b sys_fallocate
105ENDPROC(compat_sys_fallocate_wrapper) 105ENDPROC(compat_sys_fallocate_wrapper)
106 106
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 6a389dc1bd49..65d40cf6945a 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -58,7 +58,10 @@ static struct page *vectors_page[1];
58static int alloc_vectors_page(void) 58static int alloc_vectors_page(void)
59{ 59{
60 extern char __kuser_helper_start[], __kuser_helper_end[]; 60 extern char __kuser_helper_start[], __kuser_helper_end[];
61 extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
62
61 int kuser_sz = __kuser_helper_end - __kuser_helper_start; 63 int kuser_sz = __kuser_helper_end - __kuser_helper_start;
64 int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
62 unsigned long vpage; 65 unsigned long vpage;
63 66
64 vpage = get_zeroed_page(GFP_ATOMIC); 67 vpage = get_zeroed_page(GFP_ATOMIC);
@@ -72,7 +75,7 @@ static int alloc_vectors_page(void)
72 75
73 /* sigreturn code */ 76 /* sigreturn code */
74 memcpy((void *)vpage + AARCH32_KERN_SIGRET_CODE_OFFSET, 77 memcpy((void *)vpage + AARCH32_KERN_SIGRET_CODE_OFFSET,
75 aarch32_sigret_code, sizeof(aarch32_sigret_code)); 78 __aarch32_sigret_code_start, sigret_sz);
76 79
77 flush_icache_range(vpage, vpage + PAGE_SIZE); 80 flush_icache_range(vpage, vpage + PAGE_SIZE);
78 vectors_page[0] = virt_to_page(vpage); 81 vectors_page[0] = virt_to_page(vpage);
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index f8ab9d8e2ea3..5161ad992091 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -54,7 +54,6 @@ SECTIONS
54 } 54 }
55 .text : { /* Real text segment */ 55 .text : { /* Real text segment */
56 _stext = .; /* Text and read-only data */ 56 _stext = .; /* Text and read-only data */
57 *(.smp.pen.text)
58 __exception_text_start = .; 57 __exception_text_start = .;
59 *(.exception.text) 58 *(.exception.text)
60 __exception_text_end = .; 59 __exception_text_end = .;
@@ -97,30 +96,13 @@ SECTIONS
97 PERCPU_SECTION(64) 96 PERCPU_SECTION(64)
98 97
99 __init_end = .; 98 __init_end = .;
100 . = ALIGN(THREAD_SIZE); 99
101 __data_loc = .; 100 . = ALIGN(PAGE_SIZE);
102 101 _data = .;
103 .data : AT(__data_loc) { 102 __data_loc = _data - LOAD_OFFSET;
104 _data = .; /* address in memory */ 103 _sdata = .;
105 _sdata = .; 104 RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE)
106 105 _edata = .;
107 /*
108 * first, the init task union, aligned
109 * to an 8192 byte boundary.
110 */
111 INIT_TASK_DATA(THREAD_SIZE)
112 NOSAVE_DATA
113 CACHELINE_ALIGNED_DATA(64)
114 READ_MOSTLY_DATA(64)
115
116 /*
117 * and the usual data section
118 */
119 DATA_DATA
120 CONSTRUCTORS
121
122 _edata = .;
123 }
124 _edata_loc = __data_loc + SIZEOF(.data); 106 _edata_loc = __data_loc + SIZEOF(.data);
125 107
126 BSS_SECTION(0, 0, 0) 108 BSS_SECTION(0, 0, 0)
diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S
index ba84e6705e20..2b0244d65c16 100644
--- a/arch/arm64/kvm/hyp-init.S
+++ b/arch/arm64/kvm/hyp-init.S
@@ -74,7 +74,10 @@ __do_hyp_init:
74 msr mair_el2, x4 74 msr mair_el2, x4
75 isb 75 isb
76 76
77 mov x4, #SCTLR_EL2_FLAGS 77 mrs x4, sctlr_el2
78 and x4, x4, #SCTLR_EL2_EE // preserve endianness of EL2
79 ldr x5, =SCTLR_EL2_FLAGS
80 orr x4, x4, x5
78 msr sctlr_el2, x4 81 msr sctlr_el2, x4
79 isb 82 isb
80 83
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index 1ac0bbbdddb2..3b47c36e10ff 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -403,6 +403,14 @@ __kvm_hyp_code_start:
403 ldr w9, [x2, #GICH_ELRSR0] 403 ldr w9, [x2, #GICH_ELRSR0]
404 ldr w10, [x2, #GICH_ELRSR1] 404 ldr w10, [x2, #GICH_ELRSR1]
405 ldr w11, [x2, #GICH_APR] 405 ldr w11, [x2, #GICH_APR]
406CPU_BE( rev w4, w4 )
407CPU_BE( rev w5, w5 )
408CPU_BE( rev w6, w6 )
409CPU_BE( rev w7, w7 )
410CPU_BE( rev w8, w8 )
411CPU_BE( rev w9, w9 )
412CPU_BE( rev w10, w10 )
413CPU_BE( rev w11, w11 )
406 414
407 str w4, [x3, #VGIC_CPU_HCR] 415 str w4, [x3, #VGIC_CPU_HCR]
408 str w5, [x3, #VGIC_CPU_VMCR] 416 str w5, [x3, #VGIC_CPU_VMCR]
@@ -421,6 +429,7 @@ __kvm_hyp_code_start:
421 ldr w4, [x3, #VGIC_CPU_NR_LR] 429 ldr w4, [x3, #VGIC_CPU_NR_LR]
422 add x3, x3, #VGIC_CPU_LR 430 add x3, x3, #VGIC_CPU_LR
4231: ldr w5, [x2], #4 4311: ldr w5, [x2], #4
432CPU_BE( rev w5, w5 )
424 str w5, [x3], #4 433 str w5, [x3], #4
425 sub w4, w4, #1 434 sub w4, w4, #1
426 cbnz w4, 1b 435 cbnz w4, 1b
@@ -446,6 +455,9 @@ __kvm_hyp_code_start:
446 ldr w4, [x3, #VGIC_CPU_HCR] 455 ldr w4, [x3, #VGIC_CPU_HCR]
447 ldr w5, [x3, #VGIC_CPU_VMCR] 456 ldr w5, [x3, #VGIC_CPU_VMCR]
448 ldr w6, [x3, #VGIC_CPU_APR] 457 ldr w6, [x3, #VGIC_CPU_APR]
458CPU_BE( rev w4, w4 )
459CPU_BE( rev w5, w5 )
460CPU_BE( rev w6, w6 )
449 461
450 str w4, [x2, #GICH_HCR] 462 str w4, [x2, #GICH_HCR]
451 str w5, [x2, #GICH_VMCR] 463 str w5, [x2, #GICH_VMCR]
@@ -456,6 +468,7 @@ __kvm_hyp_code_start:
456 ldr w4, [x3, #VGIC_CPU_NR_LR] 468 ldr w4, [x3, #VGIC_CPU_NR_LR]
457 add x3, x3, #VGIC_CPU_LR 469 add x3, x3, #VGIC_CPU_LR
4581: ldr w5, [x3], #4 4701: ldr w5, [x3], #4
471CPU_BE( rev w5, w5 )
459 str w5, [x2], #4 472 str w5, [x2], #4
460 sub w4, w4, #1 473 sub w4, w4, #1
461 cbnz w4, 1b 474 cbnz w4, 1b
diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
index 1725cd6db37a..2bb1d586664c 100644
--- a/arch/arm64/mm/ioremap.c
+++ b/arch/arm64/mm/ioremap.c
@@ -77,8 +77,24 @@ EXPORT_SYMBOL(__ioremap);
77 77
78void __iounmap(volatile void __iomem *io_addr) 78void __iounmap(volatile void __iomem *io_addr)
79{ 79{
80 void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr); 80 unsigned long addr = (unsigned long)io_addr & PAGE_MASK;
81 81
82 vunmap(addr); 82 /*
83 * We could get an address outside vmalloc range in case
84 * of ioremap_cache() reusing a RAM mapping.
85 */
86 if (VMALLOC_START <= addr && addr < VMALLOC_END)
87 vunmap((void *)addr);
83} 88}
84EXPORT_SYMBOL(__iounmap); 89EXPORT_SYMBOL(__iounmap);
90
91void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size)
92{
93 /* For normal memory we already have a cacheable mapping. */
94 if (pfn_valid(__phys_to_pfn(phys_addr)))
95 return (void __iomem *)__phys_to_virt(phys_addr);
96
97 return __ioremap_caller(phys_addr, size, __pgprot(PROT_NORMAL),
98 __builtin_return_address(0));
99}
100EXPORT_SYMBOL(ioremap_cache);
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index b1b31bbc967b..421b99fd635d 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -162,9 +162,9 @@ ENDPROC(__cpu_setup)
162 * CE0 XWHW CZ ME TEEA S 162 * CE0 XWHW CZ ME TEEA S
163 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM 163 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
164 * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved 164 * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved
165 * .... .100 .... 01.1 11.1 ..01 0001 1101 < software settings 165 * .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings
166 */ 166 */
167 .type crval, #object 167 .type crval, #object
168crval: 168crval:
169 .word 0x030802e2 // clear 169 .word 0x000802e2 // clear
170 .word 0x0405d11d // set 170 .word 0x0405d11d // set
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-14 22:25:49 -0400