aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-01-20 18:40:44 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2014-01-20 18:40:44 -0500
commit82b51734b4f228c76b6064b6e899d9d3d4c17c1a (patch)
tree0f8735944ab146713dba402261b4c7cc5629d02f /arch/arm64
parent15c81026204da897a05424c79263aea861a782cc (diff)
parent883c057367014d20a14b5054e4eb0d81ce3bea5c (diff)
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull ARM64 updates from Catalin Marinas: - CPU suspend support on top of PSCI (firmware Power State Coordination Interface) - jump label support - CMA can now be enabled on arm64 - HWCAP bits for crypto and CRC32 extensions - optimised percpu using tpidr_el1 register - code cleanup * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (42 commits) arm64: fix typo in entry.S arm64: kernel: restore HW breakpoint registers in cpu_suspend jump_label: use defined macros instead of hard-coding for better readability arm64, jump label: optimize jump label implementation arm64, jump label: detect %c support for ARM64 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64: move encode_insn_immediate() from module.c to insn.c arm64: introduce interfaces to hotpatch kernel and module code arm64: introduce basic aarch64 instruction decoding helpers arm64: dts: Reduce size of virtio block device for foundation model arm64: Remove unused __data_loc variable arm64: Enable CMA arm64: Warn on NULL device structure for dma APIs arm64: Add hwcaps for crypto and CRC32 extensions. arm64: drop redundant macros from read_cpuid() arm64: Remove outdated comment arm64: cmpxchg: update macros to prevent warnings arm64: support single-step and breakpoint handler hooks ARM64: fix framepointer check in unwind_frame ARM64: check stack pointer in get_wchan ...
Diffstat (limited to 'arch/arm64')
-rw-r--r--arch/arm64/Kconfig27
-rw-r--r--arch/arm64/boot/dts/foundation-v8.dts2
-rw-r--r--arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi6
-rw-r--r--arch/arm64/include/asm/Kbuild1
-rw-r--r--arch/arm64/include/asm/cmpxchg.h28
-rw-r--r--arch/arm64/include/asm/cpu_ops.h6
-rw-r--r--arch/arm64/include/asm/cputype.h28
-rw-r--r--arch/arm64/include/asm/debug-monitors.h21
-rw-r--r--arch/arm64/include/asm/dma-contiguous.h29
-rw-r--r--arch/arm64/include/asm/futex.h1
-rw-r--r--arch/arm64/include/asm/hardirq.h2
-rw-r--r--arch/arm64/include/asm/insn.h108
-rw-r--r--arch/arm64/include/asm/jump_label.h52
-rw-r--r--arch/arm64/include/asm/memory.h3
-rw-r--r--arch/arm64/include/asm/percpu.h41
-rw-r--r--arch/arm64/include/asm/proc-fns.h3
-rw-r--r--arch/arm64/include/asm/smp_plat.h13
-rw-r--r--arch/arm64/include/asm/suspend.h27
-rw-r--r--arch/arm64/include/asm/uaccess.h25
-rw-r--r--arch/arm64/include/asm/word-at-a-time.h94
-rw-r--r--arch/arm64/include/uapi/asm/hwcap.h6
-rw-r--r--arch/arm64/kernel/Makefile4
-rw-r--r--arch/arm64/kernel/arm64ksyms.c5
-rw-r--r--arch/arm64/kernel/asm-offsets.c11
-rw-r--r--arch/arm64/kernel/debug-monitors.c88
-rw-r--r--arch/arm64/kernel/entry.S4
-rw-r--r--arch/arm64/kernel/fpsimd.c36
-rw-r--r--arch/arm64/kernel/head.S10
-rw-r--r--arch/arm64/kernel/hw_breakpoint.c203
-rw-r--r--arch/arm64/kernel/insn.c304
-rw-r--r--arch/arm64/kernel/jump_label.c58
-rw-r--r--arch/arm64/kernel/module.c157
-rw-r--r--arch/arm64/kernel/perf_event.c108
-rw-r--r--arch/arm64/kernel/process.c14
-rw-r--r--arch/arm64/kernel/setup.c122
-rw-r--r--arch/arm64/kernel/sleep.S184
-rw-r--r--arch/arm64/kernel/smp.c23
-rw-r--r--arch/arm64/kernel/stacktrace.c2
-rw-r--r--arch/arm64/kernel/suspend.c132
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S3
-rw-r--r--arch/arm64/lib/Makefile8
-rw-r--r--arch/arm64/lib/strncpy_from_user.S50
-rw-r--r--arch/arm64/lib/strnlen_user.S47
-rw-r--r--arch/arm64/mm/dma-mapping.c35
-rw-r--r--arch/arm64/mm/init.c3
-rw-r--r--arch/arm64/mm/proc.S69
46 files changed, 1808 insertions, 395 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 6d4dd22ee4b7..dd4327f09ba4 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2,6 +2,7 @@ config ARM64
2 def_bool y 2 def_bool y
3 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE 3 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
4 select ARCH_USE_CMPXCHG_LOCKREF 4 select ARCH_USE_CMPXCHG_LOCKREF
5 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
5 select ARCH_WANT_OPTIONAL_GPIOLIB 6 select ARCH_WANT_OPTIONAL_GPIOLIB
6 select ARCH_WANT_COMPAT_IPC_PARSE_VERSION 7 select ARCH_WANT_COMPAT_IPC_PARSE_VERSION
7 select ARCH_WANT_FRAME_POINTERS 8 select ARCH_WANT_FRAME_POINTERS
@@ -11,19 +12,27 @@ config ARM64
11 select BUILDTIME_EXTABLE_SORT 12 select BUILDTIME_EXTABLE_SORT
12 select CLONE_BACKWARDS 13 select CLONE_BACKWARDS
13 select COMMON_CLK 14 select COMMON_CLK
15 select CPU_PM if (SUSPEND || CPU_IDLE)
16 select DCACHE_WORD_ACCESS
14 select GENERIC_CLOCKEVENTS 17 select GENERIC_CLOCKEVENTS
18 select GENERIC_CLOCKEVENTS_BROADCAST if SMP
15 select GENERIC_IOMAP 19 select GENERIC_IOMAP
16 select GENERIC_IRQ_PROBE 20 select GENERIC_IRQ_PROBE
17 select GENERIC_IRQ_SHOW 21 select GENERIC_IRQ_SHOW
18 select GENERIC_SCHED_CLOCK 22 select GENERIC_SCHED_CLOCK
19 select GENERIC_SMP_IDLE_THREAD 23 select GENERIC_SMP_IDLE_THREAD
24 select GENERIC_STRNCPY_FROM_USER
25 select GENERIC_STRNLEN_USER
20 select GENERIC_TIME_VSYSCALL 26 select GENERIC_TIME_VSYSCALL
21 select HARDIRQS_SW_RESEND 27 select HARDIRQS_SW_RESEND
28 select HAVE_ARCH_JUMP_LABEL
22 select HAVE_ARCH_TRACEHOOK 29 select HAVE_ARCH_TRACEHOOK
23 select HAVE_DEBUG_BUGVERBOSE 30 select HAVE_DEBUG_BUGVERBOSE
24 select HAVE_DEBUG_KMEMLEAK 31 select HAVE_DEBUG_KMEMLEAK
25 select HAVE_DMA_API_DEBUG 32 select HAVE_DMA_API_DEBUG
26 select HAVE_DMA_ATTRS 33 select HAVE_DMA_ATTRS
34 select HAVE_DMA_CONTIGUOUS
35 select HAVE_EFFICIENT_UNALIGNED_ACCESS
27 select HAVE_GENERIC_DMA_COHERENT 36 select HAVE_GENERIC_DMA_COHERENT
28 select HAVE_HW_BREAKPOINT if PERF_EVENTS 37 select HAVE_HW_BREAKPOINT if PERF_EVENTS
29 select HAVE_MEMBLOCK 38 select HAVE_MEMBLOCK
@@ -275,6 +284,24 @@ config SYSVIPC_COMPAT
275 284
276endmenu 285endmenu
277 286
287menu "Power management options"
288
289source "kernel/power/Kconfig"
290
291config ARCH_SUSPEND_POSSIBLE
292 def_bool y
293
294config ARM64_CPU_SUSPEND
295 def_bool PM_SLEEP
296
297endmenu
298
299menu "CPU Power Management"
300
301source "drivers/cpuidle/Kconfig"
302
303endmenu
304
278source "net/Kconfig" 305source "net/Kconfig"
279 306
280source "drivers/Kconfig" 307source "drivers/Kconfig"
diff --git a/arch/arm64/boot/dts/foundation-v8.dts b/arch/arm64/boot/dts/foundation-v8.dts
index 519c4b2c0687..4a060906809d 100644
--- a/arch/arm64/boot/dts/foundation-v8.dts
+++ b/arch/arm64/boot/dts/foundation-v8.dts
@@ -224,7 +224,7 @@
224 224
225 virtio_block@0130000 { 225 virtio_block@0130000 {
226 compatible = "virtio,mmio"; 226 compatible = "virtio,mmio";
227 reg = <0x130000 0x1000>; 227 reg = <0x130000 0x200>;
228 interrupts = <42>; 228 interrupts = <42>;
229 }; 229 };
230 }; 230 };
diff --git a/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi b/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi
index b45e5f39f577..2f2ecd217363 100644
--- a/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi
+++ b/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi
@@ -183,6 +183,12 @@
183 clocks = <&v2m_oscclk1>, <&v2m_clk24mhz>; 183 clocks = <&v2m_oscclk1>, <&v2m_clk24mhz>;
184 clock-names = "clcdclk", "apb_pclk"; 184 clock-names = "clcdclk", "apb_pclk";
185 }; 185 };
186
187 virtio_block@0130000 {
188 compatible = "virtio,mmio";
189 reg = <0x130000 0x200>;
190 interrupts = <42>;
191 };
186 }; 192 };
187 193
188 v2m_fixed_3v3: fixedregulator@0 { 194 v2m_fixed_3v3: fixedregulator@0 {
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild
index 519f89f5b6a3..d0ff25de67ca 100644
--- a/arch/arm64/include/asm/Kbuild
+++ b/arch/arm64/include/asm/Kbuild
@@ -26,7 +26,6 @@ generic-y += mman.h
26generic-y += msgbuf.h 26generic-y += msgbuf.h
27generic-y += mutex.h 27generic-y += mutex.h
28generic-y += pci.h 28generic-y += pci.h
29generic-y += percpu.h
30generic-y += poll.h 29generic-y += poll.h
31generic-y += posix_types.h 30generic-y += posix_types.h
32generic-y += resource.h 31generic-y += resource.h
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 3914c0dcd09c..56166d7f4a25 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -158,17 +158,23 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
158 return ret; 158 return ret;
159} 159}
160 160
161#define cmpxchg(ptr,o,n) \ 161#define cmpxchg(ptr, o, n) \
162 ((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ 162({ \
163 (unsigned long)(o), \ 163 __typeof__(*(ptr)) __ret; \
164 (unsigned long)(n), \ 164 __ret = (__typeof__(*(ptr))) \
165 sizeof(*(ptr)))) 165 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
166 166 sizeof(*(ptr))); \
167#define cmpxchg_local(ptr,o,n) \ 167 __ret; \
168 ((__typeof__(*(ptr)))__cmpxchg((ptr), \ 168})
169 (unsigned long)(o), \ 169
170 (unsigned long)(n), \ 170#define cmpxchg_local(ptr, o, n) \
171 sizeof(*(ptr)))) 171({ \
172 __typeof__(*(ptr)) __ret; \
173 __ret = (__typeof__(*(ptr))) \
174 __cmpxchg((ptr), (unsigned long)(o), \
175 (unsigned long)(n), sizeof(*(ptr))); \
176 __ret; \
177})
172 178
173#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) 179#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
174#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 180#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h
index c4cdb5e5b73d..152413076503 100644
--- a/arch/arm64/include/asm/cpu_ops.h
+++ b/arch/arm64/include/asm/cpu_ops.h
@@ -39,6 +39,9 @@ struct device_node;
39 * from the cpu to be killed. 39 * from the cpu to be killed.
40 * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the 40 * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
41 * cpu being killed. 41 * cpu being killed.
42 * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing
43 * to wrong parameters or error conditions. Called from the
44 * CPU being suspended. Must be called with IRQs disabled.
42 */ 45 */
43struct cpu_operations { 46struct cpu_operations {
44 const char *name; 47 const char *name;
@@ -50,6 +53,9 @@ struct cpu_operations {
50 int (*cpu_disable)(unsigned int cpu); 53 int (*cpu_disable)(unsigned int cpu);
51 void (*cpu_die)(unsigned int cpu); 54 void (*cpu_die)(unsigned int cpu);
52#endif 55#endif
56#ifdef CONFIG_ARM64_CPU_SUSPEND
57 int (*cpu_suspend)(unsigned long);
58#endif
53}; 59};
54 60
55extern const struct cpu_operations *cpu_ops[NR_CPUS]; 61extern const struct cpu_operations *cpu_ops[NR_CPUS];
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index 5fe138e0b828..c404fb0df3a6 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -16,23 +16,23 @@
16#ifndef __ASM_CPUTYPE_H 16#ifndef __ASM_CPUTYPE_H
17#define __ASM_CPUTYPE_H 17#define __ASM_CPUTYPE_H
18 18
19#define ID_MIDR_EL1 "midr_el1"
20#define ID_MPIDR_EL1 "mpidr_el1"
21#define ID_CTR_EL0 "ctr_el0"
22
23#define ID_AA64PFR0_EL1 "id_aa64pfr0_el1"
24#define ID_AA64DFR0_EL1 "id_aa64dfr0_el1"
25#define ID_AA64AFR0_EL1 "id_aa64afr0_el1"
26#define ID_AA64ISAR0_EL1 "id_aa64isar0_el1"
27#define ID_AA64MMFR0_EL1 "id_aa64mmfr0_el1"
28
29#define INVALID_HWID ULONG_MAX 19#define INVALID_HWID ULONG_MAX
30 20
31#define MPIDR_HWID_BITMASK 0xff00ffffff 21#define MPIDR_HWID_BITMASK 0xff00ffffff
32 22
23#define MPIDR_LEVEL_BITS_SHIFT 3
24#define MPIDR_LEVEL_BITS (1 << MPIDR_LEVEL_BITS_SHIFT)
25#define MPIDR_LEVEL_MASK ((1 << MPIDR_LEVEL_BITS) - 1)
26
27#define MPIDR_LEVEL_SHIFT(level) \
28 (((1 << level) >> 1) << MPIDR_LEVEL_BITS_SHIFT)
29
30#define MPIDR_AFFINITY_LEVEL(mpidr, level) \
31 ((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK)
32
33#define read_cpuid(reg) ({ \ 33#define read_cpuid(reg) ({ \
34 u64 __val; \ 34 u64 __val; \
35 asm("mrs %0, " reg : "=r" (__val)); \ 35 asm("mrs %0, " #reg : "=r" (__val)); \
36 __val; \ 36 __val; \
37}) 37})
38 38
@@ -54,12 +54,12 @@
54 */ 54 */
55static inline u32 __attribute_const__ read_cpuid_id(void) 55static inline u32 __attribute_const__ read_cpuid_id(void)
56{ 56{
57 return read_cpuid(ID_MIDR_EL1); 57 return read_cpuid(MIDR_EL1);
58} 58}
59 59
60static inline u64 __attribute_const__ read_cpuid_mpidr(void) 60static inline u64 __attribute_const__ read_cpuid_mpidr(void)
61{ 61{
62 return read_cpuid(ID_MPIDR_EL1); 62 return read_cpuid(MPIDR_EL1);
63} 63}
64 64
65static inline unsigned int __attribute_const__ read_cpuid_implementor(void) 65static inline unsigned int __attribute_const__ read_cpuid_implementor(void)
@@ -74,7 +74,7 @@ static inline unsigned int __attribute_const__ read_cpuid_part_number(void)
74 74
75static inline u32 __attribute_const__ read_cpuid_cachetype(void) 75static inline u32 __attribute_const__ read_cpuid_cachetype(void)
76{ 76{
77 return read_cpuid(ID_CTR_EL0); 77 return read_cpuid(CTR_EL0);
78} 78}
79 79
80#endif /* __ASSEMBLY__ */ 80#endif /* __ASSEMBLY__ */
diff --git a/arch/arm64/include/asm/debug-monitors.h b/arch/arm64/include/asm/debug-monitors.h
index a2232d07be9d..62314791570c 100644
--- a/arch/arm64/include/asm/debug-monitors.h
+++ b/arch/arm64/include/asm/debug-monitors.h
@@ -62,6 +62,27 @@ struct task_struct;
62 62
63#define DBG_ARCH_ID_RESERVED 0 /* In case of ptrace ABI updates. */ 63#define DBG_ARCH_ID_RESERVED 0 /* In case of ptrace ABI updates. */
64 64
65#define DBG_HOOK_HANDLED 0
66#define DBG_HOOK_ERROR 1
67
68struct step_hook {
69 struct list_head node;
70 int (*fn)(struct pt_regs *regs, unsigned int esr);
71};
72
73void register_step_hook(struct step_hook *hook);
74void unregister_step_hook(struct step_hook *hook);
75
76struct break_hook {
77 struct list_head node;
78 u32 esr_val;
79 u32 esr_mask;
80 int (*fn)(struct pt_regs *regs, unsigned int esr);
81};
82
83void register_break_hook(struct break_hook *hook);
84void unregister_break_hook(struct break_hook *hook);
85
65u8 debug_monitors_arch(void); 86u8 debug_monitors_arch(void);
66 87
67void enable_debug_monitors(enum debug_el el); 88void enable_debug_monitors(enum debug_el el);
diff --git a/arch/arm64/include/asm/dma-contiguous.h b/arch/arm64/include/asm/dma-contiguous.h
new file mode 100644
index 000000000000..d6aacb61ff4a
--- /dev/null
+++ b/arch/arm64/include/asm/dma-contiguous.h
@@ -0,0 +1,29 @@
1/*
2 * Copyright (c) 2013, The Linux Foundation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 and
6 * only version 2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#ifndef _ASM_DMA_CONTIGUOUS_H
15#define _ASM_DMA_CONTIGUOUS_H
16
17#ifdef __KERNEL__
18#ifdef CONFIG_DMA_CMA
19
20#include <linux/types.h>
21#include <asm-generic/dma-contiguous.h>
22
23static inline void
24dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) { }
25
26#endif
27#endif
28
29#endif
diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index c582fa316366..78cc3aba5d69 100644
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -30,6 +30,7 @@
30" cbnz %w3, 1b\n" \ 30" cbnz %w3, 1b\n" \
31"3:\n" \ 31"3:\n" \
32" .pushsection .fixup,\"ax\"\n" \ 32" .pushsection .fixup,\"ax\"\n" \
33" .align 2\n" \
33"4: mov %w0, %w5\n" \ 34"4: mov %w0, %w5\n" \
34" b 3b\n" \ 35" b 3b\n" \
35" .popsection\n" \ 36" .popsection\n" \
diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index 990c051e7829..ae4801d77514 100644
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -20,7 +20,7 @@
20#include <linux/threads.h> 20#include <linux/threads.h>
21#include <asm/irq.h> 21#include <asm/irq.h>
22 22
23#define NR_IPI 4 23#define NR_IPI 5
24 24
25typedef struct { 25typedef struct {
26 unsigned int __softirq_pending; 26 unsigned int __softirq_pending;
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
new file mode 100644
index 000000000000..c44ad39ed310
--- /dev/null
+++ b/arch/arm64/include/asm/insn.h
@@ -0,0 +1,108 @@
1/*
2 * Copyright (C) 2013 Huawei Ltd.
3 * Author: Jiang Liu <liuj97@gmail.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17#ifndef __ASM_INSN_H
18#define __ASM_INSN_H
19#include <linux/types.h>
20
21/* A64 instructions are always 32 bits. */
22#define AARCH64_INSN_SIZE 4
23
24/*
25 * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
26 * Section C3.1 "A64 instruction index by encoding":
27 * AArch64 main encoding table
28 * Bit position
29 * 28 27 26 25 Encoding Group
30 * 0 0 - - Unallocated
31 * 1 0 0 - Data processing, immediate
32 * 1 0 1 - Branch, exception generation and system instructions
33 * - 1 - 0 Loads and stores
34 * - 1 0 1 Data processing - register
35 * 0 1 1 1 Data processing - SIMD and floating point
36 * 1 1 1 1 Data processing - SIMD and floating point
37 * "-" means "don't care"
38 */
39enum aarch64_insn_encoding_class {
40 AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
41 AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
42 AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
43 AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
44 AARCH64_INSN_CLS_LDST, /* Loads and stores */
45 AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and
46 * system instructions */
47};
48
49enum aarch64_insn_hint_op {
50 AARCH64_INSN_HINT_NOP = 0x0 << 5,
51 AARCH64_INSN_HINT_YIELD = 0x1 << 5,
52 AARCH64_INSN_HINT_WFE = 0x2 << 5,
53 AARCH64_INSN_HINT_WFI = 0x3 << 5,
54 AARCH64_INSN_HINT_SEV = 0x4 << 5,
55 AARCH64_INSN_HINT_SEVL = 0x5 << 5,
56};
57
58enum aarch64_insn_imm_type {
59 AARCH64_INSN_IMM_ADR,
60 AARCH64_INSN_IMM_26,
61 AARCH64_INSN_IMM_19,
62 AARCH64_INSN_IMM_16,
63 AARCH64_INSN_IMM_14,
64 AARCH64_INSN_IMM_12,
65 AARCH64_INSN_IMM_9,
66 AARCH64_INSN_IMM_MAX
67};
68
69enum aarch64_insn_branch_type {
70 AARCH64_INSN_BRANCH_NOLINK,
71 AARCH64_INSN_BRANCH_LINK,
72};
73
74#define __AARCH64_INSN_FUNCS(abbr, mask, val) \
75static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
76{ return (code & (mask)) == (val); } \
77static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \
78{ return (val); }
79
80__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000)
81__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000)
82__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001)
83__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002)
84__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003)
85__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000)
86__AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F)
87
88#undef __AARCH64_INSN_FUNCS
89
90bool aarch64_insn_is_nop(u32 insn);
91
92int aarch64_insn_read(void *addr, u32 *insnp);
93int aarch64_insn_write(void *addr, u32 insn);
94enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
95u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
96 u32 insn, u64 imm);
97u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
98 enum aarch64_insn_branch_type type);
99u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op);
100u32 aarch64_insn_gen_nop(void);
101
102bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
103
104int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
105int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
106int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
107
108#endif /* __ASM_INSN_H */
diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
new file mode 100644
index 000000000000..076a1c714049
--- /dev/null
+++ b/arch/arm64/include/asm/jump_label.h
@@ -0,0 +1,52 @@
1/*
2 * Copyright (C) 2013 Huawei Ltd.
3 * Author: Jiang Liu <liuj97@gmail.com>
4 *
5 * Based on arch/arm/include/asm/jump_label.h
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19#ifndef __ASM_JUMP_LABEL_H
20#define __ASM_JUMP_LABEL_H
21#include <linux/types.h>
22#include <asm/insn.h>
23
24#ifdef __KERNEL__
25
26#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
27
28static __always_inline bool arch_static_branch(struct static_key *key)
29{
30 asm goto("1: nop\n\t"
31 ".pushsection __jump_table, \"aw\"\n\t"
32 ".align 3\n\t"
33 ".quad 1b, %l[l_yes], %c0\n\t"
34 ".popsection\n\t"
35 : : "i"(key) : : l_yes);
36
37 return false;
38l_yes:
39 return true;
40}
41
42#endif /* __KERNEL__ */
43
44typedef u64 jump_label_t;
45
46struct jump_entry {
47 jump_label_t code;
48 jump_label_t target;
49 jump_label_t key;
50};
51
52#endif /* __ASM_JUMP_LABEL_H */
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 37762175896f..9dc5dc39fded 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -146,8 +146,7 @@ static inline void *phys_to_virt(phys_addr_t x)
146#define ARCH_PFN_OFFSET PHYS_PFN_OFFSET 146#define ARCH_PFN_OFFSET PHYS_PFN_OFFSET
147 147
148#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) 148#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
149#define virt_addr_valid(kaddr) (((void *)(kaddr) >= (void *)PAGE_OFFSET) && \ 149#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
150 ((void *)(kaddr) < (void *)high_memory))
151 150
152#endif 151#endif
153 152
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
new file mode 100644
index 000000000000..13fb0b3efc5f
--- /dev/null
+++ b/arch/arm64/include/asm/percpu.h
@@ -0,0 +1,41 @@
1/*
2 * Copyright (C) 2013 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_PERCPU_H
17#define __ASM_PERCPU_H
18
19static inline void set_my_cpu_offset(unsigned long off)
20{
21 asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
22}
23
24static inline unsigned long __my_cpu_offset(void)
25{
26 unsigned long off;
27 register unsigned long *sp asm ("sp");
28
29 /*
30 * We want to allow caching the value, so avoid using volatile and
31 * instead use a fake stack read to hazard against barrier().
32 */
33 asm("mrs %0, tpidr_el1" : "=r" (off) : "Q" (*sp));
34
35 return off;
36}
37#define __my_cpu_offset __my_cpu_offset()
38
39#include <asm-generic/percpu.h>
40
41#endif /* __ASM_PERCPU_H */
diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h
index 7cdf466fd0c5..0c657bb54597 100644
--- a/arch/arm64/include/asm/proc-fns.h
+++ b/arch/arm64/include/asm/proc-fns.h
@@ -26,11 +26,14 @@
26#include <asm/page.h> 26#include <asm/page.h>
27 27
28struct mm_struct; 28struct mm_struct;
29struct cpu_suspend_ctx;
29 30
30extern void cpu_cache_off(void); 31extern void cpu_cache_off(void);
31extern void cpu_do_idle(void); 32extern void cpu_do_idle(void);
32extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); 33extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
33extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); 34extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
35extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr);
36extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr);
34 37
35#include <asm/memory.h> 38#include <asm/memory.h>
36 39
diff --git a/arch/arm64/include/asm/smp_plat.h b/arch/arm64/include/asm/smp_plat.h
index ed43a0d2b1b2..59e282311b58 100644
--- a/arch/arm64/include/asm/smp_plat.h
+++ b/arch/arm64/include/asm/smp_plat.h
@@ -21,6 +21,19 @@
21 21
22#include <asm/types.h> 22#include <asm/types.h>
23 23
24struct mpidr_hash {
25 u64 mask;
26 u32 shift_aff[4];
27 u32 bits;
28};
29
30extern struct mpidr_hash mpidr_hash;
31
32static inline u32 mpidr_hash_size(void)
33{
34 return 1 << mpidr_hash.bits;
35}
36
24/* 37/*
25 * Logical CPU mapping. 38 * Logical CPU mapping.
26 */ 39 */
diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h
new file mode 100644
index 000000000000..e9c149c042e0
--- /dev/null
+++ b/arch/arm64/include/asm/suspend.h
@@ -0,0 +1,27 @@
1#ifndef __ASM_SUSPEND_H
2#define __ASM_SUSPEND_H
3
4#define NR_CTX_REGS 11
5
6/*
7 * struct cpu_suspend_ctx must be 16-byte aligned since it is allocated on
8 * the stack, which must be 16-byte aligned on v8
9 */
10struct cpu_suspend_ctx {
11 /*
12 * This struct must be kept in sync with
13 * cpu_do_{suspend/resume} in mm/proc.S
14 */
15 u64 ctx_regs[NR_CTX_REGS];
16 u64 sp;
17} __aligned(16);
18
19struct sleep_save_sp {
20 phys_addr_t *save_ptr_stash;
21 phys_addr_t save_ptr_stash_phys;
22};
23
24extern void cpu_resume(void);
25extern int cpu_suspend(unsigned long);
26
27#endif
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 7ecc2b23882e..6c0f684aca81 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -100,6 +100,7 @@ static inline void set_fs(mm_segment_t fs)
100}) 100})
101 101
102#define access_ok(type, addr, size) __range_ok(addr, size) 102#define access_ok(type, addr, size) __range_ok(addr, size)
103#define user_addr_max get_fs
103 104
104/* 105/*
105 * The "__xxx" versions of the user access functions do not verify the address 106 * The "__xxx" versions of the user access functions do not verify the address
@@ -240,9 +241,6 @@ extern unsigned long __must_check __copy_to_user(void __user *to, const void *fr
240extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n); 241extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);
241extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); 242extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
242 243
243extern unsigned long __must_check __strncpy_from_user(char *to, const char __user *from, unsigned long count);
244extern unsigned long __must_check __strnlen_user(const char __user *s, long n);
245
246static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) 244static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
247{ 245{
248 if (access_ok(VERIFY_READ, from, n)) 246 if (access_ok(VERIFY_READ, from, n))
@@ -276,24 +274,9 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo
276 return n; 274 return n;
277} 275}
278 276
279static inline long __must_check strncpy_from_user(char *dst, const char __user *src, long count) 277extern long strncpy_from_user(char *dest, const char __user *src, long count);
280{
281 long res = -EFAULT;
282 if (access_ok(VERIFY_READ, src, 1))
283 res = __strncpy_from_user(dst, src, count);
284 return res;
285}
286
287#define strlen_user(s) strnlen_user(s, ~0UL >> 1)
288 278
289static inline long __must_check strnlen_user(const char __user *s, long n) 279extern __must_check long strlen_user(const char __user *str);
290{ 280extern __must_check long strnlen_user(const char __user *str, long n);
291 unsigned long res = 0;
292
293 if (__addr_ok(s))
294 res = __strnlen_user(s, n);
295
296 return res;
297}
298 281
299#endif /* __ASM_UACCESS_H */ 282#endif /* __ASM_UACCESS_H */
diff --git a/arch/arm64/include/asm/word-at-a-time.h b/arch/arm64/include/asm/word-at-a-time.h
new file mode 100644
index 000000000000..aab5bf09e9d9
--- /dev/null
+++ b/arch/arm64/include/asm/word-at-a-time.h
@@ -0,0 +1,94 @@
1/*
2 * Copyright (C) 2013 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_WORD_AT_A_TIME_H
17#define __ASM_WORD_AT_A_TIME_H
18
19#ifndef __AARCH64EB__
20
21#include <linux/kernel.h>
22
23struct word_at_a_time {
24 const unsigned long one_bits, high_bits;
25};
26
27#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
28
29static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
30 const struct word_at_a_time *c)
31{
32 unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
33 *bits = mask;
34 return mask;
35}
36
37#define prep_zero_mask(a, bits, c) (bits)
38
39static inline unsigned long create_zero_mask(unsigned long bits)
40{
41 bits = (bits - 1) & ~bits;
42 return bits >> 7;
43}
44
45static inline unsigned long find_zero(unsigned long mask)
46{
47 return fls64(mask) >> 3;
48}
49
50#define zero_bytemask(mask) (mask)
51
52#else /* __AARCH64EB__ */
53#include <asm-generic/word-at-a-time.h>
54#endif
55
56/*
57 * Load an unaligned word from kernel space.
58 *
59 * In the (very unlikely) case of the word being a page-crosser
60 * and the next page not being mapped, take the exception and
61 * return zeroes in the non-existing part.
62 */
63static inline unsigned long load_unaligned_zeropad(const void *addr)
64{
65 unsigned long ret, offset;
66
67 /* Load word from unaligned pointer addr */
68 asm(
69 "1: ldr %0, %3\n"
70 "2:\n"
71 " .pushsection .fixup,\"ax\"\n"
72 " .align 2\n"
73 "3: and %1, %2, #0x7\n"
74 " bic %2, %2, #0x7\n"
75 " ldr %0, [%2]\n"
76 " lsl %1, %1, #0x3\n"
77#ifndef __AARCH64EB__
78 " lsr %0, %0, %1\n"
79#else
80 " lsl %0, %0, %1\n"
81#endif
82 " b 2b\n"
83 " .popsection\n"
84 " .pushsection __ex_table,\"a\"\n"
85 " .align 3\n"
86 " .quad 1b, 3b\n"
87 " .popsection"
88 : "=&r" (ret), "=&r" (offset)
89 : "r" (addr), "Q" (*(unsigned long *)addr));
90
91 return ret;
92}
93
94#endif /* __ASM_WORD_AT_A_TIME_H */
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 9b12476e9c85..73cf0f54d57c 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -22,6 +22,10 @@
22#define HWCAP_FP (1 << 0) 22#define HWCAP_FP (1 << 0)
23#define HWCAP_ASIMD (1 << 1) 23#define HWCAP_ASIMD (1 << 1)
24#define HWCAP_EVTSTRM (1 << 2) 24#define HWCAP_EVTSTRM (1 << 2)
25 25#define HWCAP_AES (1 << 3)
26#define HWCAP_PMULL (1 << 4)
27#define HWCAP_SHA1 (1 << 5)
28#define HWCAP_SHA2 (1 << 6)
29#define HWCAP_CRC32 (1 << 7)
26 30
27#endif /* _UAPI__ASM_HWCAP_H */ 31#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 5ba2fd43a75b..2d4554b13410 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
9arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ 9arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
10 entry-fpsimd.o process.o ptrace.o setup.o signal.o \ 10 entry-fpsimd.o process.o ptrace.o setup.o signal.o \
11 sys.o stacktrace.o time.o traps.o io.o vdso.o \ 11 sys.o stacktrace.o time.o traps.o io.o vdso.o \
12 hyp-stub.o psci.o cpu_ops.o 12 hyp-stub.o psci.o cpu_ops.o insn.o
13 13
14arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ 14arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
15 sys_compat.o 15 sys_compat.o
@@ -18,6 +18,8 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o
18arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o 18arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
19arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o 19arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
20arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o 20arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
21arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o
22arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
21 23
22obj-y += $(arm64-obj-y) vdso/ 24obj-y += $(arm64-obj-y) vdso/
23obj-m += $(arm64-obj-m) 25obj-m += $(arm64-obj-m)
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index e7ee770c0697..338b568cd8ae 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -29,13 +29,10 @@
29 29
30#include <asm/checksum.h> 30#include <asm/checksum.h>
31 31
32 /* user mem (segment) */
33EXPORT_SYMBOL(__strnlen_user);
34EXPORT_SYMBOL(__strncpy_from_user);
35
36EXPORT_SYMBOL(copy_page); 32EXPORT_SYMBOL(copy_page);
37EXPORT_SYMBOL(clear_page); 33EXPORT_SYMBOL(clear_page);
38 34
35 /* user mem (segment) */
39EXPORT_SYMBOL(__copy_from_user); 36EXPORT_SYMBOL(__copy_from_user);
40EXPORT_SYMBOL(__copy_to_user); 37EXPORT_SYMBOL(__copy_to_user);
41EXPORT_SYMBOL(__clear_user); 38EXPORT_SYMBOL(__clear_user);
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 666e231d410b..646f888387cd 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -25,6 +25,8 @@
25#include <asm/thread_info.h> 25#include <asm/thread_info.h>
26#include <asm/memory.h> 26#include <asm/memory.h>
27#include <asm/cputable.h> 27#include <asm/cputable.h>
28#include <asm/smp_plat.h>
29#include <asm/suspend.h>
28#include <asm/vdso_datapage.h> 30#include <asm/vdso_datapage.h>
29#include <linux/kbuild.h> 31#include <linux/kbuild.h>
30 32
@@ -138,5 +140,14 @@ int main(void)
138 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); 140 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
139 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); 141 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
140#endif 142#endif
143#ifdef CONFIG_ARM64_CPU_SUSPEND
144 DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
145 DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
146 DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));
147 DEFINE(MPIDR_HASH_SHIFTS, offsetof(struct mpidr_hash, shift_aff));
148 DEFINE(SLEEP_SAVE_SP_SZ, sizeof(struct sleep_save_sp));
149 DEFINE(SLEEP_SAVE_SP_PHYS, offsetof(struct sleep_save_sp, save_ptr_stash_phys));
150 DEFINE(SLEEP_SAVE_SP_VIRT, offsetof(struct sleep_save_sp, save_ptr_stash));
151#endif
141 return 0; 152 return 0;
142} 153}
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 4ae68579031d..636ba8b6240b 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -187,6 +187,48 @@ static void clear_regs_spsr_ss(struct pt_regs *regs)
187 regs->pstate = spsr; 187 regs->pstate = spsr;
188} 188}
189 189
190/* EL1 Single Step Handler hooks */
191static LIST_HEAD(step_hook);
192DEFINE_RWLOCK(step_hook_lock);
193
194void register_step_hook(struct step_hook *hook)
195{
196 write_lock(&step_hook_lock);
197 list_add(&hook->node, &step_hook);
198 write_unlock(&step_hook_lock);
199}
200
201void unregister_step_hook(struct step_hook *hook)
202{
203 write_lock(&step_hook_lock);
204 list_del(&hook->node);
205 write_unlock(&step_hook_lock);
206}
207
208/*
209 * Call registered single step handers
210 * There is no Syndrome info to check for determining the handler.
211 * So we call all the registered handlers, until the right handler is
212 * found which returns zero.
213 */
214static int call_step_hook(struct pt_regs *regs, unsigned int esr)
215{
216 struct step_hook *hook;
217 int retval = DBG_HOOK_ERROR;
218
219 read_lock(&step_hook_lock);
220
221 list_for_each_entry(hook, &step_hook, node) {
222 retval = hook->fn(regs, esr);
223 if (retval == DBG_HOOK_HANDLED)
224 break;
225 }
226
227 read_unlock(&step_hook_lock);
228
229 return retval;
230}
231
190static int single_step_handler(unsigned long addr, unsigned int esr, 232static int single_step_handler(unsigned long addr, unsigned int esr,
191 struct pt_regs *regs) 233 struct pt_regs *regs)
192{ 234{
@@ -214,7 +256,9 @@ static int single_step_handler(unsigned long addr, unsigned int esr,
214 */ 256 */
215 user_rewind_single_step(current); 257 user_rewind_single_step(current);
216 } else { 258 } else {
217 /* TODO: route to KGDB */ 259 if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED)
260 return 0;
261
218 pr_warning("Unexpected kernel single-step exception at EL1\n"); 262 pr_warning("Unexpected kernel single-step exception at EL1\n");
219 /* 263 /*
220 * Re-enable stepping since we know that we will be 264 * Re-enable stepping since we know that we will be
@@ -226,11 +270,53 @@ static int single_step_handler(unsigned long addr, unsigned int esr,
226 return 0; 270 return 0;
227} 271}
228 272
273/*
274 * Breakpoint handler is re-entrant as another breakpoint can
275 * hit within breakpoint handler, especically in kprobes.
276 * Use reader/writer locks instead of plain spinlock.
277 */
278static LIST_HEAD(break_hook);
279DEFINE_RWLOCK(break_hook_lock);
280
281void register_break_hook(struct break_hook *hook)
282{
283 write_lock(&break_hook_lock);
284 list_add(&hook->node, &break_hook);
285 write_unlock(&break_hook_lock);
286}
287
288void unregister_break_hook(struct break_hook *hook)
289{
290 write_lock(&break_hook_lock);
291 list_del(&hook->node);
292 write_unlock(&break_hook_lock);
293}
294
295static int call_break_hook(struct pt_regs *regs, unsigned int esr)
296{
297 struct break_hook *hook;
298 int (*fn)(struct pt_regs *regs, unsigned int esr) = NULL;
299
300 read_lock(&break_hook_lock);
301 list_for_each_entry(hook, &break_hook, node)
302 if ((esr & hook->esr_mask) == hook->esr_val)
303 fn = hook->fn;
304 read_unlock(&break_hook_lock);
305
306 return fn ? fn(regs, esr) : DBG_HOOK_ERROR;
307}
308
229static int brk_handler(unsigned long addr, unsigned int esr, 309static int brk_handler(unsigned long addr, unsigned int esr,
230 struct pt_regs *regs) 310 struct pt_regs *regs)
231{ 311{
232 siginfo_t info; 312 siginfo_t info;
233 313
314 if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED)
315 return 0;
316
317 pr_warn("unexpected brk exception at %lx, esr=0x%x\n",
318 (long)instruction_pointer(regs), esr);
319
234 if (!user_mode(regs)) 320 if (!user_mode(regs))
235 return -EFAULT; 321 return -EFAULT;
236 322
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 4d2c6f3f0c41..39ac630d83de 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -288,6 +288,8 @@ el1_dbg:
288 /* 288 /*
289 * Debug exception handling 289 * Debug exception handling
290 */ 290 */
291 cmp x24, #ESR_EL1_EC_BRK64 // if BRK64
292 cinc x24, x24, eq // set bit '0'
291 tbz x24, #0, el1_inv // EL1 only 293 tbz x24, #0, el1_inv // EL1 only
292 mrs x0, far_el1 294 mrs x0, far_el1
293 mov x2, sp // struct pt_regs 295 mov x2, sp // struct pt_regs
@@ -314,7 +316,7 @@ el1_irq:
314 316
315#ifdef CONFIG_PREEMPT 317#ifdef CONFIG_PREEMPT
316 get_thread_info tsk 318 get_thread_info tsk
317 ldr w24, [tsk, #TI_PREEMPT] // restore preempt count 319 ldr w24, [tsk, #TI_PREEMPT] // get preempt count
318 cbnz w24, 1f // preempt count != 0 320 cbnz w24, 1f // preempt count != 0
319 ldr x0, [tsk, #TI_FLAGS] // get flags 321 ldr x0, [tsk, #TI_FLAGS] // get flags
320 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? 322 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index bb785d23dbde..4aef42a04bdc 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -17,6 +17,7 @@
17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */ 18 */
19 19
20#include <linux/cpu_pm.h>
20#include <linux/kernel.h> 21#include <linux/kernel.h>
21#include <linux/init.h> 22#include <linux/init.h>
22#include <linux/sched.h> 23#include <linux/sched.h>
@@ -113,6 +114,39 @@ EXPORT_SYMBOL(kernel_neon_end);
113 114
114#endif /* CONFIG_KERNEL_MODE_NEON */ 115#endif /* CONFIG_KERNEL_MODE_NEON */
115 116
117#ifdef CONFIG_CPU_PM
118static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
119 unsigned long cmd, void *v)
120{
121 switch (cmd) {
122 case CPU_PM_ENTER:
123 if (current->mm)
124 fpsimd_save_state(&current->thread.fpsimd_state);
125 break;
126 case CPU_PM_EXIT:
127 if (current->mm)
128 fpsimd_load_state(&current->thread.fpsimd_state);
129 break;
130 case CPU_PM_ENTER_FAILED:
131 default:
132 return NOTIFY_DONE;
133 }
134 return NOTIFY_OK;
135}
136
137static struct notifier_block fpsimd_cpu_pm_notifier_block = {
138 .notifier_call = fpsimd_cpu_pm_notifier,
139};
140
141static void fpsimd_pm_init(void)
142{
143 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
144}
145
146#else
147static inline void fpsimd_pm_init(void) { }
148#endif /* CONFIG_CPU_PM */
149
116/* 150/*
117 * FP/SIMD support code initialisation. 151 * FP/SIMD support code initialisation.
118 */ 152 */
@@ -131,6 +165,8 @@ static int __init fpsimd_init(void)
131 else 165 else
132 elf_hwcap |= HWCAP_ASIMD; 166 elf_hwcap |= HWCAP_ASIMD;
133 167
168 fpsimd_pm_init();
169
134 return 0; 170 return 0;
135} 171}
136late_initcall(fpsimd_init); 172late_initcall(fpsimd_init);
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index c68cca5c3523..0b281fffda51 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -482,8 +482,6 @@ ENDPROC(__create_page_tables)
482 .type __switch_data, %object 482 .type __switch_data, %object
483__switch_data: 483__switch_data:
484 .quad __mmap_switched 484 .quad __mmap_switched
485 .quad __data_loc // x4
486 .quad _data // x5
487 .quad __bss_start // x6 485 .quad __bss_start // x6
488 .quad _end // x7 486 .quad _end // x7
489 .quad processor_id // x4 487 .quad processor_id // x4
@@ -498,15 +496,7 @@ __switch_data:
498__mmap_switched: 496__mmap_switched:
499 adr x3, __switch_data + 8 497 adr x3, __switch_data + 8
500 498
501 ldp x4, x5, [x3], #16
502 ldp x6, x7, [x3], #16 499 ldp x6, x7, [x3], #16
503 cmp x4, x5 // Copy data segment if needed
5041: ccmp x5, x6, #4, ne
505 b.eq 2f
506 ldr x16, [x4], #8
507 str x16, [x5], #8
508 b 1b
5092:
5101: cmp x6, x7 5001: cmp x6, x7
511 b.hs 2f 501 b.hs 2f
512 str xzr, [x6], #8 // Clear BSS 502 str xzr, [x6], #8 // Clear BSS
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index ff516f6691e4..f17f581116fc 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -20,6 +20,7 @@
20 20
21#define pr_fmt(fmt) "hw-breakpoint: " fmt 21#define pr_fmt(fmt) "hw-breakpoint: " fmt
22 22
23#include <linux/cpu_pm.h>
23#include <linux/errno.h> 24#include <linux/errno.h>
24#include <linux/hw_breakpoint.h> 25#include <linux/hw_breakpoint.h>
25#include <linux/perf_event.h> 26#include <linux/perf_event.h>
@@ -169,15 +170,68 @@ static enum debug_el debug_exception_level(int privilege)
169 } 170 }
170} 171}
171 172
172/* 173enum hw_breakpoint_ops {
173 * Install a perf counter breakpoint. 174 HW_BREAKPOINT_INSTALL,
175 HW_BREAKPOINT_UNINSTALL,
176 HW_BREAKPOINT_RESTORE
177};
178
179/**
180 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
181 * operations
182 *
183 * @slots: pointer to array of slots
184 * @max_slots: max number of slots
185 * @bp: perf_event to setup
186 * @ops: operation to be carried out on the slot
187 *
188 * Return:
189 * slot index on success
190 * -ENOSPC if no slot is available/matches
191 * -EINVAL on wrong operations parameter
174 */ 192 */
175int arch_install_hw_breakpoint(struct perf_event *bp) 193static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
194 struct perf_event *bp,
195 enum hw_breakpoint_ops ops)
196{
197 int i;
198 struct perf_event **slot;
199
200 for (i = 0; i < max_slots; ++i) {
201 slot = &slots[i];
202 switch (ops) {
203 case HW_BREAKPOINT_INSTALL:
204 if (!*slot) {
205 *slot = bp;
206 return i;
207 }
208 break;
209 case HW_BREAKPOINT_UNINSTALL:
210 if (*slot == bp) {
211 *slot = NULL;
212 return i;
213 }
214 break;
215 case HW_BREAKPOINT_RESTORE:
216 if (*slot == bp)
217 return i;
218 break;
219 default:
220 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
221 return -EINVAL;
222 }
223 }
224 return -ENOSPC;
225}
226
227static int hw_breakpoint_control(struct perf_event *bp,
228 enum hw_breakpoint_ops ops)
176{ 229{
177 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 230 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
178 struct perf_event **slot, **slots; 231 struct perf_event **slots;
179 struct debug_info *debug_info = &current->thread.debug; 232 struct debug_info *debug_info = &current->thread.debug;
180 int i, max_slots, ctrl_reg, val_reg, reg_enable; 233 int i, max_slots, ctrl_reg, val_reg, reg_enable;
234 enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege);
181 u32 ctrl; 235 u32 ctrl;
182 236
183 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 237 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
@@ -196,67 +250,54 @@ int arch_install_hw_breakpoint(struct perf_event *bp)
196 reg_enable = !debug_info->wps_disabled; 250 reg_enable = !debug_info->wps_disabled;
197 } 251 }
198 252
199 for (i = 0; i < max_slots; ++i) { 253 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
200 slot = &slots[i];
201
202 if (!*slot) {
203 *slot = bp;
204 break;
205 }
206 }
207
208 if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
209 return -ENOSPC;
210 254
211 /* Ensure debug monitors are enabled at the correct exception level. */ 255 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
212 enable_debug_monitors(debug_exception_level(info->ctrl.privilege)); 256 return i;
213 257
214 /* Setup the address register. */ 258 switch (ops) {
215 write_wb_reg(val_reg, i, info->address); 259 case HW_BREAKPOINT_INSTALL:
260 /*
261 * Ensure debug monitors are enabled at the correct exception
262 * level.
263 */
264 enable_debug_monitors(dbg_el);
265 /* Fall through */
266 case HW_BREAKPOINT_RESTORE:
267 /* Setup the address register. */
268 write_wb_reg(val_reg, i, info->address);
269
270 /* Setup the control register. */
271 ctrl = encode_ctrl_reg(info->ctrl);
272 write_wb_reg(ctrl_reg, i,
273 reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
274 break;
275 case HW_BREAKPOINT_UNINSTALL:
276 /* Reset the control register. */
277 write_wb_reg(ctrl_reg, i, 0);
216 278
217 /* Setup the control register. */ 279 /*
218 ctrl = encode_ctrl_reg(info->ctrl); 280 * Release the debug monitors for the correct exception
219 write_wb_reg(ctrl_reg, i, reg_enable ? ctrl | 0x1 : ctrl & ~0x1); 281 * level.
282 */
283 disable_debug_monitors(dbg_el);
284 break;
285 }
220 286
221 return 0; 287 return 0;
222} 288}
223 289
224void arch_uninstall_hw_breakpoint(struct perf_event *bp) 290/*
291 * Install a perf counter breakpoint.
292 */
293int arch_install_hw_breakpoint(struct perf_event *bp)
225{ 294{
226 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 295 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
227 struct perf_event **slot, **slots; 296}
228 int i, max_slots, base;
229
230 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
231 /* Breakpoint */
232 base = AARCH64_DBG_REG_BCR;
233 slots = this_cpu_ptr(bp_on_reg);
234 max_slots = core_num_brps;
235 } else {
236 /* Watchpoint */
237 base = AARCH64_DBG_REG_WCR;
238 slots = this_cpu_ptr(wp_on_reg);
239 max_slots = core_num_wrps;
240 }
241
242 /* Remove the breakpoint. */
243 for (i = 0; i < max_slots; ++i) {
244 slot = &slots[i];
245
246 if (*slot == bp) {
247 *slot = NULL;
248 break;
249 }
250 }
251
252 if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
253 return;
254
255 /* Reset the control register. */
256 write_wb_reg(base, i, 0);
257 297
258 /* Release the debug monitors for the correct exception level. */ 298void arch_uninstall_hw_breakpoint(struct perf_event *bp)
259 disable_debug_monitors(debug_exception_level(info->ctrl.privilege)); 299{
300 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
260} 301}
261 302
262static int get_hbp_len(u8 hbp_len) 303static int get_hbp_len(u8 hbp_len)
@@ -806,18 +847,36 @@ void hw_breakpoint_thread_switch(struct task_struct *next)
806/* 847/*
807 * CPU initialisation. 848 * CPU initialisation.
808 */ 849 */
809static void reset_ctrl_regs(void *unused) 850static void hw_breakpoint_reset(void *unused)
810{ 851{
811 int i; 852 int i;
812 853 struct perf_event **slots;
813 for (i = 0; i < core_num_brps; ++i) { 854 /*
814 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); 855 * When a CPU goes through cold-boot, it does not have any installed
815 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); 856 * slot, so it is safe to share the same function for restoring and
857 * resetting breakpoints; when a CPU is hotplugged in, it goes
858 * through the slots, which are all empty, hence it just resets control
859 * and value for debug registers.
860 * When this function is triggered on warm-boot through a CPU PM
861 * notifier some slots might be initialized; if so they are
862 * reprogrammed according to the debug slots content.
863 */
864 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
865 if (slots[i]) {
866 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
867 } else {
868 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
869 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
870 }
816 } 871 }
817 872
818 for (i = 0; i < core_num_wrps; ++i) { 873 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
819 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); 874 if (slots[i]) {
820 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); 875 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
876 } else {
877 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
878 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
879 }
821 } 880 }
822} 881}
823 882
@@ -827,7 +886,7 @@ static int hw_breakpoint_reset_notify(struct notifier_block *self,
827{ 886{
828 int cpu = (long)hcpu; 887 int cpu = (long)hcpu;
829 if (action == CPU_ONLINE) 888 if (action == CPU_ONLINE)
830 smp_call_function_single(cpu, reset_ctrl_regs, NULL, 1); 889 smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1);
831 return NOTIFY_OK; 890 return NOTIFY_OK;
832} 891}
833 892
@@ -835,6 +894,14 @@ static struct notifier_block hw_breakpoint_reset_nb = {
835 .notifier_call = hw_breakpoint_reset_notify, 894 .notifier_call = hw_breakpoint_reset_notify,
836}; 895};
837 896
897#ifdef CONFIG_ARM64_CPU_SUSPEND
898extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
899#else
900static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
901{
902}
903#endif
904
838/* 905/*
839 * One-time initialisation. 906 * One-time initialisation.
840 */ 907 */
@@ -850,8 +917,8 @@ static int __init arch_hw_breakpoint_init(void)
850 * Reset the breakpoint resources. We assume that a halting 917 * Reset the breakpoint resources. We assume that a halting
851 * debugger will leave the world in a nice state for us. 918 * debugger will leave the world in a nice state for us.
852 */ 919 */
853 smp_call_function(reset_ctrl_regs, NULL, 1); 920 smp_call_function(hw_breakpoint_reset, NULL, 1);
854 reset_ctrl_regs(NULL); 921 hw_breakpoint_reset(NULL);
855 922
856 /* Register debug fault handlers. */ 923 /* Register debug fault handlers. */
857 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, 924 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
@@ -861,6 +928,8 @@ static int __init arch_hw_breakpoint_init(void)
861 928
862 /* Register hotplug notifier. */ 929 /* Register hotplug notifier. */
863 register_cpu_notifier(&hw_breakpoint_reset_nb); 930 register_cpu_notifier(&hw_breakpoint_reset_nb);
931 /* Register cpu_suspend hw breakpoint restore hook */
932 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
864 933
865 return 0; 934 return 0;
866} 935}
diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
new file mode 100644
index 000000000000..92f36835486b
--- /dev/null
+++ b/arch/arm64/kernel/insn.c
@@ -0,0 +1,304 @@
1/*
2 * Copyright (C) 2013 Huawei Ltd.
3 * Author: Jiang Liu <liuj97@gmail.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17#include <linux/bitops.h>
18#include <linux/compiler.h>
19#include <linux/kernel.h>
20#include <linux/smp.h>
21#include <linux/stop_machine.h>
22#include <linux/uaccess.h>
23#include <asm/cacheflush.h>
24#include <asm/insn.h>
25
26static int aarch64_insn_encoding_class[] = {
27 AARCH64_INSN_CLS_UNKNOWN,
28 AARCH64_INSN_CLS_UNKNOWN,
29 AARCH64_INSN_CLS_UNKNOWN,
30 AARCH64_INSN_CLS_UNKNOWN,
31 AARCH64_INSN_CLS_LDST,
32 AARCH64_INSN_CLS_DP_REG,
33 AARCH64_INSN_CLS_LDST,
34 AARCH64_INSN_CLS_DP_FPSIMD,
35 AARCH64_INSN_CLS_DP_IMM,
36 AARCH64_INSN_CLS_DP_IMM,
37 AARCH64_INSN_CLS_BR_SYS,
38 AARCH64_INSN_CLS_BR_SYS,
39 AARCH64_INSN_CLS_LDST,
40 AARCH64_INSN_CLS_DP_REG,
41 AARCH64_INSN_CLS_LDST,
42 AARCH64_INSN_CLS_DP_FPSIMD,
43};
44
45enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
46{
47 return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
48}
49
50/* NOP is an alias of HINT */
51bool __kprobes aarch64_insn_is_nop(u32 insn)
52{
53 if (!aarch64_insn_is_hint(insn))
54 return false;
55
56 switch (insn & 0xFE0) {
57 case AARCH64_INSN_HINT_YIELD:
58 case AARCH64_INSN_HINT_WFE:
59 case AARCH64_INSN_HINT_WFI:
60 case AARCH64_INSN_HINT_SEV:
61 case AARCH64_INSN_HINT_SEVL:
62 return false;
63 default:
64 return true;
65 }
66}
67
68/*
69 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
70 * little-endian.
71 */
72int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
73{
74 int ret;
75 u32 val;
76
77 ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE);
78 if (!ret)
79 *insnp = le32_to_cpu(val);
80
81 return ret;
82}
83
84int __kprobes aarch64_insn_write(void *addr, u32 insn)
85{
86 insn = cpu_to_le32(insn);
87 return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
88}
89
90static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
91{
92 if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
93 return false;
94
95 return aarch64_insn_is_b(insn) ||
96 aarch64_insn_is_bl(insn) ||
97 aarch64_insn_is_svc(insn) ||
98 aarch64_insn_is_hvc(insn) ||
99 aarch64_insn_is_smc(insn) ||
100 aarch64_insn_is_brk(insn) ||
101 aarch64_insn_is_nop(insn);
102}
103
104/*
105 * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
106 * Section B2.6.5 "Concurrent modification and execution of instructions":
107 * Concurrent modification and execution of instructions can lead to the
108 * resulting instruction performing any behavior that can be achieved by
109 * executing any sequence of instructions that can be executed from the
110 * same Exception level, except where the instruction before modification
111 * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
112 * or SMC instruction.
113 */
114bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
115{
116 return __aarch64_insn_hotpatch_safe(old_insn) &&
117 __aarch64_insn_hotpatch_safe(new_insn);
118}
119
120int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
121{
122 u32 *tp = addr;
123 int ret;
124
125 /* A64 instructions must be word aligned */
126 if ((uintptr_t)tp & 0x3)
127 return -EINVAL;
128
129 ret = aarch64_insn_write(tp, insn);
130 if (ret == 0)
131 flush_icache_range((uintptr_t)tp,
132 (uintptr_t)tp + AARCH64_INSN_SIZE);
133
134 return ret;
135}
136
137struct aarch64_insn_patch {
138 void **text_addrs;
139 u32 *new_insns;
140 int insn_cnt;
141 atomic_t cpu_count;
142};
143
144static int __kprobes aarch64_insn_patch_text_cb(void *arg)
145{
146 int i, ret = 0;
147 struct aarch64_insn_patch *pp = arg;
148
149 /* The first CPU becomes master */
150 if (atomic_inc_return(&pp->cpu_count) == 1) {
151 for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
152 ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
153 pp->new_insns[i]);
154 /*
155 * aarch64_insn_patch_text_nosync() calls flush_icache_range(),
156 * which ends with "dsb; isb" pair guaranteeing global
157 * visibility.
158 */
159 atomic_set(&pp->cpu_count, -1);
160 } else {
161 while (atomic_read(&pp->cpu_count) != -1)
162 cpu_relax();
163 isb();
164 }
165
166 return ret;
167}
168
169int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
170{
171 struct aarch64_insn_patch patch = {
172 .text_addrs = addrs,
173 .new_insns = insns,
174 .insn_cnt = cnt,
175 .cpu_count = ATOMIC_INIT(0),
176 };
177
178 if (cnt <= 0)
179 return -EINVAL;
180
181 return stop_machine(aarch64_insn_patch_text_cb, &patch,
182 cpu_online_mask);
183}
184
185int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
186{
187 int ret;
188 u32 insn;
189
190 /* Unsafe to patch multiple instructions without synchronizaiton */
191 if (cnt == 1) {
192 ret = aarch64_insn_read(addrs[0], &insn);
193 if (ret)
194 return ret;
195
196 if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
197 /*
198 * ARMv8 architecture doesn't guarantee all CPUs see
199 * the new instruction after returning from function
200 * aarch64_insn_patch_text_nosync(). So send IPIs to
201 * all other CPUs to achieve instruction
202 * synchronization.
203 */
204 ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
205 kick_all_cpus_sync();
206 return ret;
207 }
208 }
209
210 return aarch64_insn_patch_text_sync(addrs, insns, cnt);
211}
212
213u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
214 u32 insn, u64 imm)
215{
216 u32 immlo, immhi, lomask, himask, mask;
217 int shift;
218
219 switch (type) {
220 case AARCH64_INSN_IMM_ADR:
221 lomask = 0x3;
222 himask = 0x7ffff;
223 immlo = imm & lomask;
224 imm >>= 2;
225 immhi = imm & himask;
226 imm = (immlo << 24) | (immhi);
227 mask = (lomask << 24) | (himask);
228 shift = 5;
229 break;
230 case AARCH64_INSN_IMM_26:
231 mask = BIT(26) - 1;
232 shift = 0;
233 break;
234 case AARCH64_INSN_IMM_19:
235 mask = BIT(19) - 1;
236 shift = 5;
237 break;
238 case AARCH64_INSN_IMM_16:
239 mask = BIT(16) - 1;
240 shift = 5;
241 break;
242 case AARCH64_INSN_IMM_14:
243 mask = BIT(14) - 1;
244 shift = 5;
245 break;
246 case AARCH64_INSN_IMM_12:
247 mask = BIT(12) - 1;
248 shift = 10;
249 break;
250 case AARCH64_INSN_IMM_9:
251 mask = BIT(9) - 1;
252 shift = 12;
253 break;
254 default:
255 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
256 type);
257 return 0;
258 }
259
260 /* Update the immediate field. */
261 insn &= ~(mask << shift);
262 insn |= (imm & mask) << shift;
263
264 return insn;
265}
266
267u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
268 enum aarch64_insn_branch_type type)
269{
270 u32 insn;
271 long offset;
272
273 /*
274 * PC: A 64-bit Program Counter holding the address of the current
275 * instruction. A64 instructions must be word-aligned.
276 */
277 BUG_ON((pc & 0x3) || (addr & 0x3));
278
279 /*
280 * B/BL support [-128M, 128M) offset
281 * ARM64 virtual address arrangement guarantees all kernel and module
282 * texts are within +/-128M.
283 */
284 offset = ((long)addr - (long)pc);
285 BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
286
287 if (type == AARCH64_INSN_BRANCH_LINK)
288 insn = aarch64_insn_get_bl_value();
289 else
290 insn = aarch64_insn_get_b_value();
291
292 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
293 offset >> 2);
294}
295
296u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
297{
298 return aarch64_insn_get_hint_value() | op;
299}
300
301u32 __kprobes aarch64_insn_gen_nop(void)
302{
303 return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
304}
diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c
new file mode 100644
index 000000000000..263a166291fb
--- /dev/null
+++ b/arch/arm64/kernel/jump_label.c
@@ -0,0 +1,58 @@
1/*
2 * Copyright (C) 2013 Huawei Ltd.
3 * Author: Jiang Liu <liuj97@gmail.com>
4 *
5 * Based on arch/arm/kernel/jump_label.c
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19#include <linux/kernel.h>
20#include <linux/jump_label.h>
21#include <asm/insn.h>
22
23#ifdef HAVE_JUMP_LABEL
24
25static void __arch_jump_label_transform(struct jump_entry *entry,
26 enum jump_label_type type,
27 bool is_static)
28{
29 void *addr = (void *)entry->code;
30 u32 insn;
31
32 if (type == JUMP_LABEL_ENABLE) {
33 insn = aarch64_insn_gen_branch_imm(entry->code,
34 entry->target,
35 AARCH64_INSN_BRANCH_NOLINK);
36 } else {
37 insn = aarch64_insn_gen_nop();
38 }
39
40 if (is_static)
41 aarch64_insn_patch_text_nosync(addr, insn);
42 else
43 aarch64_insn_patch_text(&addr, &insn, 1);
44}
45
46void arch_jump_label_transform(struct jump_entry *entry,
47 enum jump_label_type type)
48{
49 __arch_jump_label_transform(entry, type, false);
50}
51
52void arch_jump_label_transform_static(struct jump_entry *entry,
53 enum jump_label_type type)
54{
55 __arch_jump_label_transform(entry, type, true);
56}
57
58#endif /* HAVE_JUMP_LABEL */
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index e2ad0d87721f..1eb1cc955139 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -25,6 +25,10 @@
25#include <linux/mm.h> 25#include <linux/mm.h>
26#include <linux/moduleloader.h> 26#include <linux/moduleloader.h>
27#include <linux/vmalloc.h> 27#include <linux/vmalloc.h>
28#include <asm/insn.h>
29
30#define AARCH64_INSN_IMM_MOVNZ AARCH64_INSN_IMM_MAX
31#define AARCH64_INSN_IMM_MOVK AARCH64_INSN_IMM_16
28 32
29void *module_alloc(unsigned long size) 33void *module_alloc(unsigned long size)
30{ 34{
@@ -94,28 +98,18 @@ static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len)
94 return 0; 98 return 0;
95} 99}
96 100
97enum aarch64_imm_type { 101static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
98 INSN_IMM_MOVNZ, 102 int lsb, enum aarch64_insn_imm_type imm_type)
99 INSN_IMM_MOVK,
100 INSN_IMM_ADR,
101 INSN_IMM_26,
102 INSN_IMM_19,
103 INSN_IMM_16,
104 INSN_IMM_14,
105 INSN_IMM_12,
106 INSN_IMM_9,
107};
108
109static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm)
110{ 103{
111 u32 immlo, immhi, lomask, himask, mask; 104 u64 imm, limit = 0;
112 int shift; 105 s64 sval;
106 u32 insn = le32_to_cpu(*(u32 *)place);
113 107
114 /* The instruction stream is always little endian. */ 108 sval = do_reloc(op, place, val);
115 insn = le32_to_cpu(insn); 109 sval >>= lsb;
110 imm = sval & 0xffff;
116 111
117 switch (type) { 112 if (imm_type == AARCH64_INSN_IMM_MOVNZ) {
118 case INSN_IMM_MOVNZ:
119 /* 113 /*
120 * For signed MOVW relocations, we have to manipulate the 114 * For signed MOVW relocations, we have to manipulate the
121 * instruction encoding depending on whether or not the 115 * instruction encoding depending on whether or not the
@@ -134,70 +128,12 @@ static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm)
134 */ 128 */
135 imm = ~imm; 129 imm = ~imm;
136 } 130 }
137 case INSN_IMM_MOVK: 131 imm_type = AARCH64_INSN_IMM_MOVK;
138 mask = BIT(16) - 1;
139 shift = 5;
140 break;
141 case INSN_IMM_ADR:
142 lomask = 0x3;
143 himask = 0x7ffff;
144 immlo = imm & lomask;
145 imm >>= 2;
146 immhi = imm & himask;
147 imm = (immlo << 24) | (immhi);
148 mask = (lomask << 24) | (himask);
149 shift = 5;
150 break;
151 case INSN_IMM_26:
152 mask = BIT(26) - 1;
153 shift = 0;
154 break;
155 case INSN_IMM_19:
156 mask = BIT(19) - 1;
157 shift = 5;
158 break;
159 case INSN_IMM_16:
160 mask = BIT(16) - 1;
161 shift = 5;
162 break;
163 case INSN_IMM_14:
164 mask = BIT(14) - 1;
165 shift = 5;
166 break;
167 case INSN_IMM_12:
168 mask = BIT(12) - 1;
169 shift = 10;
170 break;
171 case INSN_IMM_9:
172 mask = BIT(9) - 1;
173 shift = 12;
174 break;
175 default:
176 pr_err("encode_insn_immediate: unknown immediate encoding %d\n",
177 type);
178 return 0;
179 } 132 }
180 133
181 /* Update the immediate field. */
182 insn &= ~(mask << shift);
183 insn |= (imm & mask) << shift;
184
185 return cpu_to_le32(insn);
186}
187
188static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
189 int lsb, enum aarch64_imm_type imm_type)
190{
191 u64 imm, limit = 0;
192 s64 sval;
193 u32 insn = *(u32 *)place;
194
195 sval = do_reloc(op, place, val);
196 sval >>= lsb;
197 imm = sval & 0xffff;
198
199 /* Update the instruction with the new encoding. */ 134 /* Update the instruction with the new encoding. */
200 *(u32 *)place = encode_insn_immediate(imm_type, insn, imm); 135 insn = aarch64_insn_encode_immediate(imm_type, insn, imm);
136 *(u32 *)place = cpu_to_le32(insn);
201 137
202 /* Shift out the immediate field. */ 138 /* Shift out the immediate field. */
203 sval >>= 16; 139 sval >>= 16;
@@ -206,9 +142,9 @@ static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
206 * For unsigned immediates, the overflow check is straightforward. 142 * For unsigned immediates, the overflow check is straightforward.
207 * For signed immediates, the sign bit is actually the bit past the 143 * For signed immediates, the sign bit is actually the bit past the
208 * most significant bit of the field. 144 * most significant bit of the field.
209 * The INSN_IMM_16 immediate type is unsigned. 145 * The AARCH64_INSN_IMM_16 immediate type is unsigned.
210 */ 146 */
211 if (imm_type != INSN_IMM_16) { 147 if (imm_type != AARCH64_INSN_IMM_16) {
212 sval++; 148 sval++;
213 limit++; 149 limit++;
214 } 150 }
@@ -221,11 +157,11 @@ static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
221} 157}
222 158
223static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val, 159static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val,
224 int lsb, int len, enum aarch64_imm_type imm_type) 160 int lsb, int len, enum aarch64_insn_imm_type imm_type)
225{ 161{
226 u64 imm, imm_mask; 162 u64 imm, imm_mask;
227 s64 sval; 163 s64 sval;
228 u32 insn = *(u32 *)place; 164 u32 insn = le32_to_cpu(*(u32 *)place);
229 165
230 /* Calculate the relocation value. */ 166 /* Calculate the relocation value. */
231 sval = do_reloc(op, place, val); 167 sval = do_reloc(op, place, val);
@@ -236,7 +172,8 @@ static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val,
236 imm = sval & imm_mask; 172 imm = sval & imm_mask;
237 173
238 /* Update the instruction's immediate field. */ 174 /* Update the instruction's immediate field. */
239 *(u32 *)place = encode_insn_immediate(imm_type, insn, imm); 175 insn = aarch64_insn_encode_immediate(imm_type, insn, imm);
176 *(u32 *)place = cpu_to_le32(insn);
240 177
241 /* 178 /*
242 * Extract the upper value bits (including the sign bit) and 179 * Extract the upper value bits (including the sign bit) and
@@ -318,125 +255,125 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
318 overflow_check = false; 255 overflow_check = false;
319 case R_AARCH64_MOVW_UABS_G0: 256 case R_AARCH64_MOVW_UABS_G0:
320 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 257 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
321 INSN_IMM_16); 258 AARCH64_INSN_IMM_16);
322 break; 259 break;
323 case R_AARCH64_MOVW_UABS_G1_NC: 260 case R_AARCH64_MOVW_UABS_G1_NC:
324 overflow_check = false; 261 overflow_check = false;
325 case R_AARCH64_MOVW_UABS_G1: 262 case R_AARCH64_MOVW_UABS_G1:
326 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 263 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
327 INSN_IMM_16); 264 AARCH64_INSN_IMM_16);
328 break; 265 break;
329 case R_AARCH64_MOVW_UABS_G2_NC: 266 case R_AARCH64_MOVW_UABS_G2_NC:
330 overflow_check = false; 267 overflow_check = false;
331 case R_AARCH64_MOVW_UABS_G2: 268 case R_AARCH64_MOVW_UABS_G2:
332 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 269 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
333 INSN_IMM_16); 270 AARCH64_INSN_IMM_16);
334 break; 271 break;
335 case R_AARCH64_MOVW_UABS_G3: 272 case R_AARCH64_MOVW_UABS_G3:
336 /* We're using the top bits so we can't overflow. */ 273 /* We're using the top bits so we can't overflow. */
337 overflow_check = false; 274 overflow_check = false;
338 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, 275 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48,
339 INSN_IMM_16); 276 AARCH64_INSN_IMM_16);
340 break; 277 break;
341 case R_AARCH64_MOVW_SABS_G0: 278 case R_AARCH64_MOVW_SABS_G0:
342 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 279 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
343 INSN_IMM_MOVNZ); 280 AARCH64_INSN_IMM_MOVNZ);
344 break; 281 break;
345 case R_AARCH64_MOVW_SABS_G1: 282 case R_AARCH64_MOVW_SABS_G1:
346 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 283 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
347 INSN_IMM_MOVNZ); 284 AARCH64_INSN_IMM_MOVNZ);
348 break; 285 break;
349 case R_AARCH64_MOVW_SABS_G2: 286 case R_AARCH64_MOVW_SABS_G2:
350 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 287 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
351 INSN_IMM_MOVNZ); 288 AARCH64_INSN_IMM_MOVNZ);
352 break; 289 break;
353 case R_AARCH64_MOVW_PREL_G0_NC: 290 case R_AARCH64_MOVW_PREL_G0_NC:
354 overflow_check = false; 291 overflow_check = false;
355 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 292 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
356 INSN_IMM_MOVK); 293 AARCH64_INSN_IMM_MOVK);
357 break; 294 break;
358 case R_AARCH64_MOVW_PREL_G0: 295 case R_AARCH64_MOVW_PREL_G0:
359 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 296 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
360 INSN_IMM_MOVNZ); 297 AARCH64_INSN_IMM_MOVNZ);
361 break; 298 break;
362 case R_AARCH64_MOVW_PREL_G1_NC: 299 case R_AARCH64_MOVW_PREL_G1_NC:
363 overflow_check = false; 300 overflow_check = false;
364 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 301 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
365 INSN_IMM_MOVK); 302 AARCH64_INSN_IMM_MOVK);
366 break; 303 break;
367 case R_AARCH64_MOVW_PREL_G1: 304 case R_AARCH64_MOVW_PREL_G1:
368 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 305 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
369 INSN_IMM_MOVNZ); 306 AARCH64_INSN_IMM_MOVNZ);
370 break; 307 break;
371 case R_AARCH64_MOVW_PREL_G2_NC: 308 case R_AARCH64_MOVW_PREL_G2_NC:
372 overflow_check = false; 309 overflow_check = false;
373 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 310 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
374 INSN_IMM_MOVK); 311 AARCH64_INSN_IMM_MOVK);
375 break; 312 break;
376 case R_AARCH64_MOVW_PREL_G2: 313 case R_AARCH64_MOVW_PREL_G2:
377 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 314 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
378 INSN_IMM_MOVNZ); 315 AARCH64_INSN_IMM_MOVNZ);
379 break; 316 break;
380 case R_AARCH64_MOVW_PREL_G3: 317 case R_AARCH64_MOVW_PREL_G3:
381 /* We're using the top bits so we can't overflow. */ 318 /* We're using the top bits so we can't overflow. */
382 overflow_check = false; 319 overflow_check = false;
383 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48, 320 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48,
384 INSN_IMM_MOVNZ); 321 AARCH64_INSN_IMM_MOVNZ);
385 break; 322 break;
386 323
387 /* Immediate instruction relocations. */ 324 /* Immediate instruction relocations. */
388 case R_AARCH64_LD_PREL_LO19: 325 case R_AARCH64_LD_PREL_LO19:
389 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 326 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
390 INSN_IMM_19); 327 AARCH64_INSN_IMM_19);
391 break; 328 break;
392 case R_AARCH64_ADR_PREL_LO21: 329 case R_AARCH64_ADR_PREL_LO21:
393 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21, 330 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21,
394 INSN_IMM_ADR); 331 AARCH64_INSN_IMM_ADR);
395 break; 332 break;
396 case R_AARCH64_ADR_PREL_PG_HI21_NC: 333 case R_AARCH64_ADR_PREL_PG_HI21_NC:
397 overflow_check = false; 334 overflow_check = false;
398 case R_AARCH64_ADR_PREL_PG_HI21: 335 case R_AARCH64_ADR_PREL_PG_HI21:
399 ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21, 336 ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21,
400 INSN_IMM_ADR); 337 AARCH64_INSN_IMM_ADR);
401 break; 338 break;
402 case R_AARCH64_ADD_ABS_LO12_NC: 339 case R_AARCH64_ADD_ABS_LO12_NC:
403 case R_AARCH64_LDST8_ABS_LO12_NC: 340 case R_AARCH64_LDST8_ABS_LO12_NC:
404 overflow_check = false; 341 overflow_check = false;
405 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12, 342 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12,
406 INSN_IMM_12); 343 AARCH64_INSN_IMM_12);
407 break; 344 break;
408 case R_AARCH64_LDST16_ABS_LO12_NC: 345 case R_AARCH64_LDST16_ABS_LO12_NC:
409 overflow_check = false; 346 overflow_check = false;
410 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11, 347 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11,
411 INSN_IMM_12); 348 AARCH64_INSN_IMM_12);
412 break; 349 break;
413 case R_AARCH64_LDST32_ABS_LO12_NC: 350 case R_AARCH64_LDST32_ABS_LO12_NC:
414 overflow_check = false; 351 overflow_check = false;
415 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10, 352 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10,
416 INSN_IMM_12); 353 AARCH64_INSN_IMM_12);
417 break; 354 break;
418 case R_AARCH64_LDST64_ABS_LO12_NC: 355 case R_AARCH64_LDST64_ABS_LO12_NC:
419 overflow_check = false; 356 overflow_check = false;
420 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, 357 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9,
421 INSN_IMM_12); 358 AARCH64_INSN_IMM_12);
422 break; 359 break;
423 case R_AARCH64_LDST128_ABS_LO12_NC: 360 case R_AARCH64_LDST128_ABS_LO12_NC:
424 overflow_check = false; 361 overflow_check = false;
425 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8, 362 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8,
426 INSN_IMM_12); 363 AARCH64_INSN_IMM_12);
427 break; 364 break;
428 case R_AARCH64_TSTBR14: 365 case R_AARCH64_TSTBR14:
429 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14, 366 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14,
430 INSN_IMM_14); 367 AARCH64_INSN_IMM_14);
431 break; 368 break;
432 case R_AARCH64_CONDBR19: 369 case R_AARCH64_CONDBR19:
433 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 370 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
434 INSN_IMM_19); 371 AARCH64_INSN_IMM_19);
435 break; 372 break;
436 case R_AARCH64_JUMP26: 373 case R_AARCH64_JUMP26:
437 case R_AARCH64_CALL26: 374 case R_AARCH64_CALL26:
438 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, 375 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26,
439 INSN_IMM_26); 376 AARCH64_INSN_IMM_26);
440 break; 377 break;
441 378
442 default: 379 default:
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 0e63c98d224c..5b1cd792274a 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -22,6 +22,7 @@
22 22
23#include <linux/bitmap.h> 23#include <linux/bitmap.h>
24#include <linux/interrupt.h> 24#include <linux/interrupt.h>
25#include <linux/irq.h>
25#include <linux/kernel.h> 26#include <linux/kernel.h>
26#include <linux/export.h> 27#include <linux/export.h>
27#include <linux/perf_event.h> 28#include <linux/perf_event.h>
@@ -363,26 +364,53 @@ validate_group(struct perf_event *event)
363} 364}
364 365
365static void 366static void
367armpmu_disable_percpu_irq(void *data)
368{
369 unsigned int irq = *(unsigned int *)data;
370 disable_percpu_irq(irq);
371}
372
373static void
366armpmu_release_hardware(struct arm_pmu *armpmu) 374armpmu_release_hardware(struct arm_pmu *armpmu)
367{ 375{
368 int i, irq, irqs; 376 int irq;
377 unsigned int i, irqs;
369 struct platform_device *pmu_device = armpmu->plat_device; 378 struct platform_device *pmu_device = armpmu->plat_device;
370 379
371 irqs = min(pmu_device->num_resources, num_possible_cpus()); 380 irqs = min(pmu_device->num_resources, num_possible_cpus());
381 if (!irqs)
382 return;
372 383
373 for (i = 0; i < irqs; ++i) { 384 irq = platform_get_irq(pmu_device, 0);
374 if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs)) 385 if (irq <= 0)
375 continue; 386 return;
376 irq = platform_get_irq(pmu_device, i); 387
377 if (irq >= 0) 388 if (irq_is_percpu(irq)) {
378 free_irq(irq, armpmu); 389 on_each_cpu(armpmu_disable_percpu_irq, &irq, 1);
390 free_percpu_irq(irq, &cpu_hw_events);
391 } else {
392 for (i = 0; i < irqs; ++i) {
393 if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs))
394 continue;
395 irq = platform_get_irq(pmu_device, i);
396 if (irq > 0)
397 free_irq(irq, armpmu);
398 }
379 } 399 }
380} 400}
381 401
402static void
403armpmu_enable_percpu_irq(void *data)
404{
405 unsigned int irq = *(unsigned int *)data;
406 enable_percpu_irq(irq, IRQ_TYPE_NONE);
407}
408
382static int 409static int
383armpmu_reserve_hardware(struct arm_pmu *armpmu) 410armpmu_reserve_hardware(struct arm_pmu *armpmu)
384{ 411{
385 int i, err, irq, irqs; 412 int err, irq;
413 unsigned int i, irqs;
386 struct platform_device *pmu_device = armpmu->plat_device; 414 struct platform_device *pmu_device = armpmu->plat_device;
387 415
388 if (!pmu_device) { 416 if (!pmu_device) {
@@ -391,39 +419,59 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu)
391 } 419 }
392 420
393 irqs = min(pmu_device->num_resources, num_possible_cpus()); 421 irqs = min(pmu_device->num_resources, num_possible_cpus());
394 if (irqs < 1) { 422 if (!irqs) {
395 pr_err("no irqs for PMUs defined\n"); 423 pr_err("no irqs for PMUs defined\n");
396 return -ENODEV; 424 return -ENODEV;
397 } 425 }
398 426
399 for (i = 0; i < irqs; ++i) { 427 irq = platform_get_irq(pmu_device, 0);
400 err = 0; 428 if (irq <= 0) {
401 irq = platform_get_irq(pmu_device, i); 429 pr_err("failed to get valid irq for PMU device\n");
402 if (irq < 0) 430 return -ENODEV;
403 continue; 431 }
404 432
405 /* 433 if (irq_is_percpu(irq)) {
406 * If we have a single PMU interrupt that we can't shift, 434 err = request_percpu_irq(irq, armpmu->handle_irq,
407 * assume that we're running on a uniprocessor machine and 435 "arm-pmu", &cpu_hw_events);
408 * continue. Otherwise, continue without this interrupt.
409 */
410 if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) {
411 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
412 irq, i);
413 continue;
414 }
415 436
416 err = request_irq(irq, armpmu->handle_irq,
417 IRQF_NOBALANCING,
418 "arm-pmu", armpmu);
419 if (err) { 437 if (err) {
420 pr_err("unable to request IRQ%d for ARM PMU counters\n", 438 pr_err("unable to request percpu IRQ%d for ARM PMU counters\n",
421 irq); 439 irq);
422 armpmu_release_hardware(armpmu); 440 armpmu_release_hardware(armpmu);
423 return err; 441 return err;
424 } 442 }
425 443
426 cpumask_set_cpu(i, &armpmu->active_irqs); 444 on_each_cpu(armpmu_enable_percpu_irq, &irq, 1);
445 } else {
446 for (i = 0; i < irqs; ++i) {
447 err = 0;
448 irq = platform_get_irq(pmu_device, i);
449 if (irq <= 0)
450 continue;
451
452 /*
453 * If we have a single PMU interrupt that we can't shift,
454 * assume that we're running on a uniprocessor machine and
455 * continue. Otherwise, continue without this interrupt.
456 */
457 if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) {
458 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
459 irq, i);
460 continue;
461 }
462
463 err = request_irq(irq, armpmu->handle_irq,
464 IRQF_NOBALANCING,
465 "arm-pmu", armpmu);
466 if (err) {
467 pr_err("unable to request IRQ%d for ARM PMU counters\n",
468 irq);
469 armpmu_release_hardware(armpmu);
470 return err;
471 }
472
473 cpumask_set_cpu(i, &armpmu->active_irqs);
474 }
427 } 475 }
428 476
429 return 0; 477 return 0;
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index de17c89985db..248a15db37f2 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -33,6 +33,7 @@
33#include <linux/kallsyms.h> 33#include <linux/kallsyms.h>
34#include <linux/init.h> 34#include <linux/init.h>
35#include <linux/cpu.h> 35#include <linux/cpu.h>
36#include <linux/cpuidle.h>
36#include <linux/elfcore.h> 37#include <linux/elfcore.h>
37#include <linux/pm.h> 38#include <linux/pm.h>
38#include <linux/tick.h> 39#include <linux/tick.h>
@@ -98,8 +99,10 @@ void arch_cpu_idle(void)
98 * This should do all the clock switching and wait for interrupt 99 * This should do all the clock switching and wait for interrupt
99 * tricks 100 * tricks
100 */ 101 */
101 cpu_do_idle(); 102 if (cpuidle_idle_call()) {
102 local_irq_enable(); 103 cpu_do_idle();
104 local_irq_enable();
105 }
103} 106}
104 107
105#ifdef CONFIG_HOTPLUG_CPU 108#ifdef CONFIG_HOTPLUG_CPU
@@ -308,6 +311,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
308unsigned long get_wchan(struct task_struct *p) 311unsigned long get_wchan(struct task_struct *p)
309{ 312{
310 struct stackframe frame; 313 struct stackframe frame;
314 unsigned long stack_page;
311 int count = 0; 315 int count = 0;
312 if (!p || p == current || p->state == TASK_RUNNING) 316 if (!p || p == current || p->state == TASK_RUNNING)
313 return 0; 317 return 0;
@@ -315,9 +319,11 @@ unsigned long get_wchan(struct task_struct *p)
315 frame.fp = thread_saved_fp(p); 319 frame.fp = thread_saved_fp(p);
316 frame.sp = thread_saved_sp(p); 320 frame.sp = thread_saved_sp(p);
317 frame.pc = thread_saved_pc(p); 321 frame.pc = thread_saved_pc(p);
322 stack_page = (unsigned long)task_stack_page(p);
318 do { 323 do {
319 int ret = unwind_frame(&frame); 324 if (frame.sp < stack_page ||
320 if (ret < 0) 325 frame.sp >= stack_page + THREAD_SIZE ||
326 unwind_frame(&frame))
321 return 0; 327 return 0;
322 if (!in_sched_functions(frame.pc)) 328 if (!in_sched_functions(frame.pc))
323 return frame.pc; 329 return frame.pc;
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index bd9bbd0e44ed..c8e9effe52e1 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -108,20 +108,95 @@ void __init early_print(const char *str, ...)
108 printk("%s", buf); 108 printk("%s", buf);
109} 109}
110 110
111void __init smp_setup_processor_id(void)
112{
113 /*
114 * clear __my_cpu_offset on boot CPU to avoid hang caused by
115 * using percpu variable early, for example, lockdep will
116 * access percpu variable inside lock_release
117 */
118 set_my_cpu_offset(0);
119}
120
111bool arch_match_cpu_phys_id(int cpu, u64 phys_id) 121bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
112{ 122{
113 return phys_id == cpu_logical_map(cpu); 123 return phys_id == cpu_logical_map(cpu);
114} 124}
115 125
126struct mpidr_hash mpidr_hash;
127#ifdef CONFIG_SMP
128/**
129 * smp_build_mpidr_hash - Pre-compute shifts required at each affinity
130 * level in order to build a linear index from an
131 * MPIDR value. Resulting algorithm is a collision
132 * free hash carried out through shifting and ORing
133 */
134static void __init smp_build_mpidr_hash(void)
135{
136 u32 i, affinity, fs[4], bits[4], ls;
137 u64 mask = 0;
138 /*
139 * Pre-scan the list of MPIDRS and filter out bits that do
140 * not contribute to affinity levels, ie they never toggle.
141 */
142 for_each_possible_cpu(i)
143 mask |= (cpu_logical_map(i) ^ cpu_logical_map(0));
144 pr_debug("mask of set bits %#llx\n", mask);
145 /*
146 * Find and stash the last and first bit set at all affinity levels to
147 * check how many bits are required to represent them.
148 */
149 for (i = 0; i < 4; i++) {
150 affinity = MPIDR_AFFINITY_LEVEL(mask, i);
151 /*
152 * Find the MSB bit and LSB bits position
153 * to determine how many bits are required
154 * to express the affinity level.
155 */
156 ls = fls(affinity);
157 fs[i] = affinity ? ffs(affinity) - 1 : 0;
158 bits[i] = ls - fs[i];
159 }
160 /*
161 * An index can be created from the MPIDR_EL1 by isolating the
162 * significant bits at each affinity level and by shifting
163 * them in order to compress the 32 bits values space to a
164 * compressed set of values. This is equivalent to hashing
165 * the MPIDR_EL1 through shifting and ORing. It is a collision free
166 * hash though not minimal since some levels might contain a number
167 * of CPUs that is not an exact power of 2 and their bit
168 * representation might contain holes, eg MPIDR_EL1[7:0] = {0x2, 0x80}.
169 */
170 mpidr_hash.shift_aff[0] = MPIDR_LEVEL_SHIFT(0) + fs[0];
171 mpidr_hash.shift_aff[1] = MPIDR_LEVEL_SHIFT(1) + fs[1] - bits[0];
172 mpidr_hash.shift_aff[2] = MPIDR_LEVEL_SHIFT(2) + fs[2] -
173 (bits[1] + bits[0]);
174 mpidr_hash.shift_aff[3] = MPIDR_LEVEL_SHIFT(3) +
175 fs[3] - (bits[2] + bits[1] + bits[0]);
176 mpidr_hash.mask = mask;
177 mpidr_hash.bits = bits[3] + bits[2] + bits[1] + bits[0];
178 pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] aff3[%u] mask[%#llx] bits[%u]\n",
179 mpidr_hash.shift_aff[0],
180 mpidr_hash.shift_aff[1],
181 mpidr_hash.shift_aff[2],
182 mpidr_hash.shift_aff[3],
183 mpidr_hash.mask,
184 mpidr_hash.bits);
185 /*
186 * 4x is an arbitrary value used to warn on a hash table much bigger
187 * than expected on most systems.
188 */
189 if (mpidr_hash_size() > 4 * num_possible_cpus())
190 pr_warn("Large number of MPIDR hash buckets detected\n");
191 __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
192}
193#endif
194
116static void __init setup_processor(void) 195static void __init setup_processor(void)
117{ 196{
118 struct cpu_info *cpu_info; 197 struct cpu_info *cpu_info;
198 u64 features, block;
119 199
120 /*
121 * locate processor in the list of supported processor
122 * types. The linker builds this table for us from the
123 * entries in arch/arm/mm/proc.S
124 */
125 cpu_info = lookup_processor_type(read_cpuid_id()); 200 cpu_info = lookup_processor_type(read_cpuid_id());
126 if (!cpu_info) { 201 if (!cpu_info) {
127 printk("CPU configuration botched (ID %08x), unable to continue.\n", 202 printk("CPU configuration botched (ID %08x), unable to continue.\n",
@@ -136,6 +211,37 @@ static void __init setup_processor(void)
136 211
137 sprintf(init_utsname()->machine, ELF_PLATFORM); 212 sprintf(init_utsname()->machine, ELF_PLATFORM);
138 elf_hwcap = 0; 213 elf_hwcap = 0;
214
215 /*
216 * ID_AA64ISAR0_EL1 contains 4-bit wide signed feature blocks.
217 * The blocks we test below represent incremental functionality
218 * for non-negative values. Negative values are reserved.
219 */
220 features = read_cpuid(ID_AA64ISAR0_EL1);
221 block = (features >> 4) & 0xf;
222 if (!(block & 0x8)) {
223 switch (block) {
224 default:
225 case 2:
226 elf_hwcap |= HWCAP_PMULL;
227 case 1:
228 elf_hwcap |= HWCAP_AES;
229 case 0:
230 break;
231 }
232 }
233
234 block = (features >> 8) & 0xf;
235 if (block && !(block & 0x8))
236 elf_hwcap |= HWCAP_SHA1;
237
238 block = (features >> 12) & 0xf;
239 if (block && !(block & 0x8))
240 elf_hwcap |= HWCAP_SHA2;
241
242 block = (features >> 16) & 0xf;
243 if (block && !(block & 0x8))
244 elf_hwcap |= HWCAP_CRC32;
139} 245}
140 246
141static void __init setup_machine_fdt(phys_addr_t dt_phys) 247static void __init setup_machine_fdt(phys_addr_t dt_phys)
@@ -236,6 +342,7 @@ void __init setup_arch(char **cmdline_p)
236 cpu_read_bootcpu_ops(); 342 cpu_read_bootcpu_ops();
237#ifdef CONFIG_SMP 343#ifdef CONFIG_SMP
238 smp_init_cpus(); 344 smp_init_cpus();
345 smp_build_mpidr_hash();
239#endif 346#endif
240 347
241#ifdef CONFIG_VT 348#ifdef CONFIG_VT
@@ -275,6 +382,11 @@ static const char *hwcap_str[] = {
275 "fp", 382 "fp",
276 "asimd", 383 "asimd",
277 "evtstrm", 384 "evtstrm",
385 "aes",
386 "pmull",
387 "sha1",
388 "sha2",
389 "crc32",
278 NULL 390 NULL
279}; 391};
280 392
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
new file mode 100644
index 000000000000..b1925729c692
--- /dev/null
+++ b/arch/arm64/kernel/sleep.S
@@ -0,0 +1,184 @@
1#include <linux/errno.h>
2#include <linux/linkage.h>
3#include <asm/asm-offsets.h>
4#include <asm/assembler.h>
5
6 .text
7/*
8 * Implementation of MPIDR_EL1 hash algorithm through shifting
9 * and OR'ing.
10 *
11 * @dst: register containing hash result
12 * @rs0: register containing affinity level 0 bit shift
13 * @rs1: register containing affinity level 1 bit shift
14 * @rs2: register containing affinity level 2 bit shift
15 * @rs3: register containing affinity level 3 bit shift
16 * @mpidr: register containing MPIDR_EL1 value
17 * @mask: register containing MPIDR mask
18 *
19 * Pseudo C-code:
20 *
21 *u32 dst;
22 *
23 *compute_mpidr_hash(u32 rs0, u32 rs1, u32 rs2, u32 rs3, u64 mpidr, u64 mask) {
24 * u32 aff0, aff1, aff2, aff3;
25 * u64 mpidr_masked = mpidr & mask;
26 * aff0 = mpidr_masked & 0xff;
27 * aff1 = mpidr_masked & 0xff00;
28 * aff2 = mpidr_masked & 0xff0000;
29 * aff2 = mpidr_masked & 0xff00000000;
30 * dst = (aff0 >> rs0 | aff1 >> rs1 | aff2 >> rs2 | aff3 >> rs3);
31 *}
32 * Input registers: rs0, rs1, rs2, rs3, mpidr, mask
33 * Output register: dst
34 * Note: input and output registers must be disjoint register sets
35 (eg: a macro instance with mpidr = x1 and dst = x1 is invalid)
36 */
37 .macro compute_mpidr_hash dst, rs0, rs1, rs2, rs3, mpidr, mask
38 and \mpidr, \mpidr, \mask // mask out MPIDR bits
39 and \dst, \mpidr, #0xff // mask=aff0
40 lsr \dst ,\dst, \rs0 // dst=aff0>>rs0
41 and \mask, \mpidr, #0xff00 // mask = aff1
42 lsr \mask ,\mask, \rs1
43 orr \dst, \dst, \mask // dst|=(aff1>>rs1)
44 and \mask, \mpidr, #0xff0000 // mask = aff2
45 lsr \mask ,\mask, \rs2
46 orr \dst, \dst, \mask // dst|=(aff2>>rs2)
47 and \mask, \mpidr, #0xff00000000 // mask = aff3
48 lsr \mask ,\mask, \rs3
49 orr \dst, \dst, \mask // dst|=(aff3>>rs3)
50 .endm
51/*
52 * Save CPU state for a suspend. This saves callee registers, and allocates
53 * space on the kernel stack to save the CPU specific registers + some
54 * other data for resume.
55 *
56 * x0 = suspend finisher argument
57 */
58ENTRY(__cpu_suspend)
59 stp x29, lr, [sp, #-96]!
60 stp x19, x20, [sp,#16]
61 stp x21, x22, [sp,#32]
62 stp x23, x24, [sp,#48]
63 stp x25, x26, [sp,#64]
64 stp x27, x28, [sp,#80]
65 mov x2, sp
66 sub sp, sp, #CPU_SUSPEND_SZ // allocate cpu_suspend_ctx
67 mov x1, sp
68 /*
69 * x1 now points to struct cpu_suspend_ctx allocated on the stack
70 */
71 str x2, [x1, #CPU_CTX_SP]
72 ldr x2, =sleep_save_sp
73 ldr x2, [x2, #SLEEP_SAVE_SP_VIRT]
74#ifdef CONFIG_SMP
75 mrs x7, mpidr_el1
76 ldr x9, =mpidr_hash
77 ldr x10, [x9, #MPIDR_HASH_MASK]
78 /*
79 * Following code relies on the struct mpidr_hash
80 * members size.
81 */
82 ldp w3, w4, [x9, #MPIDR_HASH_SHIFTS]
83 ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
84 compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
85 add x2, x2, x8, lsl #3
86#endif
87 bl __cpu_suspend_finisher
88 /*
89 * Never gets here, unless suspend fails.
90 * Successful cpu_suspend should return from cpu_resume, returning
91 * through this code path is considered an error
92 * If the return value is set to 0 force x0 = -EOPNOTSUPP
93 * to make sure a proper error condition is propagated
94 */
95 cmp x0, #0
96 mov x3, #-EOPNOTSUPP
97 csel x0, x3, x0, eq
98 add sp, sp, #CPU_SUSPEND_SZ // rewind stack pointer
99 ldp x19, x20, [sp, #16]
100 ldp x21, x22, [sp, #32]
101 ldp x23, x24, [sp, #48]
102 ldp x25, x26, [sp, #64]
103 ldp x27, x28, [sp, #80]
104 ldp x29, lr, [sp], #96
105 ret
106ENDPROC(__cpu_suspend)
107 .ltorg
108
109/*
110 * x0 must contain the sctlr value retrieved from restored context
111 */
112ENTRY(cpu_resume_mmu)
113 ldr x3, =cpu_resume_after_mmu
114 msr sctlr_el1, x0 // restore sctlr_el1
115 isb
116 br x3 // global jump to virtual address
117ENDPROC(cpu_resume_mmu)
118cpu_resume_after_mmu:
119 mov x0, #0 // return zero on success
120 ldp x19, x20, [sp, #16]
121 ldp x21, x22, [sp, #32]
122 ldp x23, x24, [sp, #48]
123 ldp x25, x26, [sp, #64]
124 ldp x27, x28, [sp, #80]
125 ldp x29, lr, [sp], #96
126 ret
127ENDPROC(cpu_resume_after_mmu)
128
129 .data
130ENTRY(cpu_resume)
131 bl el2_setup // if in EL2 drop to EL1 cleanly
132#ifdef CONFIG_SMP
133 mrs x1, mpidr_el1
134 adr x4, mpidr_hash_ptr
135 ldr x5, [x4]
136 add x8, x4, x5 // x8 = struct mpidr_hash phys address
137 /* retrieve mpidr_hash members to compute the hash */
138 ldr x2, [x8, #MPIDR_HASH_MASK]
139 ldp w3, w4, [x8, #MPIDR_HASH_SHIFTS]
140 ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
141 compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
142 /* x7 contains hash index, let's use it to grab context pointer */
143#else
144 mov x7, xzr
145#endif
146 adr x0, sleep_save_sp
147 ldr x0, [x0, #SLEEP_SAVE_SP_PHYS]
148 ldr x0, [x0, x7, lsl #3]
149 /* load sp from context */
150 ldr x2, [x0, #CPU_CTX_SP]
151 adr x1, sleep_idmap_phys
152 /* load physical address of identity map page table in x1 */
153 ldr x1, [x1]
154 mov sp, x2
155 /*
156 * cpu_do_resume expects x0 to contain context physical address
157 * pointer and x1 to contain physical address of 1:1 page tables
158 */
159 bl cpu_do_resume // PC relative jump, MMU off
160 b cpu_resume_mmu // Resume MMU, never returns
161ENDPROC(cpu_resume)
162
163 .align 3
164mpidr_hash_ptr:
165 /*
166 * offset of mpidr_hash symbol from current location
167 * used to obtain run-time mpidr_hash address with MMU off
168 */
169 .quad mpidr_hash - .
170/*
171 * physical address of identity mapped page tables
172 */
173 .type sleep_idmap_phys, #object
174ENTRY(sleep_idmap_phys)
175 .quad 0
176/*
177 * struct sleep_save_sp {
178 * phys_addr_t *save_ptr_stash;
179 * phys_addr_t save_ptr_stash_phys;
180 * };
181 */
182 .type sleep_save_sp, #object
183ENTRY(sleep_save_sp)
184 .space SLEEP_SAVE_SP_SZ // struct sleep_save_sp
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index a0c2ca602cf8..1b7617ab499b 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -61,6 +61,7 @@ enum ipi_msg_type {
61 IPI_CALL_FUNC, 61 IPI_CALL_FUNC,
62 IPI_CALL_FUNC_SINGLE, 62 IPI_CALL_FUNC_SINGLE,
63 IPI_CPU_STOP, 63 IPI_CPU_STOP,
64 IPI_TIMER,
64}; 65};
65 66
66/* 67/*
@@ -122,8 +123,6 @@ asmlinkage void secondary_start_kernel(void)
122 struct mm_struct *mm = &init_mm; 123 struct mm_struct *mm = &init_mm;
123 unsigned int cpu = smp_processor_id(); 124 unsigned int cpu = smp_processor_id();
124 125
125 printk("CPU%u: Booted secondary processor\n", cpu);
126
127 /* 126 /*
128 * All kernel threads share the same mm context; grab a 127 * All kernel threads share the same mm context; grab a
129 * reference and switch to it. 128 * reference and switch to it.
@@ -132,6 +131,9 @@ asmlinkage void secondary_start_kernel(void)
132 current->active_mm = mm; 131 current->active_mm = mm;
133 cpumask_set_cpu(cpu, mm_cpumask(mm)); 132 cpumask_set_cpu(cpu, mm_cpumask(mm));
134 133
134 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
135 printk("CPU%u: Booted secondary processor\n", cpu);
136
135 /* 137 /*
136 * TTBR0 is only used for the identity mapping at this stage. Make it 138 * TTBR0 is only used for the identity mapping at this stage. Make it
137 * point to zero page to avoid speculatively fetching new entries. 139 * point to zero page to avoid speculatively fetching new entries.
@@ -271,6 +273,7 @@ void __init smp_cpus_done(unsigned int max_cpus)
271 273
272void __init smp_prepare_boot_cpu(void) 274void __init smp_prepare_boot_cpu(void)
273{ 275{
276 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
274} 277}
275 278
276static void (*smp_cross_call)(const struct cpumask *, unsigned int); 279static void (*smp_cross_call)(const struct cpumask *, unsigned int);
@@ -447,6 +450,7 @@ static const char *ipi_types[NR_IPI] = {
447 S(IPI_CALL_FUNC, "Function call interrupts"), 450 S(IPI_CALL_FUNC, "Function call interrupts"),
448 S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), 451 S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
449 S(IPI_CPU_STOP, "CPU stop interrupts"), 452 S(IPI_CPU_STOP, "CPU stop interrupts"),
453 S(IPI_TIMER, "Timer broadcast interrupts"),
450}; 454};
451 455
452void show_ipi_list(struct seq_file *p, int prec) 456void show_ipi_list(struct seq_file *p, int prec)
@@ -532,6 +536,14 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
532 irq_exit(); 536 irq_exit();
533 break; 537 break;
534 538
539#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
540 case IPI_TIMER:
541 irq_enter();
542 tick_receive_broadcast();
543 irq_exit();
544 break;
545#endif
546
535 default: 547 default:
536 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); 548 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
537 break; 549 break;
@@ -544,6 +556,13 @@ void smp_send_reschedule(int cpu)
544 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); 556 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
545} 557}
546 558
559#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
560void tick_broadcast(const struct cpumask *mask)
561{
562 smp_cross_call(mask, IPI_TIMER);
563}
564#endif
565
547void smp_send_stop(void) 566void smp_send_stop(void)
548{ 567{
549 unsigned long timeout; 568 unsigned long timeout;
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index d25459ff57fc..c3b6c63ea5fb 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -43,7 +43,7 @@ int unwind_frame(struct stackframe *frame)
43 low = frame->sp; 43 low = frame->sp;
44 high = ALIGN(low, THREAD_SIZE); 44 high = ALIGN(low, THREAD_SIZE);
45 45
46 if (fp < low || fp > high || fp & 0xf) 46 if (fp < low || fp > high - 0x18 || fp & 0xf)
47 return -EINVAL; 47 return -EINVAL;
48 48
49 frame->sp = fp + 0x10; 49 frame->sp = fp + 0x10;
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
new file mode 100644
index 000000000000..430344e2c989
--- /dev/null
+++ b/arch/arm64/kernel/suspend.c
@@ -0,0 +1,132 @@
1#include <linux/slab.h>
2#include <asm/cacheflush.h>
3#include <asm/cpu_ops.h>
4#include <asm/debug-monitors.h>
5#include <asm/pgtable.h>
6#include <asm/memory.h>
7#include <asm/smp_plat.h>
8#include <asm/suspend.h>
9#include <asm/tlbflush.h>
10
11extern int __cpu_suspend(unsigned long);
12/*
13 * This is called by __cpu_suspend() to save the state, and do whatever
14 * flushing is required to ensure that when the CPU goes to sleep we have
15 * the necessary data available when the caches are not searched.
16 *
17 * @arg: Argument to pass to suspend operations
18 * @ptr: CPU context virtual address
19 * @save_ptr: address of the location where the context physical address
20 * must be saved
21 */
22int __cpu_suspend_finisher(unsigned long arg, struct cpu_suspend_ctx *ptr,
23 phys_addr_t *save_ptr)
24{
25 int cpu = smp_processor_id();
26
27 *save_ptr = virt_to_phys(ptr);
28
29 cpu_do_suspend(ptr);
30 /*
31 * Only flush the context that must be retrieved with the MMU
32 * off. VA primitives ensure the flush is applied to all
33 * cache levels so context is pushed to DRAM.
34 */
35 __flush_dcache_area(ptr, sizeof(*ptr));
36 __flush_dcache_area(save_ptr, sizeof(*save_ptr));
37
38 return cpu_ops[cpu]->cpu_suspend(arg);
39}
40
41/*
42 * This hook is provided so that cpu_suspend code can restore HW
43 * breakpoints as early as possible in the resume path, before reenabling
44 * debug exceptions. Code cannot be run from a CPU PM notifier since by the
45 * time the notifier runs debug exceptions might have been enabled already,
46 * with HW breakpoints registers content still in an unknown state.
47 */
48void (*hw_breakpoint_restore)(void *);
49void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
50{
51 /* Prevent multiple restore hook initializations */
52 if (WARN_ON(hw_breakpoint_restore))
53 return;
54 hw_breakpoint_restore = hw_bp_restore;
55}
56
57/**
58 * cpu_suspend
59 *
60 * @arg: argument to pass to the finisher function
61 */
62int cpu_suspend(unsigned long arg)
63{
64 struct mm_struct *mm = current->active_mm;
65 int ret, cpu = smp_processor_id();
66 unsigned long flags;
67
68 /*
69 * If cpu_ops have not been registered or suspend
70 * has not been initialized, cpu_suspend call fails early.
71 */
72 if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
73 return -EOPNOTSUPP;
74
75 /*
76 * From this point debug exceptions are disabled to prevent
77 * updates to mdscr register (saved and restored along with
78 * general purpose registers) from kernel debuggers.
79 */
80 local_dbg_save(flags);
81
82 /*
83 * mm context saved on the stack, it will be restored when
84 * the cpu comes out of reset through the identity mapped
85 * page tables, so that the thread address space is properly
86 * set-up on function return.
87 */
88 ret = __cpu_suspend(arg);
89 if (ret == 0) {
90 cpu_switch_mm(mm->pgd, mm);
91 flush_tlb_all();
92 /*
93 * Restore HW breakpoint registers to sane values
94 * before debug exceptions are possibly reenabled
95 * through local_dbg_restore.
96 */
97 if (hw_breakpoint_restore)
98 hw_breakpoint_restore(NULL);
99 }
100
101 /*
102 * Restore pstate flags. OS lock and mdscr have been already
103 * restored, so from this point onwards, debugging is fully
104 * renabled if it was enabled when core started shutdown.
105 */
106 local_dbg_restore(flags);
107
108 return ret;
109}
110
111extern struct sleep_save_sp sleep_save_sp;
112extern phys_addr_t sleep_idmap_phys;
113
114static int cpu_suspend_init(void)
115{
116 void *ctx_ptr;
117
118 /* ctx_ptr is an array of physical addresses */
119 ctx_ptr = kcalloc(mpidr_hash_size(), sizeof(phys_addr_t), GFP_KERNEL);
120
121 if (WARN_ON(!ctx_ptr))
122 return -ENOMEM;
123
124 sleep_save_sp.save_ptr_stash = ctx_ptr;
125 sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr);
126 sleep_idmap_phys = virt_to_phys(idmap_pg_dir);
127 __flush_dcache_area(&sleep_save_sp, sizeof(struct sleep_save_sp));
128 __flush_dcache_area(&sleep_idmap_phys, sizeof(sleep_idmap_phys));
129
130 return 0;
131}
132early_initcall(cpu_suspend_init);
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 5161ad992091..4ba7a55b49c7 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -99,17 +99,14 @@ SECTIONS
99 99
100 . = ALIGN(PAGE_SIZE); 100 . = ALIGN(PAGE_SIZE);
101 _data = .; 101 _data = .;
102 __data_loc = _data - LOAD_OFFSET;
103 _sdata = .; 102 _sdata = .;
104 RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE) 103 RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE)
105 _edata = .; 104 _edata = .;
106 _edata_loc = __data_loc + SIZEOF(.data);
107 105
108 BSS_SECTION(0, 0, 0) 106 BSS_SECTION(0, 0, 0)
109 _end = .; 107 _end = .;
110 108
111 STABS_DEBUG 109 STABS_DEBUG
112 .comment 0 : { *(.comment) }
113} 110}
114 111
115/* 112/*
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 59acc0ef0462..328ce1a99daa 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -1,6 +1,4 @@
1lib-y := bitops.o delay.o \ 1lib-y := bitops.o clear_user.o delay.o copy_from_user.o \
2 strncpy_from_user.o strnlen_user.o clear_user.o \ 2 copy_to_user.o copy_in_user.o copy_page.o \
3 copy_from_user.o copy_to_user.o copy_in_user.o \ 3 clear_page.o memchr.o memcpy.o memmove.o memset.o \
4 copy_page.o clear_page.o \
5 memchr.o memcpy.o memmove.o memset.o \
6 strchr.o strrchr.o 4 strchr.o strrchr.o
diff --git a/arch/arm64/lib/strncpy_from_user.S b/arch/arm64/lib/strncpy_from_user.S
deleted file mode 100644
index 56e448a831a0..000000000000
--- a/arch/arm64/lib/strncpy_from_user.S
+++ /dev/null
@@ -1,50 +0,0 @@
1/*
2 * Based on arch/arm/lib/strncpy_from_user.S
3 *
4 * Copyright (C) 1995-2000 Russell King
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/linkage.h>
21#include <asm/assembler.h>
22#include <asm/errno.h>
23
24 .text
25 .align 5
26
27/*
28 * Copy a string from user space to kernel space.
29 * x0 = dst, x1 = src, x2 = byte length
30 * returns the number of characters copied (strlen of copied string),
31 * -EFAULT on exception, or "len" if we fill the whole buffer
32 */
33ENTRY(__strncpy_from_user)
34 mov x4, x1
351: subs x2, x2, #1
36 bmi 2f
37USER(9f, ldrb w3, [x1], #1 )
38 strb w3, [x0], #1
39 cbnz w3, 1b
40 sub x1, x1, #1 // take NUL character out of count
412: sub x0, x1, x4
42 ret
43ENDPROC(__strncpy_from_user)
44
45 .section .fixup,"ax"
46 .align 0
479: strb wzr, [x0] // null terminate
48 mov x0, #-EFAULT
49 ret
50 .previous
diff --git a/arch/arm64/lib/strnlen_user.S b/arch/arm64/lib/strnlen_user.S
deleted file mode 100644
index 7f7b176a5646..000000000000
--- a/arch/arm64/lib/strnlen_user.S
+++ /dev/null
@@ -1,47 +0,0 @@
1/*
2 * Based on arch/arm/lib/strnlen_user.S
3 *
4 * Copyright (C) 1995-2000 Russell King
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/linkage.h>
21#include <asm/assembler.h>
22#include <asm/errno.h>
23
24 .text
25 .align 5
26
27/* Prototype: unsigned long __strnlen_user(const char *str, long n)
28 * Purpose : get length of a string in user memory
29 * Params : str - address of string in user memory
30 * Returns : length of string *including terminator*
31 * or zero on exception, or n if too long
32 */
33ENTRY(__strnlen_user)
34 mov x2, x0
351: subs x1, x1, #1
36 b.mi 2f
37USER(9f, ldrb w3, [x0], #1 )
38 cbnz w3, 1b
392: sub x0, x0, x2
40 ret
41ENDPROC(__strnlen_user)
42
43 .section .fixup,"ax"
44 .align 0
459: mov x0, #0
46 ret
47 .previous
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 4bd7579ec9e6..45b5ab54c9ee 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -21,6 +21,7 @@
21#include <linux/export.h> 21#include <linux/export.h>
22#include <linux/slab.h> 22#include <linux/slab.h>
23#include <linux/dma-mapping.h> 23#include <linux/dma-mapping.h>
24#include <linux/dma-contiguous.h>
24#include <linux/vmalloc.h> 25#include <linux/vmalloc.h>
25#include <linux/swiotlb.h> 26#include <linux/swiotlb.h>
26 27
@@ -33,17 +34,47 @@ static void *arm64_swiotlb_alloc_coherent(struct device *dev, size_t size,
33 dma_addr_t *dma_handle, gfp_t flags, 34 dma_addr_t *dma_handle, gfp_t flags,
34 struct dma_attrs *attrs) 35 struct dma_attrs *attrs)
35{ 36{
37 if (dev == NULL) {
38 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
39 return NULL;
40 }
41
36 if (IS_ENABLED(CONFIG_ZONE_DMA32) && 42 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
37 dev->coherent_dma_mask <= DMA_BIT_MASK(32)) 43 dev->coherent_dma_mask <= DMA_BIT_MASK(32))
38 flags |= GFP_DMA32; 44 flags |= GFP_DMA32;
39 return swiotlb_alloc_coherent(dev, size, dma_handle, flags); 45 if (IS_ENABLED(CONFIG_DMA_CMA)) {
46 struct page *page;
47
48 page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
49 get_order(size));
50 if (!page)
51 return NULL;
52
53 *dma_handle = phys_to_dma(dev, page_to_phys(page));
54 return page_address(page);
55 } else {
56 return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
57 }
40} 58}
41 59
42static void arm64_swiotlb_free_coherent(struct device *dev, size_t size, 60static void arm64_swiotlb_free_coherent(struct device *dev, size_t size,
43 void *vaddr, dma_addr_t dma_handle, 61 void *vaddr, dma_addr_t dma_handle,
44 struct dma_attrs *attrs) 62 struct dma_attrs *attrs)
45{ 63{
46 swiotlb_free_coherent(dev, size, vaddr, dma_handle); 64 if (dev == NULL) {
65 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
66 return;
67 }
68
69 if (IS_ENABLED(CONFIG_DMA_CMA)) {
70 phys_addr_t paddr = dma_to_phys(dev, dma_handle);
71
72 dma_release_from_contiguous(dev,
73 phys_to_page(paddr),
74 size >> PAGE_SHIFT);
75 } else {
76 swiotlb_free_coherent(dev, size, vaddr, dma_handle);
77 }
47} 78}
48 79
49static struct dma_map_ops arm64_swiotlb_dma_ops = { 80static struct dma_map_ops arm64_swiotlb_dma_ops = {
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 0cb8742de4f2..d0b4c2efda90 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -30,6 +30,7 @@
30#include <linux/memblock.h> 30#include <linux/memblock.h>
31#include <linux/sort.h> 31#include <linux/sort.h>
32#include <linux/of_fdt.h> 32#include <linux/of_fdt.h>
33#include <linux/dma-contiguous.h>
33 34
34#include <asm/sections.h> 35#include <asm/sections.h>
35#include <asm/setup.h> 36#include <asm/setup.h>
@@ -159,6 +160,8 @@ void __init arm64_memblock_init(void)
159 memblock_reserve(base, size); 160 memblock_reserve(base, size);
160 } 161 }
161 162
163 dma_contiguous_reserve(0);
164
162 memblock_allow_resize(); 165 memblock_allow_resize();
163 memblock_dump_all(); 166 memblock_dump_all();
164} 167}
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 0f7fec52c7f8..bed1f1de1caf 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -80,6 +80,75 @@ ENTRY(cpu_do_idle)
80 ret 80 ret
81ENDPROC(cpu_do_idle) 81ENDPROC(cpu_do_idle)
82 82
83#ifdef CONFIG_ARM64_CPU_SUSPEND
84/**
85 * cpu_do_suspend - save CPU registers context
86 *
87 * x0: virtual address of context pointer
88 */
89ENTRY(cpu_do_suspend)
90 mrs x2, tpidr_el0
91 mrs x3, tpidrro_el0
92 mrs x4, contextidr_el1
93 mrs x5, mair_el1
94 mrs x6, cpacr_el1
95 mrs x7, ttbr1_el1
96 mrs x8, tcr_el1
97 mrs x9, vbar_el1
98 mrs x10, mdscr_el1
99 mrs x11, oslsr_el1
100 mrs x12, sctlr_el1
101 stp x2, x3, [x0]
102 stp x4, x5, [x0, #16]
103 stp x6, x7, [x0, #32]
104 stp x8, x9, [x0, #48]
105 stp x10, x11, [x0, #64]
106 str x12, [x0, #80]
107 ret
108ENDPROC(cpu_do_suspend)
109
110/**
111 * cpu_do_resume - restore CPU register context
112 *
113 * x0: Physical address of context pointer
114 * x1: ttbr0_el1 to be restored
115 *
116 * Returns:
117 * sctlr_el1 value in x0
118 */
119ENTRY(cpu_do_resume)
120 /*
121 * Invalidate local tlb entries before turning on MMU
122 */
123 tlbi vmalle1
124 ldp x2, x3, [x0]
125 ldp x4, x5, [x0, #16]
126 ldp x6, x7, [x0, #32]
127 ldp x8, x9, [x0, #48]
128 ldp x10, x11, [x0, #64]
129 ldr x12, [x0, #80]
130 msr tpidr_el0, x2
131 msr tpidrro_el0, x3
132 msr contextidr_el1, x4
133 msr mair_el1, x5
134 msr cpacr_el1, x6
135 msr ttbr0_el1, x1
136 msr ttbr1_el1, x7
137 msr tcr_el1, x8
138 msr vbar_el1, x9
139 msr mdscr_el1, x10
140 /*
141 * Restore oslsr_el1 by writing oslar_el1
142 */
143 ubfx x11, x11, #1, #1
144 msr oslar_el1, x11
145 mov x0, x12
146 dsb nsh // Make sure local tlb invalidation completed
147 isb
148 ret
149ENDPROC(cpu_do_resume)
150#endif
151
83/* 152/*
84 * cpu_switch_mm(pgd_phys, tsk) 153 * cpu_switch_mm(pgd_phys, tsk)
85 * 154 *
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-24 12:36:08 -0500