aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-04-16 14:58:29 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2015-04-16 14:58:29 -0400
commit714d8e7e27197dd39b2550e762a6a6fcf397a471 (patch)
treebc989a2a0e14f21912943e56d0002a26a2b7793e /arch/arm64/kernel
parentd19d5efd8c8840aa4f38a6dfbfe500d8cc27de46 (diff)
parent6d1966dfd6e0ad2f8aa4b664ae1a62e33abe1998 (diff)
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 updates from Will Deacon: "Here are the core arm64 updates for 4.1. Highlights include a significant rework to head.S (allowing us to boot on machines with physical memory at a really high address), an AES performance boost on Cortex-A57 and the ability to run a 32-bit userspace with 64k pages (although this requires said userspace to be built with a recent binutils). The head.S rework spilt over into KVM, so there are some changes under arch/arm/ which have been acked by Marc Zyngier (KVM co-maintainer). In particular, the linker script changes caused us some issues in -next, so there are a few merge commits where we had to apply fixes on top of a stable branch. Other changes include: - AES performance boost for Cortex-A57 - AArch32 (compat) userspace with 64k pages - Cortex-A53 erratum workaround for #845719 - defconfig updates (new platforms, PCI, ...)" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (39 commits) arm64: fix midr range for Cortex-A57 erratum 832075 arm64: errata: add workaround for cortex-a53 erratum #845719 arm64: Use bool function return values of true/false not 1/0 arm64: defconfig: updates for 4.1 arm64: Extract feature parsing code from cpu_errata.c arm64: alternative: Allow immediate branch as alternative instruction arm64: insn: Add aarch64_insn_decode_immediate ARM: kvm: round HYP section to page size instead of log2 upper bound ARM: kvm: assert on HYP section boundaries not actual code size arm64: head.S: ensure idmap_t0sz is visible arm64: pmu: add support for interrupt-affinity property dt: pmu: extend ARM PMU binding to allow for explicit interrupt affinity arm64: head.S: ensure visibility of page tables arm64: KVM: use ID map with increased VA range if required arm64: mm: increase VA range of identity map ARM: kvm: implement replacement for ld's LOG2CEIL() arm64: proc: remove unused cpu_get_pgd macro arm64: enforce x1|x2|x3 == 0 upon kernel entry as per boot protocol arm64: remove __calc_phys_offset arm64: merge __enable_mmu and __turn_mmu_on ...
Diffstat (limited to 'arch/arm64/kernel')
-rw-r--r--arch/arm64/kernel/Makefile4
-rw-r--r--arch/arm64/kernel/alternative.c55
-rw-r--r--arch/arm64/kernel/asm-offsets.c4
-rw-r--r--arch/arm64/kernel/cpu_errata.c47
-rw-r--r--arch/arm64/kernel/cpufeature.c47
-rw-r--r--arch/arm64/kernel/cpuinfo.c1
-rw-r--r--arch/arm64/kernel/cputable.c33
-rw-r--r--arch/arm64/kernel/entry.S20
-rw-r--r--arch/arm64/kernel/entry32.S18
-rw-r--r--arch/arm64/kernel/head.S261
-rw-r--r--arch/arm64/kernel/insn.c81
-rw-r--r--arch/arm64/kernel/perf_event.c78
-rw-r--r--arch/arm64/kernel/setup.c55
-rw-r--r--arch/arm64/kernel/smp.c3
-rw-r--r--arch/arm64/kernel/sys32.c1
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S17
16 files changed, 432 insertions, 293 deletions
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 5ee07eee80c2..b12e15b80516 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -12,12 +12,12 @@ CFLAGS_REMOVE_insn.o = -pg
12CFLAGS_REMOVE_return_address.o = -pg 12CFLAGS_REMOVE_return_address.o = -pg
13 13
14# Object file lists. 14# Object file lists.
15arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ 15arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
16 entry-fpsimd.o process.o ptrace.o setup.o signal.o \ 16 entry-fpsimd.o process.o ptrace.o setup.o signal.o \
17 sys.o stacktrace.o time.o traps.o io.o vdso.o \ 17 sys.o stacktrace.o time.o traps.o io.o vdso.o \
18 hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \ 18 hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \
19 return_address.o cpuinfo.o cpu_errata.o \ 19 return_address.o cpuinfo.o cpu_errata.o \
20 alternative.o cacheinfo.o 20 cpufeature.o alternative.o cacheinfo.o
21 21
22arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ 22arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
23 sys_compat.o entry32.o \ 23 sys_compat.o entry32.o \
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index ad7821d64a1d..21033bba9390 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -24,6 +24,7 @@
24#include <asm/cacheflush.h> 24#include <asm/cacheflush.h>
25#include <asm/alternative.h> 25#include <asm/alternative.h>
26#include <asm/cpufeature.h> 26#include <asm/cpufeature.h>
27#include <asm/insn.h>
27#include <linux/stop_machine.h> 28#include <linux/stop_machine.h>
28 29
29extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 30extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
@@ -33,6 +34,48 @@ struct alt_region {
33 struct alt_instr *end; 34 struct alt_instr *end;
34}; 35};
35 36
37/*
38 * Decode the imm field of a b/bl instruction, and return the byte
39 * offset as a signed value (so it can be used when computing a new
40 * branch target).
41 */
42static s32 get_branch_offset(u32 insn)
43{
44 s32 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
45
46 /* sign-extend the immediate before turning it into a byte offset */
47 return (imm << 6) >> 4;
48}
49
50static u32 get_alt_insn(u8 *insnptr, u8 *altinsnptr)
51{
52 u32 insn;
53
54 aarch64_insn_read(altinsnptr, &insn);
55
56 /* Stop the world on instructions we don't support... */
57 BUG_ON(aarch64_insn_is_cbz(insn));
58 BUG_ON(aarch64_insn_is_cbnz(insn));
59 BUG_ON(aarch64_insn_is_bcond(insn));
60 /* ... and there is probably more. */
61
62 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
63 enum aarch64_insn_branch_type type;
64 unsigned long target;
65
66 if (aarch64_insn_is_b(insn))
67 type = AARCH64_INSN_BRANCH_NOLINK;
68 else
69 type = AARCH64_INSN_BRANCH_LINK;
70
71 target = (unsigned long)altinsnptr + get_branch_offset(insn);
72 insn = aarch64_insn_gen_branch_imm((unsigned long)insnptr,
73 target, type);
74 }
75
76 return insn;
77}
78
36static int __apply_alternatives(void *alt_region) 79static int __apply_alternatives(void *alt_region)
37{ 80{
38 struct alt_instr *alt; 81 struct alt_instr *alt;
@@ -40,16 +83,24 @@ static int __apply_alternatives(void *alt_region)
40 u8 *origptr, *replptr; 83 u8 *origptr, *replptr;
41 84
42 for (alt = region->begin; alt < region->end; alt++) { 85 for (alt = region->begin; alt < region->end; alt++) {
86 u32 insn;
87 int i;
88
43 if (!cpus_have_cap(alt->cpufeature)) 89 if (!cpus_have_cap(alt->cpufeature))
44 continue; 90 continue;
45 91
46 BUG_ON(alt->alt_len > alt->orig_len); 92 BUG_ON(alt->alt_len != alt->orig_len);
47 93
48 pr_info_once("patching kernel code\n"); 94 pr_info_once("patching kernel code\n");
49 95
50 origptr = (u8 *)&alt->orig_offset + alt->orig_offset; 96 origptr = (u8 *)&alt->orig_offset + alt->orig_offset;
51 replptr = (u8 *)&alt->alt_offset + alt->alt_offset; 97 replptr = (u8 *)&alt->alt_offset + alt->alt_offset;
52 memcpy(origptr, replptr, alt->alt_len); 98
99 for (i = 0; i < alt->alt_len; i += sizeof(insn)) {
100 insn = get_alt_insn(origptr + i, replptr + i);
101 aarch64_insn_write(origptr + i, insn);
102 }
103
53 flush_icache_range((uintptr_t)origptr, 104 flush_icache_range((uintptr_t)origptr,
54 (uintptr_t)(origptr + alt->alt_len)); 105 (uintptr_t)(origptr + alt->alt_len));
55 } 106 }
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 56cadd3606bf..da675cc5dfae 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -24,7 +24,6 @@
24#include <linux/kvm_host.h> 24#include <linux/kvm_host.h>
25#include <asm/thread_info.h> 25#include <asm/thread_info.h>
26#include <asm/memory.h> 26#include <asm/memory.h>
27#include <asm/cputable.h>
28#include <asm/smp_plat.h> 27#include <asm/smp_plat.h>
29#include <asm/suspend.h> 28#include <asm/suspend.h>
30#include <asm/vdso_datapage.h> 29#include <asm/vdso_datapage.h>
@@ -70,9 +69,6 @@ int main(void)
70 BLANK(); 69 BLANK();
71 DEFINE(PAGE_SZ, PAGE_SIZE); 70 DEFINE(PAGE_SZ, PAGE_SIZE);
72 BLANK(); 71 BLANK();
73 DEFINE(CPU_INFO_SZ, sizeof(struct cpu_info));
74 DEFINE(CPU_INFO_SETUP, offsetof(struct cpu_info, cpu_setup));
75 BLANK();
76 DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); 72 DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL);
77 DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); 73 DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE);
78 DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); 74 DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE);
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index fa62637e63a8..6ffd91438560 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -16,8 +16,6 @@
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */ 17 */
18 18
19#define pr_fmt(fmt) "alternatives: " fmt
20
21#include <linux/types.h> 19#include <linux/types.h>
22#include <asm/cpu.h> 20#include <asm/cpu.h>
23#include <asm/cputype.h> 21#include <asm/cputype.h>
@@ -26,27 +24,11 @@
26#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) 24#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
27#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) 25#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
28 26
29/*
30 * Add a struct or another datatype to the union below if you need
31 * different means to detect an affected CPU.
32 */
33struct arm64_cpu_capabilities {
34 const char *desc;
35 u16 capability;
36 bool (*is_affected)(struct arm64_cpu_capabilities *);
37 union {
38 struct {
39 u32 midr_model;
40 u32 midr_range_min, midr_range_max;
41 };
42 };
43};
44
45#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ 27#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
46 MIDR_ARCHITECTURE_MASK) 28 MIDR_ARCHITECTURE_MASK)
47 29
48static bool __maybe_unused 30static bool __maybe_unused
49is_affected_midr_range(struct arm64_cpu_capabilities *entry) 31is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
50{ 32{
51 u32 midr = read_cpuid_id(); 33 u32 midr = read_cpuid_id();
52 34
@@ -59,12 +41,12 @@ is_affected_midr_range(struct arm64_cpu_capabilities *entry)
59} 41}
60 42
61#define MIDR_RANGE(model, min, max) \ 43#define MIDR_RANGE(model, min, max) \
62 .is_affected = is_affected_midr_range, \ 44 .matches = is_affected_midr_range, \
63 .midr_model = model, \ 45 .midr_model = model, \
64 .midr_range_min = min, \ 46 .midr_range_min = min, \
65 .midr_range_max = max 47 .midr_range_max = max
66 48
67struct arm64_cpu_capabilities arm64_errata[] = { 49const struct arm64_cpu_capabilities arm64_errata[] = {
68#if defined(CONFIG_ARM64_ERRATUM_826319) || \ 50#if defined(CONFIG_ARM64_ERRATUM_826319) || \
69 defined(CONFIG_ARM64_ERRATUM_827319) || \ 51 defined(CONFIG_ARM64_ERRATUM_827319) || \
70 defined(CONFIG_ARM64_ERRATUM_824069) 52 defined(CONFIG_ARM64_ERRATUM_824069)
@@ -88,7 +70,16 @@ struct arm64_cpu_capabilities arm64_errata[] = {
88 /* Cortex-A57 r0p0 - r1p2 */ 70 /* Cortex-A57 r0p0 - r1p2 */
89 .desc = "ARM erratum 832075", 71 .desc = "ARM erratum 832075",
90 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE, 72 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
91 MIDR_RANGE(MIDR_CORTEX_A57, 0x00, 0x12), 73 MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
74 (1 << MIDR_VARIANT_SHIFT) | 2),
75 },
76#endif
77#ifdef CONFIG_ARM64_ERRATUM_845719
78 {
79 /* Cortex-A53 r0p[01234] */
80 .desc = "ARM erratum 845719",
81 .capability = ARM64_WORKAROUND_845719,
82 MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
92 }, 83 },
93#endif 84#endif
94 { 85 {
@@ -97,15 +88,5 @@ struct arm64_cpu_capabilities arm64_errata[] = {
97 88
98void check_local_cpu_errata(void) 89void check_local_cpu_errata(void)
99{ 90{
100 struct arm64_cpu_capabilities *cpus = arm64_errata; 91 check_cpu_capabilities(arm64_errata, "enabling workaround for");
101 int i;
102
103 for (i = 0; cpus[i].desc; i++) {
104 if (!cpus[i].is_affected(&cpus[i]))
105 continue;
106
107 if (!cpus_have_cap(cpus[i].capability))
108 pr_info("enabling workaround for %s\n", cpus[i].desc);
109 cpus_set_cap(cpus[i].capability);
110 }
111} 92}
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
new file mode 100644
index 000000000000..3d9967e43d89
--- /dev/null
+++ b/arch/arm64/kernel/cpufeature.c
@@ -0,0 +1,47 @@
1/*
2 * Contains CPU feature definitions
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#define pr_fmt(fmt) "alternatives: " fmt
20
21#include <linux/types.h>
22#include <asm/cpu.h>
23#include <asm/cpufeature.h>
24
25static const struct arm64_cpu_capabilities arm64_features[] = {
26 {},
27};
28
29void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
30 const char *info)
31{
32 int i;
33
34 for (i = 0; caps[i].desc; i++) {
35 if (!caps[i].matches(&caps[i]))
36 continue;
37
38 if (!cpus_have_cap(caps[i].capability))
39 pr_info("%s %s\n", info, caps[i].desc);
40 cpus_set_cap(caps[i].capability);
41 }
42}
43
44void check_local_cpu_features(void)
45{
46 check_cpu_capabilities(arm64_features, "detected feature");
47}
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 929855691dae..75d5a867e7fb 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -236,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
236 cpuinfo_detect_icache_policy(info); 236 cpuinfo_detect_icache_policy(info);
237 237
238 check_local_cpu_errata(); 238 check_local_cpu_errata();
239 check_local_cpu_features();
239 update_cpu_features(info); 240 update_cpu_features(info);
240} 241}
241 242
diff --git a/arch/arm64/kernel/cputable.c b/arch/arm64/kernel/cputable.c
deleted file mode 100644
index fd3993cb060f..000000000000
--- a/arch/arm64/kernel/cputable.c
+++ /dev/null
@@ -1,33 +0,0 @@
1/*
2 * arch/arm64/kernel/cputable.c
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 *
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/init.h>
20
21#include <asm/cputable.h>
22
23extern unsigned long __cpu_setup(void);
24
25struct cpu_info cpu_table[] = {
26 {
27 .cpu_id_val = 0x000f0000,
28 .cpu_id_mask = 0x000f0000,
29 .cpu_name = "AArch64 Processor",
30 .cpu_setup = __cpu_setup,
31 },
32 { /* Empty */ },
33};
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index cf21bb3bf752..959fe8733560 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -21,8 +21,10 @@
21#include <linux/init.h> 21#include <linux/init.h>
22#include <linux/linkage.h> 22#include <linux/linkage.h>
23 23
24#include <asm/alternative-asm.h>
24#include <asm/assembler.h> 25#include <asm/assembler.h>
25#include <asm/asm-offsets.h> 26#include <asm/asm-offsets.h>
27#include <asm/cpufeature.h>
26#include <asm/errno.h> 28#include <asm/errno.h>
27#include <asm/esr.h> 29#include <asm/esr.h>
28#include <asm/thread_info.h> 30#include <asm/thread_info.h>
@@ -120,6 +122,24 @@
120 ct_user_enter 122 ct_user_enter
121 ldr x23, [sp, #S_SP] // load return stack pointer 123 ldr x23, [sp, #S_SP] // load return stack pointer
122 msr sp_el0, x23 124 msr sp_el0, x23
125
126#ifdef CONFIG_ARM64_ERRATUM_845719
127 alternative_insn \
128 "nop", \
129 "tbz x22, #4, 1f", \
130 ARM64_WORKAROUND_845719
131#ifdef CONFIG_PID_IN_CONTEXTIDR
132 alternative_insn \
133 "nop; nop", \
134 "mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:", \
135 ARM64_WORKAROUND_845719
136#else
137 alternative_insn \
138 "nop", \
139 "msr contextidr_el1, xzr; 1:", \
140 ARM64_WORKAROUND_845719
141#endif
142#endif
123 .endif 143 .endif
124 msr elr_el1, x21 // set up the return data 144 msr elr_el1, x21 // set up the return data
125 msr spsr_el1, x22 145 msr spsr_el1, x22
diff --git a/arch/arm64/kernel/entry32.S b/arch/arm64/kernel/entry32.S
index 9a8f6ae2530e..bd9bfaa9269b 100644
--- a/arch/arm64/kernel/entry32.S
+++ b/arch/arm64/kernel/entry32.S
@@ -19,9 +19,12 @@
19 */ 19 */
20 20
21#include <linux/linkage.h> 21#include <linux/linkage.h>
22#include <linux/const.h>
22 23
23#include <asm/assembler.h> 24#include <asm/assembler.h>
24#include <asm/asm-offsets.h> 25#include <asm/asm-offsets.h>
26#include <asm/errno.h>
27#include <asm/page.h>
25 28
26/* 29/*
27 * System call wrappers for the AArch32 compatibility layer. 30 * System call wrappers for the AArch32 compatibility layer.
@@ -54,6 +57,21 @@ ENTRY(compat_sys_fstatfs64_wrapper)
54ENDPROC(compat_sys_fstatfs64_wrapper) 57ENDPROC(compat_sys_fstatfs64_wrapper)
55 58
56/* 59/*
60 * Note: off_4k (w5) is always in units of 4K. If we can't do the
61 * requested offset because it is not page-aligned, we return -EINVAL.
62 */
63ENTRY(compat_sys_mmap2_wrapper)
64#if PAGE_SHIFT > 12
65 tst w5, #~PAGE_MASK >> 12
66 b.ne 1f
67 lsr w5, w5, #PAGE_SHIFT - 12
68#endif
69 b sys_mmap_pgoff
701: mov x0, #-EINVAL
71 ret
72ENDPROC(compat_sys_mmap2_wrapper)
73
74/*
57 * Wrappers for AArch32 syscalls that either take 64-bit parameters 75 * Wrappers for AArch32 syscalls that either take 64-bit parameters
58 * in registers or that take 32-bit parameters which require sign 76 * in registers or that take 32-bit parameters which require sign
59 * extension. 77 * extension.
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 07f930540f4a..19f915e8f6e0 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -36,7 +36,7 @@
36#include <asm/page.h> 36#include <asm/page.h>
37#include <asm/virt.h> 37#include <asm/virt.h>
38 38
39#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 39#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET)
40 40
41#if (TEXT_OFFSET & 0xfff) != 0 41#if (TEXT_OFFSET & 0xfff) != 0
42#error TEXT_OFFSET must be at least 4KB aligned 42#error TEXT_OFFSET must be at least 4KB aligned
@@ -46,13 +46,6 @@
46#error TEXT_OFFSET must be less than 2MB 46#error TEXT_OFFSET must be less than 2MB
47#endif 47#endif
48 48
49 .macro pgtbl, ttb0, ttb1, virt_to_phys
50 ldr \ttb1, =swapper_pg_dir
51 ldr \ttb0, =idmap_pg_dir
52 add \ttb1, \ttb1, \virt_to_phys
53 add \ttb0, \ttb0, \virt_to_phys
54 .endm
55
56#ifdef CONFIG_ARM64_64K_PAGES 49#ifdef CONFIG_ARM64_64K_PAGES
57#define BLOCK_SHIFT PAGE_SHIFT 50#define BLOCK_SHIFT PAGE_SHIFT
58#define BLOCK_SIZE PAGE_SIZE 51#define BLOCK_SIZE PAGE_SIZE
@@ -63,7 +56,7 @@
63#define TABLE_SHIFT PUD_SHIFT 56#define TABLE_SHIFT PUD_SHIFT
64#endif 57#endif
65 58
66#define KERNEL_START KERNEL_RAM_VADDR 59#define KERNEL_START _text
67#define KERNEL_END _end 60#define KERNEL_END _end
68 61
69/* 62/*
@@ -240,40 +233,43 @@ section_table:
240#endif 233#endif
241 234
242ENTRY(stext) 235ENTRY(stext)
243 mov x21, x0 // x21=FDT 236 bl preserve_boot_args
244 bl el2_setup // Drop to EL1, w20=cpu_boot_mode 237 bl el2_setup // Drop to EL1, w20=cpu_boot_mode
245 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 238 adrp x24, __PHYS_OFFSET
246 bl set_cpu_boot_mode_flag 239 bl set_cpu_boot_mode_flag
247 mrs x22, midr_el1 // x22=cpuid 240
248 mov x0, x22
249 bl lookup_processor_type
250 mov x23, x0 // x23=current cpu_table
251 /*
252 * __error_p may end up out of range for cbz if text areas are
253 * aligned up to section sizes.
254 */
255 cbnz x23, 1f // invalid processor (x23=0)?
256 b __error_p
2571:
258 bl __vet_fdt 241 bl __vet_fdt
259 bl __create_page_tables // x25=TTBR0, x26=TTBR1 242 bl __create_page_tables // x25=TTBR0, x26=TTBR1
260 /* 243 /*
261 * The following calls CPU specific code in a position independent 244 * The following calls CPU setup code, see arch/arm64/mm/proc.S for
262 * manner. See arch/arm64/mm/proc.S for details. x23 = base of 245 * details.
263 * cpu_info structure selected by lookup_processor_type above.
264 * On return, the CPU will be ready for the MMU to be turned on and 246 * On return, the CPU will be ready for the MMU to be turned on and
265 * the TCR will have been set. 247 * the TCR will have been set.
266 */ 248 */
267 ldr x27, __switch_data // address to jump to after 249 ldr x27, =__mmap_switched // address to jump to after
268 // MMU has been enabled 250 // MMU has been enabled
269 adrp lr, __enable_mmu // return (PIC) address 251 adr_l lr, __enable_mmu // return (PIC) address
270 add lr, lr, #:lo12:__enable_mmu 252 b __cpu_setup // initialise processor
271 ldr x12, [x23, #CPU_INFO_SETUP]
272 add x12, x12, x28 // __virt_to_phys
273 br x12 // initialise processor
274ENDPROC(stext) 253ENDPROC(stext)
275 254
276/* 255/*
256 * Preserve the arguments passed by the bootloader in x0 .. x3
257 */
258preserve_boot_args:
259 mov x21, x0 // x21=FDT
260
261 adr_l x0, boot_args // record the contents of
262 stp x21, x1, [x0] // x0 .. x3 at kernel entry
263 stp x2, x3, [x0, #16]
264
265 dmb sy // needed before dc ivac with
266 // MMU off
267
268 add x1, x0, #0x20 // 4 x 8 bytes
269 b __inval_cache_range // tail call
270ENDPROC(preserve_boot_args)
271
272/*
277 * Determine validity of the x21 FDT pointer. 273 * Determine validity of the x21 FDT pointer.
278 * The dtb must be 8-byte aligned and live in the first 512M of memory. 274 * The dtb must be 8-byte aligned and live in the first 512M of memory.
279 */ 275 */
@@ -356,7 +352,8 @@ ENDPROC(__vet_fdt)
356 * - pgd entry for fixed mappings (TTBR1) 352 * - pgd entry for fixed mappings (TTBR1)
357 */ 353 */
358__create_page_tables: 354__create_page_tables:
359 pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses 355 adrp x25, idmap_pg_dir
356 adrp x26, swapper_pg_dir
360 mov x27, lr 357 mov x27, lr
361 358
362 /* 359 /*
@@ -385,12 +382,50 @@ __create_page_tables:
385 * Create the identity mapping. 382 * Create the identity mapping.
386 */ 383 */
387 mov x0, x25 // idmap_pg_dir 384 mov x0, x25 // idmap_pg_dir
388 ldr x3, =KERNEL_START 385 adrp x3, KERNEL_START // __pa(KERNEL_START)
389 add x3, x3, x28 // __pa(KERNEL_START) 386
387#ifndef CONFIG_ARM64_VA_BITS_48
388#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3)
389#define EXTRA_PTRS (1 << (48 - EXTRA_SHIFT))
390
391 /*
392 * If VA_BITS < 48, it may be too small to allow for an ID mapping to be
393 * created that covers system RAM if that is located sufficiently high
394 * in the physical address space. So for the ID map, use an extended
395 * virtual range in that case, by configuring an additional translation
396 * level.
397 * First, we have to verify our assumption that the current value of
398 * VA_BITS was chosen such that all translation levels are fully
399 * utilised, and that lowering T0SZ will always result in an additional
400 * translation level to be configured.
401 */
402#if VA_BITS != EXTRA_SHIFT
403#error "Mismatch between VA_BITS and page size/number of translation levels"
404#endif
405
406 /*
407 * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the
408 * entire kernel image can be ID mapped. As T0SZ == (64 - #bits used),
409 * this number conveniently equals the number of leading zeroes in
410 * the physical address of KERNEL_END.
411 */
412 adrp x5, KERNEL_END
413 clz x5, x5
414 cmp x5, TCR_T0SZ(VA_BITS) // default T0SZ small enough?
415 b.ge 1f // .. then skip additional level
416
417 adr_l x6, idmap_t0sz
418 str x5, [x6]
419 dmb sy
420 dc ivac, x6 // Invalidate potentially stale cache line
421
422 create_table_entry x0, x3, EXTRA_SHIFT, EXTRA_PTRS, x5, x6
4231:
424#endif
425
390 create_pgd_entry x0, x3, x5, x6 426 create_pgd_entry x0, x3, x5, x6
391 ldr x6, =KERNEL_END
392 mov x5, x3 // __pa(KERNEL_START) 427 mov x5, x3 // __pa(KERNEL_START)
393 add x6, x6, x28 // __pa(KERNEL_END) 428 adr_l x6, KERNEL_END // __pa(KERNEL_END)
394 create_block_map x0, x7, x3, x5, x6 429 create_block_map x0, x7, x3, x5, x6
395 430
396 /* 431 /*
@@ -399,7 +434,7 @@ __create_page_tables:
399 mov x0, x26 // swapper_pg_dir 434 mov x0, x26 // swapper_pg_dir
400 mov x5, #PAGE_OFFSET 435 mov x5, #PAGE_OFFSET
401 create_pgd_entry x0, x5, x3, x6 436 create_pgd_entry x0, x5, x3, x6
402 ldr x6, =KERNEL_END 437 ldr x6, =KERNEL_END // __va(KERNEL_END)
403 mov x3, x24 // phys offset 438 mov x3, x24 // phys offset
404 create_block_map x0, x7, x3, x5, x6 439 create_block_map x0, x7, x3, x5, x6
405 440
@@ -426,6 +461,7 @@ __create_page_tables:
426 */ 461 */
427 mov x0, x25 462 mov x0, x25
428 add x1, x26, #SWAPPER_DIR_SIZE 463 add x1, x26, #SWAPPER_DIR_SIZE
464 dmb sy
429 bl __inval_cache_range 465 bl __inval_cache_range
430 466
431 mov lr, x27 467 mov lr, x27
@@ -433,37 +469,22 @@ __create_page_tables:
433ENDPROC(__create_page_tables) 469ENDPROC(__create_page_tables)
434 .ltorg 470 .ltorg
435 471
436 .align 3
437 .type __switch_data, %object
438__switch_data:
439 .quad __mmap_switched
440 .quad __bss_start // x6
441 .quad __bss_stop // x7
442 .quad processor_id // x4
443 .quad __fdt_pointer // x5
444 .quad memstart_addr // x6
445 .quad init_thread_union + THREAD_START_SP // sp
446
447/* 472/*
448 * The following fragment of code is executed with the MMU on in MMU mode, and 473 * The following fragment of code is executed with the MMU enabled.
449 * uses absolute addresses; this is not position independent.
450 */ 474 */
475 .set initial_sp, init_thread_union + THREAD_START_SP
451__mmap_switched: 476__mmap_switched:
452 adr x3, __switch_data + 8 477 adr_l x6, __bss_start
478 adr_l x7, __bss_stop
453 479
454 ldp x6, x7, [x3], #16
4551: cmp x6, x7 4801: cmp x6, x7
456 b.hs 2f 481 b.hs 2f
457 str xzr, [x6], #8 // Clear BSS 482 str xzr, [x6], #8 // Clear BSS
458 b 1b 483 b 1b
4592: 4842:
460 ldp x4, x5, [x3], #16 485 adr_l sp, initial_sp, x4
461 ldr x6, [x3], #8 486 str_l x21, __fdt_pointer, x5 // Save FDT pointer
462 ldr x16, [x3] 487 str_l x24, memstart_addr, x6 // Save PHYS_OFFSET
463 mov sp, x16
464 str x22, [x4] // Save processor ID
465 str x21, [x5] // Save FDT pointer
466 str x24, [x6] // Save PHYS_OFFSET
467 mov x29, #0 488 mov x29, #0
468 b start_kernel 489 b start_kernel
469ENDPROC(__mmap_switched) 490ENDPROC(__mmap_switched)
@@ -566,8 +587,7 @@ ENDPROC(el2_setup)
566 * in x20. See arch/arm64/include/asm/virt.h for more info. 587 * in x20. See arch/arm64/include/asm/virt.h for more info.
567 */ 588 */
568ENTRY(set_cpu_boot_mode_flag) 589ENTRY(set_cpu_boot_mode_flag)
569 ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode 590 adr_l x1, __boot_cpu_mode
570 add x1, x1, x28
571 cmp w20, #BOOT_CPU_MODE_EL2 591 cmp w20, #BOOT_CPU_MODE_EL2
572 b.ne 1f 592 b.ne 1f
573 add x1, x1, #4 593 add x1, x1, #4
@@ -588,29 +608,21 @@ ENDPROC(set_cpu_boot_mode_flag)
588 .align L1_CACHE_SHIFT 608 .align L1_CACHE_SHIFT
589ENTRY(__boot_cpu_mode) 609ENTRY(__boot_cpu_mode)
590 .long BOOT_CPU_MODE_EL2 610 .long BOOT_CPU_MODE_EL2
591 .long 0 611 .long BOOT_CPU_MODE_EL1
592 .popsection 612 .popsection
593 613
594#ifdef CONFIG_SMP 614#ifdef CONFIG_SMP
595 .align 3
5961: .quad .
597 .quad secondary_holding_pen_release
598
599 /* 615 /*
600 * This provides a "holding pen" for platforms to hold all secondary 616 * This provides a "holding pen" for platforms to hold all secondary
601 * cores are held until we're ready for them to initialise. 617 * cores are held until we're ready for them to initialise.
602 */ 618 */
603ENTRY(secondary_holding_pen) 619ENTRY(secondary_holding_pen)
604 bl el2_setup // Drop to EL1, w20=cpu_boot_mode 620 bl el2_setup // Drop to EL1, w20=cpu_boot_mode
605 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
606 bl set_cpu_boot_mode_flag 621 bl set_cpu_boot_mode_flag
607 mrs x0, mpidr_el1 622 mrs x0, mpidr_el1
608 ldr x1, =MPIDR_HWID_BITMASK 623 ldr x1, =MPIDR_HWID_BITMASK
609 and x0, x0, x1 624 and x0, x0, x1
610 adr x1, 1b 625 adr_l x3, secondary_holding_pen_release
611 ldp x2, x3, [x1]
612 sub x1, x1, x2
613 add x3, x3, x1
614pen: ldr x4, [x3] 626pen: ldr x4, [x3]
615 cmp x4, x0 627 cmp x4, x0
616 b.eq secondary_startup 628 b.eq secondary_startup
@@ -624,7 +636,6 @@ ENDPROC(secondary_holding_pen)
624 */ 636 */
625ENTRY(secondary_entry) 637ENTRY(secondary_entry)
626 bl el2_setup // Drop to EL1 638 bl el2_setup // Drop to EL1
627 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
628 bl set_cpu_boot_mode_flag 639 bl set_cpu_boot_mode_flag
629 b secondary_startup 640 b secondary_startup
630ENDPROC(secondary_entry) 641ENDPROC(secondary_entry)
@@ -633,16 +644,9 @@ ENTRY(secondary_startup)
633 /* 644 /*
634 * Common entry point for secondary CPUs. 645 * Common entry point for secondary CPUs.
635 */ 646 */
636 mrs x22, midr_el1 // x22=cpuid 647 adrp x25, idmap_pg_dir
637 mov x0, x22 648 adrp x26, swapper_pg_dir
638 bl lookup_processor_type 649 bl __cpu_setup // initialise processor
639 mov x23, x0 // x23=current cpu_table
640 cbz x23, __error_p // invalid processor (x23=0)?
641
642 pgtbl x25, x26, x28 // x25=TTBR0, x26=TTBR1
643 ldr x12, [x23, #CPU_INFO_SETUP]
644 add x12, x12, x28 // __virt_to_phys
645 blr x12 // initialise processor
646 650
647 ldr x21, =secondary_data 651 ldr x21, =secondary_data
648 ldr x27, =__secondary_switched // address to jump to after enabling the MMU 652 ldr x27, =__secondary_switched // address to jump to after enabling the MMU
@@ -658,11 +662,12 @@ ENDPROC(__secondary_switched)
658#endif /* CONFIG_SMP */ 662#endif /* CONFIG_SMP */
659 663
660/* 664/*
661 * Setup common bits before finally enabling the MMU. Essentially this is just 665 * Enable the MMU.
662 * loading the page table pointer and vector base registers.
663 * 666 *
664 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on 667 * x0 = SCTLR_EL1 value for turning on the MMU.
665 * the MMU. 668 * x27 = *virtual* address to jump to upon completion
669 *
670 * other registers depend on the function called upon completion
666 */ 671 */
667__enable_mmu: 672__enable_mmu:
668 ldr x5, =vectors 673 ldr x5, =vectors
@@ -670,89 +675,7 @@ __enable_mmu:
670 msr ttbr0_el1, x25 // load TTBR0 675 msr ttbr0_el1, x25 // load TTBR0
671 msr ttbr1_el1, x26 // load TTBR1 676 msr ttbr1_el1, x26 // load TTBR1
672 isb 677 isb
673 b __turn_mmu_on
674ENDPROC(__enable_mmu)
675
676/*
677 * Enable the MMU. This completely changes the structure of the visible memory
678 * space. You will not be able to trace execution through this.
679 *
680 * x0 = system control register
681 * x27 = *virtual* address to jump to upon completion
682 *
683 * other registers depend on the function called upon completion
684 *
685 * We align the entire function to the smallest power of two larger than it to
686 * ensure it fits within a single block map entry. Otherwise were PHYS_OFFSET
687 * close to the end of a 512MB or 1GB block we might require an additional
688 * table to map the entire function.
689 */
690 .align 4
691__turn_mmu_on:
692 msr sctlr_el1, x0 678 msr sctlr_el1, x0
693 isb 679 isb
694 br x27 680 br x27
695ENDPROC(__turn_mmu_on) 681ENDPROC(__enable_mmu)
696
697/*
698 * Calculate the start of physical memory.
699 */
700__calc_phys_offset:
701 adr x0, 1f
702 ldp x1, x2, [x0]
703 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET
704 add x24, x2, x28 // x24 = PHYS_OFFSET
705 ret
706ENDPROC(__calc_phys_offset)
707
708 .align 3
7091: .quad .
710 .quad PAGE_OFFSET
711
712/*
713 * Exception handling. Something went wrong and we can't proceed. We ought to
714 * tell the user, but since we don't have any guarantee that we're even
715 * running on the right architecture, we do virtually nothing.
716 */
717__error_p:
718ENDPROC(__error_p)
719
720__error:
7211: nop
722 b 1b
723ENDPROC(__error)
724
725/*
726 * This function gets the processor ID in w0 and searches the cpu_table[] for
727 * a match. It returns a pointer to the struct cpu_info it found. The
728 * cpu_table[] must end with an empty (all zeros) structure.
729 *
730 * This routine can be called via C code and it needs to work with the MMU
731 * both disabled and enabled (the offset is calculated automatically).
732 */
733ENTRY(lookup_processor_type)
734 adr x1, __lookup_processor_type_data
735 ldp x2, x3, [x1]
736 sub x1, x1, x2 // get offset between VA and PA
737 add x3, x3, x1 // convert VA to PA
7381:
739 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask
740 cbz w5, 2f // end of list?
741 and w6, w6, w0
742 cmp w5, w6
743 b.eq 3f
744 add x3, x3, #CPU_INFO_SZ
745 b 1b
7462:
747 mov x3, #0 // unknown processor
7483:
749 mov x0, x3
750 ret
751ENDPROC(lookup_processor_type)
752
753 .align 3
754 .type __lookup_processor_type_data, %object
755__lookup_processor_type_data:
756 .quad .
757 .quad cpu_table
758 .size __lookup_processor_type_data, . - __lookup_processor_type_data
diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
index c8eca88f12e6..924902083e47 100644
--- a/arch/arm64/kernel/insn.c
+++ b/arch/arm64/kernel/insn.c
@@ -265,23 +265,13 @@ int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
265 return aarch64_insn_patch_text_sync(addrs, insns, cnt); 265 return aarch64_insn_patch_text_sync(addrs, insns, cnt);
266} 266}
267 267
268u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, 268static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
269 u32 insn, u64 imm) 269 u32 *maskp, int *shiftp)
270{ 270{
271 u32 immlo, immhi, lomask, himask, mask; 271 u32 mask;
272 int shift; 272 int shift;
273 273
274 switch (type) { 274 switch (type) {
275 case AARCH64_INSN_IMM_ADR:
276 lomask = 0x3;
277 himask = 0x7ffff;
278 immlo = imm & lomask;
279 imm >>= 2;
280 immhi = imm & himask;
281 imm = (immlo << 24) | (immhi);
282 mask = (lomask << 24) | (himask);
283 shift = 5;
284 break;
285 case AARCH64_INSN_IMM_26: 275 case AARCH64_INSN_IMM_26:
286 mask = BIT(26) - 1; 276 mask = BIT(26) - 1;
287 shift = 0; 277 shift = 0;
@@ -320,9 +310,68 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
320 shift = 16; 310 shift = 16;
321 break; 311 break;
322 default: 312 default:
323 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", 313 return -EINVAL;
324 type); 314 }
325 return 0; 315
316 *maskp = mask;
317 *shiftp = shift;
318
319 return 0;
320}
321
322#define ADR_IMM_HILOSPLIT 2
323#define ADR_IMM_SIZE SZ_2M
324#define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1)
325#define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
326#define ADR_IMM_LOSHIFT 29
327#define ADR_IMM_HISHIFT 5
328
329u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
330{
331 u32 immlo, immhi, mask;
332 int shift;
333
334 switch (type) {
335 case AARCH64_INSN_IMM_ADR:
336 shift = 0;
337 immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
338 immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
339 insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
340 mask = ADR_IMM_SIZE - 1;
341 break;
342 default:
343 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
344 pr_err("aarch64_insn_decode_immediate: unknown immediate encoding %d\n",
345 type);
346 return 0;
347 }
348 }
349
350 return (insn >> shift) & mask;
351}
352
353u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
354 u32 insn, u64 imm)
355{
356 u32 immlo, immhi, mask;
357 int shift;
358
359 switch (type) {
360 case AARCH64_INSN_IMM_ADR:
361 shift = 0;
362 immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
363 imm >>= ADR_IMM_HILOSPLIT;
364 immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
365 imm = immlo | immhi;
366 mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
367 (ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
368 break;
369 default:
370 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
371 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
372 type);
373 return 0;
374 }
326 } 375 }
327 376
328 /* Update the immediate field. */ 377 /* Update the immediate field. */
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 25a5308744b1..195991dadc37 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -25,8 +25,10 @@
25#include <linux/irq.h> 25#include <linux/irq.h>
26#include <linux/kernel.h> 26#include <linux/kernel.h>
27#include <linux/export.h> 27#include <linux/export.h>
28#include <linux/of.h>
28#include <linux/perf_event.h> 29#include <linux/perf_event.h>
29#include <linux/platform_device.h> 30#include <linux/platform_device.h>
31#include <linux/slab.h>
30#include <linux/spinlock.h> 32#include <linux/spinlock.h>
31#include <linux/uaccess.h> 33#include <linux/uaccess.h>
32 34
@@ -322,22 +324,31 @@ out:
322} 324}
323 325
324static int 326static int
325validate_event(struct pmu_hw_events *hw_events, 327validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events,
326 struct perf_event *event) 328 struct perf_event *event)
327{ 329{
328 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 330 struct arm_pmu *armpmu;
329 struct hw_perf_event fake_event = event->hw; 331 struct hw_perf_event fake_event = event->hw;
330 struct pmu *leader_pmu = event->group_leader->pmu; 332 struct pmu *leader_pmu = event->group_leader->pmu;
331 333
332 if (is_software_event(event)) 334 if (is_software_event(event))
333 return 1; 335 return 1;
334 336
337 /*
338 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
339 * core perf code won't check that the pmu->ctx == leader->ctx
340 * until after pmu->event_init(event).
341 */
342 if (event->pmu != pmu)
343 return 0;
344
335 if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF) 345 if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF)
336 return 1; 346 return 1;
337 347
338 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 348 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
339 return 1; 349 return 1;
340 350
351 armpmu = to_arm_pmu(event->pmu);
341 return armpmu->get_event_idx(hw_events, &fake_event) >= 0; 352 return armpmu->get_event_idx(hw_events, &fake_event) >= 0;
342} 353}
343 354
@@ -355,15 +366,15 @@ validate_group(struct perf_event *event)
355 memset(fake_used_mask, 0, sizeof(fake_used_mask)); 366 memset(fake_used_mask, 0, sizeof(fake_used_mask));
356 fake_pmu.used_mask = fake_used_mask; 367 fake_pmu.used_mask = fake_used_mask;
357 368
358 if (!validate_event(&fake_pmu, leader)) 369 if (!validate_event(event->pmu, &fake_pmu, leader))
359 return -EINVAL; 370 return -EINVAL;
360 371
361 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 372 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
362 if (!validate_event(&fake_pmu, sibling)) 373 if (!validate_event(event->pmu, &fake_pmu, sibling))
363 return -EINVAL; 374 return -EINVAL;
364 } 375 }
365 376
366 if (!validate_event(&fake_pmu, event)) 377 if (!validate_event(event->pmu, &fake_pmu, event))
367 return -EINVAL; 378 return -EINVAL;
368 379
369 return 0; 380 return 0;
@@ -396,7 +407,12 @@ armpmu_release_hardware(struct arm_pmu *armpmu)
396 free_percpu_irq(irq, &cpu_hw_events); 407 free_percpu_irq(irq, &cpu_hw_events);
397 } else { 408 } else {
398 for (i = 0; i < irqs; ++i) { 409 for (i = 0; i < irqs; ++i) {
399 if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs)) 410 int cpu = i;
411
412 if (armpmu->irq_affinity)
413 cpu = armpmu->irq_affinity[i];
414
415 if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
400 continue; 416 continue;
401 irq = platform_get_irq(pmu_device, i); 417 irq = platform_get_irq(pmu_device, i);
402 if (irq > 0) 418 if (irq > 0)
@@ -450,19 +466,24 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu)
450 on_each_cpu(armpmu_enable_percpu_irq, &irq, 1); 466 on_each_cpu(armpmu_enable_percpu_irq, &irq, 1);
451 } else { 467 } else {
452 for (i = 0; i < irqs; ++i) { 468 for (i = 0; i < irqs; ++i) {
469 int cpu = i;
470
453 err = 0; 471 err = 0;
454 irq = platform_get_irq(pmu_device, i); 472 irq = platform_get_irq(pmu_device, i);
455 if (irq <= 0) 473 if (irq <= 0)
456 continue; 474 continue;
457 475
476 if (armpmu->irq_affinity)
477 cpu = armpmu->irq_affinity[i];
478
458 /* 479 /*
459 * If we have a single PMU interrupt that we can't shift, 480 * If we have a single PMU interrupt that we can't shift,
460 * assume that we're running on a uniprocessor machine and 481 * assume that we're running on a uniprocessor machine and
461 * continue. Otherwise, continue without this interrupt. 482 * continue. Otherwise, continue without this interrupt.
462 */ 483 */
463 if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { 484 if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {
464 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", 485 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
465 irq, i); 486 irq, cpu);
466 continue; 487 continue;
467 } 488 }
468 489
@@ -476,7 +497,7 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu)
476 return err; 497 return err;
477 } 498 }
478 499
479 cpumask_set_cpu(i, &armpmu->active_irqs); 500 cpumask_set_cpu(cpu, &armpmu->active_irqs);
480 } 501 }
481 } 502 }
482 503
@@ -1289,9 +1310,46 @@ static const struct of_device_id armpmu_of_device_ids[] = {
1289 1310
1290static int armpmu_device_probe(struct platform_device *pdev) 1311static int armpmu_device_probe(struct platform_device *pdev)
1291{ 1312{
1313 int i, *irqs;
1314
1292 if (!cpu_pmu) 1315 if (!cpu_pmu)
1293 return -ENODEV; 1316 return -ENODEV;
1294 1317
1318 irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
1319 if (!irqs)
1320 return -ENOMEM;
1321
1322 for (i = 0; i < pdev->num_resources; ++i) {
1323 struct device_node *dn;
1324 int cpu;
1325
1326 dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",
1327 i);
1328 if (!dn) {
1329 pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",
1330 of_node_full_name(dn), i);
1331 break;
1332 }
1333
1334 for_each_possible_cpu(cpu)
1335 if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))
1336 break;
1337
1338 of_node_put(dn);
1339 if (cpu >= nr_cpu_ids) {
1340 pr_warn("Failed to find logical CPU for %s\n",
1341 dn->name);
1342 break;
1343 }
1344
1345 irqs[i] = cpu;
1346 }
1347
1348 if (i == pdev->num_resources)
1349 cpu_pmu->irq_affinity = irqs;
1350 else
1351 kfree(irqs);
1352
1295 cpu_pmu->plat_device = pdev; 1353 cpu_pmu->plat_device = pdev;
1296 return 0; 1354 return 0;
1297} 1355}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index e8420f635bd4..51ef97274b52 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -50,7 +50,6 @@
50#include <asm/cpu.h> 50#include <asm/cpu.h>
51#include <asm/cputype.h> 51#include <asm/cputype.h>
52#include <asm/elf.h> 52#include <asm/elf.h>
53#include <asm/cputable.h>
54#include <asm/cpufeature.h> 53#include <asm/cpufeature.h>
55#include <asm/cpu_ops.h> 54#include <asm/cpu_ops.h>
56#include <asm/sections.h> 55#include <asm/sections.h>
@@ -62,9 +61,7 @@
62#include <asm/memblock.h> 61#include <asm/memblock.h>
63#include <asm/psci.h> 62#include <asm/psci.h>
64#include <asm/efi.h> 63#include <asm/efi.h>
65 64#include <asm/virt.h>
66unsigned int processor_id;
67EXPORT_SYMBOL(processor_id);
68 65
69unsigned long elf_hwcap __read_mostly; 66unsigned long elf_hwcap __read_mostly;
70EXPORT_SYMBOL_GPL(elf_hwcap); 67EXPORT_SYMBOL_GPL(elf_hwcap);
@@ -83,7 +80,6 @@ unsigned int compat_elf_hwcap2 __read_mostly;
83 80
84DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); 81DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
85 82
86static const char *cpu_name;
87phys_addr_t __fdt_pointer __initdata; 83phys_addr_t __fdt_pointer __initdata;
88 84
89/* 85/*
@@ -119,6 +115,11 @@ void __init early_print(const char *str, ...)
119 printk("%s", buf); 115 printk("%s", buf);
120} 116}
121 117
118/*
119 * The recorded values of x0 .. x3 upon kernel entry.
120 */
121u64 __cacheline_aligned boot_args[4];
122
122void __init smp_setup_processor_id(void) 123void __init smp_setup_processor_id(void)
123{ 124{
124 u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; 125 u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
@@ -207,24 +208,38 @@ static void __init smp_build_mpidr_hash(void)
207} 208}
208#endif 209#endif
209 210
211static void __init hyp_mode_check(void)
212{
213 if (is_hyp_mode_available())
214 pr_info("CPU: All CPU(s) started at EL2\n");
215 else if (is_hyp_mode_mismatched())
216 WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
217 "CPU: CPUs started in inconsistent modes");
218 else
219 pr_info("CPU: All CPU(s) started at EL1\n");
220}
221
222void __init do_post_cpus_up_work(void)
223{
224 hyp_mode_check();
225 apply_alternatives_all();
226}
227
228#ifdef CONFIG_UP_LATE_INIT
229void __init up_late_init(void)
230{
231 do_post_cpus_up_work();
232}
233#endif /* CONFIG_UP_LATE_INIT */
234
210static void __init setup_processor(void) 235static void __init setup_processor(void)
211{ 236{
212 struct cpu_info *cpu_info;
213 u64 features, block; 237 u64 features, block;
214 u32 cwg; 238 u32 cwg;
215 int cls; 239 int cls;
216 240
217 cpu_info = lookup_processor_type(read_cpuid_id()); 241 printk("CPU: AArch64 Processor [%08x] revision %d\n",
218 if (!cpu_info) { 242 read_cpuid_id(), read_cpuid_id() & 15);
219 printk("CPU configuration botched (ID %08x), unable to continue.\n",
220 read_cpuid_id());
221 while (1);
222 }
223
224 cpu_name = cpu_info->cpu_name;
225
226 printk("CPU: %s [%08x] revision %d\n",
227 cpu_name, read_cpuid_id(), read_cpuid_id() & 15);
228 243
229 sprintf(init_utsname()->machine, ELF_PLATFORM); 244 sprintf(init_utsname()->machine, ELF_PLATFORM);
230 elf_hwcap = 0; 245 elf_hwcap = 0;
@@ -402,6 +417,12 @@ void __init setup_arch(char **cmdline_p)
402 conswitchp = &dummy_con; 417 conswitchp = &dummy_con;
403#endif 418#endif
404#endif 419#endif
420 if (boot_args[1] || boot_args[2] || boot_args[3]) {
421 pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n"
422 "\tx1: %016llx\n\tx2: %016llx\n\tx3: %016llx\n"
423 "This indicates a broken bootloader or old kernel\n",
424 boot_args[1], boot_args[2], boot_args[3]);
425 }
405} 426}
406 427
407static int __init arm64_device_init(void) 428static int __init arm64_device_init(void)
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 328b8ce4b007..ffe8e1b814e0 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -151,6 +151,7 @@ asmlinkage void secondary_start_kernel(void)
151 */ 151 */
152 cpu_set_reserved_ttbr0(); 152 cpu_set_reserved_ttbr0();
153 flush_tlb_all(); 153 flush_tlb_all();
154 cpu_set_default_tcr_t0sz();
154 155
155 preempt_disable(); 156 preempt_disable();
156 trace_hardirqs_off(); 157 trace_hardirqs_off();
@@ -309,7 +310,7 @@ void cpu_die(void)
309void __init smp_cpus_done(unsigned int max_cpus) 310void __init smp_cpus_done(unsigned int max_cpus)
310{ 311{
311 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); 312 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
312 apply_alternatives_all(); 313 do_post_cpus_up_work();
313} 314}
314 315
315void __init smp_prepare_boot_cpu(void) 316void __init smp_prepare_boot_cpu(void)
diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c
index 2d5ab3c90b82..a40b1343b819 100644
--- a/arch/arm64/kernel/sys32.c
+++ b/arch/arm64/kernel/sys32.c
@@ -37,6 +37,7 @@ asmlinkage long compat_sys_readahead_wrapper(void);
37asmlinkage long compat_sys_fadvise64_64_wrapper(void); 37asmlinkage long compat_sys_fadvise64_64_wrapper(void);
38asmlinkage long compat_sys_sync_file_range2_wrapper(void); 38asmlinkage long compat_sys_sync_file_range2_wrapper(void);
39asmlinkage long compat_sys_fallocate_wrapper(void); 39asmlinkage long compat_sys_fallocate_wrapper(void);
40asmlinkage long compat_sys_mmap2_wrapper(void);
40 41
41#undef __SYSCALL 42#undef __SYSCALL
42#define __SYSCALL(nr, sym) [nr] = sym, 43#define __SYSCALL(nr, sym) [nr] = sym,
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 5d9d2dca530d..a2c29865c3fe 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -23,10 +23,14 @@ jiffies = jiffies_64;
23 23
24#define HYPERVISOR_TEXT \ 24#define HYPERVISOR_TEXT \
25 /* \ 25 /* \
26 * Force the alignment to be compatible with \ 26 * Align to 4 KB so that \
27 * the vectors requirements \ 27 * a) the HYP vector table is at its minimum \
28 * alignment of 2048 bytes \
29 * b) the HYP init code will not cross a page \
30 * boundary if its size does not exceed \
31 * 4 KB (see related ASSERT() below) \
28 */ \ 32 */ \
29 . = ALIGN(2048); \ 33 . = ALIGN(SZ_4K); \
30 VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \ 34 VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \
31 *(.hyp.idmap.text) \ 35 *(.hyp.idmap.text) \
32 VMLINUX_SYMBOL(__hyp_idmap_text_end) = .; \ 36 VMLINUX_SYMBOL(__hyp_idmap_text_end) = .; \
@@ -163,10 +167,11 @@ SECTIONS
163} 167}
164 168
165/* 169/*
166 * The HYP init code can't be more than a page long. 170 * The HYP init code can't be more than a page long,
171 * and should not cross a page boundary.
167 */ 172 */
168ASSERT(((__hyp_idmap_text_start + PAGE_SIZE) > __hyp_idmap_text_end), 173ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
169 "HYP init code too big") 174 "HYP init code too big or misaligned")
170 175
171/* 176/*
172 * If padding is applied before .head.text, virt<->phys conversions will fail. 177 * If padding is applied before .head.text, virt<->phys conversions will fail.
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-21 17:46:53 -0500