aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-05-05 15:11:37 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2017-05-05 15:11:37 -0400
commitab182e67ec99ea0c8d7435a32a4a1ed9bb02559a (patch)
treefa71bef0067a61952561552c6652d922060f5530
parent7246f60068840847bdcf595be5f0b5ca632736e0 (diff)
parent92f66f84d9695d07adf9bc987bbcce4bf9b8e87c (diff)
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 updates from Catalin Marinas: - kdump support, including two necessary memblock additions: memblock_clear_nomap() and memblock_cap_memory_range() - ARMv8.3 HWCAP bits for JavaScript conversion instructions, complex numbers and weaker release consistency - arm64 ACPI platform MSI support - arm perf updates: ACPI PMU support, L3 cache PMU in some Qualcomm SoCs, Cortex-A53 L2 cache events and DTLB refills, MAINTAINERS update for DT perf bindings - architected timer errata framework (the arch/arm64 changes only) - support for DMA_ATTR_FORCE_CONTIGUOUS in the arm64 iommu DMA API - arm64 KVM refactoring to use common system register definitions - remove support for ASID-tagged VIVT I-cache (no ARMv8 implementation using it and deprecated in the architecture) together with some I-cache handling clean-up - PE/COFF EFI header clean-up/hardening - define BUG() instruction without CONFIG_BUG * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (92 commits) arm64: Fix the DMA mmap and get_sgtable API with DMA_ATTR_FORCE_CONTIGUOUS arm64: Print DT machine model in setup_machine_fdt() arm64: pmu: Wire-up Cortex A53 L2 cache events and DTLB refills arm64: module: split core and init PLT sections arm64: pmuv3: handle pmuv3+ arm64: Add CNTFRQ_EL0 trap handler arm64: Silence spurious kbuild warning on menuconfig arm64: pmuv3: use arm_pmu ACPI framework arm64: pmuv3: handle !PMUv3 when probing drivers/perf: arm_pmu: add ACPI framework arm64: add function to get a cpu's MADT GICC table drivers/perf: arm_pmu: split out platform device probe logic drivers/perf: arm_pmu: move irq request/free into probe drivers/perf: arm_pmu: split cpu-local irq request/free drivers/perf: arm_pmu: rename irq request/free functions drivers/perf: arm_pmu: handle no platform_device drivers/perf: arm_pmu: simplify cpu_pmu_request_irqs() drivers/perf: arm_pmu: factor out pmu registration drivers/perf: arm_pmu: fold init into alloc drivers/perf: arm_pmu: define armpmu_init_fn ...
Diffstat
-rw-r--r--Documentation/arm64/cpu-feature-registers.txt12
-rw-r--r--Documentation/devicetree/bindings/chosen.txt45
-rw-r--r--Documentation/kdump/kdump.txt16
-rw-r--r--Documentation/perf/qcom_l3_pmu.txt25
-rw-r--r--MAINTAINERS1
-rw-r--r--arch/arm64/Kconfig11
-rw-r--r--arch/arm64/Kconfig.debug4
-rw-r--r--arch/arm64/Makefile4
-rw-r--r--arch/arm64/configs/defconfig1
-rw-r--r--arch/arm64/include/asm/acpi.h2
-rw-r--r--arch/arm64/include/asm/arch_gicv3.h81
-rw-r--r--arch/arm64/include/asm/bug.h32
-rw-r--r--arch/arm64/include/asm/cache.h38
-rw-r--r--arch/arm64/include/asm/cacheflush.h1
-rw-r--r--arch/arm64/include/asm/cachetype.h100
-rw-r--r--arch/arm64/include/asm/cpufeature.h4
-rw-r--r--arch/arm64/include/asm/esr.h4
-rw-r--r--arch/arm64/include/asm/hardirq.h2
-rw-r--r--arch/arm64/include/asm/hw_breakpoint.h4
-rw-r--r--arch/arm64/include/asm/kexec.h52
-rw-r--r--arch/arm64/include/asm/kvm_mmu.h13
-rw-r--r--arch/arm64/include/asm/mmu.h1
-rw-r--r--arch/arm64/include/asm/module.h14
-rw-r--r--arch/arm64/include/asm/pgtable.h10
-rw-r--r--arch/arm64/include/asm/processor.h2
-rw-r--r--arch/arm64/include/asm/sections.h2
-rw-r--r--arch/arm64/include/asm/smp.h3
-rw-r--r--arch/arm64/include/asm/sysreg.h167
-rw-r--r--arch/arm64/include/uapi/asm/hwcap.h3
-rw-r--r--arch/arm64/kernel/Makefile3
-rw-r--r--arch/arm64/kernel/alternative.c11
-rw-r--r--arch/arm64/kernel/cacheinfo.c38
-rw-r--r--arch/arm64/kernel/cpufeature.c30
-rw-r--r--arch/arm64/kernel/cpuinfo.c37
-rw-r--r--arch/arm64/kernel/crash_dump.c71
-rw-r--r--arch/arm64/kernel/debug-monitors.c2
-rw-r--r--arch/arm64/kernel/efi-header.S155
-rw-r--r--arch/arm64/kernel/head.S222
-rw-r--r--arch/arm64/kernel/hibernate.c10
-rw-r--r--arch/arm64/kernel/machine_kexec.c170
-rw-r--r--arch/arm64/kernel/module-plts.c108
-rw-r--r--arch/arm64/kernel/module.c2
-rw-r--r--arch/arm64/kernel/module.lds1
-rw-r--r--arch/arm64/kernel/perf_event.c120
-rw-r--r--arch/arm64/kernel/process.c2
-rw-r--r--arch/arm64/kernel/reloc_test_core.c81
-rw-r--r--arch/arm64/kernel/reloc_test_syms.S83
-rw-r--r--arch/arm64/kernel/setup.c12
-rw-r--r--arch/arm64/kernel/smp.c79
-rw-r--r--arch/arm64/kernel/traps.c14
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S27
-rw-r--r--arch/arm64/kvm/hyp/tlb.c22
-rw-r--r--arch/arm64/kvm/reset.c2
-rw-r--r--arch/arm64/kvm/sys_regs.c4
-rw-r--r--arch/arm64/mm/context.c3
-rw-r--r--arch/arm64/mm/dma-mapping.c128
-rw-r--r--arch/arm64/mm/fault.c55
-rw-r--r--arch/arm64/mm/flush.c4
-rw-r--r--arch/arm64/mm/init.c181
-rw-r--r--arch/arm64/mm/mmu.c311
-rw-r--r--arch/arm64/mm/pageattr.c15
-rw-r--r--drivers/acpi/arm64/iort.c158
-rw-r--r--drivers/acpi/glue.c6
-rw-r--r--drivers/firmware/efi/libstub/fdt.c28
-rw-r--r--drivers/perf/Kconfig14
-rw-r--r--drivers/perf/Makefile4
-rw-r--r--drivers/perf/arm_pmu.c530
-rw-r--r--drivers/perf/arm_pmu_acpi.c256
-rw-r--r--drivers/perf/arm_pmu_platform.c235
-rw-r--r--drivers/perf/qcom_l3_pmu.c849
-rw-r--r--include/linux/acpi_iort.h3
-rw-r--r--include/linux/cpuhotplug.h2
-rw-r--r--include/linux/memblock.h2
-rw-r--r--include/linux/pe.h177
-rw-r--r--include/linux/perf/arm_pmu.h29
-rw-r--r--mm/memblock.c56
-rw-r--r--virt/kvm/arm/vgic/vgic.c4
77 files changed, 3754 insertions, 1256 deletions
diff --git a/Documentation/arm64/cpu-feature-registers.txt b/Documentation/arm64/cpu-feature-registers.txt
index 61ca21ebef1a..d1c97f9f51cc 100644
--- a/Documentation/arm64/cpu-feature-registers.txt
+++ b/Documentation/arm64/cpu-feature-registers.txt
@@ -169,6 +169,18 @@ infrastructure:
169 as available on the CPU where it is fetched and is not a system 169 as available on the CPU where it is fetched and is not a system
170 wide safe value. 170 wide safe value.
171 171
172 4) ID_AA64ISAR1_EL1 - Instruction set attribute register 1
173
174 x--------------------------------------------------x
175 | Name | bits | visible |
176 |--------------------------------------------------|
177 | LRCPC | [23-20] | y |
178 |--------------------------------------------------|
179 | FCMA | [19-16] | y |
180 |--------------------------------------------------|
181 | JSCVT | [15-12] | y |
182 x--------------------------------------------------x
183
172Appendix I: Example 184Appendix I: Example
173--------------------------- 185---------------------------
174 186
diff --git a/Documentation/devicetree/bindings/chosen.txt b/Documentation/devicetree/bindings/chosen.txt
index 6ae9d82d4c37..b5e39af4ddc0 100644
--- a/Documentation/devicetree/bindings/chosen.txt
+++ b/Documentation/devicetree/bindings/chosen.txt
@@ -52,3 +52,48 @@ This property is set (currently only on PowerPC, and only needed on
52book3e) by some versions of kexec-tools to tell the new kernel that it 52book3e) by some versions of kexec-tools to tell the new kernel that it
53is being booted by kexec, as the booting environment may differ (e.g. 53is being booted by kexec, as the booting environment may differ (e.g.
54a different secondary CPU release mechanism) 54a different secondary CPU release mechanism)
55
56linux,usable-memory-range
57-------------------------
58
59This property (arm64 only) holds a base address and size, describing a
60limited region in which memory may be considered available for use by
61the kernel. Memory outside of this range is not available for use.
62
63This property describes a limitation: memory within this range is only
64valid when also described through another mechanism that the kernel
65would otherwise use to determine available memory (e.g. memory nodes
66or the EFI memory map). Valid memory may be sparse within the range.
67e.g.
68
69/ {
70 chosen {
71 linux,usable-memory-range = <0x9 0xf0000000 0x0 0x10000000>;
72 };
73};
74
75The main usage is for crash dump kernel to identify its own usable
76memory and exclude, at its boot time, any other memory areas that are
77part of the panicked kernel's memory.
78
79While this property does not represent a real hardware, the address
80and the size are expressed in #address-cells and #size-cells,
81respectively, of the root node.
82
83linux,elfcorehdr
84----------------
85
86This property (currently used only on arm64) holds the memory range,
87the address and the size, of the elf core header which mainly describes
88the panicked kernel's memory layout as PT_LOAD segments of elf format.
89e.g.
90
91/ {
92 chosen {
93 linux,elfcorehdr = <0x9 0xfffff000 0x0 0x800>;
94 };
95};
96
97While this property does not represent a real hardware, the address
98and the size are expressed in #address-cells and #size-cells,
99respectively, of the root node.
diff --git a/Documentation/kdump/kdump.txt b/Documentation/kdump/kdump.txt
index b0eb27b956d9..615434d81108 100644
--- a/Documentation/kdump/kdump.txt
+++ b/Documentation/kdump/kdump.txt
@@ -18,7 +18,7 @@ memory image to a dump file on the local disk, or across the network to
18a remote system. 18a remote system.
19 19
20Kdump and kexec are currently supported on the x86, x86_64, ppc64, ia64, 20Kdump and kexec are currently supported on the x86, x86_64, ppc64, ia64,
21s390x and arm architectures. 21s390x, arm and arm64 architectures.
22 22
23When the system kernel boots, it reserves a small section of memory for 23When the system kernel boots, it reserves a small section of memory for
24the dump-capture kernel. This ensures that ongoing Direct Memory Access 24the dump-capture kernel. This ensures that ongoing Direct Memory Access
@@ -249,6 +249,13 @@ Dump-capture kernel config options (Arch Dependent, arm)
249 249
250 AUTO_ZRELADDR=y 250 AUTO_ZRELADDR=y
251 251
252Dump-capture kernel config options (Arch Dependent, arm64)
253----------------------------------------------------------
254
255- Please note that kvm of the dump-capture kernel will not be enabled
256 on non-VHE systems even if it is configured. This is because the CPU
257 will not be reset to EL2 on panic.
258
252Extended crashkernel syntax 259Extended crashkernel syntax
253=========================== 260===========================
254 261
@@ -305,6 +312,8 @@ Boot into System Kernel
305 kernel will automatically locate the crash kernel image within the 312 kernel will automatically locate the crash kernel image within the
306 first 512MB of RAM if X is not given. 313 first 512MB of RAM if X is not given.
307 314
315 On arm64, use "crashkernel=Y[@X]". Note that the start address of
316 the kernel, X if explicitly specified, must be aligned to 2MiB (0x200000).
308 317
309Load the Dump-capture Kernel 318Load the Dump-capture Kernel
310============================ 319============================
@@ -327,6 +336,8 @@ For s390x:
327 - Use image or bzImage 336 - Use image or bzImage
328For arm: 337For arm:
329 - Use zImage 338 - Use zImage
339For arm64:
340 - Use vmlinux or Image
330 341
331If you are using a uncompressed vmlinux image then use following command 342If you are using a uncompressed vmlinux image then use following command
332to load dump-capture kernel. 343to load dump-capture kernel.
@@ -370,6 +381,9 @@ For s390x:
370For arm: 381For arm:
371 "1 maxcpus=1 reset_devices" 382 "1 maxcpus=1 reset_devices"
372 383
384For arm64:
385 "1 maxcpus=1 reset_devices"
386
373Notes on loading the dump-capture kernel: 387Notes on loading the dump-capture kernel:
374 388
375* By default, the ELF headers are stored in ELF64 format to support 389* By default, the ELF headers are stored in ELF64 format to support
diff --git a/Documentation/perf/qcom_l3_pmu.txt b/Documentation/perf/qcom_l3_pmu.txt
new file mode 100644
index 000000000000..96b3a9444a0d
--- /dev/null
+++ b/Documentation/perf/qcom_l3_pmu.txt
@@ -0,0 +1,25 @@
1Qualcomm Datacenter Technologies L3 Cache Performance Monitoring Unit (PMU)
2===========================================================================
3
4This driver supports the L3 cache PMUs found in Qualcomm Datacenter Technologies
5Centriq SoCs. The L3 cache on these SOCs is composed of multiple slices, shared
6by all cores within a socket. Each slice is exposed as a separate uncore perf
7PMU with device name l3cache_<socket>_<instance>. User space is responsible
8for aggregating across slices.
9
10The driver provides a description of its available events and configuration
11options in sysfs, see /sys/devices/l3cache*. Given that these are uncore PMUs
12the driver also exposes a "cpumask" sysfs attribute which contains a mask
13consisting of one CPU per socket which will be used to handle all the PMU
14events on that socket.
15
16The hardware implements 32bit event counters and has a flat 8bit event space
17exposed via the "event" format attribute. In addition to the 32bit physical
18counters the driver supports virtual 64bit hardware counters by using hardware
19counter chaining. This feature is exposed via the "lc" (long counter) format
20flag. E.g.:
21
22 perf stat -e l3cache_0_0/read-miss,lc/
23
24Given that these are uncore PMUs the driver does not support sampling, therefore
25"perf record" will not work. Per-task perf sessions are not supported.
diff --git a/MAINTAINERS b/MAINTAINERS
index b1f9b45f061f..ac0ce262150c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -983,6 +983,7 @@ F: arch/arm*/include/asm/perf_event.h
983F: drivers/perf/* 983F: drivers/perf/*
984F: include/linux/perf/arm_pmu.h 984F: include/linux/perf/arm_pmu.h
985F: Documentation/devicetree/bindings/arm/pmu.txt 985F: Documentation/devicetree/bindings/arm/pmu.txt
986F: Documentation/devicetree/bindings/perf/
986 987
987ARM PORT 988ARM PORT
988M: Russell King <linux@armlinux.org.uk> 989M: Russell King <linux@armlinux.org.uk>
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 67695fadae96..3dcd7ec69bca 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -736,6 +736,17 @@ config KEXEC
736 but it is independent of the system firmware. And like a reboot 736 but it is independent of the system firmware. And like a reboot
737 you can start any kernel with it, not just Linux. 737 you can start any kernel with it, not just Linux.
738 738
739config CRASH_DUMP
740 bool "Build kdump crash kernel"
741 help
742 Generate crash dump after being started by kexec. This should
743 be normally only set in special crash dump kernels which are
744 loaded in the main kernel with kexec-tools into a specially
745 reserved region and then later executed after a crash by
746 kdump/kexec.
747
748 For more details see Documentation/kdump/kdump.txt
749
739config XEN_DOM0 750config XEN_DOM0
740 def_bool y 751 def_bool y
741 depends on XEN 752 depends on XEN
diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug
index fca2f02cde68..cc6bd559af85 100644
--- a/arch/arm64/Kconfig.debug
+++ b/arch/arm64/Kconfig.debug
@@ -92,6 +92,10 @@ config DEBUG_EFI
92 the kernel that are only useful when using a debug build of the 92 the kernel that are only useful when using a debug build of the
93 UEFI firmware 93 UEFI firmware
94 94
95config ARM64_RELOC_TEST
96 depends on m
97 tristate "Relocation testing module"
98
95source "drivers/hwtracing/coresight/Kconfig" 99source "drivers/hwtracing/coresight/Kconfig"
96 100
97endmenu 101endmenu
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index b9a4a934ca05..7dedf2d8494e 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -37,10 +37,12 @@ $(warning LSE atomics not supported by binutils)
37 endif 37 endif
38endif 38endif
39 39
40ifeq ($(CONFIG_ARM64), y)
40brokengasinst := $(call as-instr,1:\n.inst 0\n.rept . - 1b\n\nnop\n.endr\n,,-DCONFIG_BROKEN_GAS_INST=1) 41brokengasinst := $(call as-instr,1:\n.inst 0\n.rept . - 1b\n\nnop\n.endr\n,,-DCONFIG_BROKEN_GAS_INST=1)
41 42
42ifneq ($(brokengasinst),) 43 ifneq ($(brokengasinst),)
43$(warning Detected assembler with broken .inst; disassembly will be unreliable) 44$(warning Detected assembler with broken .inst; disassembly will be unreliable)
45 endif
44endif 46endif
45 47
46KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) $(brokengasinst) 48KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) $(brokengasinst)
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 7c48028ec64a..927ee18bbdf2 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -82,6 +82,7 @@ CONFIG_CMA=y
82CONFIG_SECCOMP=y 82CONFIG_SECCOMP=y
83CONFIG_XEN=y 83CONFIG_XEN=y
84CONFIG_KEXEC=y 84CONFIG_KEXEC=y
85CONFIG_CRASH_DUMP=y
85# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set 86# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
86CONFIG_COMPAT=y 87CONFIG_COMPAT=y
87CONFIG_CPU_IDLE=y 88CONFIG_CPU_IDLE=y
diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h
index c1976c0adca7..0e99978da3f0 100644
--- a/arch/arm64/include/asm/acpi.h
+++ b/arch/arm64/include/asm/acpi.h
@@ -85,6 +85,8 @@ static inline bool acpi_has_cpu_in_madt(void)
85 return true; 85 return true;
86} 86}
87 87
88struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu);
89
88static inline void arch_fix_phys_package_id(int num, u32 slot) { } 90static inline void arch_fix_phys_package_id(int num, u32 slot) { }
89void __init acpi_init_cpus(void); 91void __init acpi_init_cpus(void);
90 92
diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h
index f37e3a21f6e7..1a98bc8602a2 100644
--- a/arch/arm64/include/asm/arch_gicv3.h
+++ b/arch/arm64/include/asm/arch_gicv3.h
@@ -20,69 +20,14 @@
20 20
21#include <asm/sysreg.h> 21#include <asm/sysreg.h>
22 22
23#define ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
24#define ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
25#define ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
26#define ICC_SGI1R_EL1 sys_reg(3, 0, 12, 11, 5)
27#define ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
28#define ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
29#define ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
30#define ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
31#define ICC_BPR1_EL1 sys_reg(3, 0, 12, 12, 3)
32
33#define ICC_SRE_EL2 sys_reg(3, 4, 12, 9, 5)
34
35/*
36 * System register definitions
37 */
38#define ICH_VSEIR_EL2 sys_reg(3, 4, 12, 9, 4)
39#define ICH_HCR_EL2 sys_reg(3, 4, 12, 11, 0)
40#define ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1)
41#define ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2)
42#define ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3)
43#define ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5)
44#define ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7)
45
46#define __LR0_EL2(x) sys_reg(3, 4, 12, 12, x)
47#define __LR8_EL2(x) sys_reg(3, 4, 12, 13, x)
48
49#define ICH_LR0_EL2 __LR0_EL2(0)
50#define ICH_LR1_EL2 __LR0_EL2(1)
51#define ICH_LR2_EL2 __LR0_EL2(2)
52#define ICH_LR3_EL2 __LR0_EL2(3)
53#define ICH_LR4_EL2 __LR0_EL2(4)
54#define ICH_LR5_EL2 __LR0_EL2(5)
55#define ICH_LR6_EL2 __LR0_EL2(6)
56#define ICH_LR7_EL2 __LR0_EL2(7)
57#define ICH_LR8_EL2 __LR8_EL2(0)
58#define ICH_LR9_EL2 __LR8_EL2(1)
59#define ICH_LR10_EL2 __LR8_EL2(2)
60#define ICH_LR11_EL2 __LR8_EL2(3)
61#define ICH_LR12_EL2 __LR8_EL2(4)
62#define ICH_LR13_EL2 __LR8_EL2(5)
63#define ICH_LR14_EL2 __LR8_EL2(6)
64#define ICH_LR15_EL2 __LR8_EL2(7)
65
66#define __AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x)
67#define ICH_AP0R0_EL2 __AP0Rx_EL2(0)
68#define ICH_AP0R1_EL2 __AP0Rx_EL2(1)
69#define ICH_AP0R2_EL2 __AP0Rx_EL2(2)
70#define ICH_AP0R3_EL2 __AP0Rx_EL2(3)
71
72#define __AP1Rx_EL2(x) sys_reg(3, 4, 12, 9, x)
73#define ICH_AP1R0_EL2 __AP1Rx_EL2(0)
74#define ICH_AP1R1_EL2 __AP1Rx_EL2(1)
75#define ICH_AP1R2_EL2 __AP1Rx_EL2(2)
76#define ICH_AP1R3_EL2 __AP1Rx_EL2(3)
77
78#ifndef __ASSEMBLY__ 23#ifndef __ASSEMBLY__
79 24
80#include <linux/stringify.h> 25#include <linux/stringify.h>
81#include <asm/barrier.h> 26#include <asm/barrier.h>
82#include <asm/cacheflush.h> 27#include <asm/cacheflush.h>
83 28
84#define read_gicreg read_sysreg_s 29#define read_gicreg(r) read_sysreg_s(SYS_ ## r)
85#define write_gicreg write_sysreg_s 30#define write_gicreg(v, r) write_sysreg_s(v, SYS_ ## r)
86 31
87/* 32/*
88 * Low-level accessors 33 * Low-level accessors
@@ -93,13 +38,13 @@
93 38
94static inline void gic_write_eoir(u32 irq) 39static inline void gic_write_eoir(u32 irq)
95{ 40{
96 write_sysreg_s(irq, ICC_EOIR1_EL1); 41 write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
97 isb(); 42 isb();
98} 43}
99 44
100static inline void gic_write_dir(u32 irq) 45static inline void gic_write_dir(u32 irq)
101{ 46{
102 write_sysreg_s(irq, ICC_DIR_EL1); 47 write_sysreg_s(irq, SYS_ICC_DIR_EL1);
103 isb(); 48 isb();
104} 49}
105 50
@@ -107,7 +52,7 @@ static inline u64 gic_read_iar_common(void)
107{ 52{
108 u64 irqstat; 53 u64 irqstat;
109 54
110 irqstat = read_sysreg_s(ICC_IAR1_EL1); 55 irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
111 dsb(sy); 56 dsb(sy);
112 return irqstat; 57 return irqstat;
113} 58}
@@ -124,7 +69,7 @@ static inline u64 gic_read_iar_cavium_thunderx(void)
124 u64 irqstat; 69 u64 irqstat;
125 70
126 nops(8); 71 nops(8);
127 irqstat = read_sysreg_s(ICC_IAR1_EL1); 72 irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
128 nops(4); 73 nops(4);
129 mb(); 74 mb();
130 75
@@ -133,40 +78,40 @@ static inline u64 gic_read_iar_cavium_thunderx(void)
133 78
134static inline void gic_write_pmr(u32 val) 79static inline void gic_write_pmr(u32 val)
135{ 80{
136 write_sysreg_s(val, ICC_PMR_EL1); 81 write_sysreg_s(val, SYS_ICC_PMR_EL1);
137} 82}
138 83
139static inline void gic_write_ctlr(u32 val) 84static inline void gic_write_ctlr(u32 val)
140{ 85{
141 write_sysreg_s(val, ICC_CTLR_EL1); 86 write_sysreg_s(val, SYS_ICC_CTLR_EL1);
142 isb(); 87 isb();
143} 88}
144 89
145static inline void gic_write_grpen1(u32 val) 90static inline void gic_write_grpen1(u32 val)
146{ 91{
147 write_sysreg_s(val, ICC_GRPEN1_EL1); 92 write_sysreg_s(val, SYS_ICC_GRPEN1_EL1);
148 isb(); 93 isb();
149} 94}
150 95
151static inline void gic_write_sgi1r(u64 val) 96static inline void gic_write_sgi1r(u64 val)
152{ 97{
153 write_sysreg_s(val, ICC_SGI1R_EL1); 98 write_sysreg_s(val, SYS_ICC_SGI1R_EL1);
154} 99}
155 100
156static inline u32 gic_read_sre(void) 101static inline u32 gic_read_sre(void)
157{ 102{
158 return read_sysreg_s(ICC_SRE_EL1); 103 return read_sysreg_s(SYS_ICC_SRE_EL1);
159} 104}
160 105
161static inline void gic_write_sre(u32 val) 106static inline void gic_write_sre(u32 val)
162{ 107{
163 write_sysreg_s(val, ICC_SRE_EL1); 108 write_sysreg_s(val, SYS_ICC_SRE_EL1);
164 isb(); 109 isb();
165} 110}
166 111
167static inline void gic_write_bpr1(u32 val) 112static inline void gic_write_bpr1(u32 val)
168{ 113{
169 asm volatile("msr_s " __stringify(ICC_BPR1_EL1) ", %0" : : "r" (val)); 114 write_sysreg_s(val, SYS_ICC_BPR1_EL1);
170} 115}
171 116
172#define gic_read_typer(c) readq_relaxed(c) 117#define gic_read_typer(c) readq_relaxed(c)
diff --git a/arch/arm64/include/asm/bug.h b/arch/arm64/include/asm/bug.h
index a9be1072933c..366448eb0fb7 100644
--- a/arch/arm64/include/asm/bug.h
+++ b/arch/arm64/include/asm/bug.h
@@ -20,9 +20,6 @@
20 20
21#include <asm/brk-imm.h> 21#include <asm/brk-imm.h>
22 22
23#ifdef CONFIG_GENERIC_BUG
24#define HAVE_ARCH_BUG
25
26#ifdef CONFIG_DEBUG_BUGVERBOSE 23#ifdef CONFIG_DEBUG_BUGVERBOSE
27#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) 24#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
28#define __BUGVERBOSE_LOCATION(file, line) \ 25#define __BUGVERBOSE_LOCATION(file, line) \
@@ -36,28 +33,35 @@
36#define _BUGVERBOSE_LOCATION(file, line) 33#define _BUGVERBOSE_LOCATION(file, line)
37#endif 34#endif
38 35
39#define _BUG_FLAGS(flags) __BUG_FLAGS(flags) 36#ifdef CONFIG_GENERIC_BUG
40 37
41#define __BUG_FLAGS(flags) asm volatile ( \ 38#define __BUG_ENTRY(flags) \
42 ".pushsection __bug_table,\"a\"\n\t" \ 39 ".pushsection __bug_table,\"a\"\n\t" \
43 ".align 2\n\t" \ 40 ".align 2\n\t" \
44 "0: .long 1f - 0b\n\t" \ 41 "0: .long 1f - 0b\n\t" \
45_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ 42_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
46 ".short " #flags "\n\t" \ 43 ".short " #flags "\n\t" \
47 ".popsection\n" \ 44 ".popsection\n" \
48 \ 45 "1: "
49 "1: brk %[imm]" \ 46#else
50 :: [imm] "i" (BUG_BRK_IMM) \ 47#define __BUG_ENTRY(flags) ""
51) 48#endif
49
50#define __BUG_FLAGS(flags) \
51 asm volatile ( \
52 __BUG_ENTRY(flags) \
53 "brk %[imm]" :: [imm] "i" (BUG_BRK_IMM) \
54 );
52 55
53#define BUG() do { \ 56
54 _BUG_FLAGS(0); \ 57#define BUG() do { \
55 unreachable(); \ 58 __BUG_FLAGS(0); \
59 unreachable(); \
56} while (0) 60} while (0)
57 61
58#define __WARN_FLAGS(flags) _BUG_FLAGS(BUGFLAG_WARNING|(flags)) 62#define __WARN_FLAGS(flags) __BUG_FLAGS(BUGFLAG_WARNING|(flags))
59 63
60#endif /* ! CONFIG_GENERIC_BUG */ 64#define HAVE_ARCH_BUG
61 65
62#include <asm-generic/bug.h> 66#include <asm-generic/bug.h>
63 67
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index 5082b30bc2c0..ea9bb4e0e9bb 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -16,7 +16,18 @@
16#ifndef __ASM_CACHE_H 16#ifndef __ASM_CACHE_H
17#define __ASM_CACHE_H 17#define __ASM_CACHE_H
18 18
19#include <asm/cachetype.h> 19#include <asm/cputype.h>
20
21#define CTR_L1IP_SHIFT 14
22#define CTR_L1IP_MASK 3
23#define CTR_CWG_SHIFT 24
24#define CTR_CWG_MASK 15
25
26#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
27
28#define ICACHE_POLICY_VPIPT 0
29#define ICACHE_POLICY_VIPT 2
30#define ICACHE_POLICY_PIPT 3
20 31
21#define L1_CACHE_SHIFT 7 32#define L1_CACHE_SHIFT 7
22#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) 33#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
@@ -32,6 +43,31 @@
32 43
33#ifndef __ASSEMBLY__ 44#ifndef __ASSEMBLY__
34 45
46#include <linux/bitops.h>
47
48#define ICACHEF_ALIASING 0
49#define ICACHEF_VPIPT 1
50extern unsigned long __icache_flags;
51
52/*
53 * Whilst the D-side always behaves as PIPT on AArch64, aliasing is
54 * permitted in the I-cache.
55 */
56static inline int icache_is_aliasing(void)
57{
58 return test_bit(ICACHEF_ALIASING, &__icache_flags);
59}
60
61static inline int icache_is_vpipt(void)
62{
63 return test_bit(ICACHEF_VPIPT, &__icache_flags);
64}
65
66static inline u32 cache_type_cwg(void)
67{
68 return (read_cpuid_cachetype() >> CTR_CWG_SHIFT) & CTR_CWG_MASK;
69}
70
35#define __read_mostly __attribute__((__section__(".data..read_mostly"))) 71#define __read_mostly __attribute__((__section__(".data..read_mostly")))
36 72
37static inline int cache_line_size(void) 73static inline int cache_line_size(void)
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 5a2a6ee65f65..728f933cef8c 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -154,5 +154,6 @@ int set_memory_ro(unsigned long addr, int numpages);
154int set_memory_rw(unsigned long addr, int numpages); 154int set_memory_rw(unsigned long addr, int numpages);
155int set_memory_x(unsigned long addr, int numpages); 155int set_memory_x(unsigned long addr, int numpages);
156int set_memory_nx(unsigned long addr, int numpages); 156int set_memory_nx(unsigned long addr, int numpages);
157int set_memory_valid(unsigned long addr, unsigned long size, int enable);
157 158
158#endif 159#endif
diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h
deleted file mode 100644
index f5588692f1d4..000000000000
--- a/arch/arm64/include/asm/cachetype.h
+++ /dev/null
@@ -1,100 +0,0 @@
1/*
2 * Copyright (C) 2012 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_CACHETYPE_H
17#define __ASM_CACHETYPE_H
18
19#include <asm/cputype.h>
20
21#define CTR_L1IP_SHIFT 14
22#define CTR_L1IP_MASK 3
23#define CTR_CWG_SHIFT 24
24#define CTR_CWG_MASK 15
25
26#define ICACHE_POLICY_RESERVED 0
27#define ICACHE_POLICY_AIVIVT 1
28#define ICACHE_POLICY_VIPT 2
29#define ICACHE_POLICY_PIPT 3
30
31#ifndef __ASSEMBLY__
32
33#include <linux/bitops.h>
34
35#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
36
37#define ICACHEF_ALIASING 0
38#define ICACHEF_AIVIVT 1
39
40extern unsigned long __icache_flags;
41
42/*
43 * NumSets, bits[27:13] - (Number of sets in cache) - 1
44 * Associativity, bits[12:3] - (Associativity of cache) - 1
45 * LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2
46 */
47#define CCSIDR_EL1_WRITE_THROUGH BIT(31)
48#define CCSIDR_EL1_WRITE_BACK BIT(30)
49#define CCSIDR_EL1_READ_ALLOCATE BIT(29)
50#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28)
51#define CCSIDR_EL1_LINESIZE_MASK 0x7
52#define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK)
53#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3
54#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff
55#define CCSIDR_EL1_ASSOCIATIVITY(x) \
56 (((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK)
57#define CCSIDR_EL1_NUMSETS_SHIFT 13
58#define CCSIDR_EL1_NUMSETS_MASK 0x7fff
59#define CCSIDR_EL1_NUMSETS(x) \
60 (((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK)
61
62#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x))
63#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1)
64#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1)
65
66extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr);
67
68/* Helpers for Level 1 Instruction cache csselr = 1L */
69static inline int icache_get_linesize(void)
70{
71 return CACHE_LINESIZE(cache_get_ccsidr(1L));
72}
73
74static inline int icache_get_numsets(void)
75{
76 return CACHE_NUMSETS(cache_get_ccsidr(1L));
77}
78
79/*
80 * Whilst the D-side always behaves as PIPT on AArch64, aliasing is
81 * permitted in the I-cache.
82 */
83static inline int icache_is_aliasing(void)
84{
85 return test_bit(ICACHEF_ALIASING, &__icache_flags);
86}
87
88static inline int icache_is_aivivt(void)
89{
90 return test_bit(ICACHEF_AIVIVT, &__icache_flags);
91}
92
93static inline u32 cache_type_cwg(void)
94{
95 return (read_cpuid_cachetype() >> CTR_CWG_SHIFT) & CTR_CWG_MASK;
96}
97
98#endif /* __ASSEMBLY__ */
99
100#endif /* __ASM_CACHETYPE_H */
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index f31c48d0cd68..e7f84a7b4465 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -226,7 +226,7 @@ void update_cpu_errata_workarounds(void);
226void __init enable_errata_workarounds(void); 226void __init enable_errata_workarounds(void);
227void verify_local_cpu_errata_workarounds(void); 227void verify_local_cpu_errata_workarounds(void);
228 228
229u64 read_system_reg(u32 id); 229u64 read_sanitised_ftr_reg(u32 id);
230 230
231static inline bool cpu_supports_mixed_endian_el0(void) 231static inline bool cpu_supports_mixed_endian_el0(void)
232{ 232{
@@ -240,7 +240,7 @@ static inline bool system_supports_32bit_el0(void)
240 240
241static inline bool system_supports_mixed_endian_el0(void) 241static inline bool system_supports_mixed_endian_el0(void)
242{ 242{
243 return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1)); 243 return id_aa64mmfr0_mixed_endian_el0(read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1));
244} 244}
245 245
246static inline bool system_supports_fpsimd(void) 246static inline bool system_supports_fpsimd(void)
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index ad42e79a5d4d..85997c0e5443 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -177,6 +177,10 @@
177 177
178#define ESR_ELx_SYS64_ISS_SYS_CNTVCT (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 2, 14, 0) | \ 178#define ESR_ELx_SYS64_ISS_SYS_CNTVCT (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 2, 14, 0) | \
179 ESR_ELx_SYS64_ISS_DIR_READ) 179 ESR_ELx_SYS64_ISS_DIR_READ)
180
181#define ESR_ELx_SYS64_ISS_SYS_CNTFRQ (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 0, 14, 0) | \
182 ESR_ELx_SYS64_ISS_DIR_READ)
183
180#ifndef __ASSEMBLY__ 184#ifndef __ASSEMBLY__
181#include <asm/types.h> 185#include <asm/types.h>
182 186
diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index 8740297dac77..1473fc2f7ab7 100644
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -20,7 +20,7 @@
20#include <linux/threads.h> 20#include <linux/threads.h>
21#include <asm/irq.h> 21#include <asm/irq.h>
22 22
23#define NR_IPI 6 23#define NR_IPI 7
24 24
25typedef struct { 25typedef struct {
26 unsigned int __softirq_pending; 26 unsigned int __softirq_pending;
diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
index b6b167ac082b..41770766d964 100644
--- a/arch/arm64/include/asm/hw_breakpoint.h
+++ b/arch/arm64/include/asm/hw_breakpoint.h
@@ -149,7 +149,7 @@ static inline void ptrace_hw_copy_thread(struct task_struct *task)
149/* Determine number of BRP registers available. */ 149/* Determine number of BRP registers available. */
150static inline int get_num_brps(void) 150static inline int get_num_brps(void)
151{ 151{
152 u64 dfr0 = read_system_reg(SYS_ID_AA64DFR0_EL1); 152 u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
153 return 1 + 153 return 1 +
154 cpuid_feature_extract_unsigned_field(dfr0, 154 cpuid_feature_extract_unsigned_field(dfr0,
155 ID_AA64DFR0_BRPS_SHIFT); 155 ID_AA64DFR0_BRPS_SHIFT);
@@ -158,7 +158,7 @@ static inline int get_num_brps(void)
158/* Determine number of WRP registers available. */ 158/* Determine number of WRP registers available. */
159static inline int get_num_wrps(void) 159static inline int get_num_wrps(void)
160{ 160{
161 u64 dfr0 = read_system_reg(SYS_ID_AA64DFR0_EL1); 161 u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
162 return 1 + 162 return 1 +
163 cpuid_feature_extract_unsigned_field(dfr0, 163 cpuid_feature_extract_unsigned_field(dfr0,
164 ID_AA64DFR0_WRPS_SHIFT); 164 ID_AA64DFR0_WRPS_SHIFT);
diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index 04744dc5fb61..e17f0529a882 100644
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -40,9 +40,59 @@
40static inline void crash_setup_regs(struct pt_regs *newregs, 40static inline void crash_setup_regs(struct pt_regs *newregs,
41 struct pt_regs *oldregs) 41 struct pt_regs *oldregs)
42{ 42{
43 /* Empty routine needed to avoid build errors. */ 43 if (oldregs) {
44 memcpy(newregs, oldregs, sizeof(*newregs));
45 } else {
46 u64 tmp1, tmp2;
47
48 __asm__ __volatile__ (
49 "stp x0, x1, [%2, #16 * 0]\n"
50 "stp x2, x3, [%2, #16 * 1]\n"
51 "stp x4, x5, [%2, #16 * 2]\n"
52 "stp x6, x7, [%2, #16 * 3]\n"
53 "stp x8, x9, [%2, #16 * 4]\n"
54 "stp x10, x11, [%2, #16 * 5]\n"
55 "stp x12, x13, [%2, #16 * 6]\n"
56 "stp x14, x15, [%2, #16 * 7]\n"
57 "stp x16, x17, [%2, #16 * 8]\n"
58 "stp x18, x19, [%2, #16 * 9]\n"
59 "stp x20, x21, [%2, #16 * 10]\n"
60 "stp x22, x23, [%2, #16 * 11]\n"
61 "stp x24, x25, [%2, #16 * 12]\n"
62 "stp x26, x27, [%2, #16 * 13]\n"
63 "stp x28, x29, [%2, #16 * 14]\n"
64 "mov %0, sp\n"
65 "stp x30, %0, [%2, #16 * 15]\n"
66
67 "/* faked current PSTATE */\n"
68 "mrs %0, CurrentEL\n"
69 "mrs %1, SPSEL\n"
70 "orr %0, %0, %1\n"
71 "mrs %1, DAIF\n"
72 "orr %0, %0, %1\n"
73 "mrs %1, NZCV\n"
74 "orr %0, %0, %1\n"
75 /* pc */
76 "adr %1, 1f\n"
77 "1:\n"
78 "stp %1, %0, [%2, #16 * 16]\n"
79 : "=&r" (tmp1), "=&r" (tmp2)
80 : "r" (newregs)
81 : "memory"
82 );
83 }
44} 84}
45 85
86#if defined(CONFIG_KEXEC_CORE) && defined(CONFIG_HIBERNATION)
87extern bool crash_is_nosave(unsigned long pfn);
88extern void crash_prepare_suspend(void);
89extern void crash_post_resume(void);
90#else
91static inline bool crash_is_nosave(unsigned long pfn) {return false; }
92static inline void crash_prepare_suspend(void) {}
93static inline void crash_post_resume(void) {}
94#endif
95
46#endif /* __ASSEMBLY__ */ 96#endif /* __ASSEMBLY__ */
47 97
48#endif 98#endif
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index ed1246014901..2bc6ffa7b89b 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -108,7 +108,7 @@ alternative_else_nop_endif
108#else 108#else
109 109
110#include <asm/pgalloc.h> 110#include <asm/pgalloc.h>
111#include <asm/cachetype.h> 111#include <asm/cache.h>
112#include <asm/cacheflush.h> 112#include <asm/cacheflush.h>
113#include <asm/mmu_context.h> 113#include <asm/mmu_context.h>
114#include <asm/pgtable.h> 114#include <asm/pgtable.h>
@@ -242,12 +242,13 @@ static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
242 242
243 kvm_flush_dcache_to_poc(va, size); 243 kvm_flush_dcache_to_poc(va, size);
244 244
245 if (!icache_is_aliasing()) { /* PIPT */ 245 if (icache_is_aliasing()) {
246 flush_icache_range((unsigned long)va,
247 (unsigned long)va + size);
248 } else if (!icache_is_aivivt()) { /* non ASID-tagged VIVT */
249 /* any kind of VIPT cache */ 246 /* any kind of VIPT cache */
250 __flush_icache_all(); 247 __flush_icache_all();
248 } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
249 /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
250 flush_icache_range((unsigned long)va,
251 (unsigned long)va + size);
251 } 252 }
252} 253}
253 254
@@ -307,7 +308,7 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
307 308
308static inline unsigned int kvm_get_vmid_bits(void) 309static inline unsigned int kvm_get_vmid_bits(void)
309{ 310{
310 int reg = read_system_reg(SYS_ID_AA64MMFR1_EL1); 311 int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
311 312
312 return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8; 313 return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
313} 314}
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 47619411f0ff..5468c834b072 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -37,5 +37,6 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
37 unsigned long virt, phys_addr_t size, 37 unsigned long virt, phys_addr_t size,
38 pgprot_t prot, bool page_mappings_only); 38 pgprot_t prot, bool page_mappings_only);
39extern void *fixmap_remap_fdt(phys_addr_t dt_phys); 39extern void *fixmap_remap_fdt(phys_addr_t dt_phys);
40extern void mark_linear_text_alias_ro(void);
40 41
41#endif 42#endif
diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h
index 06ff7fd9e81f..d57693f5d4ec 100644
--- a/arch/arm64/include/asm/module.h
+++ b/arch/arm64/include/asm/module.h
@@ -17,26 +17,26 @@
17#define __ASM_MODULE_H 17#define __ASM_MODULE_H
18 18
19#include <asm-generic/module.h> 19#include <asm-generic/module.h>
20#include <asm/memory.h>
21 20
22#define MODULE_ARCH_VERMAGIC "aarch64" 21#define MODULE_ARCH_VERMAGIC "aarch64"
23 22
24#ifdef CONFIG_ARM64_MODULE_PLTS 23#ifdef CONFIG_ARM64_MODULE_PLTS
25struct mod_arch_specific { 24struct mod_plt_sec {
26 struct elf64_shdr *plt; 25 struct elf64_shdr *plt;
27 int plt_num_entries; 26 int plt_num_entries;
28 int plt_max_entries; 27 int plt_max_entries;
29}; 28};
29
30struct mod_arch_specific {
31 struct mod_plt_sec core;
32 struct mod_plt_sec init;
33};
30#endif 34#endif
31 35
32u64 module_emit_plt_entry(struct module *mod, const Elf64_Rela *rela, 36u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
33 Elf64_Sym *sym); 37 Elf64_Sym *sym);
34 38
35#ifdef CONFIG_RANDOMIZE_BASE 39#ifdef CONFIG_RANDOMIZE_BASE
36#ifdef CONFIG_MODVERSIONS
37#define ARCH_RELOCATES_KCRCTAB
38#define reloc_start (kimage_vaddr - KIMAGE_VADDR)
39#endif
40extern u64 module_alloc_base; 40extern u64 module_alloc_base;
41#else 41#else
42#define module_alloc_base ((u64)_etext - MODULES_VSIZE) 42#define module_alloc_base ((u64)_etext - MODULES_VSIZE)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 0eef6064bf3b..c213fdbd056c 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -74,6 +74,16 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
74#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) 74#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))
75#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) 75#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
76 76
77#define pte_cont_addr_end(addr, end) \
78({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \
79 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
80})
81
82#define pmd_cont_addr_end(addr, end) \
83({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \
84 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
85})
86
77#ifdef CONFIG_ARM64_HW_AFDBM 87#ifdef CONFIG_ARM64_HW_AFDBM
78#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) 88#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
79#else 89#else
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index c97b8bd2acba..9428b93fefb2 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -50,6 +50,7 @@ extern phys_addr_t arm64_dma_phys_limit;
50#define ARCH_LOW_ADDRESS_LIMIT (arm64_dma_phys_limit - 1) 50#define ARCH_LOW_ADDRESS_LIMIT (arm64_dma_phys_limit - 1)
51 51
52struct debug_info { 52struct debug_info {
53#ifdef CONFIG_HAVE_HW_BREAKPOINT
53 /* Have we suspended stepping by a debugger? */ 54 /* Have we suspended stepping by a debugger? */
54 int suspended_step; 55 int suspended_step;
55 /* Allow breakpoints and watchpoints to be disabled for this thread. */ 56 /* Allow breakpoints and watchpoints to be disabled for this thread. */
@@ -58,6 +59,7 @@ struct debug_info {
58 /* Hardware breakpoints pinned to this task. */ 59 /* Hardware breakpoints pinned to this task. */
59 struct perf_event *hbp_break[ARM_MAX_BRP]; 60 struct perf_event *hbp_break[ARM_MAX_BRP];
60 struct perf_event *hbp_watch[ARM_MAX_WRP]; 61 struct perf_event *hbp_watch[ARM_MAX_WRP];
62#endif
61}; 63};
62 64
63struct cpu_context { 65struct cpu_context {
diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 4e7e7067afdb..941267caa39c 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -24,6 +24,8 @@ extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
24extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; 24extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
25extern char __hyp_text_start[], __hyp_text_end[]; 25extern char __hyp_text_start[], __hyp_text_end[];
26extern char __idmap_text_start[], __idmap_text_end[]; 26extern char __idmap_text_start[], __idmap_text_end[];
27extern char __initdata_begin[], __initdata_end[];
28extern char __inittext_begin[], __inittext_end[];
27extern char __irqentry_text_start[], __irqentry_text_end[]; 29extern char __irqentry_text_start[], __irqentry_text_end[];
28extern char __mmuoff_data_start[], __mmuoff_data_end[]; 30extern char __mmuoff_data_start[], __mmuoff_data_end[];
29 31
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index d050d720a1b4..55f08c5acfad 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -148,6 +148,9 @@ static inline void cpu_panic_kernel(void)
148 */ 148 */
149bool cpus_are_stuck_in_kernel(void); 149bool cpus_are_stuck_in_kernel(void);
150 150
151extern void smp_send_crash_stop(void);
152extern bool smp_crash_stop_failed(void);
153
151#endif /* ifndef __ASSEMBLY__ */ 154#endif /* ifndef __ASSEMBLY__ */
152 155
153#endif /* ifndef __ASM_SMP_H */ 156#endif /* ifndef __ASM_SMP_H */
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index ac24b6e798b1..15c142ce991c 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -48,6 +48,8 @@
48 ((crn) << CRn_shift) | ((crm) << CRm_shift) | \ 48 ((crn) << CRn_shift) | ((crm) << CRm_shift) | \
49 ((op2) << Op2_shift)) 49 ((op2) << Op2_shift))
50 50
51#define sys_insn sys_reg
52
51#define sys_reg_Op0(id) (((id) >> Op0_shift) & Op0_mask) 53#define sys_reg_Op0(id) (((id) >> Op0_shift) & Op0_mask)
52#define sys_reg_Op1(id) (((id) >> Op1_shift) & Op1_mask) 54#define sys_reg_Op1(id) (((id) >> Op1_shift) & Op1_mask)
53#define sys_reg_CRn(id) (((id) >> CRn_shift) & CRn_mask) 55#define sys_reg_CRn(id) (((id) >> CRn_shift) & CRn_mask)
@@ -81,6 +83,41 @@
81 83
82#endif /* CONFIG_BROKEN_GAS_INST */ 84#endif /* CONFIG_BROKEN_GAS_INST */
83 85
86#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
87#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3)
88
89#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM | \
90 (!!x)<<8 | 0x1f)
91#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM | \
92 (!!x)<<8 | 0x1f)
93
94#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
95#define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2)
96#define SYS_DC_CISW sys_insn(1, 0, 7, 14, 2)
97
98#define SYS_OSDTRRX_EL1 sys_reg(2, 0, 0, 0, 2)
99#define SYS_MDCCINT_EL1 sys_reg(2, 0, 0, 2, 0)
100#define SYS_MDSCR_EL1 sys_reg(2, 0, 0, 2, 2)
101#define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2)
102#define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2)
103#define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4)
104#define SYS_DBGBCRn_EL1(n) sys_reg(2, 0, 0, n, 5)
105#define SYS_DBGWVRn_EL1(n) sys_reg(2, 0, 0, n, 6)
106#define SYS_DBGWCRn_EL1(n) sys_reg(2, 0, 0, n, 7)
107#define SYS_MDRAR_EL1 sys_reg(2, 0, 1, 0, 0)
108#define SYS_OSLAR_EL1 sys_reg(2, 0, 1, 0, 4)
109#define SYS_OSLSR_EL1 sys_reg(2, 0, 1, 1, 4)
110#define SYS_OSDLR_EL1 sys_reg(2, 0, 1, 3, 4)
111#define SYS_DBGPRCR_EL1 sys_reg(2, 0, 1, 4, 4)
112#define SYS_DBGCLAIMSET_EL1 sys_reg(2, 0, 7, 8, 6)
113#define SYS_DBGCLAIMCLR_EL1 sys_reg(2, 0, 7, 9, 6)
114#define SYS_DBGAUTHSTATUS_EL1 sys_reg(2, 0, 7, 14, 6)
115#define SYS_MDCCSR_EL0 sys_reg(2, 3, 0, 1, 0)
116#define SYS_DBGDTR_EL0 sys_reg(2, 3, 0, 4, 0)
117#define SYS_DBGDTRRX_EL0 sys_reg(2, 3, 0, 5, 0)
118#define SYS_DBGDTRTX_EL0 sys_reg(2, 3, 0, 5, 0)
119#define SYS_DBGVCR32_EL2 sys_reg(2, 4, 0, 7, 0)
120
84#define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0) 121#define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0)
85#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5) 122#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
86#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6) 123#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
@@ -88,6 +125,7 @@
88#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0) 125#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
89#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1) 126#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
90#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2) 127#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
128#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
91#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4) 129#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
92#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5) 130#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
93#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6) 131#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
@@ -118,17 +156,127 @@
118#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1) 156#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
119#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2) 157#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2)
120 158
121#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0) 159#define SYS_SCTLR_EL1 sys_reg(3, 0, 1, 0, 0)
160#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
161#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)
162
163#define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0)
164#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)
165#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)
166
167#define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
168
169#define SYS_AFSR0_EL1 sys_reg(3, 0, 5, 1, 0)
170#define SYS_AFSR1_EL1 sys_reg(3, 0, 5, 1, 1)
171#define SYS_ESR_EL1 sys_reg(3, 0, 5, 2, 0)
172#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
173#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
174
175#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)
176#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)
177
178#define SYS_MAIR_EL1 sys_reg(3, 0, 10, 2, 0)
179#define SYS_AMAIR_EL1 sys_reg(3, 0, 10, 3, 0)
180
181#define SYS_VBAR_EL1 sys_reg(3, 0, 12, 0, 0)
182
183#define SYS_ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
184#define SYS_ICC_SGI1R_EL1 sys_reg(3, 0, 12, 11, 5)
185#define SYS_ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
186#define SYS_ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
187#define SYS_ICC_BPR1_EL1 sys_reg(3, 0, 12, 12, 3)
188#define SYS_ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
189#define SYS_ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
190#define SYS_ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
191
192#define SYS_CONTEXTIDR_EL1 sys_reg(3, 0, 13, 0, 1)
193#define SYS_TPIDR_EL1 sys_reg(3, 0, 13, 0, 4)
194
195#define SYS_CNTKCTL_EL1 sys_reg(3, 0, 14, 1, 0)
196
197#define SYS_CLIDR_EL1 sys_reg(3, 1, 0, 0, 1)
198#define SYS_AIDR_EL1 sys_reg(3, 1, 0, 0, 7)
199
200#define SYS_CSSELR_EL1 sys_reg(3, 2, 0, 0, 0)
201
122#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1) 202#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1)
123#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7) 203#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7)
124 204
125#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) 205#define SYS_PMCR_EL0 sys_reg(3, 3, 9, 12, 0)
126#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) 206#define SYS_PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1)
207#define SYS_PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2)
208#define SYS_PMOVSCLR_EL0 sys_reg(3, 3, 9, 12, 3)
209#define SYS_PMSWINC_EL0 sys_reg(3, 3, 9, 12, 4)
210#define SYS_PMSELR_EL0 sys_reg(3, 3, 9, 12, 5)
211#define SYS_PMCEID0_EL0 sys_reg(3, 3, 9, 12, 6)
212#define SYS_PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7)
213#define SYS_PMCCNTR_EL0 sys_reg(3, 3, 9, 13, 0)
214#define SYS_PMXEVTYPER_EL0 sys_reg(3, 3, 9, 13, 1)
215#define SYS_PMXEVCNTR_EL0 sys_reg(3, 3, 9, 13, 2)
216#define SYS_PMUSERENR_EL0 sys_reg(3, 3, 9, 14, 0)
217#define SYS_PMOVSSET_EL0 sys_reg(3, 3, 9, 14, 3)
218
219#define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2)
220#define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3)
127 221
128#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM | \ 222#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
129 (!!x)<<8 | 0x1f) 223
130#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM | \ 224#define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0)
131 (!!x)<<8 | 0x1f) 225#define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1)
226#define SYS_CNTP_CVAL_EL0 sys_reg(3, 3, 14, 2, 2)
227
228#define __PMEV_op2(n) ((n) & 0x7)
229#define __CNTR_CRm(n) (0x8 | (((n) >> 3) & 0x3))
230#define SYS_PMEVCNTRn_EL0(n) sys_reg(3, 3, 14, __CNTR_CRm(n), __PMEV_op2(n))
231#define __TYPER_CRm(n) (0xc | (((n) >> 3) & 0x3))
232#define SYS_PMEVTYPERn_EL0(n) sys_reg(3, 3, 14, __TYPER_CRm(n), __PMEV_op2(n))
233
234#define SYS_PMCCFILTR_EL0 sys_reg (3, 3, 14, 15, 7)
235
236#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)
237#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)
238#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)
239
240#define __SYS__AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x)
241#define SYS_ICH_AP0R0_EL2 __SYS__AP0Rx_EL2(0)
242#define SYS_ICH_AP0R1_EL2 __SYS__AP0Rx_EL2(1)
243#define SYS_ICH_AP0R2_EL2 __SYS__AP0Rx_EL2(2)
244#define SYS_ICH_AP0R3_EL2 __SYS__AP0Rx_EL2(3)
245
246#define __SYS__AP1Rx_EL2(x) sys_reg(3, 4, 12, 9, x)
247#define SYS_ICH_AP1R0_EL2 __SYS__AP1Rx_EL2(0)
248#define SYS_ICH_AP1R1_EL2 __SYS__AP1Rx_EL2(1)
249#define SYS_ICH_AP1R2_EL2 __SYS__AP1Rx_EL2(2)
250#define SYS_ICH_AP1R3_EL2 __SYS__AP1Rx_EL2(3)
251
252#define SYS_ICH_VSEIR_EL2 sys_reg(3, 4, 12, 9, 4)
253#define SYS_ICC_SRE_EL2 sys_reg(3, 4, 12, 9, 5)
254#define SYS_ICH_HCR_EL2 sys_reg(3, 4, 12, 11, 0)
255#define SYS_ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1)
256#define SYS_ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2)
257#define SYS_ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3)
258#define SYS_ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5)
259#define SYS_ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7)
260
261#define __SYS__LR0_EL2(x) sys_reg(3, 4, 12, 12, x)
262#define SYS_ICH_LR0_EL2 __SYS__LR0_EL2(0)
263#define SYS_ICH_LR1_EL2 __SYS__LR0_EL2(1)
264#define SYS_ICH_LR2_EL2 __SYS__LR0_EL2(2)
265#define SYS_ICH_LR3_EL2 __SYS__LR0_EL2(3)
266#define SYS_ICH_LR4_EL2 __SYS__LR0_EL2(4)
267#define SYS_ICH_LR5_EL2 __SYS__LR0_EL2(5)
268#define SYS_ICH_LR6_EL2 __SYS__LR0_EL2(6)
269#define SYS_ICH_LR7_EL2 __SYS__LR0_EL2(7)
270
271#define __SYS__LR8_EL2(x) sys_reg(3, 4, 12, 13, x)
272#define SYS_ICH_LR8_EL2 __SYS__LR8_EL2(0)
273#define SYS_ICH_LR9_EL2 __SYS__LR8_EL2(1)
274#define SYS_ICH_LR10_EL2 __SYS__LR8_EL2(2)
275#define SYS_ICH_LR11_EL2 __SYS__LR8_EL2(3)
276#define SYS_ICH_LR12_EL2 __SYS__LR8_EL2(4)
277#define SYS_ICH_LR13_EL2 __SYS__LR8_EL2(5)
278#define SYS_ICH_LR14_EL2 __SYS__LR8_EL2(6)
279#define SYS_ICH_LR15_EL2 __SYS__LR8_EL2(7)
132 280
133/* Common SCTLR_ELx flags. */ 281/* Common SCTLR_ELx flags. */
134#define SCTLR_ELx_EE (1 << 25) 282#define SCTLR_ELx_EE (1 << 25)
@@ -156,6 +304,11 @@
156#define ID_AA64ISAR0_SHA1_SHIFT 8 304#define ID_AA64ISAR0_SHA1_SHIFT 8
157#define ID_AA64ISAR0_AES_SHIFT 4 305#define ID_AA64ISAR0_AES_SHIFT 4
158 306
307/* id_aa64isar1 */
308#define ID_AA64ISAR1_LRCPC_SHIFT 20
309#define ID_AA64ISAR1_FCMA_SHIFT 16
310#define ID_AA64ISAR1_JSCVT_SHIFT 12
311
159/* id_aa64pfr0 */ 312/* id_aa64pfr0 */
160#define ID_AA64PFR0_GIC_SHIFT 24 313#define ID_AA64PFR0_GIC_SHIFT 24
161#define ID_AA64PFR0_ASIMD_SHIFT 20 314#define ID_AA64PFR0_ASIMD_SHIFT 20
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 61c263cba272..4e187ce2a811 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -32,5 +32,8 @@
32#define HWCAP_ASIMDHP (1 << 10) 32#define HWCAP_ASIMDHP (1 << 10)
33#define HWCAP_CPUID (1 << 11) 33#define HWCAP_CPUID (1 << 11)
34#define HWCAP_ASIMDRDM (1 << 12) 34#define HWCAP_ASIMDRDM (1 << 12)
35#define HWCAP_JSCVT (1 << 13)
36#define HWCAP_FCMA (1 << 14)
37#define HWCAP_LRCPC (1 << 15)
35 38
36#endif /* _UAPI__ASM_HWCAP_H */ 39#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 1606c6b2a280..1dcb69d3d0e5 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -50,6 +50,9 @@ arm64-obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
50arm64-obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o 50arm64-obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o
51arm64-obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o \ 51arm64-obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o \
52 cpu-reset.o 52 cpu-reset.o
53arm64-obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o
54arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o
55arm64-obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
53 56
54obj-y += $(arm64-obj-y) vdso/ probes/ 57obj-y += $(arm64-obj-y) vdso/ probes/
55obj-m += $(arm64-obj-m) 58obj-m += $(arm64-obj-m)
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 06d650f61da7..8840c109c5d6 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -105,11 +105,11 @@ static u32 get_alt_insn(struct alt_instr *alt, u32 *insnptr, u32 *altinsnptr)
105 return insn; 105 return insn;
106} 106}
107 107
108static void __apply_alternatives(void *alt_region) 108static void __apply_alternatives(void *alt_region, bool use_linear_alias)
109{ 109{
110 struct alt_instr *alt; 110 struct alt_instr *alt;
111 struct alt_region *region = alt_region; 111 struct alt_region *region = alt_region;
112 u32 *origptr, *replptr; 112 u32 *origptr, *replptr, *updptr;
113 113
114 for (alt = region->begin; alt < region->end; alt++) { 114 for (alt = region->begin; alt < region->end; alt++) {
115 u32 insn; 115 u32 insn;
@@ -124,11 +124,12 @@ static void __apply_alternatives(void *alt_region)
124 124
125 origptr = ALT_ORIG_PTR(alt); 125 origptr = ALT_ORIG_PTR(alt);
126 replptr = ALT_REPL_PTR(alt); 126 replptr = ALT_REPL_PTR(alt);
127 updptr = use_linear_alias ? (u32 *)lm_alias(origptr) : origptr;
127 nr_inst = alt->alt_len / sizeof(insn); 128 nr_inst = alt->alt_len / sizeof(insn);
128 129
129 for (i = 0; i < nr_inst; i++) { 130 for (i = 0; i < nr_inst; i++) {
130 insn = get_alt_insn(alt, origptr + i, replptr + i); 131 insn = get_alt_insn(alt, origptr + i, replptr + i);
131 *(origptr + i) = cpu_to_le32(insn); 132 updptr[i] = cpu_to_le32(insn);
132 } 133 }
133 134
134 flush_icache_range((uintptr_t)origptr, 135 flush_icache_range((uintptr_t)origptr,
@@ -155,7 +156,7 @@ static int __apply_alternatives_multi_stop(void *unused)
155 isb(); 156 isb();
156 } else { 157 } else {
157 BUG_ON(patched); 158 BUG_ON(patched);
158 __apply_alternatives(&region); 159 __apply_alternatives(&region, true);
159 /* Barriers provided by the cache flushing */ 160 /* Barriers provided by the cache flushing */
160 WRITE_ONCE(patched, 1); 161 WRITE_ONCE(patched, 1);
161 } 162 }
@@ -176,5 +177,5 @@ void apply_alternatives(void *start, size_t length)
176 .end = start + length, 177 .end = start + length,
177 }; 178 };
178 179
179 __apply_alternatives(&region); 180 __apply_alternatives(&region, false);
180} 181}
diff --git a/arch/arm64/kernel/cacheinfo.c b/arch/arm64/kernel/cacheinfo.c
index 3f2250fc391b..380f2e2fbed5 100644
--- a/arch/arm64/kernel/cacheinfo.c
+++ b/arch/arm64/kernel/cacheinfo.c
@@ -17,15 +17,9 @@
17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */ 18 */
19 19
20#include <linux/bitops.h>
21#include <linux/cacheinfo.h> 20#include <linux/cacheinfo.h>
22#include <linux/cpu.h>
23#include <linux/compiler.h>
24#include <linux/of.h> 21#include <linux/of.h>
25 22
26#include <asm/cachetype.h>
27#include <asm/processor.h>
28
29#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */ 23#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */
30/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */ 24/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */
31#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1)) 25#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1))
@@ -43,43 +37,11 @@ static inline enum cache_type get_cache_type(int level)
43 return CLIDR_CTYPE(clidr, level); 37 return CLIDR_CTYPE(clidr, level);
44} 38}
45 39
46/*
47 * Cache Size Selection Register(CSSELR) selects which Cache Size ID
48 * Register(CCSIDR) is accessible by specifying the required cache
49 * level and the cache type. We need to ensure that no one else changes
50 * CSSELR by calling this in non-preemtible context
51 */
52u64 __attribute_const__ cache_get_ccsidr(u64 csselr)
53{
54 u64 ccsidr;
55
56 WARN_ON(preemptible());
57
58 write_sysreg(csselr, csselr_el1);
59 isb();
60 ccsidr = read_sysreg(ccsidr_el1);
61
62 return ccsidr;
63}
64
65static void ci_leaf_init(struct cacheinfo *this_leaf, 40static void ci_leaf_init(struct cacheinfo *this_leaf,
66 enum cache_type type, unsigned int level) 41 enum cache_type type, unsigned int level)
67{ 42{
68 bool is_icache = type & CACHE_TYPE_INST;
69 u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache);
70
71 this_leaf->level = level; 43 this_leaf->level = level;
72 this_leaf->type = type; 44 this_leaf->type = type;
73 this_leaf->coherency_line_size = CACHE_LINESIZE(tmp);
74 this_leaf->number_of_sets = CACHE_NUMSETS(tmp);
75 this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp);
76 this_leaf->size = this_leaf->number_of_sets *
77 this_leaf->coherency_line_size * this_leaf->ways_of_associativity;
78 this_leaf->attributes =
79 ((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) |
80 ((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) |
81 ((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) |
82 ((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0);
83} 45}
84 46
85static int __init_cache_level(unsigned int cpu) 47static int __init_cache_level(unsigned int cpu)
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 6eb77ae99b79..94b8f7fc3310 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -97,6 +97,13 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
97 ARM64_FTR_END, 97 ARM64_FTR_END,
98}; 98};
99 99
100static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
101 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
102 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
103 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
104 ARM64_FTR_END,
105};
106
100static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { 107static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
101 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0), 108 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0),
102 S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI), 109 S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
@@ -153,9 +160,9 @@ static const struct arm64_ftr_bits ftr_ctr[] = {
153 /* 160 /*
154 * Linux can handle differing I-cache policies. Userspace JITs will 161 * Linux can handle differing I-cache policies. Userspace JITs will
155 * make use of *minLine. 162 * make use of *minLine.
156 * If we have differing I-cache policies, report it as the weakest - AIVIVT. 163 * If we have differing I-cache policies, report it as the weakest - VIPT.
157 */ 164 */
158 ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_AIVIVT), /* L1Ip */ 165 ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_VIPT), /* L1Ip */
159 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */ 166 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */
160 ARM64_FTR_END, 167 ARM64_FTR_END,
161}; 168};
@@ -314,7 +321,7 @@ static const struct __ftr_reg_entry {
314 321
315 /* Op1 = 0, CRn = 0, CRm = 6 */ 322 /* Op1 = 0, CRn = 0, CRm = 6 */
316 ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0), 323 ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
317 ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_raz), 324 ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1),
318 325
319 /* Op1 = 0, CRn = 0, CRm = 7 */ 326 /* Op1 = 0, CRn = 0, CRm = 7 */
320 ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), 327 ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
@@ -585,7 +592,7 @@ void update_cpu_features(int cpu,
585 * If we have AArch32, we care about 32-bit features for compat. 592 * If we have AArch32, we care about 32-bit features for compat.
586 * If the system doesn't support AArch32, don't update them. 593 * If the system doesn't support AArch32, don't update them.
587 */ 594 */
588 if (id_aa64pfr0_32bit_el0(read_system_reg(SYS_ID_AA64PFR0_EL1)) && 595 if (id_aa64pfr0_32bit_el0(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
589 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { 596 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
590 597
591 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu, 598 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
@@ -636,7 +643,7 @@ void update_cpu_features(int cpu,
636 "Unsupported CPU feature variation.\n"); 643 "Unsupported CPU feature variation.\n");
637} 644}
638 645
639u64 read_system_reg(u32 id) 646u64 read_sanitised_ftr_reg(u32 id)
640{ 647{
641 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id); 648 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
642 649
@@ -649,10 +656,10 @@ u64 read_system_reg(u32 id)
649 case r: return read_sysreg_s(r) 656 case r: return read_sysreg_s(r)
650 657
651/* 658/*
652 * __raw_read_system_reg() - Used by a STARTING cpu before cpuinfo is populated. 659 * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
653 * Read the system register on the current CPU 660 * Read the system register on the current CPU
654 */ 661 */
655static u64 __raw_read_system_reg(u32 sys_id) 662static u64 __read_sysreg_by_encoding(u32 sys_id)
656{ 663{
657 switch (sys_id) { 664 switch (sys_id) {
658 read_sysreg_case(SYS_ID_PFR0_EL1); 665 read_sysreg_case(SYS_ID_PFR0_EL1);
@@ -709,9 +716,9 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
709 716
710 WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); 717 WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
711 if (scope == SCOPE_SYSTEM) 718 if (scope == SCOPE_SYSTEM)
712 val = read_system_reg(entry->sys_reg); 719 val = read_sanitised_ftr_reg(entry->sys_reg);
713 else 720 else
714 val = __raw_read_system_reg(entry->sys_reg); 721 val = __read_sysreg_by_encoding(entry->sys_reg);
715 722
716 return feature_matches(val, entry); 723 return feature_matches(val, entry);
717} 724}
@@ -761,7 +768,7 @@ static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry,
761 768
762static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused) 769static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
763{ 770{
764 u64 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1); 771 u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
765 772
766 return cpuid_feature_extract_signed_field(pfr0, 773 return cpuid_feature_extract_signed_field(pfr0,
767 ID_AA64PFR0_FP_SHIFT) < 0; 774 ID_AA64PFR0_FP_SHIFT) < 0;
@@ -888,6 +895,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
888 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP), 895 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
889 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD), 896 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
890 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP), 897 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
898 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
899 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
900 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
891 {}, 901 {},
892}; 902};
893 903
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 5b22c687f02a..68b1f364c515 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -15,7 +15,7 @@
15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17#include <asm/arch_timer.h> 17#include <asm/arch_timer.h>
18#include <asm/cachetype.h> 18#include <asm/cache.h>
19#include <asm/cpu.h> 19#include <asm/cpu.h>
20#include <asm/cputype.h> 20#include <asm/cputype.h>
21#include <asm/cpufeature.h> 21#include <asm/cpufeature.h>
@@ -43,10 +43,10 @@ DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
43static struct cpuinfo_arm64 boot_cpu_data; 43static struct cpuinfo_arm64 boot_cpu_data;
44 44
45static char *icache_policy_str[] = { 45static char *icache_policy_str[] = {
46 [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", 46 [0 ... ICACHE_POLICY_PIPT] = "RESERVED/UNKNOWN",
47 [ICACHE_POLICY_AIVIVT] = "AIVIVT", 47 [ICACHE_POLICY_VIPT] = "VIPT",
48 [ICACHE_POLICY_VIPT] = "VIPT", 48 [ICACHE_POLICY_PIPT] = "PIPT",
49 [ICACHE_POLICY_PIPT] = "PIPT", 49 [ICACHE_POLICY_VPIPT] = "VPIPT",
50}; 50};
51 51
52unsigned long __icache_flags; 52unsigned long __icache_flags;
@@ -65,6 +65,9 @@ static const char *const hwcap_str[] = {
65 "asimdhp", 65 "asimdhp",
66 "cpuid", 66 "cpuid",
67 "asimdrdm", 67 "asimdrdm",
68 "jscvt",
69 "fcma",
70 "lrcpc",
68 NULL 71 NULL
69}; 72};
70 73
@@ -289,20 +292,18 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
289 unsigned int cpu = smp_processor_id(); 292 unsigned int cpu = smp_processor_id();
290 u32 l1ip = CTR_L1IP(info->reg_ctr); 293 u32 l1ip = CTR_L1IP(info->reg_ctr);
291 294
292 if (l1ip != ICACHE_POLICY_PIPT) { 295 switch (l1ip) {
293 /* 296 case ICACHE_POLICY_PIPT:
294 * VIPT caches are non-aliasing if the VA always equals the PA 297 break;
295 * in all bit positions that are covered by the index. This is 298 case ICACHE_POLICY_VPIPT:
296 * the case if the size of a way (# of sets * line size) does 299 set_bit(ICACHEF_VPIPT, &__icache_flags);
297 * not exceed PAGE_SIZE. 300 break;
298 */ 301 default:
299 u32 waysize = icache_get_numsets() * icache_get_linesize(); 302 /* Fallthrough */
300 303 case ICACHE_POLICY_VIPT:
301 if (l1ip != ICACHE_POLICY_VIPT || waysize > PAGE_SIZE) 304 /* Assume aliasing */
302 set_bit(ICACHEF_ALIASING, &__icache_flags); 305 set_bit(ICACHEF_ALIASING, &__icache_flags);
303 } 306 }
304 if (l1ip == ICACHE_POLICY_AIVIVT)
305 set_bit(ICACHEF_AIVIVT, &__icache_flags);
306 307
307 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); 308 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
308} 309}
diff --git a/arch/arm64/kernel/crash_dump.c b/arch/arm64/kernel/crash_dump.c
new file mode 100644
index 000000000000..f46d57c31443
--- /dev/null
+++ b/arch/arm64/kernel/crash_dump.c
@@ -0,0 +1,71 @@
1/*
2 * Routines for doing kexec-based kdump
3 *
4 * Copyright (C) 2017 Linaro Limited
5 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/crash_dump.h>
13#include <linux/errno.h>
14#include <linux/io.h>
15#include <linux/memblock.h>
16#include <linux/uaccess.h>
17#include <asm/memory.h>
18
19/**
20 * copy_oldmem_page() - copy one page from old kernel memory
21 * @pfn: page frame number to be copied
22 * @buf: buffer where the copied page is placed
23 * @csize: number of bytes to copy
24 * @offset: offset in bytes into the page
25 * @userbuf: if set, @buf is in a user address space
26 *
27 * This function copies one page from old kernel memory into buffer pointed by
28 * @buf. If @buf is in userspace, set @userbuf to %1. Returns number of bytes
29 * copied or negative error in case of failure.
30 */
31ssize_t copy_oldmem_page(unsigned long pfn, char *buf,
32 size_t csize, unsigned long offset,
33 int userbuf)
34{
35 void *vaddr;
36
37 if (!csize)
38 return 0;
39
40 vaddr = memremap(__pfn_to_phys(pfn), PAGE_SIZE, MEMREMAP_WB);
41 if (!vaddr)
42 return -ENOMEM;
43
44 if (userbuf) {
45 if (copy_to_user((char __user *)buf, vaddr + offset, csize)) {
46 memunmap(vaddr);
47 return -EFAULT;
48 }
49 } else {
50 memcpy(buf, vaddr + offset, csize);
51 }
52
53 memunmap(vaddr);
54
55 return csize;
56}
57
58/**
59 * elfcorehdr_read - read from ELF core header
60 * @buf: buffer where the data is placed
61 * @csize: number of bytes to read
62 * @ppos: address in the memory
63 *
64 * This function reads @count bytes from elf core header which exists
65 * on crash dump kernel's memory.
66 */
67ssize_t elfcorehdr_read(char *buf, size_t count, u64 *ppos)
68{
69 memcpy(buf, phys_to_virt((phys_addr_t)*ppos), count);
70 return count;
71}
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 32913567da08..d618e25c3de1 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -36,7 +36,7 @@
36/* Determine debug architecture. */ 36/* Determine debug architecture. */
37u8 debug_monitors_arch(void) 37u8 debug_monitors_arch(void)
38{ 38{
39 return cpuid_feature_extract_unsigned_field(read_system_reg(SYS_ID_AA64DFR0_EL1), 39 return cpuid_feature_extract_unsigned_field(read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1),
40 ID_AA64DFR0_DEBUGVER_SHIFT); 40 ID_AA64DFR0_DEBUGVER_SHIFT);
41} 41}
42 42
diff --git a/arch/arm64/kernel/efi-header.S b/arch/arm64/kernel/efi-header.S
new file mode 100644
index 000000000000..613fc3000677
--- /dev/null
+++ b/arch/arm64/kernel/efi-header.S
@@ -0,0 +1,155 @@
1/*
2 * Copyright (C) 2013 - 2017 Linaro, Ltd.
3 * Copyright (C) 2013, 2014 Red Hat, Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 */
9
10#include <linux/pe.h>
11#include <linux/sizes.h>
12
13 .macro __EFI_PE_HEADER
14 .long PE_MAGIC
15coff_header:
16 .short IMAGE_FILE_MACHINE_ARM64 // Machine
17 .short section_count // NumberOfSections
18 .long 0 // TimeDateStamp
19 .long 0 // PointerToSymbolTable
20 .long 0 // NumberOfSymbols
21 .short section_table - optional_header // SizeOfOptionalHeader
22 .short IMAGE_FILE_DEBUG_STRIPPED | \
23 IMAGE_FILE_EXECUTABLE_IMAGE | \
24 IMAGE_FILE_LINE_NUMS_STRIPPED // Characteristics
25
26optional_header:
27 .short PE_OPT_MAGIC_PE32PLUS // PE32+ format
28 .byte 0x02 // MajorLinkerVersion
29 .byte 0x14 // MinorLinkerVersion
30 .long __initdata_begin - efi_header_end // SizeOfCode
31 .long __pecoff_data_size // SizeOfInitializedData
32 .long 0 // SizeOfUninitializedData
33 .long __efistub_entry - _head // AddressOfEntryPoint
34 .long efi_header_end - _head // BaseOfCode
35
36extra_header_fields:
37 .quad 0 // ImageBase
38 .long SZ_4K // SectionAlignment
39 .long PECOFF_FILE_ALIGNMENT // FileAlignment
40 .short 0 // MajorOperatingSystemVersion
41 .short 0 // MinorOperatingSystemVersion
42 .short 0 // MajorImageVersion
43 .short 0 // MinorImageVersion
44 .short 0 // MajorSubsystemVersion
45 .short 0 // MinorSubsystemVersion
46 .long 0 // Win32VersionValue
47
48 .long _end - _head // SizeOfImage
49
50 // Everything before the kernel image is considered part of the header
51 .long efi_header_end - _head // SizeOfHeaders
52 .long 0 // CheckSum
53 .short IMAGE_SUBSYSTEM_EFI_APPLICATION // Subsystem
54 .short 0 // DllCharacteristics
55 .quad 0 // SizeOfStackReserve
56 .quad 0 // SizeOfStackCommit
57 .quad 0 // SizeOfHeapReserve
58 .quad 0 // SizeOfHeapCommit
59 .long 0 // LoaderFlags
60 .long (section_table - .) / 8 // NumberOfRvaAndSizes
61
62 .quad 0 // ExportTable
63 .quad 0 // ImportTable
64 .quad 0 // ResourceTable
65 .quad 0 // ExceptionTable
66 .quad 0 // CertificationTable
67 .quad 0 // BaseRelocationTable
68
69#ifdef CONFIG_DEBUG_EFI
70 .long efi_debug_table - _head // DebugTable
71 .long efi_debug_table_size
72#endif
73
74 // Section table
75section_table:
76 .ascii ".text\0\0\0"
77 .long __initdata_begin - efi_header_end // VirtualSize
78 .long efi_header_end - _head // VirtualAddress
79 .long __initdata_begin - efi_header_end // SizeOfRawData
80 .long efi_header_end - _head // PointerToRawData
81
82 .long 0 // PointerToRelocations
83 .long 0 // PointerToLineNumbers
84 .short 0 // NumberOfRelocations
85 .short 0 // NumberOfLineNumbers
86 .long IMAGE_SCN_CNT_CODE | \
87 IMAGE_SCN_MEM_READ | \
88 IMAGE_SCN_MEM_EXECUTE // Characteristics
89
90 .ascii ".data\0\0\0"
91 .long __pecoff_data_size // VirtualSize
92 .long __initdata_begin - _head // VirtualAddress
93 .long __pecoff_data_rawsize // SizeOfRawData
94 .long __initdata_begin - _head // PointerToRawData
95
96 .long 0 // PointerToRelocations
97 .long 0 // PointerToLineNumbers
98 .short 0 // NumberOfRelocations
99 .short 0 // NumberOfLineNumbers
100 .long IMAGE_SCN_CNT_INITIALIZED_DATA | \
101 IMAGE_SCN_MEM_READ | \
102 IMAGE_SCN_MEM_WRITE // Characteristics
103
104 .set section_count, (. - section_table) / 40
105
106#ifdef CONFIG_DEBUG_EFI
107 /*
108 * The debug table is referenced via its Relative Virtual Address (RVA),
109 * which is only defined for those parts of the image that are covered
110 * by a section declaration. Since this header is not covered by any
111 * section, the debug table must be emitted elsewhere. So stick it in
112 * the .init.rodata section instead.
113 *
114 * Note that the EFI debug entry itself may legally have a zero RVA,
115 * which means we can simply put it right after the section headers.
116 */
117 __INITRODATA
118
119 .align 2
120efi_debug_table:
121 // EFI_IMAGE_DEBUG_DIRECTORY_ENTRY
122 .long 0 // Characteristics
123 .long 0 // TimeDateStamp
124 .short 0 // MajorVersion
125 .short 0 // MinorVersion
126 .long IMAGE_DEBUG_TYPE_CODEVIEW // Type
127 .long efi_debug_entry_size // SizeOfData
128 .long 0 // RVA
129 .long efi_debug_entry - _head // FileOffset
130
131 .set efi_debug_table_size, . - efi_debug_table
132 .previous
133
134efi_debug_entry:
135 // EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY
136 .ascii "NB10" // Signature
137 .long 0 // Unknown
138 .long 0 // Unknown2
139 .long 0 // Unknown3
140
141 .asciz VMLINUX_PATH
142
143 .set efi_debug_entry_size, . - efi_debug_entry
144#endif
145
146 /*
147 * EFI will load .text onwards at the 4k section alignment
148 * described in the PE/COFF header. To ensure that instruction
149 * sequences using an adrp and a :lo12: immediate will function
150 * correctly at this alignment, we must ensure that .text is
151 * placed at a 4k boundary in the Image to begin with.
152 */
153 .align 12
154efi_header_end:
155 .endm
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 4fb6ccd886d1..973df7de7bf8 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -42,6 +42,8 @@
42#include <asm/thread_info.h> 42#include <asm/thread_info.h>
43#include <asm/virt.h> 43#include <asm/virt.h>
44 44
45#include "efi-header.S"
46
45#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET) 47#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET)
46 48
47#if (TEXT_OFFSET & 0xfff) != 0 49#if (TEXT_OFFSET & 0xfff) != 0
@@ -89,166 +91,14 @@ _head:
89 .quad 0 // reserved 91 .quad 0 // reserved
90 .quad 0 // reserved 92 .quad 0 // reserved
91 .quad 0 // reserved 93 .quad 0 // reserved
92 .byte 0x41 // Magic number, "ARM\x64" 94 .ascii "ARM\x64" // Magic number
93 .byte 0x52
94 .byte 0x4d
95 .byte 0x64
96#ifdef CONFIG_EFI 95#ifdef CONFIG_EFI
97 .long pe_header - _head // Offset to the PE header. 96 .long pe_header - _head // Offset to the PE header.
98#else
99 .word 0 // reserved
100#endif
101 97
102#ifdef CONFIG_EFI
103 .align 3
104pe_header: 98pe_header:
105 .ascii "PE" 99 __EFI_PE_HEADER
106 .short 0 100#else
107coff_header: 101 .long 0 // reserved
108 .short 0xaa64 // AArch64
109 .short 2 // nr_sections
110 .long 0 // TimeDateStamp
111 .long 0 // PointerToSymbolTable
112 .long 1 // NumberOfSymbols
113 .short section_table - optional_header // SizeOfOptionalHeader
114 .short 0x206 // Characteristics.
115 // IMAGE_FILE_DEBUG_STRIPPED |
116 // IMAGE_FILE_EXECUTABLE_IMAGE |
117 // IMAGE_FILE_LINE_NUMS_STRIPPED
118optional_header:
119 .short 0x20b // PE32+ format
120 .byte 0x02 // MajorLinkerVersion
121 .byte 0x14 // MinorLinkerVersion
122 .long _end - efi_header_end // SizeOfCode
123 .long 0 // SizeOfInitializedData
124 .long 0 // SizeOfUninitializedData
125 .long __efistub_entry - _head // AddressOfEntryPoint
126 .long efi_header_end - _head // BaseOfCode
127
128extra_header_fields:
129 .quad 0 // ImageBase
130 .long 0x1000 // SectionAlignment
131 .long PECOFF_FILE_ALIGNMENT // FileAlignment
132 .short 0 // MajorOperatingSystemVersion
133 .short 0 // MinorOperatingSystemVersion
134 .short 0 // MajorImageVersion
135 .short 0 // MinorImageVersion
136 .short 0 // MajorSubsystemVersion
137 .short 0 // MinorSubsystemVersion
138 .long 0 // Win32VersionValue
139
140 .long _end - _head // SizeOfImage
141
142 // Everything before the kernel image is considered part of the header
143 .long efi_header_end - _head // SizeOfHeaders
144 .long 0 // CheckSum
145 .short 0xa // Subsystem (EFI application)
146 .short 0 // DllCharacteristics
147 .quad 0 // SizeOfStackReserve
148 .quad 0 // SizeOfStackCommit
149 .quad 0 // SizeOfHeapReserve
150 .quad 0 // SizeOfHeapCommit
151 .long 0 // LoaderFlags
152 .long (section_table - .) / 8 // NumberOfRvaAndSizes
153
154 .quad 0 // ExportTable
155 .quad 0 // ImportTable
156 .quad 0 // ResourceTable
157 .quad 0 // ExceptionTable
158 .quad 0 // CertificationTable
159 .quad 0 // BaseRelocationTable
160
161#ifdef CONFIG_DEBUG_EFI
162 .long efi_debug_table - _head // DebugTable
163 .long efi_debug_table_size
164#endif
165
166 // Section table
167section_table:
168
169 /*
170 * The EFI application loader requires a relocation section
171 * because EFI applications must be relocatable. This is a
172 * dummy section as far as we are concerned.
173 */
174 .ascii ".reloc"
175 .byte 0
176 .byte 0 // end of 0 padding of section name
177 .long 0
178 .long 0
179 .long 0 // SizeOfRawData
180 .long 0 // PointerToRawData
181 .long 0 // PointerToRelocations
182 .long 0 // PointerToLineNumbers
183 .short 0 // NumberOfRelocations
184 .short 0 // NumberOfLineNumbers
185 .long 0x42100040 // Characteristics (section flags)
186
187
188 .ascii ".text"
189 .byte 0
190 .byte 0
191 .byte 0 // end of 0 padding of section name
192 .long _end - efi_header_end // VirtualSize
193 .long efi_header_end - _head // VirtualAddress
194 .long _edata - efi_header_end // SizeOfRawData
195 .long efi_header_end - _head // PointerToRawData
196
197 .long 0 // PointerToRelocations (0 for executables)
198 .long 0 // PointerToLineNumbers (0 for executables)
199 .short 0 // NumberOfRelocations (0 for executables)
200 .short 0 // NumberOfLineNumbers (0 for executables)
201 .long 0xe0500020 // Characteristics (section flags)
202
203#ifdef CONFIG_DEBUG_EFI
204 /*
205 * The debug table is referenced via its Relative Virtual Address (RVA),
206 * which is only defined for those parts of the image that are covered
207 * by a section declaration. Since this header is not covered by any
208 * section, the debug table must be emitted elsewhere. So stick it in
209 * the .init.rodata section instead.
210 *
211 * Note that the EFI debug entry itself may legally have a zero RVA,
212 * which means we can simply put it right after the section headers.
213 */
214 __INITRODATA
215
216 .align 2
217efi_debug_table:
218 // EFI_IMAGE_DEBUG_DIRECTORY_ENTRY
219 .long 0 // Characteristics
220 .long 0 // TimeDateStamp
221 .short 0 // MajorVersion
222 .short 0 // MinorVersion
223 .long 2 // Type == EFI_IMAGE_DEBUG_TYPE_CODEVIEW
224 .long efi_debug_entry_size // SizeOfData
225 .long 0 // RVA
226 .long efi_debug_entry - _head // FileOffset
227
228 .set efi_debug_table_size, . - efi_debug_table
229 .previous
230
231efi_debug_entry:
232 // EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY
233 .ascii "NB10" // Signature
234 .long 0 // Unknown
235 .long 0 // Unknown2
236 .long 0 // Unknown3
237
238 .asciz VMLINUX_PATH
239
240 .set efi_debug_entry_size, . - efi_debug_entry
241#endif
242
243 /*
244 * EFI will load .text onwards at the 4k section alignment
245 * described in the PE/COFF header. To ensure that instruction
246 * sequences using an adrp and a :lo12: immediate will function
247 * correctly at this alignment, we must ensure that .text is
248 * placed at a 4k boundary in the Image to begin with.
249 */
250 .align 12
251efi_header_end:
252#endif 102#endif
253 103
254 __INIT 104 __INIT
@@ -534,13 +384,8 @@ ENTRY(kimage_vaddr)
534ENTRY(el2_setup) 384ENTRY(el2_setup)
535 mrs x0, CurrentEL 385 mrs x0, CurrentEL
536 cmp x0, #CurrentEL_EL2 386 cmp x0, #CurrentEL_EL2
537 b.ne 1f 387 b.eq 1f
538 mrs x0, sctlr_el2 388 mrs x0, sctlr_el1
539CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2
540CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2
541 msr sctlr_el2, x0
542 b 2f
5431: mrs x0, sctlr_el1
544CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1 389CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1
545CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1 390CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1
546 msr sctlr_el1, x0 391 msr sctlr_el1, x0
@@ -548,7 +393,11 @@ CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1
548 isb 393 isb
549 ret 394 ret
550 395
5512: 3961: mrs x0, sctlr_el2
397CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2
398CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2
399 msr sctlr_el2, x0
400
552#ifdef CONFIG_ARM64_VHE 401#ifdef CONFIG_ARM64_VHE
553 /* 402 /*
554 * Check for VHE being present. For the rest of the EL2 setup, 403 * Check for VHE being present. For the rest of the EL2 setup,
@@ -594,14 +443,14 @@ set_hcr:
594 cmp x0, #1 443 cmp x0, #1
595 b.ne 3f 444 b.ne 3f
596 445
597 mrs_s x0, ICC_SRE_EL2 446 mrs_s x0, SYS_ICC_SRE_EL2
598 orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1 447 orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1
599 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1 448 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1
600 msr_s ICC_SRE_EL2, x0 449 msr_s SYS_ICC_SRE_EL2, x0
601 isb // Make sure SRE is now set 450 isb // Make sure SRE is now set
602 mrs_s x0, ICC_SRE_EL2 // Read SRE back, 451 mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back,
603 tbz x0, #0, 3f // and check that it sticks 452 tbz x0, #0, 3f // and check that it sticks
604 msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults 453 msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
605 454
6063: 4553:
607#endif 456#endif
@@ -612,26 +461,6 @@ set_hcr:
612 msr vpidr_el2, x0 461 msr vpidr_el2, x0
613 msr vmpidr_el2, x1 462 msr vmpidr_el2, x1
614 463
615 /*
616 * When VHE is not in use, early init of EL2 and EL1 needs to be
617 * done here.
618 * When VHE _is_ in use, EL1 will not be used in the host and
619 * requires no configuration, and all non-hyp-specific EL2 setup
620 * will be done via the _EL1 system register aliases in __cpu_setup.
621 */
622 cbnz x2, 1f
623
624 /* sctlr_el1 */
625 mov x0, #0x0800 // Set/clear RES{1,0} bits
626CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems
627CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems
628 msr sctlr_el1, x0
629
630 /* Coprocessor traps. */
631 mov x0, #0x33ff
632 msr cptr_el2, x0 // Disable copro. traps to EL2
6331:
634
635#ifdef CONFIG_COMPAT 464#ifdef CONFIG_COMPAT
636 msr hstr_el2, xzr // Disable CP15 traps to EL2 465 msr hstr_el2, xzr // Disable CP15 traps to EL2
637#endif 466#endif
@@ -668,6 +497,23 @@ CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems
668 ret 497 ret
669 498
670install_el2_stub: 499install_el2_stub:
500 /*
501 * When VHE is not in use, early init of EL2 and EL1 needs to be
502 * done here.
503 * When VHE _is_ in use, EL1 will not be used in the host and
504 * requires no configuration, and all non-hyp-specific EL2 setup
505 * will be done via the _EL1 system register aliases in __cpu_setup.
506 */
507 /* sctlr_el1 */
508 mov x0, #0x0800 // Set/clear RES{1,0} bits
509CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems
510CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems
511 msr sctlr_el1, x0
512
513 /* Coprocessor traps. */
514 mov x0, #0x33ff
515 msr cptr_el2, x0 // Disable copro. traps to EL2
516
671 /* Hypervisor stub */ 517 /* Hypervisor stub */
672 adr_l x0, __hyp_stub_vectors 518 adr_l x0, __hyp_stub_vectors
673 msr vbar_el2, x0 519 msr vbar_el2, x0
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 97a7384100f3..a44e13942d30 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -28,6 +28,7 @@
28#include <asm/cacheflush.h> 28#include <asm/cacheflush.h>
29#include <asm/cputype.h> 29#include <asm/cputype.h>
30#include <asm/irqflags.h> 30#include <asm/irqflags.h>
31#include <asm/kexec.h>
31#include <asm/memory.h> 32#include <asm/memory.h>
32#include <asm/mmu_context.h> 33#include <asm/mmu_context.h>
33#include <asm/pgalloc.h> 34#include <asm/pgalloc.h>
@@ -102,7 +103,8 @@ int pfn_is_nosave(unsigned long pfn)
102 unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); 103 unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
103 unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); 104 unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
104 105
105 return (pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn); 106 return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) ||
107 crash_is_nosave(pfn);
106} 108}
107 109
108void notrace save_processor_state(void) 110void notrace save_processor_state(void)
@@ -286,6 +288,9 @@ int swsusp_arch_suspend(void)
286 local_dbg_save(flags); 288 local_dbg_save(flags);
287 289
288 if (__cpu_suspend_enter(&state)) { 290 if (__cpu_suspend_enter(&state)) {
291 /* make the crash dump kernel image visible/saveable */
292 crash_prepare_suspend();
293
289 sleep_cpu = smp_processor_id(); 294 sleep_cpu = smp_processor_id();
290 ret = swsusp_save(); 295 ret = swsusp_save();
291 } else { 296 } else {
@@ -297,6 +302,9 @@ int swsusp_arch_suspend(void)
297 if (el2_reset_needed()) 302 if (el2_reset_needed())
298 dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); 303 dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end);
299 304
305 /* make the crash dump kernel image protected again */
306 crash_post_resume();
307
300 /* 308 /*
301 * Tell the hibernation core that we've just restored 309 * Tell the hibernation core that we've just restored
302 * the memory 310 * the memory
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index bc96c8a7fc79..481f54a866c5 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -9,12 +9,19 @@
9 * published by the Free Software Foundation. 9 * published by the Free Software Foundation.
10 */ 10 */
11 11
12#include <linux/interrupt.h>
13#include <linux/irq.h>
14#include <linux/kernel.h>
12#include <linux/kexec.h> 15#include <linux/kexec.h>
16#include <linux/page-flags.h>
13#include <linux/smp.h> 17#include <linux/smp.h>
14 18
15#include <asm/cacheflush.h> 19#include <asm/cacheflush.h>
16#include <asm/cpu_ops.h> 20#include <asm/cpu_ops.h>
21#include <asm/memory.h>
22#include <asm/mmu.h>
17#include <asm/mmu_context.h> 23#include <asm/mmu_context.h>
24#include <asm/page.h>
18 25
19#include "cpu-reset.h" 26#include "cpu-reset.h"
20 27
@@ -22,8 +29,6 @@
22extern const unsigned char arm64_relocate_new_kernel[]; 29extern const unsigned char arm64_relocate_new_kernel[];
23extern const unsigned long arm64_relocate_new_kernel_size; 30extern const unsigned long arm64_relocate_new_kernel_size;
24 31
25static unsigned long kimage_start;
26
27/** 32/**
28 * kexec_image_info - For debugging output. 33 * kexec_image_info - For debugging output.
29 */ 34 */
@@ -64,8 +69,6 @@ void machine_kexec_cleanup(struct kimage *kimage)
64 */ 69 */
65int machine_kexec_prepare(struct kimage *kimage) 70int machine_kexec_prepare(struct kimage *kimage)
66{ 71{
67 kimage_start = kimage->start;
68
69 kexec_image_info(kimage); 72 kexec_image_info(kimage);
70 73
71 if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { 74 if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) {
@@ -144,11 +147,15 @@ void machine_kexec(struct kimage *kimage)
144{ 147{
145 phys_addr_t reboot_code_buffer_phys; 148 phys_addr_t reboot_code_buffer_phys;
146 void *reboot_code_buffer; 149 void *reboot_code_buffer;
150 bool in_kexec_crash = (kimage == kexec_crash_image);
151 bool stuck_cpus = cpus_are_stuck_in_kernel();
147 152
148 /* 153 /*
149 * New cpus may have become stuck_in_kernel after we loaded the image. 154 * New cpus may have become stuck_in_kernel after we loaded the image.
150 */ 155 */
151 BUG_ON(cpus_are_stuck_in_kernel() || (num_online_cpus() > 1)); 156 BUG_ON(!in_kexec_crash && (stuck_cpus || (num_online_cpus() > 1)));
157 WARN(in_kexec_crash && (stuck_cpus || smp_crash_stop_failed()),
158 "Some CPUs may be stale, kdump will be unreliable.\n");
152 159
153 reboot_code_buffer_phys = page_to_phys(kimage->control_code_page); 160 reboot_code_buffer_phys = page_to_phys(kimage->control_code_page);
154 reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys); 161 reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys);
@@ -183,7 +190,7 @@ void machine_kexec(struct kimage *kimage)
183 kexec_list_flush(kimage); 190 kexec_list_flush(kimage);
184 191
185 /* Flush the new image if already in place. */ 192 /* Flush the new image if already in place. */
186 if (kimage->head & IND_DONE) 193 if ((kimage != kexec_crash_image) && (kimage->head & IND_DONE))
187 kexec_segment_flush(kimage); 194 kexec_segment_flush(kimage);
188 195
189 pr_info("Bye!\n"); 196 pr_info("Bye!\n");
@@ -200,13 +207,158 @@ void machine_kexec(struct kimage *kimage)
200 * relocation is complete. 207 * relocation is complete.
201 */ 208 */
202 209
203 cpu_soft_restart(1, reboot_code_buffer_phys, kimage->head, 210 cpu_soft_restart(kimage != kexec_crash_image,
204 kimage_start, 0); 211 reboot_code_buffer_phys, kimage->head, kimage->start, 0);
205 212
206 BUG(); /* Should never get here. */ 213 BUG(); /* Should never get here. */
207} 214}
208 215
216static void machine_kexec_mask_interrupts(void)
217{
218 unsigned int i;
219 struct irq_desc *desc;
220
221 for_each_irq_desc(i, desc) {
222 struct irq_chip *chip;
223 int ret;
224
225 chip = irq_desc_get_chip(desc);
226 if (!chip)
227 continue;
228
229 /*
230 * First try to remove the active state. If this
231 * fails, try to EOI the interrupt.
232 */
233 ret = irq_set_irqchip_state(i, IRQCHIP_STATE_ACTIVE, false);
234
235 if (ret && irqd_irq_inprogress(&desc->irq_data) &&
236 chip->irq_eoi)
237 chip->irq_eoi(&desc->irq_data);
238
239 if (chip->irq_mask)
240 chip->irq_mask(&desc->irq_data);
241
242 if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data))
243 chip->irq_disable(&desc->irq_data);
244 }
245}
246
247/**
248 * machine_crash_shutdown - shutdown non-crashing cpus and save registers
249 */
209void machine_crash_shutdown(struct pt_regs *regs) 250void machine_crash_shutdown(struct pt_regs *regs)
210{ 251{
211 /* Empty routine needed to avoid build errors. */ 252 local_irq_disable();
253
254 /* shutdown non-crashing cpus */
255 smp_send_crash_stop();
256
257 /* for crashing cpu */
258 crash_save_cpu(regs, smp_processor_id());
259 machine_kexec_mask_interrupts();
260
261 pr_info("Starting crashdump kernel...\n");
262}
263
264void arch_kexec_protect_crashkres(void)
265{
266 int i;
267
268 kexec_segment_flush(kexec_crash_image);
269
270 for (i = 0; i < kexec_crash_image->nr_segments; i++)
271 set_memory_valid(
272 __phys_to_virt(kexec_crash_image->segment[i].mem),
273 kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 0);
274}
275
276void arch_kexec_unprotect_crashkres(void)
277{
278 int i;
279
280 for (i = 0; i < kexec_crash_image->nr_segments; i++)
281 set_memory_valid(
282 __phys_to_virt(kexec_crash_image->segment[i].mem),
283 kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 1);
284}
285
286#ifdef CONFIG_HIBERNATION
287/*
288 * To preserve the crash dump kernel image, the relevant memory segments
289 * should be mapped again around the hibernation.
290 */
291void crash_prepare_suspend(void)
292{
293 if (kexec_crash_image)
294 arch_kexec_unprotect_crashkres();
295}
296
297void crash_post_resume(void)
298{
299 if (kexec_crash_image)
300 arch_kexec_protect_crashkres();
301}
302
303/*
304 * crash_is_nosave
305 *
306 * Return true only if a page is part of reserved memory for crash dump kernel,
307 * but does not hold any data of loaded kernel image.
308 *
309 * Note that all the pages in crash dump kernel memory have been initially
310 * marked as Reserved in kexec_reserve_crashkres_pages().
311 *
312 * In hibernation, the pages which are Reserved and yet "nosave" are excluded
313 * from the hibernation iamge. crash_is_nosave() does thich check for crash
314 * dump kernel and will reduce the total size of hibernation image.
315 */
316
317bool crash_is_nosave(unsigned long pfn)
318{
319 int i;
320 phys_addr_t addr;
321
322 if (!crashk_res.end)
323 return false;
324
325 /* in reserved memory? */
326 addr = __pfn_to_phys(pfn);
327 if ((addr < crashk_res.start) || (crashk_res.end < addr))
328 return false;
329
330 if (!kexec_crash_image)
331 return true;
332
333 /* not part of loaded kernel image? */
334 for (i = 0; i < kexec_crash_image->nr_segments; i++)
335 if (addr >= kexec_crash_image->segment[i].mem &&
336 addr < (kexec_crash_image->segment[i].mem +
337 kexec_crash_image->segment[i].memsz))
338 return false;
339
340 return true;
341}
342
343void crash_free_reserved_phys_range(unsigned long begin, unsigned long end)
344{
345 unsigned long addr;
346 struct page *page;
347
348 for (addr = begin; addr < end; addr += PAGE_SIZE) {
349 page = phys_to_page(addr);
350 ClearPageReserved(page);
351 free_reserved_page(page);
352 }
353}
354#endif /* CONFIG_HIBERNATION */
355
356void arch_crash_save_vmcoreinfo(void)
357{
358 VMCOREINFO_NUMBER(VA_BITS);
359 /* Please note VMCOREINFO_NUMBER() uses "%d", not "%x" */
360 vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n",
361 kimage_voffset);
362 vmcoreinfo_append_str("NUMBER(PHYS_OFFSET)=0x%llx\n",
363 PHYS_OFFSET);
212} 364}
diff --git a/arch/arm64/kernel/module-plts.c b/arch/arm64/kernel/module-plts.c
index 1ce90d8450ae..d05dbe658409 100644
--- a/arch/arm64/kernel/module-plts.c
+++ b/arch/arm64/kernel/module-plts.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2014-2016 Linaro Ltd. <ard.biesheuvel@linaro.org> 2 * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -26,35 +26,21 @@ struct plt_entry {
26 __le32 br; /* br x16 */ 26 __le32 br; /* br x16 */
27}; 27};
28 28
29u64 module_emit_plt_entry(struct module *mod, const Elf64_Rela *rela, 29static bool in_init(const struct module *mod, void *loc)
30{
31 return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
32}
33
34u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
30 Elf64_Sym *sym) 35 Elf64_Sym *sym)
31{ 36{
32 struct plt_entry *plt = (struct plt_entry *)mod->arch.plt->sh_addr; 37 struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
33 int i = mod->arch.plt_num_entries; 38 &mod->arch.init;
39 struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
40 int i = pltsec->plt_num_entries;
34 u64 val = sym->st_value + rela->r_addend; 41 u64 val = sym->st_value + rela->r_addend;
35 42
36 /* 43 /*
37 * We only emit PLT entries against undefined (SHN_UNDEF) symbols,
38 * which are listed in the ELF symtab section, but without a type
39 * or a size.
40 * So, similar to how the module loader uses the Elf64_Sym::st_value
41 * field to store the resolved addresses of undefined symbols, let's
42 * borrow the Elf64_Sym::st_size field (whose value is never used by
43 * the module loader, even for symbols that are defined) to record
44 * the address of a symbol's associated PLT entry as we emit it for a
45 * zero addend relocation (which is the only kind we have to deal with
46 * in practice). This allows us to find duplicates without having to
47 * go through the table every time.
48 */
49 if (rela->r_addend == 0 && sym->st_size != 0) {
50 BUG_ON(sym->st_size < (u64)plt || sym->st_size >= (u64)&plt[i]);
51 return sym->st_size;
52 }
53
54 mod->arch.plt_num_entries++;
55 BUG_ON(mod->arch.plt_num_entries > mod->arch.plt_max_entries);
56
57 /*
58 * MOVK/MOVN/MOVZ opcode: 44 * MOVK/MOVN/MOVZ opcode:
59 * +--------+------------+--------+-----------+-------------+---------+ 45 * +--------+------------+--------+-----------+-------------+---------+
60 * | sf[31] | opc[30:29] | 100101 | hw[22:21] | imm16[20:5] | Rd[4:0] | 46 * | sf[31] | opc[30:29] | 100101 | hw[22:21] | imm16[20:5] | Rd[4:0] |
@@ -72,8 +58,19 @@ u64 module_emit_plt_entry(struct module *mod, const Elf64_Rela *rela,
72 cpu_to_le32(0xd61f0200) 58 cpu_to_le32(0xd61f0200)
73 }; 59 };
74 60
75 if (rela->r_addend == 0) 61 /*
76 sym->st_size = (u64)&plt[i]; 62 * Check if the entry we just created is a duplicate. Given that the
63 * relocations are sorted, this will be the last entry we allocated.
64 * (if one exists).
65 */
66 if (i > 0 &&
67 plt[i].mov0 == plt[i - 1].mov0 &&
68 plt[i].mov1 == plt[i - 1].mov1 &&
69 plt[i].mov2 == plt[i - 1].mov2)
70 return (u64)&plt[i - 1];
71
72 pltsec->plt_num_entries++;
73 BUG_ON(pltsec->plt_num_entries > pltsec->plt_max_entries);
77 74
78 return (u64)&plt[i]; 75 return (u64)&plt[i];
79} 76}
@@ -104,7 +101,8 @@ static bool duplicate_rel(const Elf64_Rela *rela, int num)
104 return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0; 101 return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
105} 102}
106 103
107static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num) 104static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num,
105 Elf64_Word dstidx)
108{ 106{
109 unsigned int ret = 0; 107 unsigned int ret = 0;
110 Elf64_Sym *s; 108 Elf64_Sym *s;
@@ -116,13 +114,17 @@ static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num)
116 case R_AARCH64_CALL26: 114 case R_AARCH64_CALL26:
117 /* 115 /*
118 * We only have to consider branch targets that resolve 116 * We only have to consider branch targets that resolve
119 * to undefined symbols. This is not simply a heuristic, 117 * to symbols that are defined in a different section.
120 * it is a fundamental limitation, since the PLT itself 118 * This is not simply a heuristic, it is a fundamental
121 * is part of the module, and needs to be within 128 MB 119 * limitation, since there is no guaranteed way to emit
122 * as well, so modules can never grow beyond that limit. 120 * PLT entries sufficiently close to the branch if the
121 * section size exceeds the range of a branch
122 * instruction. So ignore relocations against defined
123 * symbols if they live in the same section as the
124 * relocation target.
123 */ 125 */
124 s = syms + ELF64_R_SYM(rela[i].r_info); 126 s = syms + ELF64_R_SYM(rela[i].r_info);
125 if (s->st_shndx != SHN_UNDEF) 127 if (s->st_shndx == dstidx)
126 break; 128 break;
127 129
128 /* 130 /*
@@ -149,7 +151,8 @@ static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num)
149int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, 151int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
150 char *secstrings, struct module *mod) 152 char *secstrings, struct module *mod)
151{ 153{
152 unsigned long plt_max_entries = 0; 154 unsigned long core_plts = 0;
155 unsigned long init_plts = 0;
153 Elf64_Sym *syms = NULL; 156 Elf64_Sym *syms = NULL;
154 int i; 157 int i;
155 158
@@ -158,14 +161,16 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
158 * entries. Record the symtab address as well. 161 * entries. Record the symtab address as well.
159 */ 162 */
160 for (i = 0; i < ehdr->e_shnum; i++) { 163 for (i = 0; i < ehdr->e_shnum; i++) {
161 if (strcmp(".plt", secstrings + sechdrs[i].sh_name) == 0) 164 if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
162 mod->arch.plt = sechdrs + i; 165 mod->arch.core.plt = sechdrs + i;
166 else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
167 mod->arch.init.plt = sechdrs + i;
163 else if (sechdrs[i].sh_type == SHT_SYMTAB) 168 else if (sechdrs[i].sh_type == SHT_SYMTAB)
164 syms = (Elf64_Sym *)sechdrs[i].sh_addr; 169 syms = (Elf64_Sym *)sechdrs[i].sh_addr;
165 } 170 }
166 171
167 if (!mod->arch.plt) { 172 if (!mod->arch.core.plt || !mod->arch.init.plt) {
168 pr_err("%s: module PLT section missing\n", mod->name); 173 pr_err("%s: module PLT section(s) missing\n", mod->name);
169 return -ENOEXEC; 174 return -ENOEXEC;
170 } 175 }
171 if (!syms) { 176 if (!syms) {
@@ -188,14 +193,27 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
188 /* sort by type, symbol index and addend */ 193 /* sort by type, symbol index and addend */
189 sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL); 194 sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
190 195
191 plt_max_entries += count_plts(syms, rels, numrels); 196 if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)
197 core_plts += count_plts(syms, rels, numrels,
198 sechdrs[i].sh_info);
199 else
200 init_plts += count_plts(syms, rels, numrels,
201 sechdrs[i].sh_info);
192 } 202 }
193 203
194 mod->arch.plt->sh_type = SHT_NOBITS; 204 mod->arch.core.plt->sh_type = SHT_NOBITS;
195 mod->arch.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 205 mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
196 mod->arch.plt->sh_addralign = L1_CACHE_BYTES; 206 mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
197 mod->arch.plt->sh_size = plt_max_entries * sizeof(struct plt_entry); 207 mod->arch.core.plt->sh_size = (core_plts + 1) * sizeof(struct plt_entry);
198 mod->arch.plt_num_entries = 0; 208 mod->arch.core.plt_num_entries = 0;
199 mod->arch.plt_max_entries = plt_max_entries; 209 mod->arch.core.plt_max_entries = core_plts;
210
211 mod->arch.init.plt->sh_type = SHT_NOBITS;
212 mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
213 mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
214 mod->arch.init.plt->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
215 mod->arch.init.plt_num_entries = 0;
216 mod->arch.init.plt_max_entries = init_plts;
217
200 return 0; 218 return 0;
201} 219}
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 7f316982ce00..c9a2ab446dc6 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -380,7 +380,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
380 380
381 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && 381 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
382 ovf == -ERANGE) { 382 ovf == -ERANGE) {
383 val = module_emit_plt_entry(me, &rel[i], sym); 383 val = module_emit_plt_entry(me, loc, &rel[i], sym);
384 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 384 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2,
385 26, AARCH64_INSN_IMM_26); 385 26, AARCH64_INSN_IMM_26);
386 } 386 }
diff --git a/arch/arm64/kernel/module.lds b/arch/arm64/kernel/module.lds
index 8949f6c6f729..f7c9781a9d48 100644
--- a/arch/arm64/kernel/module.lds
+++ b/arch/arm64/kernel/module.lds
@@ -1,3 +1,4 @@
1SECTIONS { 1SECTIONS {
2 .plt (NOLOAD) : { BYTE(0) } 2 .plt (NOLOAD) : { BYTE(0) }
3 .init.plt (NOLOAD) : { BYTE(0) }
3} 4}
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 57ae9d9ed9bb..bcc79471b38e 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -290,6 +290,12 @@ static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
290 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, 290 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
291 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, 291 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
292 292
293 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE,
294 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL,
295 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE,
296 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL,
297
298 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
293 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, 299 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
294 300
295 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, 301 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
@@ -957,10 +963,26 @@ static int armv8_vulcan_map_event(struct perf_event *event)
957 ARMV8_PMU_EVTYPE_EVENT); 963 ARMV8_PMU_EVTYPE_EVENT);
958} 964}
959 965
966struct armv8pmu_probe_info {
967 struct arm_pmu *pmu;
968 bool present;
969};
970
960static void __armv8pmu_probe_pmu(void *info) 971static void __armv8pmu_probe_pmu(void *info)
961{ 972{
962 struct arm_pmu *cpu_pmu = info; 973 struct armv8pmu_probe_info *probe = info;
974 struct arm_pmu *cpu_pmu = probe->pmu;
975 u64 dfr0;
963 u32 pmceid[2]; 976 u32 pmceid[2];
977 int pmuver;
978
979 dfr0 = read_sysreg(id_aa64dfr0_el1);
980 pmuver = cpuid_feature_extract_signed_field(dfr0,
981 ID_AA64DFR0_PMUVER_SHIFT);
982 if (pmuver < 1)
983 return;
984
985 probe->present = true;
964 986
965 /* Read the nb of CNTx counters supported from PMNC */ 987 /* Read the nb of CNTx counters supported from PMNC */
966 cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT) 988 cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)
@@ -979,13 +1001,27 @@ static void __armv8pmu_probe_pmu(void *info)
979 1001
980static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu) 1002static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
981{ 1003{
982 return smp_call_function_any(&cpu_pmu->supported_cpus, 1004 struct armv8pmu_probe_info probe = {
1005 .pmu = cpu_pmu,
1006 .present = false,
1007 };
1008 int ret;
1009
1010 ret = smp_call_function_any(&cpu_pmu->supported_cpus,
983 __armv8pmu_probe_pmu, 1011 __armv8pmu_probe_pmu,
984 cpu_pmu, 1); 1012 &probe, 1);
1013 if (ret)
1014 return ret;
1015
1016 return probe.present ? 0 : -ENODEV;
985} 1017}
986 1018
987static void armv8_pmu_init(struct arm_pmu *cpu_pmu) 1019static int armv8_pmu_init(struct arm_pmu *cpu_pmu)
988{ 1020{
1021 int ret = armv8pmu_probe_pmu(cpu_pmu);
1022 if (ret)
1023 return ret;
1024
989 cpu_pmu->handle_irq = armv8pmu_handle_irq, 1025 cpu_pmu->handle_irq = armv8pmu_handle_irq,
990 cpu_pmu->enable = armv8pmu_enable_event, 1026 cpu_pmu->enable = armv8pmu_enable_event,
991 cpu_pmu->disable = armv8pmu_disable_event, 1027 cpu_pmu->disable = armv8pmu_disable_event,
@@ -997,78 +1033,104 @@ static void armv8_pmu_init(struct arm_pmu *cpu_pmu)
997 cpu_pmu->reset = armv8pmu_reset, 1033 cpu_pmu->reset = armv8pmu_reset,
998 cpu_pmu->max_period = (1LLU << 32) - 1, 1034 cpu_pmu->max_period = (1LLU << 32) - 1,
999 cpu_pmu->set_event_filter = armv8pmu_set_event_filter; 1035 cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
1036
1037 return 0;
1000} 1038}
1001 1039
1002static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) 1040static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
1003{ 1041{
1004 armv8_pmu_init(cpu_pmu); 1042 int ret = armv8_pmu_init(cpu_pmu);
1043 if (ret)
1044 return ret;
1045
1005 cpu_pmu->name = "armv8_pmuv3"; 1046 cpu_pmu->name = "armv8_pmuv3";
1006 cpu_pmu->map_event = armv8_pmuv3_map_event; 1047 cpu_pmu->map_event = armv8_pmuv3_map_event;
1007 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = 1048 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1008 &armv8_pmuv3_events_attr_group; 1049 &armv8_pmuv3_events_attr_group;
1009 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = 1050 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1010 &armv8_pmuv3_format_attr_group; 1051 &armv8_pmuv3_format_attr_group;
1011 return armv8pmu_probe_pmu(cpu_pmu); 1052
1053 return 0;
1012} 1054}
1013 1055
1014static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) 1056static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
1015{ 1057{
1016 armv8_pmu_init(cpu_pmu); 1058 int ret = armv8_pmu_init(cpu_pmu);
1059 if (ret)
1060 return ret;
1061
1017 cpu_pmu->name = "armv8_cortex_a53"; 1062 cpu_pmu->name = "armv8_cortex_a53";
1018 cpu_pmu->map_event = armv8_a53_map_event; 1063 cpu_pmu->map_event = armv8_a53_map_event;
1019 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = 1064 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1020 &armv8_pmuv3_events_attr_group; 1065 &armv8_pmuv3_events_attr_group;
1021 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = 1066 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1022 &armv8_pmuv3_format_attr_group; 1067 &armv8_pmuv3_format_attr_group;
1023 return armv8pmu_probe_pmu(cpu_pmu); 1068
1069 return 0;
1024} 1070}
1025 1071
1026static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu) 1072static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
1027{ 1073{
1028 armv8_pmu_init(cpu_pmu); 1074 int ret = armv8_pmu_init(cpu_pmu);
1075 if (ret)
1076 return ret;
1077
1029 cpu_pmu->name = "armv8_cortex_a57"; 1078 cpu_pmu->name = "armv8_cortex_a57";
1030 cpu_pmu->map_event = armv8_a57_map_event; 1079 cpu_pmu->map_event = armv8_a57_map_event;
1031 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = 1080 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1032 &armv8_pmuv3_events_attr_group; 1081 &armv8_pmuv3_events_attr_group;
1033 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = 1082 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1034 &armv8_pmuv3_format_attr_group; 1083 &armv8_pmuv3_format_attr_group;
1035 return armv8pmu_probe_pmu(cpu_pmu); 1084
1085 return 0;
1036} 1086}
1037 1087
1038static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) 1088static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
1039{ 1089{
1040 armv8_pmu_init(cpu_pmu); 1090 int ret = armv8_pmu_init(cpu_pmu);
1091 if (ret)
1092 return ret;
1093
1041 cpu_pmu->name = "armv8_cortex_a72"; 1094 cpu_pmu->name = "armv8_cortex_a72";
1042 cpu_pmu->map_event = armv8_a57_map_event; 1095 cpu_pmu->map_event = armv8_a57_map_event;
1043 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = 1096 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1044 &armv8_pmuv3_events_attr_group; 1097 &armv8_pmuv3_events_attr_group;
1045 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = 1098 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1046 &armv8_pmuv3_format_attr_group; 1099 &armv8_pmuv3_format_attr_group;
1047 return armv8pmu_probe_pmu(cpu_pmu); 1100
1101 return 0;
1048} 1102}
1049 1103
1050static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) 1104static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
1051{ 1105{
1052 armv8_pmu_init(cpu_pmu); 1106 int ret = armv8_pmu_init(cpu_pmu);
1107 if (ret)
1108 return ret;
1109
1053 cpu_pmu->name = "armv8_cavium_thunder"; 1110 cpu_pmu->name = "armv8_cavium_thunder";
1054 cpu_pmu->map_event = armv8_thunder_map_event; 1111 cpu_pmu->map_event = armv8_thunder_map_event;
1055 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = 1112 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1056 &armv8_pmuv3_events_attr_group; 1113 &armv8_pmuv3_events_attr_group;
1057 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = 1114 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1058 &armv8_pmuv3_format_attr_group; 1115 &armv8_pmuv3_format_attr_group;
1059 return armv8pmu_probe_pmu(cpu_pmu); 1116
1117 return 0;
1060} 1118}
1061 1119
1062static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) 1120static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
1063{ 1121{
1064 armv8_pmu_init(cpu_pmu); 1122 int ret = armv8_pmu_init(cpu_pmu);
1123 if (ret)
1124 return ret;
1125
1065 cpu_pmu->name = "armv8_brcm_vulcan"; 1126 cpu_pmu->name = "armv8_brcm_vulcan";
1066 cpu_pmu->map_event = armv8_vulcan_map_event; 1127 cpu_pmu->map_event = armv8_vulcan_map_event;
1067 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = 1128 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1068 &armv8_pmuv3_events_attr_group; 1129 &armv8_pmuv3_events_attr_group;
1069 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = 1130 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1070 &armv8_pmuv3_format_attr_group; 1131 &armv8_pmuv3_format_attr_group;
1071 return armv8pmu_probe_pmu(cpu_pmu); 1132
1133 return 0;
1072} 1134}
1073 1135
1074static const struct of_device_id armv8_pmu_of_device_ids[] = { 1136static const struct of_device_id armv8_pmu_of_device_ids[] = {
@@ -1081,24 +1143,9 @@ static const struct of_device_id armv8_pmu_of_device_ids[] = {
1081 {}, 1143 {},
1082}; 1144};
1083 1145
1084/*
1085 * Non DT systems have their micro/arch events probed at run-time.
1086 * A fairly complete list of generic events are provided and ones that
1087 * aren't supported by the current PMU are disabled.
1088 */
1089static const struct pmu_probe_info armv8_pmu_probe_table[] = {
1090 PMU_PROBE(0, 0, armv8_pmuv3_init), /* enable all defined counters */
1091 { /* sentinel value */ }
1092};
1093
1094static int armv8_pmu_device_probe(struct platform_device *pdev) 1146static int armv8_pmu_device_probe(struct platform_device *pdev)
1095{ 1147{
1096 if (acpi_disabled) 1148 return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL);
1097 return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids,
1098 NULL);
1099
1100 return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids,
1101 armv8_pmu_probe_table);
1102} 1149}
1103 1150
1104static struct platform_driver armv8_pmu_driver = { 1151static struct platform_driver armv8_pmu_driver = {
@@ -1109,4 +1156,11 @@ static struct platform_driver armv8_pmu_driver = {
1109 .probe = armv8_pmu_device_probe, 1156 .probe = armv8_pmu_device_probe,
1110}; 1157};
1111 1158
1112builtin_platform_driver(armv8_pmu_driver); 1159static int __init armv8_pmu_driver_init(void)
1160{
1161 if (acpi_disabled)
1162 return platform_driver_register(&armv8_pmu_driver);
1163 else
1164 return arm_pmu_acpi_probe(armv8_pmuv3_init);
1165}
1166device_initcall(armv8_pmu_driver_init)
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 043d373b8369..ae2a835898d7 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -205,12 +205,10 @@ void __show_regs(struct pt_regs *regs)
205 205
206 pr_cont("\n"); 206 pr_cont("\n");
207 } 207 }
208 printk("\n");
209} 208}
210 209
211void show_regs(struct pt_regs * regs) 210void show_regs(struct pt_regs * regs)
212{ 211{
213 printk("\n");
214 __show_regs(regs); 212 __show_regs(regs);
215} 213}
216 214
diff --git a/arch/arm64/kernel/reloc_test_core.c b/arch/arm64/kernel/reloc_test_core.c
new file mode 100644
index 000000000000..c124752a8bd3
--- /dev/null
+++ b/arch/arm64/kernel/reloc_test_core.c
@@ -0,0 +1,81 @@
1/*
2 * Copyright (C) 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10#include <linux/module.h>
11
12int sym64_rel;
13
14#define SYM64_ABS_VAL 0xffff880000cccccc
15#define SYM32_ABS_VAL 0xf800cccc
16#define SYM16_ABS_VAL 0xf8cc
17
18#define __SET_ABS(name, val) asm(".globl " #name "; .set "#name ", " #val)
19#define SET_ABS(name, val) __SET_ABS(name, val)
20
21SET_ABS(sym64_abs, SYM64_ABS_VAL);
22SET_ABS(sym32_abs, SYM32_ABS_VAL);
23SET_ABS(sym16_abs, SYM16_ABS_VAL);
24
25asmlinkage u64 absolute_data64(void);
26asmlinkage u64 absolute_data32(void);
27asmlinkage u64 absolute_data16(void);
28asmlinkage u64 signed_movw(void);
29asmlinkage u64 unsigned_movw(void);
30asmlinkage u64 relative_adrp(void);
31asmlinkage u64 relative_adr(void);
32asmlinkage u64 relative_data64(void);
33asmlinkage u64 relative_data32(void);
34asmlinkage u64 relative_data16(void);
35
36static struct {
37 char name[32];
38 u64 (*f)(void);
39 u64 expect;
40} const funcs[] = {
41 { "R_AARCH64_ABS64", absolute_data64, UL(SYM64_ABS_VAL) },
42 { "R_AARCH64_ABS32", absolute_data32, UL(SYM32_ABS_VAL) },
43 { "R_AARCH64_ABS16", absolute_data16, UL(SYM16_ABS_VAL) },
44 { "R_AARCH64_MOVW_SABS_Gn", signed_movw, UL(SYM64_ABS_VAL) },
45 { "R_AARCH64_MOVW_UABS_Gn", unsigned_movw, UL(SYM64_ABS_VAL) },
46#ifndef CONFIG_ARM64_ERRATUM_843419
47 { "R_AARCH64_ADR_PREL_PG_HI21", relative_adrp, (u64)&sym64_rel },
48#endif
49 { "R_AARCH64_ADR_PREL_LO21", relative_adr, (u64)&sym64_rel },
50 { "R_AARCH64_PREL64", relative_data64, (u64)&sym64_rel },
51 { "R_AARCH64_PREL32", relative_data32, (u64)&sym64_rel },
52 { "R_AARCH64_PREL16", relative_data16, (u64)&sym64_rel },
53};
54
55static int reloc_test_init(void)
56{
57 int i;
58
59 pr_info("Relocation test:\n");
60 pr_info("-------------------------------------------------------\n");
61
62 for (i = 0; i < ARRAY_SIZE(funcs); i++) {
63 u64 ret = funcs[i].f();
64
65 pr_info("%-31s 0x%016llx %s\n", funcs[i].name, ret,
66 ret == funcs[i].expect ? "pass" : "fail");
67 if (ret != funcs[i].expect)
68 pr_err("Relocation failed, expected 0x%016llx, not 0x%016llx\n",
69 funcs[i].expect, ret);
70 }
71 return 0;
72}
73
74static void reloc_test_exit(void)
75{
76}
77
78module_init(reloc_test_init);
79module_exit(reloc_test_exit);
80
81MODULE_LICENSE("GPL v2");
diff --git a/arch/arm64/kernel/reloc_test_syms.S b/arch/arm64/kernel/reloc_test_syms.S
new file mode 100644
index 000000000000..e1edcefeb02d
--- /dev/null
+++ b/arch/arm64/kernel/reloc_test_syms.S
@@ -0,0 +1,83 @@
1/*
2 * Copyright (C) 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10#include <linux/linkage.h>
11
12ENTRY(absolute_data64)
13 ldr x0, 0f
14 ret
150: .quad sym64_abs
16ENDPROC(absolute_data64)
17
18ENTRY(absolute_data32)
19 ldr w0, 0f
20 ret
210: .long sym32_abs
22ENDPROC(absolute_data32)
23
24ENTRY(absolute_data16)
25 adr x0, 0f
26 ldrh w0, [x0]
27 ret
280: .short sym16_abs, 0
29ENDPROC(absolute_data16)
30
31ENTRY(signed_movw)
32 movz x0, #:abs_g2_s:sym64_abs
33 movk x0, #:abs_g1_nc:sym64_abs
34 movk x0, #:abs_g0_nc:sym64_abs
35 ret
36ENDPROC(signed_movw)
37
38ENTRY(unsigned_movw)
39 movz x0, #:abs_g3:sym64_abs
40 movk x0, #:abs_g2_nc:sym64_abs
41 movk x0, #:abs_g1_nc:sym64_abs
42 movk x0, #:abs_g0_nc:sym64_abs
43 ret
44ENDPROC(unsigned_movw)
45
46#ifndef CONFIG_ARM64_ERRATUM_843419
47
48ENTRY(relative_adrp)
49 adrp x0, sym64_rel
50 add x0, x0, #:lo12:sym64_rel
51 ret
52ENDPROC(relative_adrp)
53
54#endif
55
56ENTRY(relative_adr)
57 adr x0, sym64_rel
58 ret
59ENDPROC(relative_adr)
60
61ENTRY(relative_data64)
62 adr x1, 0f
63 ldr x0, [x1]
64 add x0, x0, x1
65 ret
660: .quad sym64_rel - .
67ENDPROC(relative_data64)
68
69ENTRY(relative_data32)
70 adr x1, 0f
71 ldr w0, [x1]
72 add x0, x0, x1
73 ret
740: .long sym64_rel - .
75ENDPROC(relative_data32)
76
77ENTRY(relative_data16)
78 adr x1, 0f
79 ldrsh w0, [x1]
80 add x0, x0, x1
81 ret
820: .short sym64_rel - ., 0
83ENDPROC(relative_data16)
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 42274bda0ccb..2c822ef94f34 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -31,7 +31,6 @@
31#include <linux/screen_info.h> 31#include <linux/screen_info.h>
32#include <linux/init.h> 32#include <linux/init.h>
33#include <linux/kexec.h> 33#include <linux/kexec.h>
34#include <linux/crash_dump.h>
35#include <linux/root_dev.h> 34#include <linux/root_dev.h>
36#include <linux/cpu.h> 35#include <linux/cpu.h>
37#include <linux/interrupt.h> 36#include <linux/interrupt.h>
@@ -181,6 +180,7 @@ static void __init smp_build_mpidr_hash(void)
181static void __init setup_machine_fdt(phys_addr_t dt_phys) 180static void __init setup_machine_fdt(phys_addr_t dt_phys)
182{ 181{
183 void *dt_virt = fixmap_remap_fdt(dt_phys); 182 void *dt_virt = fixmap_remap_fdt(dt_phys);
183 const char *name;
184 184
185 if (!dt_virt || !early_init_dt_scan(dt_virt)) { 185 if (!dt_virt || !early_init_dt_scan(dt_virt)) {
186 pr_crit("\n" 186 pr_crit("\n"
@@ -193,7 +193,9 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys)
193 cpu_relax(); 193 cpu_relax();
194 } 194 }
195 195
196 dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); 196 name = of_flat_dt_get_machine_name();
197 pr_info("Machine model: %s\n", name);
198 dump_stack_set_arch_desc("%s (DT)", name);
197} 199}
198 200
199static void __init request_standard_resources(void) 201static void __init request_standard_resources(void)
@@ -226,6 +228,12 @@ static void __init request_standard_resources(void)
226 if (kernel_data.start >= res->start && 228 if (kernel_data.start >= res->start &&
227 kernel_data.end <= res->end) 229 kernel_data.end <= res->end)
228 request_resource(res, &kernel_data); 230 request_resource(res, &kernel_data);
231#ifdef CONFIG_KEXEC_CORE
232 /* Userspace will find "Crash kernel" region in /proc/iomem. */
233 if (crashk_res.end && crashk_res.start >= res->start &&
234 crashk_res.end <= res->end)
235 request_resource(res, &crashk_res);
236#endif
229 } 237 }
230} 238}
231 239
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 9b1036570586..6e0e16a3a7d4 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -39,6 +39,7 @@
39#include <linux/completion.h> 39#include <linux/completion.h>
40#include <linux/of.h> 40#include <linux/of.h>
41#include <linux/irq_work.h> 41#include <linux/irq_work.h>
42#include <linux/kexec.h>
42 43
43#include <asm/alternative.h> 44#include <asm/alternative.h>
44#include <asm/atomic.h> 45#include <asm/atomic.h>
@@ -76,6 +77,7 @@ enum ipi_msg_type {
76 IPI_RESCHEDULE, 77 IPI_RESCHEDULE,
77 IPI_CALL_FUNC, 78 IPI_CALL_FUNC,
78 IPI_CPU_STOP, 79 IPI_CPU_STOP,
80 IPI_CPU_CRASH_STOP,
79 IPI_TIMER, 81 IPI_TIMER,
80 IPI_IRQ_WORK, 82 IPI_IRQ_WORK,
81 IPI_WAKEUP 83 IPI_WAKEUP
@@ -434,6 +436,7 @@ void __init smp_cpus_done(unsigned int max_cpus)
434 setup_cpu_features(); 436 setup_cpu_features();
435 hyp_mode_check(); 437 hyp_mode_check();
436 apply_alternatives_all(); 438 apply_alternatives_all();
439 mark_linear_text_alias_ro();
437} 440}
438 441
439void __init smp_prepare_boot_cpu(void) 442void __init smp_prepare_boot_cpu(void)
@@ -518,6 +521,13 @@ static bool bootcpu_valid __initdata;
518static unsigned int cpu_count = 1; 521static unsigned int cpu_count = 1;
519 522
520#ifdef CONFIG_ACPI 523#ifdef CONFIG_ACPI
524static struct acpi_madt_generic_interrupt cpu_madt_gicc[NR_CPUS];
525
526struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)
527{
528 return &cpu_madt_gicc[cpu];
529}
530
521/* 531/*
522 * acpi_map_gic_cpu_interface - parse processor MADT entry 532 * acpi_map_gic_cpu_interface - parse processor MADT entry
523 * 533 *
@@ -552,6 +562,7 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
552 return; 562 return;
553 } 563 }
554 bootcpu_valid = true; 564 bootcpu_valid = true;
565 cpu_madt_gicc[0] = *processor;
555 early_map_cpu_to_node(0, acpi_numa_get_nid(0, hwid)); 566 early_map_cpu_to_node(0, acpi_numa_get_nid(0, hwid));
556 return; 567 return;
557 } 568 }
@@ -562,6 +573,8 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
562 /* map the logical cpu id to cpu MPIDR */ 573 /* map the logical cpu id to cpu MPIDR */
563 cpu_logical_map(cpu_count) = hwid; 574 cpu_logical_map(cpu_count) = hwid;
564 575
576 cpu_madt_gicc[cpu_count] = *processor;
577
565 /* 578 /*
566 * Set-up the ACPI parking protocol cpu entries 579 * Set-up the ACPI parking protocol cpu entries
567 * while initializing the cpu_logical_map to 580 * while initializing the cpu_logical_map to
@@ -755,6 +768,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
755 S(IPI_RESCHEDULE, "Rescheduling interrupts"), 768 S(IPI_RESCHEDULE, "Rescheduling interrupts"),
756 S(IPI_CALL_FUNC, "Function call interrupts"), 769 S(IPI_CALL_FUNC, "Function call interrupts"),
757 S(IPI_CPU_STOP, "CPU stop interrupts"), 770 S(IPI_CPU_STOP, "CPU stop interrupts"),
771 S(IPI_CPU_CRASH_STOP, "CPU stop (for crash dump) interrupts"),
758 S(IPI_TIMER, "Timer broadcast interrupts"), 772 S(IPI_TIMER, "Timer broadcast interrupts"),
759 S(IPI_IRQ_WORK, "IRQ work interrupts"), 773 S(IPI_IRQ_WORK, "IRQ work interrupts"),
760 S(IPI_WAKEUP, "CPU wake-up interrupts"), 774 S(IPI_WAKEUP, "CPU wake-up interrupts"),
@@ -829,6 +843,29 @@ static void ipi_cpu_stop(unsigned int cpu)
829 cpu_relax(); 843 cpu_relax();
830} 844}
831 845
846#ifdef CONFIG_KEXEC_CORE
847static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0);
848#endif
849
850static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
851{
852#ifdef CONFIG_KEXEC_CORE
853 crash_save_cpu(regs, cpu);
854
855 atomic_dec(&waiting_for_crash_ipi);
856
857 local_irq_disable();
858
859#ifdef CONFIG_HOTPLUG_CPU
860 if (cpu_ops[cpu]->cpu_die)
861 cpu_ops[cpu]->cpu_die(cpu);
862#endif
863
864 /* just in case */
865 cpu_park_loop();
866#endif
867}
868
832/* 869/*
833 * Main handler for inter-processor interrupts 870 * Main handler for inter-processor interrupts
834 */ 871 */
@@ -859,6 +896,15 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
859 irq_exit(); 896 irq_exit();
860 break; 897 break;
861 898
899 case IPI_CPU_CRASH_STOP:
900 if (IS_ENABLED(CONFIG_KEXEC_CORE)) {
901 irq_enter();
902 ipi_cpu_crash_stop(cpu, regs);
903
904 unreachable();
905 }
906 break;
907
862#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 908#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
863 case IPI_TIMER: 909 case IPI_TIMER:
864 irq_enter(); 910 irq_enter();
@@ -931,6 +977,39 @@ void smp_send_stop(void)
931 cpumask_pr_args(cpu_online_mask)); 977 cpumask_pr_args(cpu_online_mask));
932} 978}
933 979
980#ifdef CONFIG_KEXEC_CORE
981void smp_send_crash_stop(void)
982{
983 cpumask_t mask;
984 unsigned long timeout;
985
986 if (num_online_cpus() == 1)
987 return;
988
989 cpumask_copy(&mask, cpu_online_mask);
990 cpumask_clear_cpu(smp_processor_id(), &mask);
991
992 atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1);
993
994 pr_crit("SMP: stopping secondary CPUs\n");
995 smp_cross_call(&mask, IPI_CPU_CRASH_STOP);
996
997 /* Wait up to one second for other CPUs to stop */
998 timeout = USEC_PER_SEC;
999 while ((atomic_read(&waiting_for_crash_ipi) > 0) && timeout--)
1000 udelay(1);
1001
1002 if (atomic_read(&waiting_for_crash_ipi) > 0)
1003 pr_warning("SMP: failed to stop secondary CPUs %*pbl\n",
1004 cpumask_pr_args(&mask));
1005}
1006
1007bool smp_crash_stop_failed(void)
1008{
1009 return (atomic_read(&waiting_for_crash_ipi) > 0);
1010}
1011#endif
1012
934/* 1013/*
935 * not supported here 1014 * not supported here
936 */ 1015 */
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 1de444e6c669..d4d6ae02cd55 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -513,6 +513,14 @@ static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
513 regs->pc += 4; 513 regs->pc += 4;
514} 514}
515 515
516static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
517{
518 int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
519
520 pt_regs_write_reg(regs, rt, read_sysreg(cntfrq_el0));
521 regs->pc += 4;
522}
523
516struct sys64_hook { 524struct sys64_hook {
517 unsigned int esr_mask; 525 unsigned int esr_mask;
518 unsigned int esr_val; 526 unsigned int esr_val;
@@ -537,6 +545,12 @@ static struct sys64_hook sys64_hooks[] = {
537 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, 545 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
538 .handler = cntvct_read_handler, 546 .handler = cntvct_read_handler,
539 }, 547 },
548 {
549 /* Trap read access to CNTFRQ_EL0 */
550 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
551 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
552 .handler = cntfrq_read_handler,
553 },
540 {}, 554 {},
541}; 555};
542 556
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index b8deffa9e1bf..987a00ee446c 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -143,12 +143,27 @@ SECTIONS
143 143
144 . = ALIGN(SEGMENT_ALIGN); 144 . = ALIGN(SEGMENT_ALIGN);
145 __init_begin = .; 145 __init_begin = .;
146 __inittext_begin = .;
146 147
147 INIT_TEXT_SECTION(8) 148 INIT_TEXT_SECTION(8)
148 .exit.text : { 149 .exit.text : {
149 ARM_EXIT_KEEP(EXIT_TEXT) 150 ARM_EXIT_KEEP(EXIT_TEXT)
150 } 151 }
151 152
153 . = ALIGN(4);
154 .altinstructions : {
155 __alt_instructions = .;
156 *(.altinstructions)
157 __alt_instructions_end = .;
158 }
159 .altinstr_replacement : {
160 *(.altinstr_replacement)
161 }
162
163 . = ALIGN(PAGE_SIZE);
164 __inittext_end = .;
165 __initdata_begin = .;
166
152 .init.data : { 167 .init.data : {
153 INIT_DATA 168 INIT_DATA
154 INIT_SETUP(16) 169 INIT_SETUP(16)
@@ -164,15 +179,6 @@ SECTIONS
164 179
165 PERCPU_SECTION(L1_CACHE_BYTES) 180 PERCPU_SECTION(L1_CACHE_BYTES)
166 181
167 . = ALIGN(4);
168 .altinstructions : {
169 __alt_instructions = .;
170 *(.altinstructions)
171 __alt_instructions_end = .;
172 }
173 .altinstr_replacement : {
174 *(.altinstr_replacement)
175 }
176 .rela : ALIGN(8) { 182 .rela : ALIGN(8) {
177 *(.rela .rela*) 183 *(.rela .rela*)
178 } 184 }
@@ -181,6 +187,7 @@ SECTIONS
181 __rela_size = SIZEOF(.rela); 187 __rela_size = SIZEOF(.rela);
182 188
183 . = ALIGN(SEGMENT_ALIGN); 189 . = ALIGN(SEGMENT_ALIGN);
190 __initdata_end = .;
184 __init_end = .; 191 __init_end = .;
185 192
186 _data = .; 193 _data = .;
@@ -206,6 +213,7 @@ SECTIONS
206 } 213 }
207 214
208 PECOFF_EDATA_PADDING 215 PECOFF_EDATA_PADDING
216 __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin);
209 _edata = .; 217 _edata = .;
210 218
211 BSS_SECTION(0, 0, 0) 219 BSS_SECTION(0, 0, 0)
@@ -221,6 +229,7 @@ SECTIONS
221 . += RESERVED_TTBR0_SIZE; 229 . += RESERVED_TTBR0_SIZE;
222#endif 230#endif
223 231
232 __pecoff_data_size = ABSOLUTE(. - __initdata_begin);
224 _end = .; 233 _end = .;
225 234
226 STABS_DEBUG 235 STABS_DEBUG
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 9e1d2b75eecd..73464a96c365 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -94,6 +94,28 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
94 dsb(ish); 94 dsb(ish);
95 isb(); 95 isb();
96 96
97 /*
98 * If the host is running at EL1 and we have a VPIPT I-cache,
99 * then we must perform I-cache maintenance at EL2 in order for
100 * it to have an effect on the guest. Since the guest cannot hit
101 * I-cache lines allocated with a different VMID, we don't need
102 * to worry about junk out of guest reset (we nuke the I-cache on
103 * VMID rollover), but we do need to be careful when remapping
104 * executable pages for the same guest. This can happen when KSM
105 * takes a CoW fault on an executable page, copies the page into
106 * a page that was previously mapped in the guest and then needs
107 * to invalidate the guest view of the I-cache for that page
108 * from EL1. To solve this, we invalidate the entire I-cache when
109 * unmapping a page from a guest if we have a VPIPT I-cache but
110 * the host is running at EL1. As above, we could do better if
111 * we had the VA.
112 *
113 * The moral of this story is: if you have a VPIPT I-cache, then
114 * you should be running with VHE enabled.
115 */
116 if (!has_vhe() && icache_is_vpipt())
117 __flush_icache_all();
118
97 __tlb_switch_to_host()(kvm); 119 __tlb_switch_to_host()(kvm);
98} 120}
99 121
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index d9e9697de1b2..561badf93de8 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -60,7 +60,7 @@ static bool cpu_has_32bit_el1(void)
60{ 60{
61 u64 pfr0; 61 u64 pfr0;
62 62
63 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1); 63 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
64 return !!(pfr0 & 0x20); 64 return !!(pfr0 & 0x20);
65} 65}
66 66
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 0e26f8c2b56f..26b0e77878b5 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1183,8 +1183,8 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu,
1183 if (p->is_write) { 1183 if (p->is_write) {
1184 return ignore_write(vcpu, p); 1184 return ignore_write(vcpu, p);
1185 } else { 1185 } else {
1186 u64 dfr = read_system_reg(SYS_ID_AA64DFR0_EL1); 1186 u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1187 u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1); 1187 u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1188 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT); 1188 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
1189 1189
1190 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) | 1190 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index 68634c630cdd..ab9f5f0fb2c7 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -119,9 +119,6 @@ static void flush_context(unsigned int cpu)
119 119
120 /* Queue a TLB invalidate and flush the I-cache if necessary. */ 120 /* Queue a TLB invalidate and flush the I-cache if necessary. */
121 cpumask_setall(&tlb_flush_pending); 121 cpumask_setall(&tlb_flush_pending);
122
123 if (icache_is_aivivt())
124 __flush_icache_all();
125} 122}
126 123
127static bool check_update_reserved_asid(u64 asid, u64 newasid) 124static bool check_update_reserved_asid(u64 asid, u64 newasid)
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 7f8b37e85a2b..4dac4afc95a5 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -308,24 +308,15 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
308 sg->length, dir); 308 sg->length, dir);
309} 309}
310 310
311static int __swiotlb_mmap(struct device *dev, 311static int __swiotlb_mmap_pfn(struct vm_area_struct *vma,
312 struct vm_area_struct *vma, 312 unsigned long pfn, size_t size)
313 void *cpu_addr, dma_addr_t dma_addr, size_t size,
314 unsigned long attrs)
315{ 313{
316 int ret = -ENXIO; 314 int ret = -ENXIO;
317 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> 315 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
318 PAGE_SHIFT; 316 PAGE_SHIFT;
319 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 317 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
320 unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
321 unsigned long off = vma->vm_pgoff; 318 unsigned long off = vma->vm_pgoff;
322 319
323 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
324 is_device_dma_coherent(dev));
325
326 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
327 return ret;
328
329 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 320 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
330 ret = remap_pfn_range(vma, vma->vm_start, 321 ret = remap_pfn_range(vma, vma->vm_start,
331 pfn + off, 322 pfn + off,
@@ -336,19 +327,43 @@ static int __swiotlb_mmap(struct device *dev,
336 return ret; 327 return ret;
337} 328}
338 329
339static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt, 330static int __swiotlb_mmap(struct device *dev,
340 void *cpu_addr, dma_addr_t handle, size_t size, 331 struct vm_area_struct *vma,
341 unsigned long attrs) 332 void *cpu_addr, dma_addr_t dma_addr, size_t size,
333 unsigned long attrs)
334{
335 int ret;
336 unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
337
338 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
339 is_device_dma_coherent(dev));
340
341 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
342 return ret;
343
344 return __swiotlb_mmap_pfn(vma, pfn, size);
345}
346
347static int __swiotlb_get_sgtable_page(struct sg_table *sgt,
348 struct page *page, size_t size)
342{ 349{
343 int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 350 int ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
344 351
345 if (!ret) 352 if (!ret)
346 sg_set_page(sgt->sgl, phys_to_page(dma_to_phys(dev, handle)), 353 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
347 PAGE_ALIGN(size), 0);
348 354
349 return ret; 355 return ret;
350} 356}
351 357
358static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
359 void *cpu_addr, dma_addr_t handle, size_t size,
360 unsigned long attrs)
361{
362 struct page *page = phys_to_page(dma_to_phys(dev, handle));
363
364 return __swiotlb_get_sgtable_page(sgt, page, size);
365}
366
352static int __swiotlb_dma_supported(struct device *hwdev, u64 mask) 367static int __swiotlb_dma_supported(struct device *hwdev, u64 mask)
353{ 368{
354 if (swiotlb) 369 if (swiotlb)
@@ -584,20 +599,7 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size,
584 */ 599 */
585 gfp |= __GFP_ZERO; 600 gfp |= __GFP_ZERO;
586 601
587 if (gfpflags_allow_blocking(gfp)) { 602 if (!gfpflags_allow_blocking(gfp)) {
588 struct page **pages;
589 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
590
591 pages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot,
592 handle, flush_page);
593 if (!pages)
594 return NULL;
595
596 addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,
597 __builtin_return_address(0));
598 if (!addr)
599 iommu_dma_free(dev, pages, iosize, handle);
600 } else {
601 struct page *page; 603 struct page *page;
602 /* 604 /*
603 * In atomic context we can't remap anything, so we'll only 605 * In atomic context we can't remap anything, so we'll only
@@ -621,6 +623,45 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size,
621 __free_from_pool(addr, size); 623 __free_from_pool(addr, size);
622 addr = NULL; 624 addr = NULL;
623 } 625 }
626 } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
627 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
628 struct page *page;
629
630 page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
631 get_order(size), gfp);
632 if (!page)
633 return NULL;
634
635 *handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);
636 if (iommu_dma_mapping_error(dev, *handle)) {
637 dma_release_from_contiguous(dev, page,
638 size >> PAGE_SHIFT);
639 return NULL;
640 }
641 if (!coherent)
642 __dma_flush_area(page_to_virt(page), iosize);
643
644 addr = dma_common_contiguous_remap(page, size, VM_USERMAP,
645 prot,
646 __builtin_return_address(0));
647 if (!addr) {
648 iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs);
649 dma_release_from_contiguous(dev, page,
650 size >> PAGE_SHIFT);
651 }
652 } else {
653 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
654 struct page **pages;
655
656 pages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot,
657 handle, flush_page);
658 if (!pages)
659 return NULL;
660
661 addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,
662 __builtin_return_address(0));
663 if (!addr)
664 iommu_dma_free(dev, pages, iosize, handle);
624 } 665 }
625 return addr; 666 return addr;
626} 667}
@@ -632,7 +673,8 @@ static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
632 673
633 size = PAGE_ALIGN(size); 674 size = PAGE_ALIGN(size);
634 /* 675 /*
635 * @cpu_addr will be one of 3 things depending on how it was allocated: 676 * @cpu_addr will be one of 4 things depending on how it was allocated:
677 * - A remapped array of pages for contiguous allocations.
636 * - A remapped array of pages from iommu_dma_alloc(), for all 678 * - A remapped array of pages from iommu_dma_alloc(), for all
637 * non-atomic allocations. 679 * non-atomic allocations.
638 * - A non-cacheable alias from the atomic pool, for atomic 680 * - A non-cacheable alias from the atomic pool, for atomic
@@ -644,6 +686,12 @@ static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
644 if (__in_atomic_pool(cpu_addr, size)) { 686 if (__in_atomic_pool(cpu_addr, size)) {
645 iommu_dma_unmap_page(dev, handle, iosize, 0, 0); 687 iommu_dma_unmap_page(dev, handle, iosize, 0, 0);
646 __free_from_pool(cpu_addr, size); 688 __free_from_pool(cpu_addr, size);
689 } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
690 struct page *page = vmalloc_to_page(cpu_addr);
691
692 iommu_dma_unmap_page(dev, handle, iosize, 0, attrs);
693 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
694 dma_common_free_remap(cpu_addr, size, VM_USERMAP);
647 } else if (is_vmalloc_addr(cpu_addr)){ 695 } else if (is_vmalloc_addr(cpu_addr)){
648 struct vm_struct *area = find_vm_area(cpu_addr); 696 struct vm_struct *area = find_vm_area(cpu_addr);
649 697
@@ -670,6 +718,15 @@ static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
670 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 718 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
671 return ret; 719 return ret;
672 720
721 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
722 /*
723 * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,
724 * hence in the vmalloc space.
725 */
726 unsigned long pfn = vmalloc_to_pfn(cpu_addr);
727 return __swiotlb_mmap_pfn(vma, pfn, size);
728 }
729
673 area = find_vm_area(cpu_addr); 730 area = find_vm_area(cpu_addr);
674 if (WARN_ON(!area || !area->pages)) 731 if (WARN_ON(!area || !area->pages))
675 return -ENXIO; 732 return -ENXIO;
@@ -684,6 +741,15 @@ static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
684 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 741 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
685 struct vm_struct *area = find_vm_area(cpu_addr); 742 struct vm_struct *area = find_vm_area(cpu_addr);
686 743
744 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
745 /*
746 * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,
747 * hence in the vmalloc space.
748 */
749 struct page *page = vmalloc_to_page(cpu_addr);
750 return __swiotlb_get_sgtable_page(sgt, page, size);
751 }
752
687 if (WARN_ON(!area || !area->pages)) 753 if (WARN_ON(!area || !area->pages))
688 return -ENXIO; 754 return -ENXIO;
689 755
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 1b35b8bddbfb..37b95dff0b07 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -174,12 +174,33 @@ static bool is_el1_instruction_abort(unsigned int esr)
174 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; 174 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
175} 175}
176 176
177static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs,
178 unsigned long addr)
179{
180 unsigned int ec = ESR_ELx_EC(esr);
181 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
182
183 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
184 return false;
185
186 if (fsc_type == ESR_ELx_FSC_PERM)
187 return true;
188
189 if (addr < USER_DS && system_uses_ttbr0_pan())
190 return fsc_type == ESR_ELx_FSC_FAULT &&
191 (regs->pstate & PSR_PAN_BIT);
192
193 return false;
194}
195
177/* 196/*
178 * The kernel tried to access some page that wasn't present. 197 * The kernel tried to access some page that wasn't present.
179 */ 198 */
180static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, 199static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
181 unsigned int esr, struct pt_regs *regs) 200 unsigned int esr, struct pt_regs *regs)
182{ 201{
202 const char *msg;
203
183 /* 204 /*
184 * Are we prepared to handle this kernel fault? 205 * Are we prepared to handle this kernel fault?
185 * We are almost certainly not prepared to handle instruction faults. 206 * We are almost certainly not prepared to handle instruction faults.
@@ -191,9 +212,20 @@ static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
191 * No handler, we'll have to terminate things with extreme prejudice. 212 * No handler, we'll have to terminate things with extreme prejudice.
192 */ 213 */
193 bust_spinlocks(1); 214 bust_spinlocks(1);
194 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", 215
195 (addr < PAGE_SIZE) ? "NULL pointer dereference" : 216 if (is_permission_fault(esr, regs, addr)) {
196 "paging request", addr); 217 if (esr & ESR_ELx_WNR)
218 msg = "write to read-only memory";
219 else
220 msg = "read from unreadable memory";
221 } else if (addr < PAGE_SIZE) {
222 msg = "NULL pointer dereference";
223 } else {
224 msg = "paging request";
225 }
226
227 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg,
228 addr);
197 229
198 show_pte(mm, addr); 230 show_pte(mm, addr);
199 die("Oops", regs, esr); 231 die("Oops", regs, esr);
@@ -287,21 +319,6 @@ out:
287 return fault; 319 return fault;
288} 320}
289 321
290static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs)
291{
292 unsigned int ec = ESR_ELx_EC(esr);
293 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
294
295 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
296 return false;
297
298 if (system_uses_ttbr0_pan())
299 return fsc_type == ESR_ELx_FSC_FAULT &&
300 (regs->pstate & PSR_PAN_BIT);
301 else
302 return fsc_type == ESR_ELx_FSC_PERM;
303}
304
305static bool is_el0_instruction_abort(unsigned int esr) 322static bool is_el0_instruction_abort(unsigned int esr)
306{ 323{
307 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; 324 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
@@ -339,7 +356,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
339 mm_flags |= FAULT_FLAG_WRITE; 356 mm_flags |= FAULT_FLAG_WRITE;
340 } 357 }
341 358
342 if (addr < USER_DS && is_permission_fault(esr, regs)) { 359 if (addr < USER_DS && is_permission_fault(esr, regs, addr)) {
343 /* regs->orig_addr_limit may be 0 if we entered from EL0 */ 360 /* regs->orig_addr_limit may be 0 if we entered from EL0 */
344 if (regs->orig_addr_limit == KERNEL_DS) 361 if (regs->orig_addr_limit == KERNEL_DS)
345 die("Accessing user space memory with fs=KERNEL_DS", regs, esr); 362 die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index 554a2558c12e..21a8d828cbf4 100644
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -22,7 +22,7 @@
22#include <linux/pagemap.h> 22#include <linux/pagemap.h>
23 23
24#include <asm/cacheflush.h> 24#include <asm/cacheflush.h>
25#include <asm/cachetype.h> 25#include <asm/cache.h>
26#include <asm/tlbflush.h> 26#include <asm/tlbflush.h>
27 27
28void sync_icache_aliases(void *kaddr, unsigned long len) 28void sync_icache_aliases(void *kaddr, unsigned long len)
@@ -65,8 +65,6 @@ void __sync_icache_dcache(pte_t pte, unsigned long addr)
65 if (!test_and_set_bit(PG_dcache_clean, &page->flags)) 65 if (!test_and_set_bit(PG_dcache_clean, &page->flags))
66 sync_icache_aliases(page_address(page), 66 sync_icache_aliases(page_address(page),
67 PAGE_SIZE << compound_order(page)); 67 PAGE_SIZE << compound_order(page));
68 else if (icache_is_aivivt())
69 __flush_icache_all();
70} 68}
71 69
72/* 70/*
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index e19e06593e37..5960bef0170d 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -30,6 +30,7 @@
30#include <linux/gfp.h> 30#include <linux/gfp.h>
31#include <linux/memblock.h> 31#include <linux/memblock.h>
32#include <linux/sort.h> 32#include <linux/sort.h>
33#include <linux/of.h>
33#include <linux/of_fdt.h> 34#include <linux/of_fdt.h>
34#include <linux/dma-mapping.h> 35#include <linux/dma-mapping.h>
35#include <linux/dma-contiguous.h> 36#include <linux/dma-contiguous.h>
@@ -37,6 +38,8 @@
37#include <linux/swiotlb.h> 38#include <linux/swiotlb.h>
38#include <linux/vmalloc.h> 39#include <linux/vmalloc.h>
39#include <linux/mm.h> 40#include <linux/mm.h>
41#include <linux/kexec.h>
42#include <linux/crash_dump.h>
40 43
41#include <asm/boot.h> 44#include <asm/boot.h>
42#include <asm/fixmap.h> 45#include <asm/fixmap.h>
@@ -77,6 +80,142 @@ static int __init early_initrd(char *p)
77early_param("initrd", early_initrd); 80early_param("initrd", early_initrd);
78#endif 81#endif
79 82
83#ifdef CONFIG_KEXEC_CORE
84/*
85 * reserve_crashkernel() - reserves memory for crash kernel
86 *
87 * This function reserves memory area given in "crashkernel=" kernel command
88 * line parameter. The memory reserved is used by dump capture kernel when
89 * primary kernel is crashing.
90 */
91static void __init reserve_crashkernel(void)
92{
93 unsigned long long crash_base, crash_size;
94 int ret;
95
96 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
97 &crash_size, &crash_base);
98 /* no crashkernel= or invalid value specified */
99 if (ret || !crash_size)
100 return;
101
102 crash_size = PAGE_ALIGN(crash_size);
103
104 if (crash_base == 0) {
105 /* Current arm64 boot protocol requires 2MB alignment */
106 crash_base = memblock_find_in_range(0, ARCH_LOW_ADDRESS_LIMIT,
107 crash_size, SZ_2M);
108 if (crash_base == 0) {
109 pr_warn("cannot allocate crashkernel (size:0x%llx)\n",
110 crash_size);
111 return;
112 }
113 } else {
114 /* User specifies base address explicitly. */
115 if (!memblock_is_region_memory(crash_base, crash_size)) {
116 pr_warn("cannot reserve crashkernel: region is not memory\n");
117 return;
118 }
119
120 if (memblock_is_region_reserved(crash_base, crash_size)) {
121 pr_warn("cannot reserve crashkernel: region overlaps reserved memory\n");
122 return;
123 }
124
125 if (!IS_ALIGNED(crash_base, SZ_2M)) {
126 pr_warn("cannot reserve crashkernel: base address is not 2MB aligned\n");
127 return;
128 }
129 }
130 memblock_reserve(crash_base, crash_size);
131
132 pr_info("crashkernel reserved: 0x%016llx - 0x%016llx (%lld MB)\n",
133 crash_base, crash_base + crash_size, crash_size >> 20);
134
135 crashk_res.start = crash_base;
136 crashk_res.end = crash_base + crash_size - 1;
137}
138
139static void __init kexec_reserve_crashkres_pages(void)
140{
141#ifdef CONFIG_HIBERNATION
142 phys_addr_t addr;
143 struct page *page;
144
145 if (!crashk_res.end)
146 return;
147
148 /*
149 * To reduce the size of hibernation image, all the pages are
150 * marked as Reserved initially.
151 */
152 for (addr = crashk_res.start; addr < (crashk_res.end + 1);
153 addr += PAGE_SIZE) {
154 page = phys_to_page(addr);
155 SetPageReserved(page);
156 }
157#endif
158}
159#else
160static void __init reserve_crashkernel(void)
161{
162}
163
164static void __init kexec_reserve_crashkres_pages(void)
165{
166}
167#endif /* CONFIG_KEXEC_CORE */
168
169#ifdef CONFIG_CRASH_DUMP
170static int __init early_init_dt_scan_elfcorehdr(unsigned long node,
171 const char *uname, int depth, void *data)
172{
173 const __be32 *reg;
174 int len;
175
176 if (depth != 1 || strcmp(uname, "chosen") != 0)
177 return 0;
178
179 reg = of_get_flat_dt_prop(node, "linux,elfcorehdr", &len);
180 if (!reg || (len < (dt_root_addr_cells + dt_root_size_cells)))
181 return 1;
182
183 elfcorehdr_addr = dt_mem_next_cell(dt_root_addr_cells, &reg);
184 elfcorehdr_size = dt_mem_next_cell(dt_root_size_cells, &reg);
185
186 return 1;
187}
188
189/*
190 * reserve_elfcorehdr() - reserves memory for elf core header
191 *
192 * This function reserves the memory occupied by an elf core header
193 * described in the device tree. This region contains all the
194 * information about primary kernel's core image and is used by a dump
195 * capture kernel to access the system memory on primary kernel.
196 */
197static void __init reserve_elfcorehdr(void)
198{
199 of_scan_flat_dt(early_init_dt_scan_elfcorehdr, NULL);
200
201 if (!elfcorehdr_size)
202 return;
203
204 if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) {
205 pr_warn("elfcorehdr is overlapped\n");
206 return;
207 }
208
209 memblock_reserve(elfcorehdr_addr, elfcorehdr_size);
210
211 pr_info("Reserving %lldKB of memory at 0x%llx for elfcorehdr\n",
212 elfcorehdr_size >> 10, elfcorehdr_addr);
213}
214#else
215static void __init reserve_elfcorehdr(void)
216{
217}
218#endif /* CONFIG_CRASH_DUMP */
80/* 219/*
81 * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It 220 * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
82 * currently assumes that for memory starting above 4G, 32-bit devices will 221 * currently assumes that for memory starting above 4G, 32-bit devices will
@@ -188,10 +327,45 @@ static int __init early_mem(char *p)
188} 327}
189early_param("mem", early_mem); 328early_param("mem", early_mem);
190 329
330static int __init early_init_dt_scan_usablemem(unsigned long node,
331 const char *uname, int depth, void *data)
332{
333 struct memblock_region *usablemem = data;
334 const __be32 *reg;
335 int len;
336
337 if (depth != 1 || strcmp(uname, "chosen") != 0)
338 return 0;
339
340 reg = of_get_flat_dt_prop(node, "linux,usable-memory-range", &len);
341 if (!reg || (len < (dt_root_addr_cells + dt_root_size_cells)))
342 return 1;
343
344 usablemem->base = dt_mem_next_cell(dt_root_addr_cells, &reg);
345 usablemem->size = dt_mem_next_cell(dt_root_size_cells, &reg);
346
347 return 1;
348}
349
350static void __init fdt_enforce_memory_region(void)
351{
352 struct memblock_region reg = {
353 .size = 0,
354 };
355
356 of_scan_flat_dt(early_init_dt_scan_usablemem, &reg);
357
358 if (reg.size)
359 memblock_cap_memory_range(reg.base, reg.size);
360}
361
191void __init arm64_memblock_init(void) 362void __init arm64_memblock_init(void)
192{ 363{
193 const s64 linear_region_size = -(s64)PAGE_OFFSET; 364 const s64 linear_region_size = -(s64)PAGE_OFFSET;
194 365
366 /* Handle linux,usable-memory-range property */
367 fdt_enforce_memory_region();
368
195 /* 369 /*
196 * Ensure that the linear region takes up exactly half of the kernel 370 * Ensure that the linear region takes up exactly half of the kernel
197 * virtual address space. This way, we can distinguish a linear address 371 * virtual address space. This way, we can distinguish a linear address
@@ -297,6 +471,11 @@ void __init arm64_memblock_init(void)
297 arm64_dma_phys_limit = max_zone_dma_phys(); 471 arm64_dma_phys_limit = max_zone_dma_phys();
298 else 472 else
299 arm64_dma_phys_limit = PHYS_MASK + 1; 473 arm64_dma_phys_limit = PHYS_MASK + 1;
474
475 reserve_crashkernel();
476
477 reserve_elfcorehdr();
478
300 dma_contiguous_reserve(arm64_dma_phys_limit); 479 dma_contiguous_reserve(arm64_dma_phys_limit);
301 480
302 memblock_allow_resize(); 481 memblock_allow_resize();
@@ -416,6 +595,8 @@ void __init mem_init(void)
416 /* this will put all unused low memory onto the freelists */ 595 /* this will put all unused low memory onto the freelists */
417 free_all_bootmem(); 596 free_all_bootmem();
418 597
598 kexec_reserve_crashkres_pages();
599
419 mem_init_print_info(NULL); 600 mem_init_print_info(NULL);
420 601
421#define MLK(b, t) b, t, ((t) - (b)) >> 10 602#define MLK(b, t) b, t, ((t) - (b)) >> 10
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index d28dbcf596b6..0c429ec6fde8 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -22,6 +22,8 @@
22#include <linux/kernel.h> 22#include <linux/kernel.h>
23#include <linux/errno.h> 23#include <linux/errno.h>
24#include <linux/init.h> 24#include <linux/init.h>
25#include <linux/ioport.h>
26#include <linux/kexec.h>
25#include <linux/libfdt.h> 27#include <linux/libfdt.h>
26#include <linux/mman.h> 28#include <linux/mman.h>
27#include <linux/nodemask.h> 29#include <linux/nodemask.h>
@@ -43,6 +45,9 @@
43#include <asm/mmu_context.h> 45#include <asm/mmu_context.h>
44#include <asm/ptdump.h> 46#include <asm/ptdump.h>
45 47
48#define NO_BLOCK_MAPPINGS BIT(0)
49#define NO_CONT_MAPPINGS BIT(1)
50
46u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 51u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
47 52
48u64 kimage_voffset __ro_after_init; 53u64 kimage_voffset __ro_after_init;
@@ -103,33 +108,27 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
103 */ 108 */
104 static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE; 109 static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;
105 110
106 return old == 0 || new == 0 || ((old ^ new) & ~mask) == 0; 111 /* creating or taking down mappings is always safe */
112 if (old == 0 || new == 0)
113 return true;
114
115 /* live contiguous mappings may not be manipulated at all */
116 if ((old | new) & PTE_CONT)
117 return false;
118
119 return ((old ^ new) & ~mask) == 0;
107} 120}
108 121
109static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 122static void init_pte(pmd_t *pmd, unsigned long addr, unsigned long end,
110 unsigned long end, unsigned long pfn, 123 phys_addr_t phys, pgprot_t prot)
111 pgprot_t prot,
112 phys_addr_t (*pgtable_alloc)(void))
113{ 124{
114 pte_t *pte; 125 pte_t *pte;
115 126
116 BUG_ON(pmd_sect(*pmd));
117 if (pmd_none(*pmd)) {
118 phys_addr_t pte_phys;
119 BUG_ON(!pgtable_alloc);
120 pte_phys = pgtable_alloc();
121 pte = pte_set_fixmap(pte_phys);
122 __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
123 pte_clear_fixmap();
124 }
125 BUG_ON(pmd_bad(*pmd));
126
127 pte = pte_set_fixmap_offset(pmd, addr); 127 pte = pte_set_fixmap_offset(pmd, addr);
128 do { 128 do {
129 pte_t old_pte = *pte; 129 pte_t old_pte = *pte;
130 130
131 set_pte(pte, pfn_pte(pfn, prot)); 131 set_pte(pte, pfn_pte(__phys_to_pfn(phys), prot));
132 pfn++;
133 132
134 /* 133 /*
135 * After the PTE entry has been populated once, we 134 * After the PTE entry has been populated once, we
@@ -137,32 +136,51 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
137 */ 136 */
138 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte))); 137 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));
139 138
139 phys += PAGE_SIZE;
140 } while (pte++, addr += PAGE_SIZE, addr != end); 140 } while (pte++, addr += PAGE_SIZE, addr != end);
141 141
142 pte_clear_fixmap(); 142 pte_clear_fixmap();
143} 143}
144 144
145static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 145static void alloc_init_cont_pte(pmd_t *pmd, unsigned long addr,
146 phys_addr_t phys, pgprot_t prot, 146 unsigned long end, phys_addr_t phys,
147 phys_addr_t (*pgtable_alloc)(void), 147 pgprot_t prot,
148 bool page_mappings_only) 148 phys_addr_t (*pgtable_alloc)(void),
149 int flags)
149{ 150{
150 pmd_t *pmd;
151 unsigned long next; 151 unsigned long next;
152 152
153 /* 153 BUG_ON(pmd_sect(*pmd));
154 * Check for initial section mappings in the pgd/pud and remove them. 154 if (pmd_none(*pmd)) {
155 */ 155 phys_addr_t pte_phys;
156 BUG_ON(pud_sect(*pud));
157 if (pud_none(*pud)) {
158 phys_addr_t pmd_phys;
159 BUG_ON(!pgtable_alloc); 156 BUG_ON(!pgtable_alloc);
160 pmd_phys = pgtable_alloc(); 157 pte_phys = pgtable_alloc();
161 pmd = pmd_set_fixmap(pmd_phys); 158 __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
162 __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
163 pmd_clear_fixmap();
164 } 159 }
165 BUG_ON(pud_bad(*pud)); 160 BUG_ON(pmd_bad(*pmd));
161
162 do {
163 pgprot_t __prot = prot;
164
165 next = pte_cont_addr_end(addr, end);
166
167 /* use a contiguous mapping if the range is suitably aligned */
168 if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
169 (flags & NO_CONT_MAPPINGS) == 0)
170 __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
171
172 init_pte(pmd, addr, next, phys, __prot);
173
174 phys += next - addr;
175 } while (addr = next, addr != end);
176}
177
178static void init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
179 phys_addr_t phys, pgprot_t prot,
180 phys_addr_t (*pgtable_alloc)(void), int flags)
181{
182 unsigned long next;
183 pmd_t *pmd;
166 184
167 pmd = pmd_set_fixmap_offset(pud, addr); 185 pmd = pmd_set_fixmap_offset(pud, addr);
168 do { 186 do {
@@ -172,7 +190,7 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
172 190
173 /* try section mapping first */ 191 /* try section mapping first */
174 if (((addr | next | phys) & ~SECTION_MASK) == 0 && 192 if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
175 !page_mappings_only) { 193 (flags & NO_BLOCK_MAPPINGS) == 0) {
176 pmd_set_huge(pmd, phys, prot); 194 pmd_set_huge(pmd, phys, prot);
177 195
178 /* 196 /*
@@ -182,8 +200,8 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
182 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), 200 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
183 pmd_val(*pmd))); 201 pmd_val(*pmd)));
184 } else { 202 } else {
185 alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 203 alloc_init_cont_pte(pmd, addr, next, phys, prot,
186 prot, pgtable_alloc); 204 pgtable_alloc, flags);
187 205
188 BUG_ON(pmd_val(old_pmd) != 0 && 206 BUG_ON(pmd_val(old_pmd) != 0 &&
189 pmd_val(old_pmd) != pmd_val(*pmd)); 207 pmd_val(old_pmd) != pmd_val(*pmd));
@@ -194,6 +212,41 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
194 pmd_clear_fixmap(); 212 pmd_clear_fixmap();
195} 213}
196 214
215static void alloc_init_cont_pmd(pud_t *pud, unsigned long addr,
216 unsigned long end, phys_addr_t phys,
217 pgprot_t prot,
218 phys_addr_t (*pgtable_alloc)(void), int flags)
219{
220 unsigned long next;
221
222 /*
223 * Check for initial section mappings in the pgd/pud.
224 */
225 BUG_ON(pud_sect(*pud));
226 if (pud_none(*pud)) {
227 phys_addr_t pmd_phys;
228 BUG_ON(!pgtable_alloc);
229 pmd_phys = pgtable_alloc();
230 __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
231 }
232 BUG_ON(pud_bad(*pud));
233
234 do {
235 pgprot_t __prot = prot;
236
237 next = pmd_cont_addr_end(addr, end);
238
239 /* use a contiguous mapping if the range is suitably aligned */
240 if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
241 (flags & NO_CONT_MAPPINGS) == 0)
242 __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
243
244 init_pmd(pud, addr, next, phys, __prot, pgtable_alloc, flags);
245
246 phys += next - addr;
247 } while (addr = next, addr != end);
248}
249
197static inline bool use_1G_block(unsigned long addr, unsigned long next, 250static inline bool use_1G_block(unsigned long addr, unsigned long next,
198 unsigned long phys) 251 unsigned long phys)
199{ 252{
@@ -209,7 +262,7 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next,
209static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, 262static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
210 phys_addr_t phys, pgprot_t prot, 263 phys_addr_t phys, pgprot_t prot,
211 phys_addr_t (*pgtable_alloc)(void), 264 phys_addr_t (*pgtable_alloc)(void),
212 bool page_mappings_only) 265 int flags)
213{ 266{
214 pud_t *pud; 267 pud_t *pud;
215 unsigned long next; 268 unsigned long next;
@@ -231,7 +284,8 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
231 /* 284 /*
232 * For 4K granule only, attempt to put down a 1GB block 285 * For 4K granule only, attempt to put down a 1GB block
233 */ 286 */
234 if (use_1G_block(addr, next, phys) && !page_mappings_only) { 287 if (use_1G_block(addr, next, phys) &&
288 (flags & NO_BLOCK_MAPPINGS) == 0) {
235 pud_set_huge(pud, phys, prot); 289 pud_set_huge(pud, phys, prot);
236 290
237 /* 291 /*
@@ -241,8 +295,8 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
241 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), 295 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
242 pud_val(*pud))); 296 pud_val(*pud)));
243 } else { 297 } else {
244 alloc_init_pmd(pud, addr, next, phys, prot, 298 alloc_init_cont_pmd(pud, addr, next, phys, prot,
245 pgtable_alloc, page_mappings_only); 299 pgtable_alloc, flags);
246 300
247 BUG_ON(pud_val(old_pud) != 0 && 301 BUG_ON(pud_val(old_pud) != 0 &&
248 pud_val(old_pud) != pud_val(*pud)); 302 pud_val(old_pud) != pud_val(*pud));
@@ -257,7 +311,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
257 unsigned long virt, phys_addr_t size, 311 unsigned long virt, phys_addr_t size,
258 pgprot_t prot, 312 pgprot_t prot,
259 phys_addr_t (*pgtable_alloc)(void), 313 phys_addr_t (*pgtable_alloc)(void),
260 bool page_mappings_only) 314 int flags)
261{ 315{
262 unsigned long addr, length, end, next; 316 unsigned long addr, length, end, next;
263 pgd_t *pgd = pgd_offset_raw(pgdir, virt); 317 pgd_t *pgd = pgd_offset_raw(pgdir, virt);
@@ -277,7 +331,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
277 do { 331 do {
278 next = pgd_addr_end(addr, end); 332 next = pgd_addr_end(addr, end);
279 alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, 333 alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
280 page_mappings_only); 334 flags);
281 phys += next - addr; 335 phys += next - addr;
282 } while (pgd++, addr = next, addr != end); 336 } while (pgd++, addr = next, addr != end);
283} 337}
@@ -306,82 +360,80 @@ static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
306 &phys, virt); 360 &phys, virt);
307 return; 361 return;
308 } 362 }
309 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, false); 363 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
364 NO_CONT_MAPPINGS);
310} 365}
311 366
312void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 367void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
313 unsigned long virt, phys_addr_t size, 368 unsigned long virt, phys_addr_t size,
314 pgprot_t prot, bool page_mappings_only) 369 pgprot_t prot, bool page_mappings_only)
315{ 370{
371 int flags = 0;
372
316 BUG_ON(mm == &init_mm); 373 BUG_ON(mm == &init_mm);
317 374
375 if (page_mappings_only)
376 flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
377
318 __create_pgd_mapping(mm->pgd, phys, virt, size, prot, 378 __create_pgd_mapping(mm->pgd, phys, virt, size, prot,
319 pgd_pgtable_alloc, page_mappings_only); 379 pgd_pgtable_alloc, flags);
320} 380}
321 381
322static void create_mapping_late(phys_addr_t phys, unsigned long virt, 382static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
323 phys_addr_t size, pgprot_t prot) 383 phys_addr_t size, pgprot_t prot)
324{ 384{
325 if (virt < VMALLOC_START) { 385 if (virt < VMALLOC_START) {
326 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 386 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
327 &phys, virt); 387 &phys, virt);
328 return; 388 return;
329 } 389 }
330 390
331 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, 391 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
332 NULL, debug_pagealloc_enabled()); 392 NO_CONT_MAPPINGS);
393
394 /* flush the TLBs after updating live kernel mappings */
395 flush_tlb_kernel_range(virt, virt + size);
333} 396}
334 397
335static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) 398static void __init __map_memblock(pgd_t *pgd, phys_addr_t start,
399 phys_addr_t end, pgprot_t prot, int flags)
336{ 400{
337 phys_addr_t kernel_start = __pa_symbol(_text); 401 __create_pgd_mapping(pgd, start, __phys_to_virt(start), end - start,
338 phys_addr_t kernel_end = __pa_symbol(__init_begin); 402 prot, early_pgtable_alloc, flags);
339 403}
340 /*
341 * Take care not to create a writable alias for the
342 * read-only text and rodata sections of the kernel image.
343 */
344
345 /* No overlap with the kernel text/rodata */
346 if (end < kernel_start || start >= kernel_end) {
347 __create_pgd_mapping(pgd, start, __phys_to_virt(start),
348 end - start, PAGE_KERNEL,
349 early_pgtable_alloc,
350 debug_pagealloc_enabled());
351 return;
352 }
353
354 /*
355 * This block overlaps the kernel text/rodata mappings.
356 * Map the portion(s) which don't overlap.
357 */
358 if (start < kernel_start)
359 __create_pgd_mapping(pgd, start,
360 __phys_to_virt(start),
361 kernel_start - start, PAGE_KERNEL,
362 early_pgtable_alloc,
363 debug_pagealloc_enabled());
364 if (kernel_end < end)
365 __create_pgd_mapping(pgd, kernel_end,
366 __phys_to_virt(kernel_end),
367 end - kernel_end, PAGE_KERNEL,
368 early_pgtable_alloc,
369 debug_pagealloc_enabled());
370 404
405void __init mark_linear_text_alias_ro(void)
406{
371 /* 407 /*
372 * Map the linear alias of the [_text, __init_begin) interval as 408 * Remove the write permissions from the linear alias of .text/.rodata
373 * read-only/non-executable. This makes the contents of the
374 * region accessible to subsystems such as hibernate, but
375 * protects it from inadvertent modification or execution.
376 */ 409 */
377 __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), 410 update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text),
378 kernel_end - kernel_start, PAGE_KERNEL_RO, 411 (unsigned long)__init_begin - (unsigned long)_text,
379 early_pgtable_alloc, debug_pagealloc_enabled()); 412 PAGE_KERNEL_RO);
380} 413}
381 414
382static void __init map_mem(pgd_t *pgd) 415static void __init map_mem(pgd_t *pgd)
383{ 416{
417 phys_addr_t kernel_start = __pa_symbol(_text);
418 phys_addr_t kernel_end = __pa_symbol(__init_begin);
384 struct memblock_region *reg; 419 struct memblock_region *reg;
420 int flags = 0;
421
422 if (debug_pagealloc_enabled())
423 flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
424
425 /*
426 * Take care not to create a writable alias for the
427 * read-only text and rodata sections of the kernel image.
428 * So temporarily mark them as NOMAP to skip mappings in
429 * the following for-loop
430 */
431 memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
432#ifdef CONFIG_KEXEC_CORE
433 if (crashk_res.end)
434 memblock_mark_nomap(crashk_res.start,
435 resource_size(&crashk_res));
436#endif
385 437
386 /* map all the memory banks */ 438 /* map all the memory banks */
387 for_each_memblock(memory, reg) { 439 for_each_memblock(memory, reg) {
@@ -393,33 +445,57 @@ static void __init map_mem(pgd_t *pgd)
393 if (memblock_is_nomap(reg)) 445 if (memblock_is_nomap(reg))
394 continue; 446 continue;
395 447
396 __map_memblock(pgd, start, end); 448 __map_memblock(pgd, start, end, PAGE_KERNEL, flags);
449 }
450
451 /*
452 * Map the linear alias of the [_text, __init_begin) interval
453 * as non-executable now, and remove the write permission in
454 * mark_linear_text_alias_ro() below (which will be called after
455 * alternative patching has completed). This makes the contents
456 * of the region accessible to subsystems such as hibernate,
457 * but protects it from inadvertent modification or execution.
458 * Note that contiguous mappings cannot be remapped in this way,
459 * so we should avoid them here.
460 */
461 __map_memblock(pgd, kernel_start, kernel_end,
462 PAGE_KERNEL, NO_CONT_MAPPINGS);
463 memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
464
465#ifdef CONFIG_KEXEC_CORE
466 /*
467 * Use page-level mappings here so that we can shrink the region
468 * in page granularity and put back unused memory to buddy system
469 * through /sys/kernel/kexec_crash_size interface.
470 */
471 if (crashk_res.end) {
472 __map_memblock(pgd, crashk_res.start, crashk_res.end + 1,
473 PAGE_KERNEL,
474 NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
475 memblock_clear_nomap(crashk_res.start,
476 resource_size(&crashk_res));
397 } 477 }
478#endif
398} 479}
399 480
400void mark_rodata_ro(void) 481void mark_rodata_ro(void)
401{ 482{
402 unsigned long section_size; 483 unsigned long section_size;
403 484
404 section_size = (unsigned long)_etext - (unsigned long)_text;
405 create_mapping_late(__pa_symbol(_text), (unsigned long)_text,
406 section_size, PAGE_KERNEL_ROX);
407 /* 485 /*
408 * mark .rodata as read only. Use __init_begin rather than __end_rodata 486 * mark .rodata as read only. Use __init_begin rather than __end_rodata
409 * to cover NOTES and EXCEPTION_TABLE. 487 * to cover NOTES and EXCEPTION_TABLE.
410 */ 488 */
411 section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; 489 section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
412 create_mapping_late(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, 490 update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
413 section_size, PAGE_KERNEL_RO); 491 section_size, PAGE_KERNEL_RO);
414 492
415 /* flush the TLBs after updating live kernel mappings */
416 flush_tlb_all();
417
418 debug_checkwx(); 493 debug_checkwx();
419} 494}
420 495
421static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, 496static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
422 pgprot_t prot, struct vm_struct *vma) 497 pgprot_t prot, struct vm_struct *vma,
498 int flags)
423{ 499{
424 phys_addr_t pa_start = __pa_symbol(va_start); 500 phys_addr_t pa_start = __pa_symbol(va_start);
425 unsigned long size = va_end - va_start; 501 unsigned long size = va_end - va_start;
@@ -428,7 +504,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
428 BUG_ON(!PAGE_ALIGNED(size)); 504 BUG_ON(!PAGE_ALIGNED(size));
429 505
430 __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, 506 __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
431 early_pgtable_alloc, debug_pagealloc_enabled()); 507 early_pgtable_alloc, flags);
432 508
433 vma->addr = va_start; 509 vma->addr = va_start;
434 vma->phys_addr = pa_start; 510 vma->phys_addr = pa_start;
@@ -439,18 +515,39 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
439 vm_area_add_early(vma); 515 vm_area_add_early(vma);
440} 516}
441 517
518static int __init parse_rodata(char *arg)
519{
520 return strtobool(arg, &rodata_enabled);
521}
522early_param("rodata", parse_rodata);
523
442/* 524/*
443 * Create fine-grained mappings for the kernel. 525 * Create fine-grained mappings for the kernel.
444 */ 526 */
445static void __init map_kernel(pgd_t *pgd) 527static void __init map_kernel(pgd_t *pgd)
446{ 528{
447 static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; 529 static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
530 vmlinux_initdata, vmlinux_data;
448 531
449 map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); 532 /*
450 map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); 533 * External debuggers may need to write directly to the text
451 map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, 534 * mapping to install SW breakpoints. Allow this (only) when
452 &vmlinux_init); 535 * explicitly requested with rodata=off.
453 map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); 536 */
537 pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
538
539 /*
540 * Only rodata will be remapped with different permissions later on,
541 * all other segments are allowed to use contiguous mappings.
542 */
543 map_kernel_segment(pgd, _text, _etext, text_prot, &vmlinux_text, 0);
544 map_kernel_segment(pgd, __start_rodata, __inittext_begin, PAGE_KERNEL,
545 &vmlinux_rodata, NO_CONT_MAPPINGS);
546 map_kernel_segment(pgd, __inittext_begin, __inittext_end, text_prot,
547 &vmlinux_inittext, 0);
548 map_kernel_segment(pgd, __initdata_begin, __initdata_end, PAGE_KERNEL,
549 &vmlinux_initdata, 0);
550 map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data, 0);
454 551
455 if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { 552 if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
456 /* 553 /*
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index 8def55e7249b..3212ee0558f6 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -125,20 +125,23 @@ int set_memory_x(unsigned long addr, int numpages)
125} 125}
126EXPORT_SYMBOL_GPL(set_memory_x); 126EXPORT_SYMBOL_GPL(set_memory_x);
127 127
128#ifdef CONFIG_DEBUG_PAGEALLOC 128int set_memory_valid(unsigned long addr, int numpages, int enable)
129void __kernel_map_pages(struct page *page, int numpages, int enable)
130{ 129{
131 unsigned long addr = (unsigned long) page_address(page);
132
133 if (enable) 130 if (enable)
134 __change_memory_common(addr, PAGE_SIZE * numpages, 131 return __change_memory_common(addr, PAGE_SIZE * numpages,
135 __pgprot(PTE_VALID), 132 __pgprot(PTE_VALID),
136 __pgprot(0)); 133 __pgprot(0));
137 else 134 else
138 __change_memory_common(addr, PAGE_SIZE * numpages, 135 return __change_memory_common(addr, PAGE_SIZE * numpages,
139 __pgprot(0), 136 __pgprot(0),
140 __pgprot(PTE_VALID)); 137 __pgprot(PTE_VALID));
141} 138}
139
140#ifdef CONFIG_DEBUG_PAGEALLOC
141void __kernel_map_pages(struct page *page, int numpages, int enable)
142{
143 set_memory_valid((unsigned long)page_address(page), numpages, enable);
144}
142#ifdef CONFIG_HIBERNATION 145#ifdef CONFIG_HIBERNATION
143/* 146/*
144 * When built with CONFIG_DEBUG_PAGEALLOC and CONFIG_HIBERNATION, this function 147 * When built with CONFIG_DEBUG_PAGEALLOC and CONFIG_HIBERNATION, this function
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index 4a5bb967250b..22e08d272db7 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -225,7 +225,7 @@ static struct acpi_iort_node *iort_scan_node(enum acpi_iort_node_type type,
225 225
226 if (iort_node->type == type && 226 if (iort_node->type == type &&
227 ACPI_SUCCESS(callback(iort_node, context))) 227 ACPI_SUCCESS(callback(iort_node, context)))
228 return iort_node; 228 return iort_node;
229 229
230 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node, 230 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
231 iort_node->length); 231 iort_node->length);
@@ -253,17 +253,15 @@ static acpi_status iort_match_node_callback(struct acpi_iort_node *node,
253 void *context) 253 void *context)
254{ 254{
255 struct device *dev = context; 255 struct device *dev = context;
256 acpi_status status; 256 acpi_status status = AE_NOT_FOUND;
257 257
258 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT) { 258 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT) {
259 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; 259 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
260 struct acpi_device *adev = to_acpi_device_node(dev->fwnode); 260 struct acpi_device *adev = to_acpi_device_node(dev->fwnode);
261 struct acpi_iort_named_component *ncomp; 261 struct acpi_iort_named_component *ncomp;
262 262
263 if (!adev) { 263 if (!adev)
264 status = AE_NOT_FOUND;
265 goto out; 264 goto out;
266 }
267 265
268 status = acpi_get_name(adev->handle, ACPI_FULL_PATHNAME, &buf); 266 status = acpi_get_name(adev->handle, ACPI_FULL_PATHNAME, &buf);
269 if (ACPI_FAILURE(status)) { 267 if (ACPI_FAILURE(status)) {
@@ -289,8 +287,6 @@ static acpi_status iort_match_node_callback(struct acpi_iort_node *node,
289 */ 287 */
290 status = pci_rc->pci_segment_number == pci_domain_nr(bus) ? 288 status = pci_rc->pci_segment_number == pci_domain_nr(bus) ?
291 AE_OK : AE_NOT_FOUND; 289 AE_OK : AE_NOT_FOUND;
292 } else {
293 status = AE_NOT_FOUND;
294 } 290 }
295out: 291out:
296 return status; 292 return status;
@@ -322,8 +318,7 @@ static int iort_id_map(struct acpi_iort_id_mapping *map, u8 type, u32 rid_in,
322 318
323static 319static
324struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node, 320struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
325 u32 *id_out, u8 type_mask, 321 u32 *id_out, int index)
326 int index)
327{ 322{
328 struct acpi_iort_node *parent; 323 struct acpi_iort_node *parent;
329 struct acpi_iort_id_mapping *map; 324 struct acpi_iort_id_mapping *map;
@@ -345,9 +340,6 @@ struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
345 parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table, 340 parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
346 map->output_reference); 341 map->output_reference);
347 342
348 if (!(IORT_TYPE_MASK(parent->type) & type_mask))
349 return NULL;
350
351 if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) { 343 if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
352 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT || 344 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT ||
353 node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) { 345 node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
@@ -359,11 +351,11 @@ struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
359 return NULL; 351 return NULL;
360} 352}
361 353
362static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node, 354static struct acpi_iort_node *iort_node_map_id(struct acpi_iort_node *node,
363 u32 rid_in, u32 *rid_out, 355 u32 id_in, u32 *id_out,
364 u8 type_mask) 356 u8 type_mask)
365{ 357{
366 u32 rid = rid_in; 358 u32 id = id_in;
367 359
368 /* Parse the ID mapping tree to find specified node type */ 360 /* Parse the ID mapping tree to find specified node type */
369 while (node) { 361 while (node) {
@@ -371,8 +363,8 @@ static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node,
371 int i; 363 int i;
372 364
373 if (IORT_TYPE_MASK(node->type) & type_mask) { 365 if (IORT_TYPE_MASK(node->type) & type_mask) {
374 if (rid_out) 366 if (id_out)
375 *rid_out = rid; 367 *id_out = id;
376 return node; 368 return node;
377 } 369 }
378 370
@@ -389,9 +381,9 @@ static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node,
389 goto fail_map; 381 goto fail_map;
390 } 382 }
391 383
392 /* Do the RID translation */ 384 /* Do the ID translation */
393 for (i = 0; i < node->mapping_count; i++, map++) { 385 for (i = 0; i < node->mapping_count; i++, map++) {
394 if (!iort_id_map(map, node->type, rid, &rid)) 386 if (!iort_id_map(map, node->type, id, &id))
395 break; 387 break;
396 } 388 }
397 389
@@ -403,13 +395,41 @@ static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node,
403 } 395 }
404 396
405fail_map: 397fail_map:
406 /* Map input RID to output RID unchanged on mapping failure*/ 398 /* Map input ID to output ID unchanged on mapping failure */
407 if (rid_out) 399 if (id_out)
408 *rid_out = rid_in; 400 *id_out = id_in;
409 401
410 return NULL; 402 return NULL;
411} 403}
412 404
405static
406struct acpi_iort_node *iort_node_map_platform_id(struct acpi_iort_node *node,
407 u32 *id_out, u8 type_mask,
408 int index)
409{
410 struct acpi_iort_node *parent;
411 u32 id;
412
413 /* step 1: retrieve the initial dev id */
414 parent = iort_node_get_id(node, &id, index);
415 if (!parent)
416 return NULL;
417
418 /*
419 * optional step 2: map the initial dev id if its parent is not
420 * the target type we want, map it again for the use cases such
421 * as NC (named component) -> SMMU -> ITS. If the type is matched,
422 * return the initial dev id and its parent pointer directly.
423 */
424 if (!(IORT_TYPE_MASK(parent->type) & type_mask))
425 parent = iort_node_map_id(parent, id, id_out, type_mask);
426 else
427 if (id_out)
428 *id_out = id;
429
430 return parent;
431}
432
413static struct acpi_iort_node *iort_find_dev_node(struct device *dev) 433static struct acpi_iort_node *iort_find_dev_node(struct device *dev)
414{ 434{
415 struct pci_bus *pbus; 435 struct pci_bus *pbus;
@@ -443,13 +463,38 @@ u32 iort_msi_map_rid(struct device *dev, u32 req_id)
443 if (!node) 463 if (!node)
444 return req_id; 464 return req_id;
445 465
446 iort_node_map_rid(node, req_id, &dev_id, IORT_MSI_TYPE); 466 iort_node_map_id(node, req_id, &dev_id, IORT_MSI_TYPE);
447 return dev_id; 467 return dev_id;
448} 468}
449 469
450/** 470/**
471 * iort_pmsi_get_dev_id() - Get the device id for a device
472 * @dev: The device for which the mapping is to be done.
473 * @dev_id: The device ID found.
474 *
475 * Returns: 0 for successful find a dev id, -ENODEV on error
476 */
477int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id)
478{
479 int i;
480 struct acpi_iort_node *node;
481
482 node = iort_find_dev_node(dev);
483 if (!node)
484 return -ENODEV;
485
486 for (i = 0; i < node->mapping_count; i++) {
487 if (iort_node_map_platform_id(node, dev_id, IORT_MSI_TYPE, i))
488 return 0;
489 }
490
491 return -ENODEV;
492}
493
494/**
451 * iort_dev_find_its_id() - Find the ITS identifier for a device 495 * iort_dev_find_its_id() - Find the ITS identifier for a device
452 * @dev: The device. 496 * @dev: The device.
497 * @req_id: Device's requester ID
453 * @idx: Index of the ITS identifier list. 498 * @idx: Index of the ITS identifier list.
454 * @its_id: ITS identifier. 499 * @its_id: ITS identifier.
455 * 500 *
@@ -465,7 +510,7 @@ static int iort_dev_find_its_id(struct device *dev, u32 req_id,
465 if (!node) 510 if (!node)
466 return -ENXIO; 511 return -ENXIO;
467 512
468 node = iort_node_map_rid(node, req_id, NULL, IORT_MSI_TYPE); 513 node = iort_node_map_id(node, req_id, NULL, IORT_MSI_TYPE);
469 if (!node) 514 if (!node)
470 return -ENXIO; 515 return -ENXIO;
471 516
@@ -503,6 +548,56 @@ struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id)
503 return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI); 548 return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI);
504} 549}
505 550
551/**
552 * iort_get_platform_device_domain() - Find MSI domain related to a
553 * platform device
554 * @dev: the dev pointer associated with the platform device
555 *
556 * Returns: the MSI domain for this device, NULL otherwise
557 */
558static struct irq_domain *iort_get_platform_device_domain(struct device *dev)
559{
560 struct acpi_iort_node *node, *msi_parent;
561 struct fwnode_handle *iort_fwnode;
562 struct acpi_iort_its_group *its;
563 int i;
564
565 /* find its associated iort node */
566 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
567 iort_match_node_callback, dev);
568 if (!node)
569 return NULL;
570
571 /* then find its msi parent node */
572 for (i = 0; i < node->mapping_count; i++) {
573 msi_parent = iort_node_map_platform_id(node, NULL,
574 IORT_MSI_TYPE, i);
575 if (msi_parent)
576 break;
577 }
578
579 if (!msi_parent)
580 return NULL;
581
582 /* Move to ITS specific data */
583 its = (struct acpi_iort_its_group *)msi_parent->node_data;
584
585 iort_fwnode = iort_find_domain_token(its->identifiers[0]);
586 if (!iort_fwnode)
587 return NULL;
588
589 return irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
590}
591
592void acpi_configure_pmsi_domain(struct device *dev)
593{
594 struct irq_domain *msi_domain;
595
596 msi_domain = iort_get_platform_device_domain(dev);
597 if (msi_domain)
598 dev_set_msi_domain(dev, msi_domain);
599}
600
506static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data) 601static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
507{ 602{
508 u32 *rid = data; 603 u32 *rid = data;
@@ -594,8 +689,8 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev)
594 if (!node) 689 if (!node)
595 return NULL; 690 return NULL;
596 691
597 parent = iort_node_map_rid(node, rid, &streamid, 692 parent = iort_node_map_id(node, rid, &streamid,
598 IORT_IOMMU_TYPE); 693 IORT_IOMMU_TYPE);
599 694
600 ops = iort_iommu_xlate(dev, parent, streamid); 695 ops = iort_iommu_xlate(dev, parent, streamid);
601 696
@@ -607,14 +702,15 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev)
607 if (!node) 702 if (!node)
608 return NULL; 703 return NULL;
609 704
610 parent = iort_node_get_id(node, &streamid, 705 parent = iort_node_map_platform_id(node, &streamid,
611 IORT_IOMMU_TYPE, i++); 706 IORT_IOMMU_TYPE, i++);
612 707
613 while (parent) { 708 while (parent) {
614 ops = iort_iommu_xlate(dev, parent, streamid); 709 ops = iort_iommu_xlate(dev, parent, streamid);
615 710
616 parent = iort_node_get_id(node, &streamid, 711 parent = iort_node_map_platform_id(node, &streamid,
617 IORT_IOMMU_TYPE, i++); 712 IORT_IOMMU_TYPE,
713 i++);
618 } 714 }
619 } 715 }
620 716
diff --git a/drivers/acpi/glue.c b/drivers/acpi/glue.c
index edc8663b5db3..3e7020751d34 100644
--- a/drivers/acpi/glue.c
+++ b/drivers/acpi/glue.c
@@ -6,6 +6,8 @@
6 * 6 *
7 * This file is released under the GPLv2. 7 * This file is released under the GPLv2.
8 */ 8 */
9
10#include <linux/acpi_iort.h>
9#include <linux/export.h> 11#include <linux/export.h>
10#include <linux/init.h> 12#include <linux/init.h>
11#include <linux/list.h> 13#include <linux/list.h>
@@ -14,6 +16,7 @@
14#include <linux/rwsem.h> 16#include <linux/rwsem.h>
15#include <linux/acpi.h> 17#include <linux/acpi.h>
16#include <linux/dma-mapping.h> 18#include <linux/dma-mapping.h>
19#include <linux/platform_device.h>
17 20
18#include "internal.h" 21#include "internal.h"
19 22
@@ -322,6 +325,9 @@ static int acpi_platform_notify(struct device *dev)
322 if (!adev) 325 if (!adev)
323 goto out; 326 goto out;
324 327
328 if (dev->bus == &platform_bus_type)
329 acpi_configure_pmsi_domain(dev);
330
325 if (type && type->setup) 331 if (type && type->setup)
326 type->setup(dev); 332 type->setup(dev);
327 else if (adev->handler && adev->handler->bind) 333 else if (adev->handler && adev->handler->bind)
diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c
index 41f457be64e8..8830fa601e45 100644
--- a/drivers/firmware/efi/libstub/fdt.c
+++ b/drivers/firmware/efi/libstub/fdt.c
@@ -16,6 +16,22 @@
16 16
17#include "efistub.h" 17#include "efistub.h"
18 18
19#define EFI_DT_ADDR_CELLS_DEFAULT 2
20#define EFI_DT_SIZE_CELLS_DEFAULT 2
21
22static void fdt_update_cell_size(efi_system_table_t *sys_table, void *fdt)
23{
24 int offset;
25
26 offset = fdt_path_offset(fdt, "/");
27 /* Set the #address-cells and #size-cells values for an empty tree */
28
29 fdt_setprop_u32(fdt, offset, "#address-cells",
30 EFI_DT_ADDR_CELLS_DEFAULT);
31
32 fdt_setprop_u32(fdt, offset, "#size-cells", EFI_DT_SIZE_CELLS_DEFAULT);
33}
34
19static efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, 35static efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
20 unsigned long orig_fdt_size, 36 unsigned long orig_fdt_size,
21 void *fdt, int new_fdt_size, char *cmdline_ptr, 37 void *fdt, int new_fdt_size, char *cmdline_ptr,
@@ -42,10 +58,18 @@ static efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
42 } 58 }
43 } 59 }
44 60
45 if (orig_fdt) 61 if (orig_fdt) {
46 status = fdt_open_into(orig_fdt, fdt, new_fdt_size); 62 status = fdt_open_into(orig_fdt, fdt, new_fdt_size);
47 else 63 } else {
48 status = fdt_create_empty_tree(fdt, new_fdt_size); 64 status = fdt_create_empty_tree(fdt, new_fdt_size);
65 if (status == 0) {
66 /*
67 * Any failure from the following function is non
68 * critical
69 */
70 fdt_update_cell_size(sys_table, fdt);
71 }
72 }
49 73
50 if (status != 0) 74 if (status != 0)
51 goto fdt_set_fail; 75 goto fdt_set_fail;
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 93651907874f..aa587edaf9ea 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -12,6 +12,10 @@ config ARM_PMU
12 Say y if you want to use CPU performance monitors on ARM-based 12 Say y if you want to use CPU performance monitors on ARM-based
13 systems. 13 systems.
14 14
15config ARM_PMU_ACPI
16 depends on ARM_PMU && ACPI
17 def_bool y
18
15config QCOM_L2_PMU 19config QCOM_L2_PMU
16 bool "Qualcomm Technologies L2-cache PMU" 20 bool "Qualcomm Technologies L2-cache PMU"
17 depends on ARCH_QCOM && ARM64 && PERF_EVENTS && ACPI 21 depends on ARCH_QCOM && ARM64 && PERF_EVENTS && ACPI
@@ -21,6 +25,16 @@ config QCOM_L2_PMU
21 Adds the L2 cache PMU into the perf events subsystem for 25 Adds the L2 cache PMU into the perf events subsystem for
22 monitoring L2 cache events. 26 monitoring L2 cache events.
23 27
28config QCOM_L3_PMU
29 bool "Qualcomm Technologies L3-cache PMU"
30 depends on ARCH_QCOM && ARM64 && PERF_EVENTS && ACPI
31 select QCOM_IRQ_COMBINER
32 help
33 Provides support for the L3 cache performance monitor unit (PMU)
34 in Qualcomm Technologies processors.
35 Adds the L3 cache PMU into the perf events subsystem for
36 monitoring L3 cache events.
37
24config XGENE_PMU 38config XGENE_PMU
25 depends on PERF_EVENTS && ARCH_XGENE 39 depends on PERF_EVENTS && ARCH_XGENE
26 bool "APM X-Gene SoC PMU" 40 bool "APM X-Gene SoC PMU"
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index ef24833c94a8..6420bd4394d5 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -1,3 +1,5 @@
1obj-$(CONFIG_ARM_PMU) += arm_pmu.o 1obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
2obj-$(CONFIG_ARM_PMU_ACPI) += arm_pmu_acpi.o
2obj-$(CONFIG_QCOM_L2_PMU) += qcom_l2_pmu.o 3obj-$(CONFIG_QCOM_L2_PMU) += qcom_l2_pmu.o
4obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
3obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o 5obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 9612b84bc3e0..dc459eb1246b 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -16,7 +16,6 @@
16#include <linux/cpu_pm.h> 16#include <linux/cpu_pm.h>
17#include <linux/export.h> 17#include <linux/export.h>
18#include <linux/kernel.h> 18#include <linux/kernel.h>
19#include <linux/of_device.h>
20#include <linux/perf/arm_pmu.h> 19#include <linux/perf/arm_pmu.h>
21#include <linux/platform_device.h> 20#include <linux/platform_device.h>
22#include <linux/slab.h> 21#include <linux/slab.h>
@@ -25,7 +24,6 @@
25#include <linux/irq.h> 24#include <linux/irq.h>
26#include <linux/irqdesc.h> 25#include <linux/irqdesc.h>
27 26
28#include <asm/cputype.h>
29#include <asm/irq_regs.h> 27#include <asm/irq_regs.h>
30 28
31static int 29static int
@@ -235,20 +233,15 @@ armpmu_add(struct perf_event *event, int flags)
235 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 233 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
236 struct hw_perf_event *hwc = &event->hw; 234 struct hw_perf_event *hwc = &event->hw;
237 int idx; 235 int idx;
238 int err = 0;
239 236
240 /* An event following a process won't be stopped earlier */ 237 /* An event following a process won't be stopped earlier */
241 if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 238 if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))
242 return -ENOENT; 239 return -ENOENT;
243 240
244 perf_pmu_disable(event->pmu);
245
246 /* If we don't have a space for the counter then finish early. */ 241 /* If we don't have a space for the counter then finish early. */
247 idx = armpmu->get_event_idx(hw_events, event); 242 idx = armpmu->get_event_idx(hw_events, event);
248 if (idx < 0) { 243 if (idx < 0)
249 err = idx; 244 return idx;
250 goto out;
251 }
252 245
253 /* 246 /*
254 * If there is an event in the counter we are going to use then make 247 * If there is an event in the counter we are going to use then make
@@ -265,9 +258,7 @@ armpmu_add(struct perf_event *event, int flags)
265 /* Propagate our changes to the userspace mapping. */ 258 /* Propagate our changes to the userspace mapping. */
266 perf_event_update_userpage(event); 259 perf_event_update_userpage(event);
267 260
268out: 261 return 0;
269 perf_pmu_enable(event->pmu);
270 return err;
271} 262}
272 263
273static int 264static int
@@ -323,10 +314,16 @@ validate_group(struct perf_event *event)
323 return 0; 314 return 0;
324} 315}
325 316
317static struct arm_pmu_platdata *armpmu_get_platdata(struct arm_pmu *armpmu)
318{
319 struct platform_device *pdev = armpmu->plat_device;
320
321 return pdev ? dev_get_platdata(&pdev->dev) : NULL;
322}
323
326static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) 324static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
327{ 325{
328 struct arm_pmu *armpmu; 326 struct arm_pmu *armpmu;
329 struct platform_device *plat_device;
330 struct arm_pmu_platdata *plat; 327 struct arm_pmu_platdata *plat;
331 int ret; 328 int ret;
332 u64 start_clock, finish_clock; 329 u64 start_clock, finish_clock;
@@ -338,8 +335,8 @@ static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
338 * dereference. 335 * dereference.
339 */ 336 */
340 armpmu = *(void **)dev; 337 armpmu = *(void **)dev;
341 plat_device = armpmu->plat_device; 338
342 plat = dev_get_platdata(&plat_device->dev); 339 plat = armpmu_get_platdata(armpmu);
343 340
344 start_clock = sched_clock(); 341 start_clock = sched_clock();
345 if (plat && plat->handle_irq) 342 if (plat && plat->handle_irq)
@@ -352,37 +349,6 @@ static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
352 return ret; 349 return ret;
353} 350}
354 351
355static void
356armpmu_release_hardware(struct arm_pmu *armpmu)
357{
358 armpmu->free_irq(armpmu);
359}
360
361static int
362armpmu_reserve_hardware(struct arm_pmu *armpmu)
363{
364 int err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);
365 if (err) {
366 armpmu_release_hardware(armpmu);
367 return err;
368 }
369
370 return 0;
371}
372
373static void
374hw_perf_event_destroy(struct perf_event *event)
375{
376 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
377 atomic_t *active_events = &armpmu->active_events;
378 struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
379
380 if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
381 armpmu_release_hardware(armpmu);
382 mutex_unlock(pmu_reserve_mutex);
383 }
384}
385
386static int 352static int
387event_requires_mode_exclusion(struct perf_event_attr *attr) 353event_requires_mode_exclusion(struct perf_event_attr *attr)
388{ 354{
@@ -455,8 +421,6 @@ __hw_perf_event_init(struct perf_event *event)
455static int armpmu_event_init(struct perf_event *event) 421static int armpmu_event_init(struct perf_event *event)
456{ 422{
457 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 423 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
458 int err = 0;
459 atomic_t *active_events = &armpmu->active_events;
460 424
461 /* 425 /*
462 * Reject CPU-affine events for CPUs that are of a different class to 426 * Reject CPU-affine events for CPUs that are of a different class to
@@ -476,26 +440,7 @@ static int armpmu_event_init(struct perf_event *event)
476 if (armpmu->map_event(event) == -ENOENT) 440 if (armpmu->map_event(event) == -ENOENT)
477 return -ENOENT; 441 return -ENOENT;
478 442
479 event->destroy = hw_perf_event_destroy; 443 return __hw_perf_event_init(event);
480
481 if (!atomic_inc_not_zero(active_events)) {
482 mutex_lock(&armpmu->reserve_mutex);
483 if (atomic_read(active_events) == 0)
484 err = armpmu_reserve_hardware(armpmu);
485
486 if (!err)
487 atomic_inc(active_events);
488 mutex_unlock(&armpmu->reserve_mutex);
489 }
490
491 if (err)
492 return err;
493
494 err = __hw_perf_event_init(event);
495 if (err)
496 hw_perf_event_destroy(event);
497
498 return err;
499} 444}
500 445
501static void armpmu_enable(struct pmu *pmu) 446static void armpmu_enable(struct pmu *pmu)
@@ -553,27 +498,6 @@ static struct attribute_group armpmu_common_attr_group = {
553 .attrs = armpmu_common_attrs, 498 .attrs = armpmu_common_attrs,
554}; 499};
555 500
556static void armpmu_init(struct arm_pmu *armpmu)
557{
558 atomic_set(&armpmu->active_events, 0);
559 mutex_init(&armpmu->reserve_mutex);
560
561 armpmu->pmu = (struct pmu) {
562 .pmu_enable = armpmu_enable,
563 .pmu_disable = armpmu_disable,
564 .event_init = armpmu_event_init,
565 .add = armpmu_add,
566 .del = armpmu_del,
567 .start = armpmu_start,
568 .stop = armpmu_stop,
569 .read = armpmu_read,
570 .filter_match = armpmu_filter_match,
571 .attr_groups = armpmu->attr_groups,
572 };
573 armpmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] =
574 &armpmu_common_attr_group;
575}
576
577/* Set at runtime when we know what CPU type we are. */ 501/* Set at runtime when we know what CPU type we are. */
578static struct arm_pmu *__oprofile_cpu_pmu; 502static struct arm_pmu *__oprofile_cpu_pmu;
579 503
@@ -601,113 +525,85 @@ int perf_num_counters(void)
601} 525}
602EXPORT_SYMBOL_GPL(perf_num_counters); 526EXPORT_SYMBOL_GPL(perf_num_counters);
603 527
604static void cpu_pmu_enable_percpu_irq(void *data) 528void armpmu_free_irq(struct arm_pmu *armpmu, int cpu)
605{ 529{
606 int irq = *(int *)data; 530 struct pmu_hw_events __percpu *hw_events = armpmu->hw_events;
531 int irq = per_cpu(hw_events->irq, cpu);
607 532
608 enable_percpu_irq(irq, IRQ_TYPE_NONE); 533 if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
609} 534 return;
610 535
611static void cpu_pmu_disable_percpu_irq(void *data) 536 if (irq_is_percpu(irq)) {
612{ 537 free_percpu_irq(irq, &hw_events->percpu_pmu);
613 int irq = *(int *)data; 538 cpumask_clear(&armpmu->active_irqs);
539 return;
540 }
614 541
615 disable_percpu_irq(irq); 542 free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
616} 543}
617 544
618static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu) 545void armpmu_free_irqs(struct arm_pmu *armpmu)
619{ 546{
620 int i, irq, irqs; 547 int cpu;
621 struct platform_device *pmu_device = cpu_pmu->plat_device;
622 struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;
623
624 irqs = min(pmu_device->num_resources, num_possible_cpus());
625
626 irq = platform_get_irq(pmu_device, 0);
627 if (irq > 0 && irq_is_percpu(irq)) {
628 on_each_cpu_mask(&cpu_pmu->supported_cpus,
629 cpu_pmu_disable_percpu_irq, &irq, 1);
630 free_percpu_irq(irq, &hw_events->percpu_pmu);
631 } else {
632 for (i = 0; i < irqs; ++i) {
633 int cpu = i;
634
635 if (cpu_pmu->irq_affinity)
636 cpu = cpu_pmu->irq_affinity[i];
637 548
638 if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs)) 549 for_each_cpu(cpu, &armpmu->supported_cpus)
639 continue; 550 armpmu_free_irq(armpmu, cpu);
640 irq = platform_get_irq(pmu_device, i);
641 if (irq > 0)
642 free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
643 }
644 }
645} 551}
646 552
647static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler) 553int armpmu_request_irq(struct arm_pmu *armpmu, int cpu)
648{ 554{
649 int i, err, irq, irqs; 555 int err = 0;
650 struct platform_device *pmu_device = cpu_pmu->plat_device; 556 struct pmu_hw_events __percpu *hw_events = armpmu->hw_events;
651 struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events; 557 const irq_handler_t handler = armpmu_dispatch_irq;
652 558 int irq = per_cpu(hw_events->irq, cpu);
653 if (!pmu_device) 559 if (!irq)
654 return -ENODEV;
655
656 irqs = min(pmu_device->num_resources, num_possible_cpus());
657 if (irqs < 1) {
658 pr_warn_once("perf/ARM: No irqs for PMU defined, sampling events not supported\n");
659 return 0; 560 return 0;
660 }
661 561
662 irq = platform_get_irq(pmu_device, 0); 562 if (irq_is_percpu(irq) && cpumask_empty(&armpmu->active_irqs)) {
663 if (irq > 0 && irq_is_percpu(irq)) {
664 err = request_percpu_irq(irq, handler, "arm-pmu", 563 err = request_percpu_irq(irq, handler, "arm-pmu",
665 &hw_events->percpu_pmu); 564 &hw_events->percpu_pmu);
666 if (err) { 565 } else if (irq_is_percpu(irq)) {
667 pr_err("unable to request IRQ%d for ARM PMU counters\n", 566 int other_cpu = cpumask_first(&armpmu->active_irqs);
668 irq); 567 int other_irq = per_cpu(hw_events->irq, other_cpu);
669 return err;
670 }
671 568
672 on_each_cpu_mask(&cpu_pmu->supported_cpus, 569 if (irq != other_irq) {
673 cpu_pmu_enable_percpu_irq, &irq, 1); 570 pr_warn("mismatched PPIs detected.\n");
571 err = -EINVAL;
572 }
674 } else { 573 } else {
675 for (i = 0; i < irqs; ++i) { 574 err = request_irq(irq, handler,
676 int cpu = i; 575 IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",
576 per_cpu_ptr(&hw_events->percpu_pmu, cpu));
577 }
677 578
678 err = 0; 579 if (err) {
679 irq = platform_get_irq(pmu_device, i); 580 pr_err("unable to request IRQ%d for ARM PMU counters\n",
680 if (irq < 0) 581 irq);
681 continue; 582 return err;
583 }
682 584
683 if (cpu_pmu->irq_affinity) 585 cpumask_set_cpu(cpu, &armpmu->active_irqs);
684 cpu = cpu_pmu->irq_affinity[i];
685 586
686 /* 587 return 0;
687 * If we have a single PMU interrupt that we can't shift, 588}
688 * assume that we're running on a uniprocessor machine and 589
689 * continue. Otherwise, continue without this interrupt. 590int armpmu_request_irqs(struct arm_pmu *armpmu)
690 */ 591{
691 if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) { 592 int cpu, err;
692 pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n", 593
693 irq, cpu); 594 for_each_cpu(cpu, &armpmu->supported_cpus) {
694 continue; 595 err = armpmu_request_irq(armpmu, cpu);
695 } 596 if (err)
696 597 break;
697 err = request_irq(irq, handler,
698 IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",
699 per_cpu_ptr(&hw_events->percpu_pmu, cpu));
700 if (err) {
701 pr_err("unable to request IRQ%d for ARM PMU counters\n",
702 irq);
703 return err;
704 }
705
706 cpumask_set_cpu(cpu, &cpu_pmu->active_irqs);
707 }
708 } 598 }
709 599
710 return 0; 600 return err;
601}
602
603static int armpmu_get_cpu_irq(struct arm_pmu *pmu, int cpu)
604{
605 struct pmu_hw_events __percpu *hw_events = pmu->hw_events;
606 return per_cpu(hw_events->irq, cpu);
711} 607}
712 608
713/* 609/*
@@ -719,11 +615,42 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
719static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) 615static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node)
720{ 616{
721 struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); 617 struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node);
618 int irq;
722 619
723 if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) 620 if (!cpumask_test_cpu(cpu, &pmu->supported_cpus))
724 return 0; 621 return 0;
725 if (pmu->reset) 622 if (pmu->reset)
726 pmu->reset(pmu); 623 pmu->reset(pmu);
624
625 irq = armpmu_get_cpu_irq(pmu, cpu);
626 if (irq) {
627 if (irq_is_percpu(irq)) {
628 enable_percpu_irq(irq, IRQ_TYPE_NONE);
629 return 0;
630 }
631
632 if (irq_force_affinity(irq, cpumask_of(cpu)) &&
633 num_possible_cpus() > 1) {
634 pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n",
635 irq, cpu);
636 }
637 }
638
639 return 0;
640}
641
642static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node)
643{
644 struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node);
645 int irq;
646
647 if (!cpumask_test_cpu(cpu, &pmu->supported_cpus))
648 return 0;
649
650 irq = armpmu_get_cpu_irq(pmu, cpu);
651 if (irq && irq_is_percpu(irq))
652 disable_percpu_irq(irq);
653
727 return 0; 654 return 0;
728} 655}
729 656
@@ -828,56 +755,22 @@ static inline void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) { }
828static int cpu_pmu_init(struct arm_pmu *cpu_pmu) 755static int cpu_pmu_init(struct arm_pmu *cpu_pmu)
829{ 756{
830 int err; 757 int err;
831 int cpu;
832 struct pmu_hw_events __percpu *cpu_hw_events;
833
834 cpu_hw_events = alloc_percpu(struct pmu_hw_events);
835 if (!cpu_hw_events)
836 return -ENOMEM;
837 758
838 err = cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, 759 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_STARTING,
839 &cpu_pmu->node); 760 &cpu_pmu->node);
840 if (err) 761 if (err)
841 goto out_free; 762 goto out;
842 763
843 err = cpu_pm_pmu_register(cpu_pmu); 764 err = cpu_pm_pmu_register(cpu_pmu);
844 if (err) 765 if (err)
845 goto out_unregister; 766 goto out_unregister;
846 767
847 for_each_possible_cpu(cpu) {
848 struct pmu_hw_events *events = per_cpu_ptr(cpu_hw_events, cpu);
849 raw_spin_lock_init(&events->pmu_lock);
850 events->percpu_pmu = cpu_pmu;
851 }
852
853 cpu_pmu->hw_events = cpu_hw_events;
854 cpu_pmu->request_irq = cpu_pmu_request_irq;
855 cpu_pmu->free_irq = cpu_pmu_free_irq;
856
857 /* Ensure the PMU has sane values out of reset. */
858 if (cpu_pmu->reset)
859 on_each_cpu_mask(&cpu_pmu->supported_cpus, cpu_pmu->reset,
860 cpu_pmu, 1);
861
862 /* If no interrupts available, set the corresponding capability flag */
863 if (!platform_get_irq(cpu_pmu->plat_device, 0))
864 cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
865
866 /*
867 * This is a CPU PMU potentially in a heterogeneous configuration (e.g.
868 * big.LITTLE). This is not an uncore PMU, and we have taken ctx
869 * sharing into account (e.g. with our pmu::filter_match callback and
870 * pmu::event_init group validation).
871 */
872 cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_HETEROGENEOUS_CPUS;
873
874 return 0; 768 return 0;
875 769
876out_unregister: 770out_unregister:
877 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, 771 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING,
878 &cpu_pmu->node); 772 &cpu_pmu->node);
879out_free: 773out:
880 free_percpu(cpu_hw_events);
881 return err; 774 return err;
882} 775}
883 776
@@ -886,177 +779,78 @@ static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)
886 cpu_pm_pmu_unregister(cpu_pmu); 779 cpu_pm_pmu_unregister(cpu_pmu);
887 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, 780 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING,
888 &cpu_pmu->node); 781 &cpu_pmu->node);
889 free_percpu(cpu_pmu->hw_events);
890} 782}
891 783
892/* 784struct arm_pmu *armpmu_alloc(void)
893 * CPU PMU identification and probing.
894 */
895static int probe_current_pmu(struct arm_pmu *pmu,
896 const struct pmu_probe_info *info)
897{ 785{
898 int cpu = get_cpu(); 786 struct arm_pmu *pmu;
899 unsigned int cpuid = read_cpuid_id(); 787 int cpu;
900 int ret = -ENODEV;
901
902 pr_info("probing PMU on CPU %d\n", cpu);
903 788
904 for (; info->init != NULL; info++) { 789 pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
905 if ((cpuid & info->mask) != info->cpuid) 790 if (!pmu) {
906 continue; 791 pr_info("failed to allocate PMU device!\n");
907 ret = info->init(pmu); 792 goto out;
908 break;
909 } 793 }
910 794
911 put_cpu(); 795 pmu->hw_events = alloc_percpu(struct pmu_hw_events);
912 return ret; 796 if (!pmu->hw_events) {
913} 797 pr_info("failed to allocate per-cpu PMU data.\n");
914 798 goto out_free_pmu;
915static int of_pmu_irq_cfg(struct arm_pmu *pmu) 799 }
916{
917 int *irqs, i = 0;
918 bool using_spi = false;
919 struct platform_device *pdev = pmu->plat_device;
920
921 irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
922 if (!irqs)
923 return -ENOMEM;
924
925 do {
926 struct device_node *dn;
927 int cpu, irq;
928
929 /* See if we have an affinity entry */
930 dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity", i);
931 if (!dn)
932 break;
933
934 /* Check the IRQ type and prohibit a mix of PPIs and SPIs */
935 irq = platform_get_irq(pdev, i);
936 if (irq > 0) {
937 bool spi = !irq_is_percpu(irq);
938
939 if (i > 0 && spi != using_spi) {
940 pr_err("PPI/SPI IRQ type mismatch for %s!\n",
941 dn->name);
942 of_node_put(dn);
943 kfree(irqs);
944 return -EINVAL;
945 }
946
947 using_spi = spi;
948 }
949
950 /* Now look up the logical CPU number */
951 for_each_possible_cpu(cpu) {
952 struct device_node *cpu_dn;
953
954 cpu_dn = of_cpu_device_node_get(cpu);
955 of_node_put(cpu_dn);
956
957 if (dn == cpu_dn)
958 break;
959 }
960 800
961 if (cpu >= nr_cpu_ids) { 801 pmu->pmu = (struct pmu) {
962 pr_warn("Failed to find logical CPU for %s\n", 802 .pmu_enable = armpmu_enable,
963 dn->name); 803 .pmu_disable = armpmu_disable,
964 of_node_put(dn); 804 .event_init = armpmu_event_init,
965 cpumask_setall(&pmu->supported_cpus); 805 .add = armpmu_add,
966 break; 806 .del = armpmu_del,
967 } 807 .start = armpmu_start,
968 of_node_put(dn); 808 .stop = armpmu_stop,
809 .read = armpmu_read,
810 .filter_match = armpmu_filter_match,
811 .attr_groups = pmu->attr_groups,
812 /*
813 * This is a CPU PMU potentially in a heterogeneous
814 * configuration (e.g. big.LITTLE). This is not an uncore PMU,
815 * and we have taken ctx sharing into account (e.g. with our
816 * pmu::filter_match callback and pmu::event_init group
817 * validation).
818 */
819 .capabilities = PERF_PMU_CAP_HETEROGENEOUS_CPUS,
820 };
969 821
970 /* For SPIs, we need to track the affinity per IRQ */ 822 pmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] =
971 if (using_spi) { 823 &armpmu_common_attr_group;
972 if (i >= pdev->num_resources)
973 break;
974 824
975 irqs[i] = cpu; 825 for_each_possible_cpu(cpu) {
976 } 826 struct pmu_hw_events *events;
977 827
978 /* Keep track of the CPUs containing this PMU type */ 828 events = per_cpu_ptr(pmu->hw_events, cpu);
979 cpumask_set_cpu(cpu, &pmu->supported_cpus); 829 raw_spin_lock_init(&events->pmu_lock);
980 i++; 830 events->percpu_pmu = pmu;
981 } while (1);
982
983 /* If we didn't manage to parse anything, try the interrupt affinity */
984 if (cpumask_weight(&pmu->supported_cpus) == 0) {
985 int irq = platform_get_irq(pdev, 0);
986
987 if (irq > 0 && irq_is_percpu(irq)) {
988 /* If using PPIs, check the affinity of the partition */
989 int ret;
990
991 ret = irq_get_percpu_devid_partition(irq, &pmu->supported_cpus);
992 if (ret) {
993 kfree(irqs);
994 return ret;
995 }
996 } else {
997 /* Otherwise default to all CPUs */
998 cpumask_setall(&pmu->supported_cpus);
999 }
1000 } 831 }
1001 832
1002 /* If we matched up the IRQ affinities, use them to route the SPIs */ 833 return pmu;
1003 if (using_spi && i == pdev->num_resources)
1004 pmu->irq_affinity = irqs;
1005 else
1006 kfree(irqs);
1007 834
1008 return 0; 835out_free_pmu:
836 kfree(pmu);
837out:
838 return NULL;
1009} 839}
1010 840
1011int arm_pmu_device_probe(struct platform_device *pdev, 841void armpmu_free(struct arm_pmu *pmu)
1012 const struct of_device_id *of_table,
1013 const struct pmu_probe_info *probe_table)
1014{ 842{
1015 const struct of_device_id *of_id; 843 free_percpu(pmu->hw_events);
1016 const int (*init_fn)(struct arm_pmu *); 844 kfree(pmu);
1017 struct device_node *node = pdev->dev.of_node; 845}
1018 struct arm_pmu *pmu;
1019 int ret = -ENODEV;
1020
1021 pmu = kzalloc(sizeof(struct arm_pmu), GFP_KERNEL);
1022 if (!pmu) {
1023 pr_info("failed to allocate PMU device!\n");
1024 return -ENOMEM;
1025 }
1026
1027 armpmu_init(pmu);
1028
1029 pmu->plat_device = pdev;
1030
1031 if (node && (of_id = of_match_node(of_table, pdev->dev.of_node))) {
1032 init_fn = of_id->data;
1033
1034 pmu->secure_access = of_property_read_bool(pdev->dev.of_node,
1035 "secure-reg-access");
1036
1037 /* arm64 systems boot only as non-secure */
1038 if (IS_ENABLED(CONFIG_ARM64) && pmu->secure_access) {
1039 pr_warn("ignoring \"secure-reg-access\" property for arm64\n");
1040 pmu->secure_access = false;
1041 }
1042
1043 ret = of_pmu_irq_cfg(pmu);
1044 if (!ret)
1045 ret = init_fn(pmu);
1046 } else if (probe_table) {
1047 cpumask_setall(&pmu->supported_cpus);
1048 ret = probe_current_pmu(pmu, probe_table);
1049 }
1050
1051 if (ret) {
1052 pr_info("%s: failed to probe PMU!\n", of_node_full_name(node));
1053 goto out_free;
1054 }
1055 846
847int armpmu_register(struct arm_pmu *pmu)
848{
849 int ret;
1056 850
1057 ret = cpu_pmu_init(pmu); 851 ret = cpu_pmu_init(pmu);
1058 if (ret) 852 if (ret)
1059 goto out_free; 853 return ret;
1060 854
1061 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); 855 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
1062 if (ret) 856 if (ret)
@@ -1066,17 +860,12 @@ int arm_pmu_device_probe(struct platform_device *pdev,
1066 __oprofile_cpu_pmu = pmu; 860 __oprofile_cpu_pmu = pmu;
1067 861
1068 pr_info("enabled with %s PMU driver, %d counters available\n", 862 pr_info("enabled with %s PMU driver, %d counters available\n",
1069 pmu->name, pmu->num_events); 863 pmu->name, pmu->num_events);
1070 864
1071 return 0; 865 return 0;
1072 866
1073out_destroy: 867out_destroy:
1074 cpu_pmu_destroy(pmu); 868 cpu_pmu_destroy(pmu);
1075out_free:
1076 pr_info("%s: failed to register PMU devices!\n",
1077 of_node_full_name(node));
1078 kfree(pmu->irq_affinity);
1079 kfree(pmu);
1080 return ret; 869 return ret;
1081} 870}
1082 871
@@ -1086,7 +875,8 @@ static int arm_pmu_hp_init(void)
1086 875
1087 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_STARTING, 876 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_STARTING,
1088 "perf/arm/pmu:starting", 877 "perf/arm/pmu:starting",
1089 arm_perf_starting_cpu, NULL); 878 arm_perf_starting_cpu,
879 arm_perf_teardown_cpu);
1090 if (ret) 880 if (ret)
1091 pr_err("CPU hotplug notifier for ARM PMU could not be registered: %d\n", 881 pr_err("CPU hotplug notifier for ARM PMU could not be registered: %d\n",
1092 ret); 882 ret);
diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c
new file mode 100644
index 000000000000..34c862f213c7
--- /dev/null
+++ b/drivers/perf/arm_pmu_acpi.c
@@ -0,0 +1,256 @@
1/*
2 * ACPI probing code for ARM performance counters.
3 *
4 * Copyright (C) 2017 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/acpi.h>
12#include <linux/cpumask.h>
13#include <linux/init.h>
14#include <linux/percpu.h>
15#include <linux/perf/arm_pmu.h>
16
17#include <asm/cputype.h>
18
19static DEFINE_PER_CPU(struct arm_pmu *, probed_pmus);
20static DEFINE_PER_CPU(int, pmu_irqs);
21
22static int arm_pmu_acpi_register_irq(int cpu)
23{
24 struct acpi_madt_generic_interrupt *gicc;
25 int gsi, trigger;
26
27 gicc = acpi_cpu_get_madt_gicc(cpu);
28 if (WARN_ON(!gicc))
29 return -EINVAL;
30
31 gsi = gicc->performance_interrupt;
32 if (gicc->flags & ACPI_MADT_PERFORMANCE_IRQ_MODE)
33 trigger = ACPI_EDGE_SENSITIVE;
34 else
35 trigger = ACPI_LEVEL_SENSITIVE;
36
37 /*
38 * Helpfully, the MADT GICC doesn't have a polarity flag for the
39 * "performance interrupt". Luckily, on compliant GICs the polarity is
40 * a fixed value in HW (for both SPIs and PPIs) that we cannot change
41 * from SW.
42 *
43 * Here we pass in ACPI_ACTIVE_HIGH to keep the core code happy. This
44 * may not match the real polarity, but that should not matter.
45 *
46 * Other interrupt controllers are not supported with ACPI.
47 */
48 return acpi_register_gsi(NULL, gsi, trigger, ACPI_ACTIVE_HIGH);
49}
50
51static void arm_pmu_acpi_unregister_irq(int cpu)
52{
53 struct acpi_madt_generic_interrupt *gicc;
54 int gsi;
55
56 gicc = acpi_cpu_get_madt_gicc(cpu);
57 if (!gicc)
58 return;
59
60 gsi = gicc->performance_interrupt;
61 acpi_unregister_gsi(gsi);
62}
63
64static int arm_pmu_acpi_parse_irqs(void)
65{
66 int irq, cpu, irq_cpu, err;
67
68 for_each_possible_cpu(cpu) {
69 irq = arm_pmu_acpi_register_irq(cpu);
70 if (irq < 0) {
71 err = irq;
72 pr_warn("Unable to parse ACPI PMU IRQ for CPU%d: %d\n",
73 cpu, err);
74 goto out_err;
75 } else if (irq == 0) {
76 pr_warn("No ACPI PMU IRQ for CPU%d\n", cpu);
77 }
78
79 per_cpu(pmu_irqs, cpu) = irq;
80 }
81
82 return 0;
83
84out_err:
85 for_each_possible_cpu(cpu) {
86 irq = per_cpu(pmu_irqs, cpu);
87 if (!irq)
88 continue;
89
90 arm_pmu_acpi_unregister_irq(cpu);
91
92 /*
93 * Blat all copies of the IRQ so that we only unregister the
94 * corresponding GSI once (e.g. when we have PPIs).
95 */
96 for_each_possible_cpu(irq_cpu) {
97 if (per_cpu(pmu_irqs, irq_cpu) == irq)
98 per_cpu(pmu_irqs, irq_cpu) = 0;
99 }
100 }
101
102 return err;
103}
104
105static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
106{
107 unsigned long cpuid = read_cpuid_id();
108 struct arm_pmu *pmu;
109 int cpu;
110
111 for_each_possible_cpu(cpu) {
112 pmu = per_cpu(probed_pmus, cpu);
113 if (!pmu || pmu->acpi_cpuid != cpuid)
114 continue;
115
116 return pmu;
117 }
118
119 pmu = armpmu_alloc();
120 if (!pmu) {
121 pr_warn("Unable to allocate PMU for CPU%d\n",
122 smp_processor_id());
123 return NULL;
124 }
125
126 pmu->acpi_cpuid = cpuid;
127
128 return pmu;
129}
130
131/*
132 * This must run before the common arm_pmu hotplug logic, so that we can
133 * associate a CPU and its interrupt before the common code tries to manage the
134 * affinity and so on.
135 *
136 * Note that hotplug events are serialized, so we cannot race with another CPU
137 * coming up. The perf core won't open events while a hotplug event is in
138 * progress.
139 */
140static int arm_pmu_acpi_cpu_starting(unsigned int cpu)
141{
142 struct arm_pmu *pmu;
143 struct pmu_hw_events __percpu *hw_events;
144 int irq;
145
146 /* If we've already probed this CPU, we have nothing to do */
147 if (per_cpu(probed_pmus, cpu))
148 return 0;
149
150 irq = per_cpu(pmu_irqs, cpu);
151
152 pmu = arm_pmu_acpi_find_alloc_pmu();
153 if (!pmu)
154 return -ENOMEM;
155
156 cpumask_set_cpu(cpu, &pmu->supported_cpus);
157
158 per_cpu(probed_pmus, cpu) = pmu;
159
160 /*
161 * Log and request the IRQ so the core arm_pmu code can manage it. In
162 * some situations (e.g. mismatched PPIs), we may fail to request the
163 * IRQ. However, it may be too late for us to do anything about it.
164 * The common ARM PMU code will log a warning in this case.
165 */
166 hw_events = pmu->hw_events;
167 per_cpu(hw_events->irq, cpu) = irq;
168 armpmu_request_irq(pmu, cpu);
169
170 /*
171 * Ideally, we'd probe the PMU here when we find the first matching
172 * CPU. We can't do that for several reasons; see the comment in
173 * arm_pmu_acpi_init().
174 *
175 * So for the time being, we're done.
176 */
177 return 0;
178}
179
180int arm_pmu_acpi_probe(armpmu_init_fn init_fn)
181{
182 int pmu_idx = 0;
183 int cpu, ret;
184
185 if (acpi_disabled)
186 return 0;
187
188 /*
189 * Initialise and register the set of PMUs which we know about right
190 * now. Ideally we'd do this in arm_pmu_acpi_cpu_starting() so that we
191 * could handle late hotplug, but this may lead to deadlock since we
192 * might try to register a hotplug notifier instance from within a
193 * hotplug notifier.
194 *
195 * There's also the problem of having access to the right init_fn,
196 * without tying this too deeply into the "real" PMU driver.
197 *
198 * For the moment, as with the platform/DT case, we need at least one
199 * of a PMU's CPUs to be online at probe time.
200 */
201 for_each_possible_cpu(cpu) {
202 struct arm_pmu *pmu = per_cpu(probed_pmus, cpu);
203 char *base_name;
204
205 if (!pmu || pmu->name)
206 continue;
207
208 ret = init_fn(pmu);
209 if (ret == -ENODEV) {
210 /* PMU not handled by this driver, or not present */
211 continue;
212 } else if (ret) {
213 pr_warn("Unable to initialise PMU for CPU%d\n", cpu);
214 return ret;
215 }
216
217 base_name = pmu->name;
218 pmu->name = kasprintf(GFP_KERNEL, "%s_%d", base_name, pmu_idx++);
219 if (!pmu->name) {
220 pr_warn("Unable to allocate PMU name for CPU%d\n", cpu);
221 return -ENOMEM;
222 }
223
224 ret = armpmu_register(pmu);
225 if (ret) {
226 pr_warn("Failed to register PMU for CPU%d\n", cpu);
227 return ret;
228 }
229 }
230
231 return 0;
232}
233
234static int arm_pmu_acpi_init(void)
235{
236 int ret;
237
238 if (acpi_disabled)
239 return 0;
240
241 /*
242 * We can't request IRQs yet, since we don't know the cookie value
243 * until we know which CPUs share the same logical PMU. We'll handle
244 * that in arm_pmu_acpi_cpu_starting().
245 */
246 ret = arm_pmu_acpi_parse_irqs();
247 if (ret)
248 return ret;
249
250 ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_ACPI_STARTING,
251 "perf/arm/pmu_acpi:starting",
252 arm_pmu_acpi_cpu_starting, NULL);
253
254 return ret;
255}
256subsys_initcall(arm_pmu_acpi_init)
diff --git a/drivers/perf/arm_pmu_platform.c b/drivers/perf/arm_pmu_platform.c
new file mode 100644
index 000000000000..69255f53057a
--- /dev/null
+++ b/drivers/perf/arm_pmu_platform.c
@@ -0,0 +1,235 @@
1/*
2 * platform_device probing code for ARM performance counters.
3 *
4 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
5 * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
6 */
7#define pr_fmt(fmt) "hw perfevents: " fmt
8
9#include <linux/bug.h>
10#include <linux/cpumask.h>
11#include <linux/device.h>
12#include <linux/errno.h>
13#include <linux/irq.h>
14#include <linux/irqdesc.h>
15#include <linux/kconfig.h>
16#include <linux/of.h>
17#include <linux/of_device.h>
18#include <linux/percpu.h>
19#include <linux/perf/arm_pmu.h>
20#include <linux/platform_device.h>
21#include <linux/printk.h>
22#include <linux/smp.h>
23
24static int probe_current_pmu(struct arm_pmu *pmu,
25 const struct pmu_probe_info *info)
26{
27 int cpu = get_cpu();
28 unsigned int cpuid = read_cpuid_id();
29 int ret = -ENODEV;
30
31 pr_info("probing PMU on CPU %d\n", cpu);
32
33 for (; info->init != NULL; info++) {
34 if ((cpuid & info->mask) != info->cpuid)
35 continue;
36 ret = info->init(pmu);
37 break;
38 }
39
40 put_cpu();
41 return ret;
42}
43
44static int pmu_parse_percpu_irq(struct arm_pmu *pmu, int irq)
45{
46 int cpu, ret;
47 struct pmu_hw_events __percpu *hw_events = pmu->hw_events;
48
49 ret = irq_get_percpu_devid_partition(irq, &pmu->supported_cpus);
50 if (ret)
51 return ret;
52
53 for_each_cpu(cpu, &pmu->supported_cpus)
54 per_cpu(hw_events->irq, cpu) = irq;
55
56 return 0;
57}
58
59static bool pmu_has_irq_affinity(struct device_node *node)
60{
61 return !!of_find_property(node, "interrupt-affinity", NULL);
62}
63
64static int pmu_parse_irq_affinity(struct device_node *node, int i)
65{
66 struct device_node *dn;
67 int cpu;
68
69 /*
70 * If we don't have an interrupt-affinity property, we guess irq
71 * affinity matches our logical CPU order, as we used to assume.
72 * This is fragile, so we'll warn in pmu_parse_irqs().
73 */
74 if (!pmu_has_irq_affinity(node))
75 return i;
76
77 dn = of_parse_phandle(node, "interrupt-affinity", i);
78 if (!dn) {
79 pr_warn("failed to parse interrupt-affinity[%d] for %s\n",
80 i, node->name);
81 return -EINVAL;
82 }
83
84 /* Now look up the logical CPU number */
85 for_each_possible_cpu(cpu) {
86 struct device_node *cpu_dn;
87
88 cpu_dn = of_cpu_device_node_get(cpu);
89 of_node_put(cpu_dn);
90
91 if (dn == cpu_dn)
92 break;
93 }
94
95 if (cpu >= nr_cpu_ids) {
96 pr_warn("failed to find logical CPU for %s\n", dn->name);
97 }
98
99 of_node_put(dn);
100
101 return cpu;
102}
103
104static int pmu_parse_irqs(struct arm_pmu *pmu)
105{
106 int i = 0, num_irqs;
107 struct platform_device *pdev = pmu->plat_device;
108 struct pmu_hw_events __percpu *hw_events = pmu->hw_events;
109
110 num_irqs = platform_irq_count(pdev);
111 if (num_irqs < 0) {
112 pr_err("unable to count PMU IRQs\n");
113 return num_irqs;
114 }
115
116 /*
117 * In this case we have no idea which CPUs are covered by the PMU.
118 * To match our prior behaviour, we assume all CPUs in this case.
119 */
120 if (num_irqs == 0) {
121 pr_warn("no irqs for PMU, sampling events not supported\n");
122 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
123 cpumask_setall(&pmu->supported_cpus);
124 return 0;
125 }
126
127 if (num_irqs == 1) {
128 int irq = platform_get_irq(pdev, 0);
129 if (irq && irq_is_percpu(irq))
130 return pmu_parse_percpu_irq(pmu, irq);
131 }
132
133 if (!pmu_has_irq_affinity(pdev->dev.of_node)) {
134 pr_warn("no interrupt-affinity property for %s, guessing.\n",
135 of_node_full_name(pdev->dev.of_node));
136 }
137
138 /*
139 * Some platforms have all PMU IRQs OR'd into a single IRQ, with a
140 * special platdata function that attempts to demux them.
141 */
142 if (dev_get_platdata(&pdev->dev))
143 cpumask_setall(&pmu->supported_cpus);
144
145 for (i = 0; i < num_irqs; i++) {
146 int cpu, irq;
147
148 irq = platform_get_irq(pdev, i);
149 if (WARN_ON(irq <= 0))
150 continue;
151
152 if (irq_is_percpu(irq)) {
153 pr_warn("multiple PPIs or mismatched SPI/PPI detected\n");
154 return -EINVAL;
155 }
156
157 cpu = pmu_parse_irq_affinity(pdev->dev.of_node, i);
158 if (cpu < 0)
159 return cpu;
160 if (cpu >= nr_cpu_ids)
161 continue;
162
163 if (per_cpu(hw_events->irq, cpu)) {
164 pr_warn("multiple PMU IRQs for the same CPU detected\n");
165 return -EINVAL;
166 }
167
168 per_cpu(hw_events->irq, cpu) = irq;
169 cpumask_set_cpu(cpu, &pmu->supported_cpus);
170 }
171
172 return 0;
173}
174
175int arm_pmu_device_probe(struct platform_device *pdev,
176 const struct of_device_id *of_table,
177 const struct pmu_probe_info *probe_table)
178{
179 const struct of_device_id *of_id;
180 armpmu_init_fn init_fn;
181 struct device_node *node = pdev->dev.of_node;
182 struct arm_pmu *pmu;
183 int ret = -ENODEV;
184
185 pmu = armpmu_alloc();
186 if (!pmu)
187 return -ENOMEM;
188
189 pmu->plat_device = pdev;
190
191 ret = pmu_parse_irqs(pmu);
192 if (ret)
193 goto out_free;
194
195 if (node && (of_id = of_match_node(of_table, pdev->dev.of_node))) {
196 init_fn = of_id->data;
197
198 pmu->secure_access = of_property_read_bool(pdev->dev.of_node,
199 "secure-reg-access");
200
201 /* arm64 systems boot only as non-secure */
202 if (IS_ENABLED(CONFIG_ARM64) && pmu->secure_access) {
203 pr_warn("ignoring \"secure-reg-access\" property for arm64\n");
204 pmu->secure_access = false;
205 }
206
207 ret = init_fn(pmu);
208 } else if (probe_table) {
209 cpumask_setall(&pmu->supported_cpus);
210 ret = probe_current_pmu(pmu, probe_table);
211 }
212
213 if (ret) {
214 pr_info("%s: failed to probe PMU!\n", of_node_full_name(node));
215 goto out_free;
216 }
217
218 ret = armpmu_request_irqs(pmu);
219 if (ret)
220 goto out_free_irqs;
221
222 ret = armpmu_register(pmu);
223 if (ret)
224 goto out_free;
225
226 return 0;
227
228out_free_irqs:
229 armpmu_free_irqs(pmu);
230out_free:
231 pr_info("%s: failed to register PMU devices!\n",
232 of_node_full_name(node));
233 armpmu_free(pmu);
234 return ret;
235}
diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c
new file mode 100644
index 000000000000..7f6b62b29e9d
--- /dev/null
+++ b/drivers/perf/qcom_l3_pmu.c
@@ -0,0 +1,849 @@
1/*
2 * Driver for the L3 cache PMUs in Qualcomm Technologies chips.
3 *
4 * The driver supports a distributed cache architecture where the overall
5 * cache for a socket is comprised of multiple slices each with its own PMU.
6 * Access to each individual PMU is provided even though all CPUs share all
7 * the slices. User space needs to aggregate to individual counts to provide
8 * a global picture.
9 *
10 * See Documentation/perf/qcom_l3_pmu.txt for more details.
11 *
12 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License version 2 and
16 * only version 2 as published by the Free Software Foundation.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 */
23
24#include <linux/acpi.h>
25#include <linux/bitops.h>
26#include <linux/interrupt.h>
27#include <linux/io.h>
28#include <linux/list.h>
29#include <linux/module.h>
30#include <linux/perf_event.h>
31#include <linux/platform_device.h>
32
33/*
34 * General constants
35 */
36
37/* Number of counters on each PMU */
38#define L3_NUM_COUNTERS 8
39/* Mask for the event type field within perf_event_attr.config and EVTYPE reg */
40#define L3_EVTYPE_MASK 0xFF
41/*
42 * Bit position of the 'long counter' flag within perf_event_attr.config.
43 * Reserve some space between the event type and this flag to allow expansion
44 * in the event type field.
45 */
46#define L3_EVENT_LC_BIT 32
47
48/*
49 * Register offsets
50 */
51
52/* Perfmon registers */
53#define L3_HML3_PM_CR 0x000
54#define L3_HML3_PM_EVCNTR(__cntr) (0x420 + ((__cntr) & 0x7) * 8)
55#define L3_HML3_PM_CNTCTL(__cntr) (0x120 + ((__cntr) & 0x7) * 8)
56#define L3_HML3_PM_EVTYPE(__cntr) (0x220 + ((__cntr) & 0x7) * 8)
57#define L3_HML3_PM_FILTRA 0x300
58#define L3_HML3_PM_FILTRB 0x308
59#define L3_HML3_PM_FILTRC 0x310
60#define L3_HML3_PM_FILTRAM 0x304
61#define L3_HML3_PM_FILTRBM 0x30C
62#define L3_HML3_PM_FILTRCM 0x314
63
64/* Basic counter registers */
65#define L3_M_BC_CR 0x500
66#define L3_M_BC_SATROLL_CR 0x504
67#define L3_M_BC_CNTENSET 0x508
68#define L3_M_BC_CNTENCLR 0x50C
69#define L3_M_BC_INTENSET 0x510
70#define L3_M_BC_INTENCLR 0x514
71#define L3_M_BC_GANG 0x718
72#define L3_M_BC_OVSR 0x740
73#define L3_M_BC_IRQCTL 0x96C
74
75/*
76 * Bit field definitions
77 */
78
79/* L3_HML3_PM_CR */
80#define PM_CR_RESET (0)
81
82/* L3_HML3_PM_XCNTCTL/L3_HML3_PM_CNTCTLx */
83#define PMCNT_RESET (0)
84
85/* L3_HML3_PM_EVTYPEx */
86#define EVSEL(__val) ((__val) & L3_EVTYPE_MASK)
87
88/* Reset value for all the filter registers */
89#define PM_FLTR_RESET (0)
90
91/* L3_M_BC_CR */
92#define BC_RESET (1UL << 1)
93#define BC_ENABLE (1UL << 0)
94
95/* L3_M_BC_SATROLL_CR */
96#define BC_SATROLL_CR_RESET (0)
97
98/* L3_M_BC_CNTENSET */
99#define PMCNTENSET(__cntr) (1UL << ((__cntr) & 0x7))
100
101/* L3_M_BC_CNTENCLR */
102#define PMCNTENCLR(__cntr) (1UL << ((__cntr) & 0x7))
103#define BC_CNTENCLR_RESET (0xFF)
104
105/* L3_M_BC_INTENSET */
106#define PMINTENSET(__cntr) (1UL << ((__cntr) & 0x7))
107
108/* L3_M_BC_INTENCLR */
109#define PMINTENCLR(__cntr) (1UL << ((__cntr) & 0x7))
110#define BC_INTENCLR_RESET (0xFF)
111
112/* L3_M_BC_GANG */
113#define GANG_EN(__cntr) (1UL << ((__cntr) & 0x7))
114#define BC_GANG_RESET (0)
115
116/* L3_M_BC_OVSR */
117#define PMOVSRCLR(__cntr) (1UL << ((__cntr) & 0x7))
118#define PMOVSRCLR_RESET (0xFF)
119
120/* L3_M_BC_IRQCTL */
121#define PMIRQONMSBEN(__cntr) (1UL << ((__cntr) & 0x7))
122#define BC_IRQCTL_RESET (0x0)
123
124/*
125 * Events
126 */
127
128#define L3_EVENT_CYCLES 0x01
129#define L3_EVENT_READ_HIT 0x20
130#define L3_EVENT_READ_MISS 0x21
131#define L3_EVENT_READ_HIT_D 0x22
132#define L3_EVENT_READ_MISS_D 0x23
133#define L3_EVENT_WRITE_HIT 0x24
134#define L3_EVENT_WRITE_MISS 0x25
135
136/*
137 * Decoding of settings from perf_event_attr
138 *
139 * The config format for perf events is:
140 * - config: bits 0-7: event type
141 * bit 32: HW counter size requested, 0: 32 bits, 1: 64 bits
142 */
143
144static inline u32 get_event_type(struct perf_event *event)
145{
146 return (event->attr.config) & L3_EVTYPE_MASK;
147}
148
149static inline bool event_uses_long_counter(struct perf_event *event)
150{
151 return !!(event->attr.config & BIT_ULL(L3_EVENT_LC_BIT));
152}
153
154static inline int event_num_counters(struct perf_event *event)
155{
156 return event_uses_long_counter(event) ? 2 : 1;
157}
158
159/*
160 * Main PMU, inherits from the core perf PMU type
161 */
162struct l3cache_pmu {
163 struct pmu pmu;
164 struct hlist_node node;
165 void __iomem *regs;
166 struct perf_event *events[L3_NUM_COUNTERS];
167 unsigned long used_mask[BITS_TO_LONGS(L3_NUM_COUNTERS)];
168 cpumask_t cpumask;
169};
170
171#define to_l3cache_pmu(p) (container_of(p, struct l3cache_pmu, pmu))
172
173/*
174 * Type used to group hardware counter operations
175 *
176 * Used to implement two types of hardware counters, standard (32bits) and
177 * long (64bits). The hardware supports counter chaining which we use to
178 * implement long counters. This support is exposed via the 'lc' flag field
179 * in perf_event_attr.config.
180 */
181struct l3cache_event_ops {
182 /* Called to start event monitoring */
183 void (*start)(struct perf_event *event);
184 /* Called to stop event monitoring */
185 void (*stop)(struct perf_event *event, int flags);
186 /* Called to update the perf_event */
187 void (*update)(struct perf_event *event);
188};
189
190/*
191 * Implementation of long counter operations
192 *
193 * 64bit counters are implemented by chaining two of the 32bit physical
194 * counters. The PMU only supports chaining of adjacent even/odd pairs
195 * and for simplicity the driver always configures the odd counter to
196 * count the overflows of the lower-numbered even counter. Note that since
197 * the resulting hardware counter is 64bits no IRQs are required to maintain
198 * the software counter which is also 64bits.
199 */
200
201static void qcom_l3_cache__64bit_counter_start(struct perf_event *event)
202{
203 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
204 int idx = event->hw.idx;
205 u32 evsel = get_event_type(event);
206 u32 gang;
207
208 /* Set the odd counter to count the overflows of the even counter */
209 gang = readl_relaxed(l3pmu->regs + L3_M_BC_GANG);
210 gang |= GANG_EN(idx + 1);
211 writel_relaxed(gang, l3pmu->regs + L3_M_BC_GANG);
212
213 /* Initialize the hardware counters and reset prev_count*/
214 local64_set(&event->hw.prev_count, 0);
215 writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1));
216 writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
217
218 /*
219 * Set the event types, the upper half must use zero and the lower
220 * half the actual event type
221 */
222 writel_relaxed(EVSEL(0), l3pmu->regs + L3_HML3_PM_EVTYPE(idx + 1));
223 writel_relaxed(EVSEL(evsel), l3pmu->regs + L3_HML3_PM_EVTYPE(idx));
224
225 /* Finally, enable the counters */
226 writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx + 1));
227 writel_relaxed(PMCNTENSET(idx + 1), l3pmu->regs + L3_M_BC_CNTENSET);
228 writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx));
229 writel_relaxed(PMCNTENSET(idx), l3pmu->regs + L3_M_BC_CNTENSET);
230}
231
232static void qcom_l3_cache__64bit_counter_stop(struct perf_event *event,
233 int flags)
234{
235 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
236 int idx = event->hw.idx;
237 u32 gang = readl_relaxed(l3pmu->regs + L3_M_BC_GANG);
238
239 /* Disable the counters */
240 writel_relaxed(PMCNTENCLR(idx), l3pmu->regs + L3_M_BC_CNTENCLR);
241 writel_relaxed(PMCNTENCLR(idx + 1), l3pmu->regs + L3_M_BC_CNTENCLR);
242
243 /* Disable chaining */
244 writel_relaxed(gang & ~GANG_EN(idx + 1), l3pmu->regs + L3_M_BC_GANG);
245}
246
247static void qcom_l3_cache__64bit_counter_update(struct perf_event *event)
248{
249 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
250 int idx = event->hw.idx;
251 u32 hi, lo;
252 u64 prev, new;
253
254 do {
255 prev = local64_read(&event->hw.prev_count);
256 do {
257 hi = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1));
258 lo = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
259 } while (hi != readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1)));
260 new = ((u64)hi << 32) | lo;
261 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
262
263 local64_add(new - prev, &event->count);
264}
265
266static const struct l3cache_event_ops event_ops_long = {
267 .start = qcom_l3_cache__64bit_counter_start,
268 .stop = qcom_l3_cache__64bit_counter_stop,
269 .update = qcom_l3_cache__64bit_counter_update,
270};
271
272/*
273 * Implementation of standard counter operations
274 *
275 * 32bit counters use a single physical counter and a hardware feature that
276 * asserts the overflow IRQ on the toggling of the most significant bit in
277 * the counter. This feature allows the counters to be left free-running
278 * without needing the usual reprogramming required to properly handle races
279 * during concurrent calls to update.
280 */
281
282static void qcom_l3_cache__32bit_counter_start(struct perf_event *event)
283{
284 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
285 int idx = event->hw.idx;
286 u32 evsel = get_event_type(event);
287 u32 irqctl = readl_relaxed(l3pmu->regs + L3_M_BC_IRQCTL);
288
289 /* Set the counter to assert the overflow IRQ on MSB toggling */
290 writel_relaxed(irqctl | PMIRQONMSBEN(idx), l3pmu->regs + L3_M_BC_IRQCTL);
291
292 /* Initialize the hardware counter and reset prev_count*/
293 local64_set(&event->hw.prev_count, 0);
294 writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
295
296 /* Set the event type */
297 writel_relaxed(EVSEL(evsel), l3pmu->regs + L3_HML3_PM_EVTYPE(idx));
298
299 /* Enable interrupt generation by this counter */
300 writel_relaxed(PMINTENSET(idx), l3pmu->regs + L3_M_BC_INTENSET);
301
302 /* Finally, enable the counter */
303 writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx));
304 writel_relaxed(PMCNTENSET(idx), l3pmu->regs + L3_M_BC_CNTENSET);
305}
306
307static void qcom_l3_cache__32bit_counter_stop(struct perf_event *event,
308 int flags)
309{
310 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
311 int idx = event->hw.idx;
312 u32 irqctl = readl_relaxed(l3pmu->regs + L3_M_BC_IRQCTL);
313
314 /* Disable the counter */
315 writel_relaxed(PMCNTENCLR(idx), l3pmu->regs + L3_M_BC_CNTENCLR);
316
317 /* Disable interrupt generation by this counter */
318 writel_relaxed(PMINTENCLR(idx), l3pmu->regs + L3_M_BC_INTENCLR);
319
320 /* Set the counter to not assert the overflow IRQ on MSB toggling */
321 writel_relaxed(irqctl & ~PMIRQONMSBEN(idx), l3pmu->regs + L3_M_BC_IRQCTL);
322}
323
324static void qcom_l3_cache__32bit_counter_update(struct perf_event *event)
325{
326 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
327 int idx = event->hw.idx;
328 u32 prev, new;
329
330 do {
331 prev = local64_read(&event->hw.prev_count);
332 new = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
333 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
334
335 local64_add(new - prev, &event->count);
336}
337
338static const struct l3cache_event_ops event_ops_std = {
339 .start = qcom_l3_cache__32bit_counter_start,
340 .stop = qcom_l3_cache__32bit_counter_stop,
341 .update = qcom_l3_cache__32bit_counter_update,
342};
343
344/* Retrieve the appropriate operations for the given event */
345static
346const struct l3cache_event_ops *l3cache_event_get_ops(struct perf_event *event)
347{
348 if (event_uses_long_counter(event))
349 return &event_ops_long;
350 else
351 return &event_ops_std;
352}
353
354/*
355 * Top level PMU functions.
356 */
357
358static inline void qcom_l3_cache__init(struct l3cache_pmu *l3pmu)
359{
360 int i;
361
362 writel_relaxed(BC_RESET, l3pmu->regs + L3_M_BC_CR);
363
364 /*
365 * Use writel for the first programming command to ensure the basic
366 * counter unit is stopped before proceeding
367 */
368 writel(BC_SATROLL_CR_RESET, l3pmu->regs + L3_M_BC_SATROLL_CR);
369
370 writel_relaxed(BC_CNTENCLR_RESET, l3pmu->regs + L3_M_BC_CNTENCLR);
371 writel_relaxed(BC_INTENCLR_RESET, l3pmu->regs + L3_M_BC_INTENCLR);
372 writel_relaxed(PMOVSRCLR_RESET, l3pmu->regs + L3_M_BC_OVSR);
373 writel_relaxed(BC_GANG_RESET, l3pmu->regs + L3_M_BC_GANG);
374 writel_relaxed(BC_IRQCTL_RESET, l3pmu->regs + L3_M_BC_IRQCTL);
375 writel_relaxed(PM_CR_RESET, l3pmu->regs + L3_HML3_PM_CR);
376
377 for (i = 0; i < L3_NUM_COUNTERS; ++i) {
378 writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(i));
379 writel_relaxed(EVSEL(0), l3pmu->regs + L3_HML3_PM_EVTYPE(i));
380 }
381
382 writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRA);
383 writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRAM);
384 writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRB);
385 writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRBM);
386 writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRC);
387 writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRCM);
388
389 /*
390 * Use writel here to ensure all programming commands are done
391 * before proceeding
392 */
393 writel(BC_ENABLE, l3pmu->regs + L3_M_BC_CR);
394}
395
396static irqreturn_t qcom_l3_cache__handle_irq(int irq_num, void *data)
397{
398 struct l3cache_pmu *l3pmu = data;
399 /* Read the overflow status register */
400 long status = readl_relaxed(l3pmu->regs + L3_M_BC_OVSR);
401 int idx;
402
403 if (status == 0)
404 return IRQ_NONE;
405
406 /* Clear the bits we read on the overflow status register */
407 writel_relaxed(status, l3pmu->regs + L3_M_BC_OVSR);
408
409 for_each_set_bit(idx, &status, L3_NUM_COUNTERS) {
410 struct perf_event *event;
411 const struct l3cache_event_ops *ops;
412
413 event = l3pmu->events[idx];
414 if (!event)
415 continue;
416
417 /*
418 * Since the IRQ is not enabled for events using long counters
419 * we should never see one of those here, however, be consistent
420 * and use the ops indirections like in the other operations.
421 */
422
423 ops = l3cache_event_get_ops(event);
424 ops->update(event);
425 }
426
427 return IRQ_HANDLED;
428}
429
430/*
431 * Implementation of abstract pmu functionality required by
432 * the core perf events code.
433 */
434
435static void qcom_l3_cache__pmu_enable(struct pmu *pmu)
436{
437 struct l3cache_pmu *l3pmu = to_l3cache_pmu(pmu);
438
439 /* Ensure the other programming commands are observed before enabling */
440 wmb();
441
442 writel_relaxed(BC_ENABLE, l3pmu->regs + L3_M_BC_CR);
443}
444
445static void qcom_l3_cache__pmu_disable(struct pmu *pmu)
446{
447 struct l3cache_pmu *l3pmu = to_l3cache_pmu(pmu);
448
449 writel_relaxed(0, l3pmu->regs + L3_M_BC_CR);
450
451 /* Ensure the basic counter unit is stopped before proceeding */
452 wmb();
453}
454
455/*
456 * We must NOT create groups containing events from multiple hardware PMUs,
457 * although mixing different software and hardware PMUs is allowed.
458 */
459static bool qcom_l3_cache__validate_event_group(struct perf_event *event)
460{
461 struct perf_event *leader = event->group_leader;
462 struct perf_event *sibling;
463 int counters = 0;
464
465 if (leader->pmu != event->pmu && !is_software_event(leader))
466 return false;
467
468 counters = event_num_counters(event);
469 counters += event_num_counters(leader);
470
471 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
472 if (is_software_event(sibling))
473 continue;
474 if (sibling->pmu != event->pmu)
475 return false;
476 counters += event_num_counters(sibling);
477 }
478
479 /*
480 * If the group requires more counters than the HW has, it
481 * cannot ever be scheduled.
482 */
483 return counters <= L3_NUM_COUNTERS;
484}
485
486static int qcom_l3_cache__event_init(struct perf_event *event)
487{
488 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
489 struct hw_perf_event *hwc = &event->hw;
490
491 /*
492 * Is the event for this PMU?
493 */
494 if (event->attr.type != event->pmu->type)
495 return -ENOENT;
496
497 /*
498 * There are no per-counter mode filters in the PMU.
499 */
500 if (event->attr.exclude_user || event->attr.exclude_kernel ||
501 event->attr.exclude_hv || event->attr.exclude_idle)
502 return -EINVAL;
503
504 /*
505 * Sampling not supported since these events are not core-attributable.
506 */
507 if (hwc->sample_period)
508 return -EINVAL;
509
510 /*
511 * Task mode not available, we run the counters as socket counters,
512 * not attributable to any CPU and therefore cannot attribute per-task.
513 */
514 if (event->cpu < 0)
515 return -EINVAL;
516
517 /* Validate the group */
518 if (!qcom_l3_cache__validate_event_group(event))
519 return -EINVAL;
520
521 hwc->idx = -1;
522
523 /*
524 * Many perf core operations (eg. events rotation) operate on a
525 * single CPU context. This is obvious for CPU PMUs, where one
526 * expects the same sets of events being observed on all CPUs,
527 * but can lead to issues for off-core PMUs, like this one, where
528 * each event could be theoretically assigned to a different CPU.
529 * To mitigate this, we enforce CPU assignment to one designated
530 * processor (the one described in the "cpumask" attribute exported
531 * by the PMU device). perf user space tools honor this and avoid
532 * opening more than one copy of the events.
533 */
534 event->cpu = cpumask_first(&l3pmu->cpumask);
535
536 return 0;
537}
538
539static void qcom_l3_cache__event_start(struct perf_event *event, int flags)
540{
541 struct hw_perf_event *hwc = &event->hw;
542 const struct l3cache_event_ops *ops = l3cache_event_get_ops(event);
543
544 hwc->state = 0;
545 ops->start(event);
546}
547
548static void qcom_l3_cache__event_stop(struct perf_event *event, int flags)
549{
550 struct hw_perf_event *hwc = &event->hw;
551 const struct l3cache_event_ops *ops = l3cache_event_get_ops(event);
552
553 if (hwc->state & PERF_HES_STOPPED)
554 return;
555
556 ops->stop(event, flags);
557 if (flags & PERF_EF_UPDATE)
558 ops->update(event);
559 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
560}
561
562static int qcom_l3_cache__event_add(struct perf_event *event, int flags)
563{
564 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
565 struct hw_perf_event *hwc = &event->hw;
566 int order = event_uses_long_counter(event) ? 1 : 0;
567 int idx;
568
569 /*
570 * Try to allocate a counter.
571 */
572 idx = bitmap_find_free_region(l3pmu->used_mask, L3_NUM_COUNTERS, order);
573 if (idx < 0)
574 /* The counters are all in use. */
575 return -EAGAIN;
576
577 hwc->idx = idx;
578 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
579 l3pmu->events[idx] = event;
580
581 if (flags & PERF_EF_START)
582 qcom_l3_cache__event_start(event, 0);
583
584 /* Propagate changes to the userspace mapping. */
585 perf_event_update_userpage(event);
586
587 return 0;
588}
589
590static void qcom_l3_cache__event_del(struct perf_event *event, int flags)
591{
592 struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
593 struct hw_perf_event *hwc = &event->hw;
594 int order = event_uses_long_counter(event) ? 1 : 0;
595
596 /* Stop and clean up */
597 qcom_l3_cache__event_stop(event, flags | PERF_EF_UPDATE);
598 l3pmu->events[hwc->idx] = NULL;
599 bitmap_release_region(l3pmu->used_mask, hwc->idx, order);
600
601 /* Propagate changes to the userspace mapping. */
602 perf_event_update_userpage(event);
603}
604
605static void qcom_l3_cache__event_read(struct perf_event *event)
606{
607 const struct l3cache_event_ops *ops = l3cache_event_get_ops(event);
608
609 ops->update(event);
610}
611
612/*
613 * Add sysfs attributes
614 *
615 * We export:
616 * - formats, used by perf user space and other tools to configure events
617 * - events, used by perf user space and other tools to create events
618 * symbolically, e.g.:
619 * perf stat -a -e l3cache_0_0/event=read-miss/ ls
620 * perf stat -a -e l3cache_0_0/event=0x21/ ls
621 * - cpumask, used by perf user space and other tools to know on which CPUs
622 * to open the events
623 */
624
625/* formats */
626
627static ssize_t l3cache_pmu_format_show(struct device *dev,
628 struct device_attribute *attr, char *buf)
629{
630 struct dev_ext_attribute *eattr;
631
632 eattr = container_of(attr, struct dev_ext_attribute, attr);
633 return sprintf(buf, "%s\n", (char *) eattr->var);
634}
635
636#define L3CACHE_PMU_FORMAT_ATTR(_name, _config) \
637 (&((struct dev_ext_attribute[]) { \
638 { .attr = __ATTR(_name, 0444, l3cache_pmu_format_show, NULL), \
639 .var = (void *) _config, } \
640 })[0].attr.attr)
641
642static struct attribute *qcom_l3_cache_pmu_formats[] = {
643 L3CACHE_PMU_FORMAT_ATTR(event, "config:0-7"),
644 L3CACHE_PMU_FORMAT_ATTR(lc, "config:" __stringify(L3_EVENT_LC_BIT)),
645 NULL,
646};
647
648static struct attribute_group qcom_l3_cache_pmu_format_group = {
649 .name = "format",
650 .attrs = qcom_l3_cache_pmu_formats,
651};
652
653/* events */
654
655static ssize_t l3cache_pmu_event_show(struct device *dev,
656 struct device_attribute *attr, char *page)
657{
658 struct perf_pmu_events_attr *pmu_attr;
659
660 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
661 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
662}
663
664#define L3CACHE_EVENT_ATTR(_name, _id) \
665 (&((struct perf_pmu_events_attr[]) { \
666 { .attr = __ATTR(_name, 0444, l3cache_pmu_event_show, NULL), \
667 .id = _id, } \
668 })[0].attr.attr)
669
670static struct attribute *qcom_l3_cache_pmu_events[] = {
671 L3CACHE_EVENT_ATTR(cycles, L3_EVENT_CYCLES),
672 L3CACHE_EVENT_ATTR(read-hit, L3_EVENT_READ_HIT),
673 L3CACHE_EVENT_ATTR(read-miss, L3_EVENT_READ_MISS),
674 L3CACHE_EVENT_ATTR(read-hit-d-side, L3_EVENT_READ_HIT_D),
675 L3CACHE_EVENT_ATTR(read-miss-d-side, L3_EVENT_READ_MISS_D),
676 L3CACHE_EVENT_ATTR(write-hit, L3_EVENT_WRITE_HIT),
677 L3CACHE_EVENT_ATTR(write-miss, L3_EVENT_WRITE_MISS),
678 NULL
679};
680
681static struct attribute_group qcom_l3_cache_pmu_events_group = {
682 .name = "events",
683 .attrs = qcom_l3_cache_pmu_events,
684};
685
686/* cpumask */
687
688static ssize_t qcom_l3_cache_pmu_cpumask_show(struct device *dev,
689 struct device_attribute *attr, char *buf)
690{
691 struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev));
692
693 return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask);
694}
695
696static DEVICE_ATTR(cpumask, 0444, qcom_l3_cache_pmu_cpumask_show, NULL);
697
698static struct attribute *qcom_l3_cache_pmu_cpumask_attrs[] = {
699 &dev_attr_cpumask.attr,
700 NULL,
701};
702
703static struct attribute_group qcom_l3_cache_pmu_cpumask_attr_group = {
704 .attrs = qcom_l3_cache_pmu_cpumask_attrs,
705};
706
707/*
708 * Per PMU device attribute groups
709 */
710static const struct attribute_group *qcom_l3_cache_pmu_attr_grps[] = {
711 &qcom_l3_cache_pmu_format_group,
712 &qcom_l3_cache_pmu_events_group,
713 &qcom_l3_cache_pmu_cpumask_attr_group,
714 NULL,
715};
716
717/*
718 * Probing functions and data.
719 */
720
721static int qcom_l3_cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
722{
723 struct l3cache_pmu *l3pmu = hlist_entry_safe(node, struct l3cache_pmu, node);
724
725 /* If there is not a CPU/PMU association pick this CPU */
726 if (cpumask_empty(&l3pmu->cpumask))
727 cpumask_set_cpu(cpu, &l3pmu->cpumask);
728
729 return 0;
730}
731
732static int qcom_l3_cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
733{
734 struct l3cache_pmu *l3pmu = hlist_entry_safe(node, struct l3cache_pmu, node);
735 unsigned int target;
736
737 if (!cpumask_test_and_clear_cpu(cpu, &l3pmu->cpumask))
738 return 0;
739 target = cpumask_any_but(cpu_online_mask, cpu);
740 if (target >= nr_cpu_ids)
741 return 0;
742 perf_pmu_migrate_context(&l3pmu->pmu, cpu, target);
743 cpumask_set_cpu(target, &l3pmu->cpumask);
744 return 0;
745}
746
747static int qcom_l3_cache_pmu_probe(struct platform_device *pdev)
748{
749 struct l3cache_pmu *l3pmu;
750 struct acpi_device *acpi_dev;
751 struct resource *memrc;
752 int ret;
753 char *name;
754
755 /* Initialize the PMU data structures */
756
757 acpi_dev = ACPI_COMPANION(&pdev->dev);
758 if (!acpi_dev)
759 return -ENODEV;
760
761 l3pmu = devm_kzalloc(&pdev->dev, sizeof(*l3pmu), GFP_KERNEL);
762 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "l3cache_%s_%s",
763 acpi_dev->parent->pnp.unique_id, acpi_dev->pnp.unique_id);
764 if (!l3pmu || !name)
765 return -ENOMEM;
766
767 l3pmu->pmu = (struct pmu) {
768 .task_ctx_nr = perf_invalid_context,
769
770 .pmu_enable = qcom_l3_cache__pmu_enable,
771 .pmu_disable = qcom_l3_cache__pmu_disable,
772 .event_init = qcom_l3_cache__event_init,
773 .add = qcom_l3_cache__event_add,
774 .del = qcom_l3_cache__event_del,
775 .start = qcom_l3_cache__event_start,
776 .stop = qcom_l3_cache__event_stop,
777 .read = qcom_l3_cache__event_read,
778
779 .attr_groups = qcom_l3_cache_pmu_attr_grps,
780 };
781
782 memrc = platform_get_resource(pdev, IORESOURCE_MEM, 0);
783 l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc);
784 if (IS_ERR(l3pmu->regs)) {
785 dev_err(&pdev->dev, "Can't map PMU @%pa\n", &memrc->start);
786 return PTR_ERR(l3pmu->regs);
787 }
788
789 qcom_l3_cache__init(l3pmu);
790
791 ret = platform_get_irq(pdev, 0);
792 if (ret <= 0)
793 return ret;
794
795 ret = devm_request_irq(&pdev->dev, ret, qcom_l3_cache__handle_irq, 0,
796 name, l3pmu);
797 if (ret) {
798 dev_err(&pdev->dev, "Request for IRQ failed for slice @%pa\n",
799 &memrc->start);
800 return ret;
801 }
802
803 /* Add this instance to the list used by the offline callback */
804 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE, &l3pmu->node);
805 if (ret) {
806 dev_err(&pdev->dev, "Error %d registering hotplug", ret);
807 return ret;
808 }
809
810 ret = perf_pmu_register(&l3pmu->pmu, name, -1);
811 if (ret < 0) {
812 dev_err(&pdev->dev, "Failed to register L3 cache PMU (%d)\n", ret);
813 return ret;
814 }
815
816 dev_info(&pdev->dev, "Registered %s, type: %d\n", name, l3pmu->pmu.type);
817
818 return 0;
819}
820
821static const struct acpi_device_id qcom_l3_cache_pmu_acpi_match[] = {
822 { "QCOM8081", },
823 { }
824};
825MODULE_DEVICE_TABLE(acpi, qcom_l3_cache_pmu_acpi_match);
826
827static struct platform_driver qcom_l3_cache_pmu_driver = {
828 .driver = {
829 .name = "qcom-l3cache-pmu",
830 .acpi_match_table = ACPI_PTR(qcom_l3_cache_pmu_acpi_match),
831 },
832 .probe = qcom_l3_cache_pmu_probe,
833};
834
835static int __init register_qcom_l3_cache_pmu_driver(void)
836{
837 int ret;
838
839 /* Install a hook to update the reader CPU in case it goes offline */
840 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,
841 "perf/qcom/l3cache:online",
842 qcom_l3_cache_pmu_online_cpu,
843 qcom_l3_cache_pmu_offline_cpu);
844 if (ret)
845 return ret;
846
847 return platform_driver_register(&qcom_l3_cache_pmu_driver);
848}
849device_initcall(register_qcom_l3_cache_pmu_driver);
diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h
index 77e08099e554..26e25d85eb3e 100644
--- a/include/linux/acpi_iort.h
+++ b/include/linux/acpi_iort.h
@@ -34,6 +34,8 @@ void acpi_iort_init(void);
34bool iort_node_match(u8 type); 34bool iort_node_match(u8 type);
35u32 iort_msi_map_rid(struct device *dev, u32 req_id); 35u32 iort_msi_map_rid(struct device *dev, u32 req_id);
36struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id); 36struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id);
37void acpi_configure_pmsi_domain(struct device *dev);
38int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id);
37/* IOMMU interface */ 39/* IOMMU interface */
38void iort_set_dma_mask(struct device *dev); 40void iort_set_dma_mask(struct device *dev);
39const struct iommu_ops *iort_iommu_configure(struct device *dev); 41const struct iommu_ops *iort_iommu_configure(struct device *dev);
@@ -45,6 +47,7 @@ static inline u32 iort_msi_map_rid(struct device *dev, u32 req_id)
45static inline struct irq_domain *iort_get_device_domain(struct device *dev, 47static inline struct irq_domain *iort_get_device_domain(struct device *dev,
46 u32 req_id) 48 u32 req_id)
47{ return NULL; } 49{ return NULL; }
50static inline void acpi_configure_pmsi_domain(struct device *dev) { }
48/* IOMMU interface */ 51/* IOMMU interface */
49static inline void iort_set_dma_mask(struct device *dev) { } 52static inline void iort_set_dma_mask(struct device *dev) { }
50static inline 53static inline
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 62d240e962f0..0f2a80377520 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -94,6 +94,7 @@ enum cpuhp_state {
94 CPUHP_AP_ARM_VFP_STARTING, 94 CPUHP_AP_ARM_VFP_STARTING,
95 CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING, 95 CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING,
96 CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING, 96 CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
97 CPUHP_AP_PERF_ARM_ACPI_STARTING,
97 CPUHP_AP_PERF_ARM_STARTING, 98 CPUHP_AP_PERF_ARM_STARTING,
98 CPUHP_AP_ARM_L2X0_STARTING, 99 CPUHP_AP_ARM_L2X0_STARTING,
99 CPUHP_AP_ARM_ARCH_TIMER_STARTING, 100 CPUHP_AP_ARM_ARCH_TIMER_STARTING,
@@ -137,6 +138,7 @@ enum cpuhp_state {
137 CPUHP_AP_PERF_ARM_CCN_ONLINE, 138 CPUHP_AP_PERF_ARM_CCN_ONLINE,
138 CPUHP_AP_PERF_ARM_L2X0_ONLINE, 139 CPUHP_AP_PERF_ARM_L2X0_ONLINE,
139 CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 140 CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
141 CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,
140 CPUHP_AP_WORKQUEUE_ONLINE, 142 CPUHP_AP_WORKQUEUE_ONLINE,
141 CPUHP_AP_RCUTREE_ONLINE, 143 CPUHP_AP_RCUTREE_ONLINE,
142 CPUHP_AP_ONLINE_DYN, 144 CPUHP_AP_ONLINE_DYN,
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index bdfc65af4152..4ce24a376262 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -93,6 +93,7 @@ int memblock_mark_hotplug(phys_addr_t base, phys_addr_t size);
93int memblock_clear_hotplug(phys_addr_t base, phys_addr_t size); 93int memblock_clear_hotplug(phys_addr_t base, phys_addr_t size);
94int memblock_mark_mirror(phys_addr_t base, phys_addr_t size); 94int memblock_mark_mirror(phys_addr_t base, phys_addr_t size);
95int memblock_mark_nomap(phys_addr_t base, phys_addr_t size); 95int memblock_mark_nomap(phys_addr_t base, phys_addr_t size);
96int memblock_clear_nomap(phys_addr_t base, phys_addr_t size);
96ulong choose_memblock_flags(void); 97ulong choose_memblock_flags(void);
97 98
98/* Low level functions */ 99/* Low level functions */
@@ -335,6 +336,7 @@ phys_addr_t memblock_mem_size(unsigned long limit_pfn);
335phys_addr_t memblock_start_of_DRAM(void); 336phys_addr_t memblock_start_of_DRAM(void);
336phys_addr_t memblock_end_of_DRAM(void); 337phys_addr_t memblock_end_of_DRAM(void);
337void memblock_enforce_memory_limit(phys_addr_t memory_limit); 338void memblock_enforce_memory_limit(phys_addr_t memory_limit);
339void memblock_cap_memory_range(phys_addr_t base, phys_addr_t size);
338void memblock_mem_limit_remove_map(phys_addr_t limit); 340void memblock_mem_limit_remove_map(phys_addr_t limit);
339bool memblock_is_memory(phys_addr_t addr); 341bool memblock_is_memory(phys_addr_t addr);
340int memblock_is_map_memory(phys_addr_t addr); 342int memblock_is_map_memory(phys_addr_t addr);
diff --git a/include/linux/pe.h b/include/linux/pe.h
index e170b95e763b..143ce75be5f0 100644
--- a/include/linux/pe.h
+++ b/include/linux/pe.h
@@ -23,34 +23,6 @@
23 23
24#define MZ_MAGIC 0x5a4d /* "MZ" */ 24#define MZ_MAGIC 0x5a4d /* "MZ" */
25 25
26struct mz_hdr {
27 uint16_t magic; /* MZ_MAGIC */
28 uint16_t lbsize; /* size of last used block */
29 uint16_t blocks; /* pages in file, 0x3 */
30 uint16_t relocs; /* relocations */
31 uint16_t hdrsize; /* header size in "paragraphs" */
32 uint16_t min_extra_pps; /* .bss */
33 uint16_t max_extra_pps; /* runtime limit for the arena size */
34 uint16_t ss; /* relative stack segment */
35 uint16_t sp; /* initial %sp register */
36 uint16_t checksum; /* word checksum */
37 uint16_t ip; /* initial %ip register */
38 uint16_t cs; /* initial %cs relative to load segment */
39 uint16_t reloc_table_offset; /* offset of the first relocation */
40 uint16_t overlay_num; /* overlay number. set to 0. */
41 uint16_t reserved0[4]; /* reserved */
42 uint16_t oem_id; /* oem identifier */
43 uint16_t oem_info; /* oem specific */
44 uint16_t reserved1[10]; /* reserved */
45 uint32_t peaddr; /* address of pe header */
46 char message[64]; /* message to print */
47};
48
49struct mz_reloc {
50 uint16_t offset;
51 uint16_t segment;
52};
53
54#define PE_MAGIC 0x00004550 /* "PE\0\0" */ 26#define PE_MAGIC 0x00004550 /* "PE\0\0" */
55#define PE_OPT_MAGIC_PE32 0x010b 27#define PE_OPT_MAGIC_PE32 0x010b
56#define PE_OPT_MAGIC_PE32_ROM 0x0107 28#define PE_OPT_MAGIC_PE32_ROM 0x0107
@@ -62,6 +34,7 @@ struct mz_reloc {
62#define IMAGE_FILE_MACHINE_AMD64 0x8664 34#define IMAGE_FILE_MACHINE_AMD64 0x8664
63#define IMAGE_FILE_MACHINE_ARM 0x01c0 35#define IMAGE_FILE_MACHINE_ARM 0x01c0
64#define IMAGE_FILE_MACHINE_ARMV7 0x01c4 36#define IMAGE_FILE_MACHINE_ARMV7 0x01c4
37#define IMAGE_FILE_MACHINE_ARM64 0xaa64
65#define IMAGE_FILE_MACHINE_EBC 0x0ebc 38#define IMAGE_FILE_MACHINE_EBC 0x0ebc
66#define IMAGE_FILE_MACHINE_I386 0x014c 39#define IMAGE_FILE_MACHINE_I386 0x014c
67#define IMAGE_FILE_MACHINE_IA64 0x0200 40#define IMAGE_FILE_MACHINE_IA64 0x0200
@@ -98,17 +71,6 @@ struct mz_reloc {
98#define IMAGE_FILE_UP_SYSTEM_ONLY 0x4000 71#define IMAGE_FILE_UP_SYSTEM_ONLY 0x4000
99#define IMAGE_FILE_BYTES_REVERSED_HI 0x8000 72#define IMAGE_FILE_BYTES_REVERSED_HI 0x8000
100 73
101struct pe_hdr {
102 uint32_t magic; /* PE magic */
103 uint16_t machine; /* machine type */
104 uint16_t sections; /* number of sections */
105 uint32_t timestamp; /* time_t */
106 uint32_t symbol_table; /* symbol table offset */
107 uint32_t symbols; /* number of symbols */
108 uint16_t opt_hdr_size; /* size of optional header */
109 uint16_t flags; /* flags */
110};
111
112#define IMAGE_FILE_OPT_ROM_MAGIC 0x107 74#define IMAGE_FILE_OPT_ROM_MAGIC 0x107
113#define IMAGE_FILE_OPT_PE32_MAGIC 0x10b 75#define IMAGE_FILE_OPT_PE32_MAGIC 0x10b
114#define IMAGE_FILE_OPT_PE32_PLUS_MAGIC 0x20b 76#define IMAGE_FILE_OPT_PE32_PLUS_MAGIC 0x20b
@@ -134,6 +96,95 @@ struct pe_hdr {
134#define IMAGE_DLLCHARACTERISTICS_WDM_DRIVER 0x2000 96#define IMAGE_DLLCHARACTERISTICS_WDM_DRIVER 0x2000
135#define IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE 0x8000 97#define IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE 0x8000
136 98
99/* they actually defined 0x00000000 as well, but I think we'll skip that one. */
100#define IMAGE_SCN_RESERVED_0 0x00000001
101#define IMAGE_SCN_RESERVED_1 0x00000002
102#define IMAGE_SCN_RESERVED_2 0x00000004
103#define IMAGE_SCN_TYPE_NO_PAD 0x00000008 /* don't pad - obsolete */
104#define IMAGE_SCN_RESERVED_3 0x00000010
105#define IMAGE_SCN_CNT_CODE 0x00000020 /* .text */
106#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040 /* .data */
107#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080 /* .bss */
108#define IMAGE_SCN_LNK_OTHER 0x00000100 /* reserved */
109#define IMAGE_SCN_LNK_INFO 0x00000200 /* .drectve comments */
110#define IMAGE_SCN_RESERVED_4 0x00000400
111#define IMAGE_SCN_LNK_REMOVE 0x00000800 /* .o only - scn to be rm'd*/
112#define IMAGE_SCN_LNK_COMDAT 0x00001000 /* .o only - COMDAT data */
113#define IMAGE_SCN_RESERVED_5 0x00002000 /* spec omits this */
114#define IMAGE_SCN_RESERVED_6 0x00004000 /* spec omits this */
115#define IMAGE_SCN_GPREL 0x00008000 /* global pointer referenced data */
116/* spec lists 0x20000 twice, I suspect they meant 0x10000 for one of them */
117#define IMAGE_SCN_MEM_PURGEABLE 0x00010000 /* reserved for "future" use */
118#define IMAGE_SCN_16BIT 0x00020000 /* reserved for "future" use */
119#define IMAGE_SCN_LOCKED 0x00040000 /* reserved for "future" use */
120#define IMAGE_SCN_PRELOAD 0x00080000 /* reserved for "future" use */
121/* and here they just stuck a 1-byte integer in the middle of a bitfield */
122#define IMAGE_SCN_ALIGN_1BYTES 0x00100000 /* it does what it says on the box */
123#define IMAGE_SCN_ALIGN_2BYTES 0x00200000
124#define IMAGE_SCN_ALIGN_4BYTES 0x00300000
125#define IMAGE_SCN_ALIGN_8BYTES 0x00400000
126#define IMAGE_SCN_ALIGN_16BYTES 0x00500000
127#define IMAGE_SCN_ALIGN_32BYTES 0x00600000
128#define IMAGE_SCN_ALIGN_64BYTES 0x00700000
129#define IMAGE_SCN_ALIGN_128BYTES 0x00800000
130#define IMAGE_SCN_ALIGN_256BYTES 0x00900000
131#define IMAGE_SCN_ALIGN_512BYTES 0x00a00000
132#define IMAGE_SCN_ALIGN_1024BYTES 0x00b00000
133#define IMAGE_SCN_ALIGN_2048BYTES 0x00c00000
134#define IMAGE_SCN_ALIGN_4096BYTES 0x00d00000
135#define IMAGE_SCN_ALIGN_8192BYTES 0x00e00000
136#define IMAGE_SCN_LNK_NRELOC_OVFL 0x01000000 /* extended relocations */
137#define IMAGE_SCN_MEM_DISCARDABLE 0x02000000 /* scn can be discarded */
138#define IMAGE_SCN_MEM_NOT_CACHED 0x04000000 /* cannot be cached */
139#define IMAGE_SCN_MEM_NOT_PAGED 0x08000000 /* not pageable */
140#define IMAGE_SCN_MEM_SHARED 0x10000000 /* can be shared */
141#define IMAGE_SCN_MEM_EXECUTE 0x20000000 /* can be executed as code */
142#define IMAGE_SCN_MEM_READ 0x40000000 /* readable */
143#define IMAGE_SCN_MEM_WRITE 0x80000000 /* writeable */
144
145#define IMAGE_DEBUG_TYPE_CODEVIEW 2
146
147#ifndef __ASSEMBLY__
148
149struct mz_hdr {
150 uint16_t magic; /* MZ_MAGIC */
151 uint16_t lbsize; /* size of last used block */
152 uint16_t blocks; /* pages in file, 0x3 */
153 uint16_t relocs; /* relocations */
154 uint16_t hdrsize; /* header size in "paragraphs" */
155 uint16_t min_extra_pps; /* .bss */
156 uint16_t max_extra_pps; /* runtime limit for the arena size */
157 uint16_t ss; /* relative stack segment */
158 uint16_t sp; /* initial %sp register */
159 uint16_t checksum; /* word checksum */
160 uint16_t ip; /* initial %ip register */
161 uint16_t cs; /* initial %cs relative to load segment */
162 uint16_t reloc_table_offset; /* offset of the first relocation */
163 uint16_t overlay_num; /* overlay number. set to 0. */
164 uint16_t reserved0[4]; /* reserved */
165 uint16_t oem_id; /* oem identifier */
166 uint16_t oem_info; /* oem specific */
167 uint16_t reserved1[10]; /* reserved */
168 uint32_t peaddr; /* address of pe header */
169 char message[64]; /* message to print */
170};
171
172struct mz_reloc {
173 uint16_t offset;
174 uint16_t segment;
175};
176
177struct pe_hdr {
178 uint32_t magic; /* PE magic */
179 uint16_t machine; /* machine type */
180 uint16_t sections; /* number of sections */
181 uint32_t timestamp; /* time_t */
182 uint32_t symbol_table; /* symbol table offset */
183 uint32_t symbols; /* number of symbols */
184 uint16_t opt_hdr_size; /* size of optional header */
185 uint16_t flags; /* flags */
186};
187
137/* the fact that pe32 isn't padded where pe32+ is 64-bit means union won't 188/* the fact that pe32 isn't padded where pe32+ is 64-bit means union won't
138 * work right. vomit. */ 189 * work right. vomit. */
139struct pe32_opt_hdr { 190struct pe32_opt_hdr {
@@ -243,52 +294,6 @@ struct section_header {
243 uint32_t flags; 294 uint32_t flags;
244}; 295};
245 296
246/* they actually defined 0x00000000 as well, but I think we'll skip that one. */
247#define IMAGE_SCN_RESERVED_0 0x00000001
248#define IMAGE_SCN_RESERVED_1 0x00000002
249#define IMAGE_SCN_RESERVED_2 0x00000004
250#define IMAGE_SCN_TYPE_NO_PAD 0x00000008 /* don't pad - obsolete */
251#define IMAGE_SCN_RESERVED_3 0x00000010
252#define IMAGE_SCN_CNT_CODE 0x00000020 /* .text */
253#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040 /* .data */
254#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080 /* .bss */
255#define IMAGE_SCN_LNK_OTHER 0x00000100 /* reserved */
256#define IMAGE_SCN_LNK_INFO 0x00000200 /* .drectve comments */
257#define IMAGE_SCN_RESERVED_4 0x00000400
258#define IMAGE_SCN_LNK_REMOVE 0x00000800 /* .o only - scn to be rm'd*/
259#define IMAGE_SCN_LNK_COMDAT 0x00001000 /* .o only - COMDAT data */
260#define IMAGE_SCN_RESERVED_5 0x00002000 /* spec omits this */
261#define IMAGE_SCN_RESERVED_6 0x00004000 /* spec omits this */
262#define IMAGE_SCN_GPREL 0x00008000 /* global pointer referenced data */
263/* spec lists 0x20000 twice, I suspect they meant 0x10000 for one of them */
264#define IMAGE_SCN_MEM_PURGEABLE 0x00010000 /* reserved for "future" use */
265#define IMAGE_SCN_16BIT 0x00020000 /* reserved for "future" use */
266#define IMAGE_SCN_LOCKED 0x00040000 /* reserved for "future" use */
267#define IMAGE_SCN_PRELOAD 0x00080000 /* reserved for "future" use */
268/* and here they just stuck a 1-byte integer in the middle of a bitfield */
269#define IMAGE_SCN_ALIGN_1BYTES 0x00100000 /* it does what it says on the box */
270#define IMAGE_SCN_ALIGN_2BYTES 0x00200000
271#define IMAGE_SCN_ALIGN_4BYTES 0x00300000
272#define IMAGE_SCN_ALIGN_8BYTES 0x00400000
273#define IMAGE_SCN_ALIGN_16BYTES 0x00500000
274#define IMAGE_SCN_ALIGN_32BYTES 0x00600000
275#define IMAGE_SCN_ALIGN_64BYTES 0x00700000
276#define IMAGE_SCN_ALIGN_128BYTES 0x00800000
277#define IMAGE_SCN_ALIGN_256BYTES 0x00900000
278#define IMAGE_SCN_ALIGN_512BYTES 0x00a00000
279#define IMAGE_SCN_ALIGN_1024BYTES 0x00b00000
280#define IMAGE_SCN_ALIGN_2048BYTES 0x00c00000
281#define IMAGE_SCN_ALIGN_4096BYTES 0x00d00000
282#define IMAGE_SCN_ALIGN_8192BYTES 0x00e00000
283#define IMAGE_SCN_LNK_NRELOC_OVFL 0x01000000 /* extended relocations */
284#define IMAGE_SCN_MEM_DISCARDABLE 0x02000000 /* scn can be discarded */
285#define IMAGE_SCN_MEM_NOT_CACHED 0x04000000 /* cannot be cached */
286#define IMAGE_SCN_MEM_NOT_PAGED 0x08000000 /* not pageable */
287#define IMAGE_SCN_MEM_SHARED 0x10000000 /* can be shared */
288#define IMAGE_SCN_MEM_EXECUTE 0x20000000 /* can be executed as code */
289#define IMAGE_SCN_MEM_READ 0x40000000 /* readable */
290#define IMAGE_SCN_MEM_WRITE 0x80000000 /* writeable */
291
292enum x64_coff_reloc_type { 297enum x64_coff_reloc_type {
293 IMAGE_REL_AMD64_ABSOLUTE = 0, 298 IMAGE_REL_AMD64_ABSOLUTE = 0,
294 IMAGE_REL_AMD64_ADDR64, 299 IMAGE_REL_AMD64_ADDR64,
@@ -445,4 +450,6 @@ struct win_certificate {
445 uint16_t cert_type; 450 uint16_t cert_type;
446}; 451};
447 452
453#endif /* !__ASSEMBLY__ */
454
448#endif /* __LINUX_PE_H */ 455#endif /* __LINUX_PE_H */
diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h
index 8462da266089..1360dd6d5e61 100644
--- a/include/linux/perf/arm_pmu.h
+++ b/include/linux/perf/arm_pmu.h
@@ -75,6 +75,8 @@ struct pmu_hw_events {
75 * already have to allocate this struct per cpu. 75 * already have to allocate this struct per cpu.
76 */ 76 */
77 struct arm_pmu *percpu_pmu; 77 struct arm_pmu *percpu_pmu;
78
79 int irq;
78}; 80};
79 81
80enum armpmu_attr_groups { 82enum armpmu_attr_groups {
@@ -88,7 +90,6 @@ struct arm_pmu {
88 struct pmu pmu; 90 struct pmu pmu;
89 cpumask_t active_irqs; 91 cpumask_t active_irqs;
90 cpumask_t supported_cpus; 92 cpumask_t supported_cpus;
91 int *irq_affinity;
92 char *name; 93 char *name;
93 irqreturn_t (*handle_irq)(int irq_num, void *dev); 94 irqreturn_t (*handle_irq)(int irq_num, void *dev);
94 void (*enable)(struct perf_event *event); 95 void (*enable)(struct perf_event *event);
@@ -104,12 +105,8 @@ struct arm_pmu {
104 void (*start)(struct arm_pmu *); 105 void (*start)(struct arm_pmu *);
105 void (*stop)(struct arm_pmu *); 106 void (*stop)(struct arm_pmu *);
106 void (*reset)(void *); 107 void (*reset)(void *);
107 int (*request_irq)(struct arm_pmu *, irq_handler_t handler);
108 void (*free_irq)(struct arm_pmu *);
109 int (*map_event)(struct perf_event *event); 108 int (*map_event)(struct perf_event *event);
110 int num_events; 109 int num_events;
111 atomic_t active_events;
112 struct mutex reserve_mutex;
113 u64 max_period; 110 u64 max_period;
114 bool secure_access; /* 32-bit ARM only */ 111 bool secure_access; /* 32-bit ARM only */
115#define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40 112#define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40
@@ -120,6 +117,9 @@ struct arm_pmu {
120 struct notifier_block cpu_pm_nb; 117 struct notifier_block cpu_pm_nb;
121 /* the attr_groups array must be NULL-terminated */ 118 /* the attr_groups array must be NULL-terminated */
122 const struct attribute_group *attr_groups[ARMPMU_NR_ATTR_GROUPS + 1]; 119 const struct attribute_group *attr_groups[ARMPMU_NR_ATTR_GROUPS + 1];
120
121 /* Only to be used by ACPI probing code */
122 unsigned long acpi_cpuid;
123}; 123};
124 124
125#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu)) 125#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
@@ -135,10 +135,12 @@ int armpmu_map_event(struct perf_event *event,
135 [PERF_COUNT_HW_CACHE_RESULT_MAX], 135 [PERF_COUNT_HW_CACHE_RESULT_MAX],
136 u32 raw_event_mask); 136 u32 raw_event_mask);
137 137
138typedef int (*armpmu_init_fn)(struct arm_pmu *);
139
138struct pmu_probe_info { 140struct pmu_probe_info {
139 unsigned int cpuid; 141 unsigned int cpuid;
140 unsigned int mask; 142 unsigned int mask;
141 int (*init)(struct arm_pmu *); 143 armpmu_init_fn init;
142}; 144};
143 145
144#define PMU_PROBE(_cpuid, _mask, _fn) \ 146#define PMU_PROBE(_cpuid, _mask, _fn) \
@@ -160,6 +162,21 @@ int arm_pmu_device_probe(struct platform_device *pdev,
160 const struct of_device_id *of_table, 162 const struct of_device_id *of_table,
161 const struct pmu_probe_info *probe_table); 163 const struct pmu_probe_info *probe_table);
162 164
165#ifdef CONFIG_ACPI
166int arm_pmu_acpi_probe(armpmu_init_fn init_fn);
167#else
168static inline int arm_pmu_acpi_probe(armpmu_init_fn init_fn) { return 0; }
169#endif
170
171/* Internal functions only for core arm_pmu code */
172struct arm_pmu *armpmu_alloc(void);
173void armpmu_free(struct arm_pmu *pmu);
174int armpmu_register(struct arm_pmu *pmu);
175int armpmu_request_irqs(struct arm_pmu *armpmu);
176void armpmu_free_irqs(struct arm_pmu *armpmu);
177int armpmu_request_irq(struct arm_pmu *armpmu, int cpu);
178void armpmu_free_irq(struct arm_pmu *armpmu, int cpu);
179
163#define ARMV8_PMU_PDEV_NAME "armv8-pmu" 180#define ARMV8_PMU_PDEV_NAME "armv8-pmu"
164 181
165#endif /* CONFIG_ARM_PMU */ 182#endif /* CONFIG_ARM_PMU */
diff --git a/mm/memblock.c b/mm/memblock.c
index 696f06d17c4e..b049c9b2dba8 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -805,6 +805,18 @@ int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
805} 805}
806 806
807/** 807/**
808 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
809 * @base: the base phys addr of the region
810 * @size: the size of the region
811 *
812 * Return 0 on success, -errno on failure.
813 */
814int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
815{
816 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
817}
818
819/**
808 * __next_reserved_mem_region - next function for for_each_reserved_region() 820 * __next_reserved_mem_region - next function for for_each_reserved_region()
809 * @idx: pointer to u64 loop variable 821 * @idx: pointer to u64 loop variable
810 * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL 822 * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL
@@ -1531,11 +1543,37 @@ void __init memblock_enforce_memory_limit(phys_addr_t limit)
1531 (phys_addr_t)ULLONG_MAX); 1543 (phys_addr_t)ULLONG_MAX);
1532} 1544}
1533 1545
1546void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1547{
1548 int start_rgn, end_rgn;
1549 int i, ret;
1550
1551 if (!size)
1552 return;
1553
1554 ret = memblock_isolate_range(&memblock.memory, base, size,
1555 &start_rgn, &end_rgn);
1556 if (ret)
1557 return;
1558
1559 /* remove all the MAP regions */
1560 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1561 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1562 memblock_remove_region(&memblock.memory, i);
1563
1564 for (i = start_rgn - 1; i >= 0; i--)
1565 if (!memblock_is_nomap(&memblock.memory.regions[i]))
1566 memblock_remove_region(&memblock.memory, i);
1567
1568 /* truncate the reserved regions */
1569 memblock_remove_range(&memblock.reserved, 0, base);
1570 memblock_remove_range(&memblock.reserved,
1571 base + size, (phys_addr_t)ULLONG_MAX);
1572}
1573
1534void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1574void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1535{ 1575{
1536 struct memblock_type *type = &memblock.memory;
1537 phys_addr_t max_addr; 1576 phys_addr_t max_addr;
1538 int i, ret, start_rgn, end_rgn;
1539 1577
1540 if (!limit) 1578 if (!limit)
1541 return; 1579 return;
@@ -1546,19 +1584,7 @@ void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1546 if (max_addr == (phys_addr_t)ULLONG_MAX) 1584 if (max_addr == (phys_addr_t)ULLONG_MAX)
1547 return; 1585 return;
1548 1586
1549 ret = memblock_isolate_range(type, max_addr, (phys_addr_t)ULLONG_MAX, 1587 memblock_cap_memory_range(0, max_addr);
1550 &start_rgn, &end_rgn);
1551 if (ret)
1552 return;
1553
1554 /* remove all the MAP regions above the limit */
1555 for (i = end_rgn - 1; i >= start_rgn; i--) {
1556 if (!memblock_is_nomap(&type->regions[i]))
1557 memblock_remove_region(type, i);
1558 }
1559 /* truncate the reserved regions */
1560 memblock_remove_range(&memblock.reserved, max_addr,
1561 (phys_addr_t)ULLONG_MAX);
1562} 1588}
1563 1589
1564static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1590static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 654dfd40e449..7713d96e85b7 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -29,7 +29,9 @@
29#define DEBUG_SPINLOCK_BUG_ON(p) 29#define DEBUG_SPINLOCK_BUG_ON(p)
30#endif 30#endif
31 31
32struct vgic_global __section(.hyp.text) kvm_vgic_global_state = {.gicv3_cpuif = STATIC_KEY_FALSE_INIT,}; 32struct vgic_global kvm_vgic_global_state __ro_after_init = {
33 .gicv3_cpuif = STATIC_KEY_FALSE_INIT,
34};
33 35
34/* 36/*
35 * Locking order is always: 37 * Locking order is always:
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-15 22:38:03 -0400