aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64
diff options
context:
space:
mode:
authorIngo Molnar <mingo@kernel.org>2015-02-26 06:24:50 -0500
committerIngo Molnar <mingo@kernel.org>2015-02-26 06:24:50 -0500
commite9e4e44309f866b115d08ab4a54834008c50a8a4 (patch)
treeae9f91e682a4d6592ef263f30a4a0b1a862b7987 /arch/arm64
parent8a26ce4e544659256349551283414df504889a59 (diff)
parentc517d838eb7d07bbe9507871fab3931deccff539 (diff)
Merge tag 'v4.0-rc1' into perf/core, to refresh the tree
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'arch/arm64')
-rw-r--r--arch/arm64/Kconfig74
-rw-r--r--arch/arm64/Kconfig.debug23
-rw-r--r--arch/arm64/Makefile3
-rw-r--r--arch/arm64/boot/dts/Makefile3
-rw-r--r--arch/arm64/boot/dts/exynos/Makefile5
-rw-r--r--arch/arm64/boot/dts/exynos/exynos7-espresso.dts84
-rw-r--r--arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi588
-rw-r--r--arch/arm64/boot/dts/exynos/exynos7.dtsi530
-rw-r--r--arch/arm64/boot/dts/freescale/Makefile5
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts65
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi163
-rw-r--r--arch/arm64/boot/dts/mediatek/Makefile5
-rw-r--r--arch/arm64/boot/dts/mediatek/mt8173-evb.dts38
-rw-r--r--arch/arm64/boot/dts/mediatek/mt8173.dtsi168
-rw-r--r--arch/arm64/configs/defconfig5
-rw-r--r--arch/arm64/include/asm/bitrev.h19
-rw-r--r--arch/arm64/include/asm/cacheflush.h5
-rw-r--r--arch/arm64/include/asm/cachetype.h29
-rw-r--r--arch/arm64/include/asm/compat.h1
-rw-r--r--arch/arm64/include/asm/cpu_ops.h8
-rw-r--r--arch/arm64/include/asm/cpufeature.h2
-rw-r--r--arch/arm64/include/asm/cpuidle.h6
-rw-r--r--arch/arm64/include/asm/cputype.h17
-rw-r--r--arch/arm64/include/asm/dma-mapping.h11
-rw-r--r--arch/arm64/include/asm/efi.h30
-rw-r--r--arch/arm64/include/asm/esr.h118
-rw-r--r--arch/arm64/include/asm/fixmap.h1
-rw-r--r--arch/arm64/include/asm/fpsimdmacros.h43
-rw-r--r--arch/arm64/include/asm/hardirq.h2
-rw-r--r--arch/arm64/include/asm/io.h5
-rw-r--r--arch/arm64/include/asm/kvm_arm.h73
-rw-r--r--arch/arm64/include/asm/kvm_asm.h1
-rw-r--r--arch/arm64/include/asm/kvm_emulate.h38
-rw-r--r--arch/arm64/include/asm/kvm_host.h7
-rw-r--r--arch/arm64/include/asm/kvm_mmio.h1
-rw-r--r--arch/arm64/include/asm/kvm_mmu.h21
-rw-r--r--arch/arm64/include/asm/memory.h10
-rw-r--r--arch/arm64/include/asm/mmu.h5
-rw-r--r--arch/arm64/include/asm/pgtable-hwdef.h1
-rw-r--r--arch/arm64/include/asm/pgtable.h32
-rw-r--r--arch/arm64/include/asm/ptrace.h7
-rw-r--r--arch/arm64/include/asm/suspend.h2
-rw-r--r--arch/arm64/include/asm/syscalls.h30
-rw-r--r--arch/arm64/include/asm/thread_info.h4
-rw-r--r--arch/arm64/include/asm/uaccess.h4
-rw-r--r--arch/arm64/include/asm/unistd.h3
-rw-r--r--arch/arm64/include/uapi/asm/Kbuild1
-rw-r--r--arch/arm64/include/uapi/asm/kvm.h9
-rw-r--r--arch/arm64/include/uapi/asm/ucontext.h (renamed from arch/arm64/include/asm/ucontext.h)8
-rw-r--r--arch/arm64/kernel/Makefile6
-rw-r--r--arch/arm64/kernel/armv8_deprecated.c205
-rw-r--r--arch/arm64/kernel/asm-offsets.c3
-rw-r--r--arch/arm64/kernel/cacheinfo.c128
-rw-r--r--arch/arm64/kernel/cpuidle.c20
-rw-r--r--arch/arm64/kernel/cpuinfo.c34
-rw-r--r--arch/arm64/kernel/efi-stub.c14
-rw-r--r--arch/arm64/kernel/efi.c356
-rw-r--r--arch/arm64/kernel/entry.S66
-rw-r--r--arch/arm64/kernel/entry32.S (renamed from arch/arm64/kernel/sys32.S)34
-rw-r--r--arch/arm64/kernel/hw_breakpoint.c2
-rw-r--r--arch/arm64/kernel/insn.c47
-rw-r--r--arch/arm64/kernel/module.c4
-rw-r--r--arch/arm64/kernel/pci.c22
-rw-r--r--arch/arm64/kernel/psci.c2
-rw-r--r--arch/arm64/kernel/setup.c22
-rw-r--r--arch/arm64/kernel/signal.c2
-rw-r--r--arch/arm64/kernel/signal32.c11
-rw-r--r--arch/arm64/kernel/smp.c10
-rw-r--r--arch/arm64/kernel/suspend.c21
-rw-r--r--arch/arm64/kernel/sys.c5
-rw-r--r--arch/arm64/kernel/sys32.c51
-rw-r--r--arch/arm64/kernel/traps.c50
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S17
-rw-r--r--arch/arm64/kvm/Kconfig3
-rw-r--r--arch/arm64/kvm/Makefile2
-rw-r--r--arch/arm64/kvm/emulate.c5
-rw-r--r--arch/arm64/kvm/handle_exit.c50
-rw-r--r--arch/arm64/kvm/hyp.S39
-rw-r--r--arch/arm64/kvm/inject_fault.c14
-rw-r--r--arch/arm64/kvm/sys_regs.c63
-rw-r--r--arch/arm64/kvm/trace.h55
-rw-r--r--arch/arm64/kvm/vgic-v3-switch.S14
-rw-r--r--arch/arm64/mm/dma-mapping.c116
-rw-r--r--arch/arm64/mm/dump.c30
-rw-r--r--arch/arm64/mm/fault.c2
-rw-r--r--arch/arm64/mm/hugetlbpage.c6
-rw-r--r--arch/arm64/mm/init.c25
-rw-r--r--arch/arm64/mm/ioremap.c1
-rw-r--r--arch/arm64/mm/mm.h2
-rw-r--r--arch/arm64/mm/mmu.c342
-rw-r--r--arch/arm64/mm/proc.S14
91 files changed, 3273 insertions, 955 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b1f9a20a3677..1b8e97331ffb 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -39,6 +39,7 @@ config ARM64
39 select HARDIRQS_SW_RESEND 39 select HARDIRQS_SW_RESEND
40 select HAVE_ALIGNED_STRUCT_PAGE if SLUB 40 select HAVE_ALIGNED_STRUCT_PAGE if SLUB
41 select HAVE_ARCH_AUDITSYSCALL 41 select HAVE_ARCH_AUDITSYSCALL
42 select HAVE_ARCH_BITREVERSE
42 select HAVE_ARCH_JUMP_LABEL 43 select HAVE_ARCH_JUMP_LABEL
43 select HAVE_ARCH_KGDB 44 select HAVE_ARCH_KGDB
44 select HAVE_ARCH_SECCOMP_FILTER 45 select HAVE_ARCH_SECCOMP_FILTER
@@ -148,11 +149,65 @@ source "kernel/Kconfig.freezer"
148 149
149menu "Platform selection" 150menu "Platform selection"
150 151
152config ARCH_EXYNOS
153 bool
154 help
155 This enables support for Samsung Exynos SoC family
156
157config ARCH_EXYNOS7
158 bool "ARMv8 based Samsung Exynos7"
159 select ARCH_EXYNOS
160 select COMMON_CLK_SAMSUNG
161 select HAVE_S3C2410_WATCHDOG if WATCHDOG
162 select HAVE_S3C_RTC if RTC_CLASS
163 select PINCTRL
164 select PINCTRL_EXYNOS
165
166 help
167 This enables support for Samsung Exynos7 SoC family
168
169config ARCH_FSL_LS2085A
170 bool "Freescale LS2085A SOC"
171 help
172 This enables support for Freescale LS2085A SOC.
173
174config ARCH_MEDIATEK
175 bool "Mediatek MT65xx & MT81xx ARMv8 SoC"
176 select ARM_GIC
177 help
178 Support for Mediatek MT65xx & MT81xx ARMv8 SoCs
179
151config ARCH_SEATTLE 180config ARCH_SEATTLE
152 bool "AMD Seattle SoC Family" 181 bool "AMD Seattle SoC Family"
153 help 182 help
154 This enables support for AMD Seattle SOC Family 183 This enables support for AMD Seattle SOC Family
155 184
185config ARCH_TEGRA
186 bool "NVIDIA Tegra SoC Family"
187 select ARCH_HAS_RESET_CONTROLLER
188 select ARCH_REQUIRE_GPIOLIB
189 select CLKDEV_LOOKUP
190 select CLKSRC_MMIO
191 select CLKSRC_OF
192 select GENERIC_CLOCKEVENTS
193 select HAVE_CLK
194 select PINCTRL
195 select RESET_CONTROLLER
196 help
197 This enables support for the NVIDIA Tegra SoC family.
198
199config ARCH_TEGRA_132_SOC
200 bool "NVIDIA Tegra132 SoC"
201 depends on ARCH_TEGRA
202 select PINCTRL_TEGRA124
203 select USB_ULPI if USB_PHY
204 select USB_ULPI_VIEWPORT if USB_PHY
205 help
206 Enable support for NVIDIA Tegra132 SoC, based on the Denver
207 ARMv8 CPU. The Tegra132 SoC is similar to the Tegra124 SoC,
208 but contains an NVIDIA Denver CPU complex in place of
209 Tegra124's "4+1" Cortex-A15 CPU complex.
210
156config ARCH_THUNDER 211config ARCH_THUNDER
157 bool "Cavium Inc. Thunder SoC Family" 212 bool "Cavium Inc. Thunder SoC Family"
158 help 213 help
@@ -349,7 +404,6 @@ config ARM64_VA_BITS_42
349 404
350config ARM64_VA_BITS_48 405config ARM64_VA_BITS_48
351 bool "48-bit" 406 bool "48-bit"
352 depends on !ARM_SMMU
353 407
354endchoice 408endchoice
355 409
@@ -540,6 +594,21 @@ config CP15_BARRIER_EMULATION
540 594
541 If unsure, say Y 595 If unsure, say Y
542 596
597config SETEND_EMULATION
598 bool "Emulate SETEND instruction"
599 help
600 The SETEND instruction alters the data-endianness of the
601 AArch32 EL0, and is deprecated in ARMv8.
602
603 Say Y here to enable software emulation of the instruction
604 for AArch32 userspace code.
605
606 Note: All the cpus on the system must have mixed endian support at EL0
607 for this feature to be enabled. If a new CPU - which doesn't support mixed
608 endian - is hotplugged in after this feature has been enabled, there could
609 be unexpected results in the applications.
610
611 If unsure, say Y
543endif 612endif
544 613
545endmenu 614endmenu
@@ -627,9 +696,6 @@ source "kernel/power/Kconfig"
627config ARCH_SUSPEND_POSSIBLE 696config ARCH_SUSPEND_POSSIBLE
628 def_bool y 697 def_bool y
629 698
630config ARM64_CPU_SUSPEND
631 def_bool PM_SLEEP
632
633endmenu 699endmenu
634 700
635menu "CPU Power Management" 701menu "CPU Power Management"
diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug
index 5fdd6dce8061..4a8741073c90 100644
--- a/arch/arm64/Kconfig.debug
+++ b/arch/arm64/Kconfig.debug
@@ -66,4 +66,27 @@ config DEBUG_SET_MODULE_RONX
66 against certain classes of kernel exploits. 66 against certain classes of kernel exploits.
67 If in doubt, say "N". 67 If in doubt, say "N".
68 68
69config DEBUG_RODATA
70 bool "Make kernel text and rodata read-only"
71 help
72 If this is set, kernel text and rodata will be made read-only. This
73 is to help catch accidental or malicious attempts to change the
74 kernel's executable code. Additionally splits rodata from kernel
75 text so it can be made explicitly non-executable.
76
77 If in doubt, say Y
78
79config DEBUG_ALIGN_RODATA
80 depends on DEBUG_RODATA && !ARM64_64K_PAGES
81 bool "Align linker sections up to SECTION_SIZE"
82 help
83 If this option is enabled, sections that may potentially be marked as
84 read only or non-executable will be aligned up to the section size of
85 the kernel. This prevents sections from being split into pages and
86 avoids a potential TLB penalty. The downside is an increase in
87 alignment and potentially wasted space. Turn on this option if
88 performance is more important than memory pressure.
89
90 If in doubt, say N
91
69endmenu 92endmenu
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 066688863920..69ceedc982a5 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
15OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S 15OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
16GZFLAGS :=-9 16GZFLAGS :=-9
17 17
18LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
19
20KBUILD_DEFCONFIG := defconfig 18KBUILD_DEFCONFIG := defconfig
21 19
22KBUILD_CFLAGS += -mgeneral-regs-only 20KBUILD_CFLAGS += -mgeneral-regs-only
@@ -50,7 +48,6 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/
50core-$(CONFIG_XEN) += arch/arm64/xen/ 48core-$(CONFIG_XEN) += arch/arm64/xen/
51core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ 49core-$(CONFIG_CRYPTO) += arch/arm64/crypto/
52libs-y := arch/arm64/lib/ $(libs-y) 50libs-y := arch/arm64/lib/ $(libs-y)
53libs-y += $(LIBGCC)
54libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/ 51libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/
55 52
56# Default target when executing plain make 53# Default target when executing plain make
diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile
index c62b0f4d9ef6..e0350caf049e 100644
--- a/arch/arm64/boot/dts/Makefile
+++ b/arch/arm64/boot/dts/Makefile
@@ -2,5 +2,8 @@ dts-dirs += amd
2dts-dirs += apm 2dts-dirs += apm
3dts-dirs += arm 3dts-dirs += arm
4dts-dirs += cavium 4dts-dirs += cavium
5dts-dirs += exynos
6dts-dirs += freescale
7dts-dirs += mediatek
5 8
6subdir-y := $(dts-dirs) 9subdir-y := $(dts-dirs)
diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile
new file mode 100644
index 000000000000..20310e5b6d6f
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/Makefile
@@ -0,0 +1,5 @@
1dtb-$(CONFIG_ARCH_EXYNOS7) += exynos7-espresso.dtb
2
3always := $(dtb-y)
4subdir-y := $(dts-dirs)
5clean-files := *.dtb
diff --git a/arch/arm64/boot/dts/exynos/exynos7-espresso.dts b/arch/arm64/boot/dts/exynos/exynos7-espresso.dts
new file mode 100644
index 000000000000..5424cc450f72
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos7-espresso.dts
@@ -0,0 +1,84 @@
1/*
2 * SAMSUNG Exynos7 Espresso board device tree source
3 *
4 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10*/
11
12/dts-v1/;
13#include "exynos7.dtsi"
14
15/ {
16 model = "Samsung Exynos7 Espresso board based on EXYNOS7";
17 compatible = "samsung,exynos7-espresso", "samsung,exynos7";
18
19 aliases {
20 serial0 = &serial_2;
21 mshc0 = &mmc_0;
22 mshc2 = &mmc_2;
23 };
24
25 chosen {
26 linux,stdout-path = &serial_2;
27 };
28
29 memory@40000000 {
30 device_type = "memory";
31 reg = <0x0 0x40000000 0x0 0xC0000000>;
32 };
33};
34
35&fin_pll {
36 clock-frequency = <24000000>;
37};
38
39&serial_2 {
40 status = "okay";
41};
42
43&rtc {
44 status = "okay";
45};
46
47&watchdog {
48 status = "okay";
49};
50
51&adc {
52 status = "okay";
53};
54
55&mmc_0 {
56 status = "okay";
57 num-slots = <1>;
58 broken-cd;
59 cap-mmc-highspeed;
60 non-removable;
61 card-detect-delay = <200>;
62 clock-frequency = <800000000>;
63 samsung,dw-mshc-ciu-div = <3>;
64 samsung,dw-mshc-sdr-timing = <0 4>;
65 samsung,dw-mshc-ddr-timing = <0 2>;
66 pinctrl-names = "default";
67 pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_qrdy &sd0_bus1 &sd0_bus4 &sd0_bus8>;
68 bus-width = <8>;
69};
70
71&mmc_2 {
72 status = "okay";
73 num-slots = <1>;
74 cap-sd-highspeed;
75 card-detect-delay = <200>;
76 clock-frequency = <400000000>;
77 samsung,dw-mshc-ciu-div = <3>;
78 samsung,dw-mshc-sdr-timing = <2 3>;
79 samsung,dw-mshc-ddr-timing = <1 2>;
80 pinctrl-names = "default";
81 pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_cd &sd2_bus1 &sd2_bus4>;
82 bus-width = <4>;
83 disable-wp;
84};
diff --git a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
new file mode 100644
index 000000000000..2eef4a279131
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
@@ -0,0 +1,588 @@
1/*
2 * Samsung's Exynos7 SoC pin-mux and pin-config device tree source
3 *
4 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com
6 *
7 * Samsung's Exynos7 SoC pin-mux and pin-config options are listed as
8 * device tree nodes in this file.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13*/
14
15&pinctrl_alive {
16 gpa0: gpa0 {
17 gpio-controller;
18 #gpio-cells = <2>;
19
20 interrupt-controller;
21 interrupt-parent = <&gic>;
22 #interrupt-cells = <2>;
23 interrupts = <0 0 0>, <0 1 0>, <0 2 0>, <0 3 0>,
24 <0 4 0>, <0 5 0>, <0 6 0>, <0 7 0>;
25 };
26
27 gpa1: gpa1 {
28 gpio-controller;
29 #gpio-cells = <2>;
30
31 interrupt-controller;
32 interrupt-parent = <&gic>;
33 #interrupt-cells = <2>;
34 interrupts = <0 8 0>, <0 9 0>, <0 10 0>, <0 11 0>,
35 <0 12 0>, <0 13 0>, <0 14 0>, <0 15 0>;
36 };
37
38 gpa2: gpa2 {
39 gpio-controller;
40 #gpio-cells = <2>;
41
42 interrupt-controller;
43 #interrupt-cells = <2>;
44 };
45
46 gpa3: gpa3 {
47 gpio-controller;
48 #gpio-cells = <2>;
49
50 interrupt-controller;
51 #interrupt-cells = <2>;
52 };
53};
54
55&pinctrl_bus0 {
56 gpb0: gpb0 {
57 gpio-controller;
58 #gpio-cells = <2>;
59
60 interrupt-controller;
61 #interrupt-cells = <2>;
62 };
63
64 gpc0: gpc0 {
65 gpio-controller;
66 #gpio-cells = <2>;
67
68 interrupt-controller;
69 #interrupt-cells = <2>;
70 };
71
72 gpc1: gpc1 {
73 gpio-controller;
74 #gpio-cells = <2>;
75
76 interrupt-controller;
77 #interrupt-cells = <2>;
78 };
79
80 gpc2: gpc2 {
81 gpio-controller;
82 #gpio-cells = <2>;
83
84 interrupt-controller;
85 #interrupt-cells = <2>;
86 };
87
88 gpc3: gpc3 {
89 gpio-controller;
90 #gpio-cells = <2>;
91
92 interrupt-controller;
93 #interrupt-cells = <2>;
94 };
95
96 gpd0: gpd0 {
97 gpio-controller;
98 #gpio-cells = <2>;
99
100 interrupt-controller;
101 #interrupt-cells = <2>;
102 };
103
104 gpd1: gpd1 {
105 gpio-controller;
106 #gpio-cells = <2>;
107
108 interrupt-controller;
109 #interrupt-cells = <2>;
110 };
111
112 gpd2: gpd2 {
113 gpio-controller;
114 #gpio-cells = <2>;
115
116 interrupt-controller;
117 #interrupt-cells = <2>;
118 };
119
120 gpd4: gpd4 {
121 gpio-controller;
122 #gpio-cells = <2>;
123
124 interrupt-controller;
125 #interrupt-cells = <2>;
126 };
127
128 gpd5: gpd5 {
129 gpio-controller;
130 #gpio-cells = <2>;
131
132 interrupt-controller;
133 #interrupt-cells = <2>;
134 };
135
136 gpd6: gpd6 {
137 gpio-controller;
138 #gpio-cells = <2>;
139
140 interrupt-controller;
141 #interrupt-cells = <2>;
142 };
143
144 gpd7: gpd7 {
145 gpio-controller;
146 #gpio-cells = <2>;
147
148 interrupt-controller;
149 #interrupt-cells = <2>;
150 };
151
152 gpd8: gpd8 {
153 gpio-controller;
154 #gpio-cells = <2>;
155
156 interrupt-controller;
157 #interrupt-cells = <2>;
158 };
159
160 gpg0: gpg0 {
161 gpio-controller;
162 #gpio-cells = <2>;
163
164 interrupt-controller;
165 #interrupt-cells = <2>;
166 };
167
168 gpg3: gpg3 {
169 gpio-controller;
170 #gpio-cells = <2>;
171
172 interrupt-controller;
173 #interrupt-cells = <2>;
174 };
175
176 hs_i2c10_bus: hs-i2c10-bus {
177 samsung,pins = "gpb0-1", "gpb0-0";
178 samsung,pin-function = <2>;
179 samsung,pin-pud = <3>;
180 samsung,pin-drv = <0>;
181 };
182
183 hs_i2c11_bus: hs-i2c11-bus {
184 samsung,pins = "gpb0-3", "gpb0-2";
185 samsung,pin-function = <2>;
186 samsung,pin-pud = <3>;
187 samsung,pin-drv = <0>;
188 };
189
190 hs_i2c2_bus: hs-i2c2-bus {
191 samsung,pins = "gpd0-3", "gpd0-2";
192 samsung,pin-function = <3>;
193 samsung,pin-pud = <3>;
194 samsung,pin-drv = <0>;
195 };
196
197 uart0_data: uart0-data {
198 samsung,pins = "gpd0-0", "gpd0-1";
199 samsung,pin-function = <2>;
200 samsung,pin-pud = <0>;
201 samsung,pin-drv = <0>;
202 };
203
204 uart0_fctl: uart0-fctl {
205 samsung,pins = "gpd0-2", "gpd0-3";
206 samsung,pin-function = <2>;
207 samsung,pin-pud = <0>;
208 samsung,pin-drv = <0>;
209 };
210
211 uart2_data: uart2-data {
212 samsung,pins = "gpd1-4", "gpd1-5";
213 samsung,pin-function = <2>;
214 samsung,pin-pud = <0>;
215 samsung,pin-drv = <0>;
216 };
217
218 hs_i2c3_bus: hs-i2c3-bus {
219 samsung,pins = "gpd1-3", "gpd1-2";
220 samsung,pin-function = <3>;
221 samsung,pin-pud = <3>;
222 samsung,pin-drv = <0>;
223 };
224
225 uart1_data: uart1-data {
226 samsung,pins = "gpd1-0", "gpd1-1";
227 samsung,pin-function = <2>;
228 samsung,pin-pud = <0>;
229 samsung,pin-drv = <0>;
230 };
231
232 uart1_fctl: uart1-fctl {
233 samsung,pins = "gpd1-2", "gpd1-3";
234 samsung,pin-function = <2>;
235 samsung,pin-pud = <0>;
236 samsung,pin-drv = <0>;
237 };
238
239 hs_i2c0_bus: hs-i2c0-bus {
240 samsung,pins = "gpd2-1", "gpd2-0";
241 samsung,pin-function = <2>;
242 samsung,pin-pud = <3>;
243 samsung,pin-drv = <0>;
244 };
245
246 hs_i2c1_bus: hs-i2c1-bus {
247 samsung,pins = "gpd2-3", "gpd2-2";
248 samsung,pin-function = <2>;
249 samsung,pin-pud = <3>;
250 samsung,pin-drv = <0>;
251 };
252
253 hs_i2c9_bus: hs-i2c9-bus {
254 samsung,pins = "gpd2-7", "gpd2-6";
255 samsung,pin-function = <3>;
256 samsung,pin-pud = <3>;
257 samsung,pin-drv = <0>;
258 };
259
260 pwm0_out: pwm0-out {
261 samsung,pins = "gpd2-4";
262 samsung,pin-function = <2>;
263 samsung,pin-pud = <0>;
264 samsung,pin-drv = <0>;
265 };
266
267 pwm1_out: pwm1-out {
268 samsung,pins = "gpd2-5";
269 samsung,pin-function = <2>;
270 samsung,pin-pud = <0>;
271 samsung,pin-drv = <0>;
272 };
273
274 pwm2_out: pwm2-out {
275 samsung,pins = "gpd2-6";
276 samsung,pin-function = <2>;
277 samsung,pin-pud = <0>;
278 samsung,pin-drv = <0>;
279 };
280
281 pwm3_out: pwm3-out {
282 samsung,pins = "gpd2-7";
283 samsung,pin-function = <2>;
284 samsung,pin-pud = <0>;
285 samsung,pin-drv = <0>;
286 };
287
288 hs_i2c8_bus: hs-i2c8-bus {
289 samsung,pins = "gpd5-3", "gpd5-2";
290 samsung,pin-function = <3>;
291 samsung,pin-pud = <3>;
292 samsung,pin-drv = <0>;
293 };
294
295 uart3_data: uart3-data {
296 samsung,pins = "gpd5-0", "gpd5-1";
297 samsung,pin-function = <3>;
298 samsung,pin-pud = <0>;
299 samsung,pin-drv = <0>;
300 };
301
302 spi2_bus: spi2-bus {
303 samsung,pins = "gpd5-0", "gpd5-1", "gpd5-2", "gpd5-3";
304 samsung,pin-function = <2>;
305 samsung,pin-pud = <3>;
306 samsung,pin-drv = <0>;
307 };
308
309 spi1_bus: spi1-bus {
310 samsung,pins = "gpd6-2", "gpd6-3", "gpd6-4", "gpd6-5";
311 samsung,pin-function = <2>;
312 samsung,pin-pud = <3>;
313 samsung,pin-drv = <0>;
314 };
315
316 spi0_bus: spi0-bus {
317 samsung,pins = "gpd8-0", "gpd8-1", "gpd6-0", "gpd6-1";
318 samsung,pin-function = <2>;
319 samsung,pin-pud = <3>;
320 samsung,pin-drv = <0>;
321 };
322
323 hs_i2c4_bus: hs-i2c4-bus {
324 samsung,pins = "gpg3-1", "gpg3-0";
325 samsung,pin-function = <2>;
326 samsung,pin-pud = <3>;
327 samsung,pin-drv = <0>;
328 };
329
330 hs_i2c5_bus: hs-i2c5-bus {
331 samsung,pins = "gpg3-3", "gpg3-2";
332 samsung,pin-function = <2>;
333 samsung,pin-pud = <3>;
334 samsung,pin-drv = <0>;
335 };
336};
337
338&pinctrl_nfc {
339 gpj0: gpj0 {
340 gpio-controller;
341 #gpio-cells = <2>;
342
343 interrupt-controller;
344 #interrupt-cells = <2>;
345 };
346
347 hs_i2c6_bus: hs-i2c6-bus {
348 samsung,pins = "gpj0-1", "gpj0-0";
349 samsung,pin-function = <2>;
350 samsung,pin-pud = <3>;
351 samsung,pin-drv = <0>;
352 };
353};
354
355&pinctrl_touch {
356 gpj1: gpj1 {
357 gpio-controller;
358 #gpio-cells = <2>;
359
360 interrupt-controller;
361 #interrupt-cells = <2>;
362 };
363
364 hs_i2c7_bus: hs-i2c7-bus {
365 samsung,pins = "gpj1-1", "gpj1-0";
366 samsung,pin-function = <2>;
367 samsung,pin-pud = <3>;
368 samsung,pin-drv = <0>;
369 };
370};
371
372&pinctrl_ff {
373 gpg4: gpg4 {
374 gpio-controller;
375 #gpio-cells = <2>;
376
377 interrupt-controller;
378 #interrupt-cells = <2>;
379 };
380
381 spi3_bus: spi3-bus {
382 samsung,pins = "gpg4-0", "gpg4-1", "gpg4-2", "gpg4-3";
383 samsung,pin-function = <2>;
384 samsung,pin-pud = <3>;
385 samsung,pin-drv = <0>;
386 };
387};
388
389&pinctrl_ese {
390 gpv7: gpv7 {
391 gpio-controller;
392 #gpio-cells = <2>;
393
394 interrupt-controller;
395 #interrupt-cells = <2>;
396 };
397
398 spi4_bus: spi4-bus {
399 samsung,pins = "gpv7-0", "gpv7-1", "gpv7-2", "gpv7-3";
400 samsung,pin-function = <2>;
401 samsung,pin-pud = <3>;
402 samsung,pin-drv = <0>;
403 };
404};
405
406&pinctrl_fsys0 {
407 gpr4: gpr4 {
408 gpio-controller;
409 #gpio-cells = <2>;
410
411 interrupt-controller;
412 #interrupt-cells = <2>;
413 };
414
415 sd2_clk: sd2-clk {
416 samsung,pins = "gpr4-0";
417 samsung,pin-function = <2>;
418 samsung,pin-pud = <0>;
419 samsung,pin-drv = <3>;
420 };
421
422 sd2_cmd: sd2-cmd {
423 samsung,pins = "gpr4-1";
424 samsung,pin-function = <2>;
425 samsung,pin-pud = <0>;
426 samsung,pin-drv = <3>;
427 };
428
429 sd2_cd: sd2-cd {
430 samsung,pins = "gpr4-2";
431 samsung,pin-function = <2>;
432 samsung,pin-pud = <3>;
433 samsung,pin-drv = <3>;
434 };
435
436 sd2_bus1: sd2-bus-width1 {
437 samsung,pins = "gpr4-3";
438 samsung,pin-function = <2>;
439 samsung,pin-pud = <3>;
440 samsung,pin-drv = <3>;
441 };
442
443 sd2_bus4: sd2-bus-width4 {
444 samsung,pins = "gpr4-4", "gpr4-5", "gpr4-6";
445 samsung,pin-function = <2>;
446 samsung,pin-pud = <3>;
447 samsung,pin-drv = <3>;
448 };
449};
450
451&pinctrl_fsys1 {
452 gpr0: gpr0 {
453 gpio-controller;
454 #gpio-cells = <2>;
455
456 interrupt-controller;
457 #interrupt-cells = <2>;
458 };
459
460 gpr1: gpr1 {
461 gpio-controller;
462 #gpio-cells = <2>;
463
464 interrupt-controller;
465 #interrupt-cells = <2>;
466 };
467
468 gpr2: gpr2 {
469 gpio-controller;
470 #gpio-cells = <2>;
471
472 interrupt-controller;
473 #interrupt-cells = <2>;
474 };
475
476 gpr3: gpr3 {
477 gpio-controller;
478 #gpio-cells = <2>;
479
480 interrupt-controller;
481 #interrupt-cells = <2>;
482 };
483
484 sd0_clk: sd0-clk {
485 samsung,pins = "gpr0-0";
486 samsung,pin-function = <2>;
487 samsung,pin-pud = <0>;
488 samsung,pin-drv = <3>;
489 };
490
491 sd0_cmd: sd0-cmd {
492 samsung,pins = "gpr0-1";
493 samsung,pin-function = <2>;
494 samsung,pin-pud = <3>;
495 samsung,pin-drv = <3>;
496 };
497
498 sd0_ds: sd0-ds {
499 samsung,pins = "gpr0-2";
500 samsung,pin-function = <2>;
501 samsung,pin-pud = <1>;
502 samsung,pin-drv = <3>;
503 };
504
505 sd0_qrdy: sd0-qrdy {
506 samsung,pins = "gpr0-3";
507 samsung,pin-function = <2>;
508 samsung,pin-pud = <1>;
509 samsung,pin-drv = <3>;
510 };
511
512 sd0_bus1: sd0-bus-width1 {
513 samsung,pins = "gpr1-0";
514 samsung,pin-function = <2>;
515 samsung,pin-pud = <3>;
516 samsung,pin-drv = <3>;
517 };
518
519 sd0_bus4: sd0-bus-width4 {
520 samsung,pins = "gpr1-1", "gpr1-2", "gpr1-3";
521 samsung,pin-function = <2>;
522 samsung,pin-pud = <3>;
523 samsung,pin-drv = <3>;
524 };
525
526 sd0_bus8: sd0-bus-width8 {
527 samsung,pins = "gpr1-4", "gpr1-5", "gpr1-6", "gpr1-7";
528 samsung,pin-function = <2>;
529 samsung,pin-pud = <3>;
530 samsung,pin-drv = <3>;
531 };
532
533 sd1_clk: sd1-clk {
534 samsung,pins = "gpr2-0";
535 samsung,pin-function = <2>;
536 samsung,pin-pud = <0>;
537 samsung,pin-drv = <2>;
538 };
539
540 sd1_cmd: sd1-cmd {
541 samsung,pins = "gpr2-1";
542 samsung,pin-function = <2>;
543 samsung,pin-pud = <0>;
544 samsung,pin-drv = <2>;
545 };
546
547 sd1_ds: sd1-ds {
548 samsung,pins = "gpr2-2";
549 samsung,pin-function = <2>;
550 samsung,pin-pud = <1>;
551 samsung,pin-drv = <6>;
552 };
553
554 sd1_qrdy: sd1-qrdy {
555 samsung,pins = "gpr2-3";
556 samsung,pin-function = <2>;
557 samsung,pin-pud = <1>;
558 samsung,pin-drv = <6>;
559 };
560
561 sd1_int: sd1-int {
562 samsung,pins = "gpr2-4";
563 samsung,pin-function = <2>;
564 samsung,pin-pud = <1>;
565 samsung,pin-drv = <6>;
566 };
567
568 sd1_bus1: sd1-bus-width1 {
569 samsung,pins = "gpr3-0";
570 samsung,pin-function = <2>;
571 samsung,pin-pud = <3>;
572 samsung,pin-drv = <2>;
573 };
574
575 sd1_bus4: sd1-bus-width4 {
576 samsung,pins = "gpr3-1", "gpr3-2", "gpr3-3";
577 samsung,pin-function = <2>;
578 samsung,pin-pud = <3>;
579 samsung,pin-drv = <2>;
580 };
581
582 sd1_bus8: sd1-bus-width8 {
583 samsung,pins = "gpr3-4", "gpr3-5", "gpr3-6", "gpr3-7";
584 samsung,pin-function = <2>;
585 samsung,pin-pud = <3>;
586 samsung,pin-drv = <2>;
587 };
588};
diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi
new file mode 100644
index 000000000000..d7a37c3a6b52
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi
@@ -0,0 +1,530 @@
1/*
2 * SAMSUNG EXYNOS7 SoC device tree source
3 *
4 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <dt-bindings/clock/exynos7-clk.h>
13
14/ {
15 compatible = "samsung,exynos7";
16 interrupt-parent = <&gic>;
17 #address-cells = <2>;
18 #size-cells = <2>;
19
20 aliases {
21 pinctrl0 = &pinctrl_alive;
22 pinctrl1 = &pinctrl_bus0;
23 pinctrl2 = &pinctrl_nfc;
24 pinctrl3 = &pinctrl_touch;
25 pinctrl4 = &pinctrl_ff;
26 pinctrl5 = &pinctrl_ese;
27 pinctrl6 = &pinctrl_fsys0;
28 pinctrl7 = &pinctrl_fsys1;
29 };
30
31 cpus {
32 #address-cells = <1>;
33 #size-cells = <0>;
34
35 cpu@0 {
36 device_type = "cpu";
37 compatible = "arm,cortex-a57", "arm,armv8";
38 reg = <0x0>;
39 enable-method = "psci";
40 };
41
42 cpu@1 {
43 device_type = "cpu";
44 compatible = "arm,cortex-a57", "arm,armv8";
45 reg = <0x1>;
46 enable-method = "psci";
47 };
48
49 cpu@2 {
50 device_type = "cpu";
51 compatible = "arm,cortex-a57", "arm,armv8";
52 reg = <0x2>;
53 enable-method = "psci";
54 };
55
56 cpu@3 {
57 device_type = "cpu";
58 compatible = "arm,cortex-a57", "arm,armv8";
59 reg = <0x3>;
60 enable-method = "psci";
61 };
62 };
63
64 psci {
65 compatible = "arm,psci-0.2";
66 method = "smc";
67 };
68
69 soc: soc {
70 compatible = "simple-bus";
71 #address-cells = <1>;
72 #size-cells = <1>;
73 ranges = <0 0 0 0x18000000>;
74
75 chipid@10000000 {
76 compatible = "samsung,exynos4210-chipid";
77 reg = <0x10000000 0x100>;
78 };
79
80 fin_pll: xxti {
81 compatible = "fixed-clock";
82 clock-output-names = "fin_pll";
83 #clock-cells = <0>;
84 };
85
86 gic: interrupt-controller@11001000 {
87 compatible = "arm,gic-400";
88 #interrupt-cells = <3>;
89 #address-cells = <0>;
90 interrupt-controller;
91 reg = <0x11001000 0x1000>,
92 <0x11002000 0x1000>,
93 <0x11004000 0x2000>,
94 <0x11006000 0x2000>;
95 };
96
97 clock_topc: clock-controller@10570000 {
98 compatible = "samsung,exynos7-clock-topc";
99 reg = <0x10570000 0x10000>;
100 #clock-cells = <1>;
101 };
102
103 clock_top0: clock-controller@105d0000 {
104 compatible = "samsung,exynos7-clock-top0";
105 reg = <0x105d0000 0xb000>;
106 #clock-cells = <1>;
107 clocks = <&fin_pll>, <&clock_topc DOUT_SCLK_BUS0_PLL>,
108 <&clock_topc DOUT_SCLK_BUS1_PLL>,
109 <&clock_topc DOUT_SCLK_CC_PLL>,
110 <&clock_topc DOUT_SCLK_MFC_PLL>;
111 clock-names = "fin_pll", "dout_sclk_bus0_pll",
112 "dout_sclk_bus1_pll", "dout_sclk_cc_pll",
113 "dout_sclk_mfc_pll";
114 };
115
116 clock_top1: clock-controller@105e0000 {
117 compatible = "samsung,exynos7-clock-top1";
118 reg = <0x105e0000 0xb000>;
119 #clock-cells = <1>;
120 clocks = <&fin_pll>, <&clock_topc DOUT_SCLK_BUS0_PLL>,
121 <&clock_topc DOUT_SCLK_BUS1_PLL>,
122 <&clock_topc DOUT_SCLK_CC_PLL>,
123 <&clock_topc DOUT_SCLK_MFC_PLL>;
124 clock-names = "fin_pll", "dout_sclk_bus0_pll",
125 "dout_sclk_bus1_pll", "dout_sclk_cc_pll",
126 "dout_sclk_mfc_pll";
127 };
128
129 clock_ccore: clock-controller@105b0000 {
130 compatible = "samsung,exynos7-clock-ccore";
131 reg = <0x105b0000 0xd00>;
132 #clock-cells = <1>;
133 clocks = <&fin_pll>, <&clock_topc DOUT_ACLK_CCORE_133>;
134 clock-names = "fin_pll", "dout_aclk_ccore_133";
135 };
136
137 clock_peric0: clock-controller@13610000 {
138 compatible = "samsung,exynos7-clock-peric0";
139 reg = <0x13610000 0xd00>;
140 #clock-cells = <1>;
141 clocks = <&fin_pll>, <&clock_top0 DOUT_ACLK_PERIC0>,
142 <&clock_top0 CLK_SCLK_UART0>;
143 clock-names = "fin_pll", "dout_aclk_peric0_66",
144 "sclk_uart0";
145 };
146
147 clock_peric1: clock-controller@14c80000 {
148 compatible = "samsung,exynos7-clock-peric1";
149 reg = <0x14c80000 0xd00>;
150 #clock-cells = <1>;
151 clocks = <&fin_pll>, <&clock_top0 DOUT_ACLK_PERIC1>,
152 <&clock_top0 CLK_SCLK_UART1>,
153 <&clock_top0 CLK_SCLK_UART2>,
154 <&clock_top0 CLK_SCLK_UART3>;
155 clock-names = "fin_pll", "dout_aclk_peric1_66",
156 "sclk_uart1", "sclk_uart2", "sclk_uart3";
157 };
158
159 clock_peris: clock-controller@10040000 {
160 compatible = "samsung,exynos7-clock-peris";
161 reg = <0x10040000 0xd00>;
162 #clock-cells = <1>;
163 clocks = <&fin_pll>, <&clock_topc DOUT_ACLK_PERIS>;
164 clock-names = "fin_pll", "dout_aclk_peris_66";
165 };
166
167 clock_fsys0: clock-controller@10e90000 {
168 compatible = "samsung,exynos7-clock-fsys0";
169 reg = <0x10e90000 0xd00>;
170 #clock-cells = <1>;
171 clocks = <&fin_pll>, <&clock_top1 DOUT_ACLK_FSYS0_200>,
172 <&clock_top1 DOUT_SCLK_MMC2>;
173 clock-names = "fin_pll", "dout_aclk_fsys0_200",
174 "dout_sclk_mmc2";
175 };
176
177 clock_fsys1: clock-controller@156e0000 {
178 compatible = "samsung,exynos7-clock-fsys1";
179 reg = <0x156e0000 0xd00>;
180 #clock-cells = <1>;
181 clocks = <&fin_pll>, <&clock_top1 DOUT_ACLK_FSYS1_200>,
182 <&clock_top1 DOUT_SCLK_MMC0>,
183 <&clock_top1 DOUT_SCLK_MMC1>;
184 clock-names = "fin_pll", "dout_aclk_fsys1_200",
185 "dout_sclk_mmc0", "dout_sclk_mmc1";
186 };
187
188 serial_0: serial@13630000 {
189 compatible = "samsung,exynos4210-uart";
190 reg = <0x13630000 0x100>;
191 interrupts = <0 440 0>;
192 clocks = <&clock_peric0 PCLK_UART0>,
193 <&clock_peric0 SCLK_UART0>;
194 clock-names = "uart", "clk_uart_baud0";
195 status = "disabled";
196 };
197
198 serial_1: serial@14c20000 {
199 compatible = "samsung,exynos4210-uart";
200 reg = <0x14c20000 0x100>;
201 interrupts = <0 456 0>;
202 clocks = <&clock_peric1 PCLK_UART1>,
203 <&clock_peric1 SCLK_UART1>;
204 clock-names = "uart", "clk_uart_baud0";
205 status = "disabled";
206 };
207
208 serial_2: serial@14c30000 {
209 compatible = "samsung,exynos4210-uart";
210 reg = <0x14c30000 0x100>;
211 interrupts = <0 457 0>;
212 clocks = <&clock_peric1 PCLK_UART2>,
213 <&clock_peric1 SCLK_UART2>;
214 clock-names = "uart", "clk_uart_baud0";
215 status = "disabled";
216 };
217
218 serial_3: serial@14c40000 {
219 compatible = "samsung,exynos4210-uart";
220 reg = <0x14c40000 0x100>;
221 interrupts = <0 458 0>;
222 clocks = <&clock_peric1 PCLK_UART3>,
223 <&clock_peric1 SCLK_UART3>;
224 clock-names = "uart", "clk_uart_baud0";
225 status = "disabled";
226 };
227
228 pinctrl_alive: pinctrl@10580000 {
229 compatible = "samsung,exynos7-pinctrl";
230 reg = <0x10580000 0x1000>;
231
232 wakeup-interrupt-controller {
233 compatible = "samsung,exynos7-wakeup-eint";
234 interrupt-parent = <&gic>;
235 interrupts = <0 16 0>;
236 };
237 };
238
239 pinctrl_bus0: pinctrl@13470000 {
240 compatible = "samsung,exynos7-pinctrl";
241 reg = <0x13470000 0x1000>;
242 interrupts = <0 383 0>;
243 };
244
245 pinctrl_nfc: pinctrl@14cd0000 {
246 compatible = "samsung,exynos7-pinctrl";
247 reg = <0x14cd0000 0x1000>;
248 interrupts = <0 473 0>;
249 };
250
251 pinctrl_touch: pinctrl@14ce0000 {
252 compatible = "samsung,exynos7-pinctrl";
253 reg = <0x14ce0000 0x1000>;
254 interrupts = <0 474 0>;
255 };
256
257 pinctrl_ff: pinctrl@14c90000 {
258 compatible = "samsung,exynos7-pinctrl";
259 reg = <0x14c90000 0x1000>;
260 interrupts = <0 475 0>;
261 };
262
263 pinctrl_ese: pinctrl@14ca0000 {
264 compatible = "samsung,exynos7-pinctrl";
265 reg = <0x14ca0000 0x1000>;
266 interrupts = <0 476 0>;
267 };
268
269 pinctrl_fsys0: pinctrl@10e60000 {
270 compatible = "samsung,exynos7-pinctrl";
271 reg = <0x10e60000 0x1000>;
272 interrupts = <0 221 0>;
273 };
274
275 pinctrl_fsys1: pinctrl@15690000 {
276 compatible = "samsung,exynos7-pinctrl";
277 reg = <0x15690000 0x1000>;
278 interrupts = <0 203 0>;
279 };
280
281 hsi2c_0: hsi2c@13640000 {
282 compatible = "samsung,exynos7-hsi2c";
283 reg = <0x13640000 0x1000>;
284 interrupts = <0 441 0>;
285 #address-cells = <1>;
286 #size-cells = <0>;
287 pinctrl-names = "default";
288 pinctrl-0 = <&hs_i2c0_bus>;
289 clocks = <&clock_peric0 PCLK_HSI2C0>;
290 clock-names = "hsi2c";
291 status = "disabled";
292 };
293
294 hsi2c_1: hsi2c@13650000 {
295 compatible = "samsung,exynos7-hsi2c";
296 reg = <0x13650000 0x1000>;
297 interrupts = <0 442 0>;
298 #address-cells = <1>;
299 #size-cells = <0>;
300 pinctrl-names = "default";
301 pinctrl-0 = <&hs_i2c1_bus>;
302 clocks = <&clock_peric0 PCLK_HSI2C1>;
303 clock-names = "hsi2c";
304 status = "disabled";
305 };
306
307 hsi2c_2: hsi2c@14e60000 {
308 compatible = "samsung,exynos7-hsi2c";
309 reg = <0x14e60000 0x1000>;
310 interrupts = <0 459 0>;
311 #address-cells = <1>;
312 #size-cells = <0>;
313 pinctrl-names = "default";
314 pinctrl-0 = <&hs_i2c2_bus>;
315 clocks = <&clock_peric1 PCLK_HSI2C2>;
316 clock-names = "hsi2c";
317 status = "disabled";
318 };
319
320 hsi2c_3: hsi2c@14e70000 {
321 compatible = "samsung,exynos7-hsi2c";
322 reg = <0x14e70000 0x1000>;
323 interrupts = <0 460 0>;
324 #address-cells = <1>;
325 #size-cells = <0>;
326 pinctrl-names = "default";
327 pinctrl-0 = <&hs_i2c3_bus>;
328 clocks = <&clock_peric1 PCLK_HSI2C3>;
329 clock-names = "hsi2c";
330 status = "disabled";
331 };
332
333 hsi2c_4: hsi2c@13660000 {
334 compatible = "samsung,exynos7-hsi2c";
335 reg = <0x13660000 0x1000>;
336 interrupts = <0 443 0>;
337 #address-cells = <1>;
338 #size-cells = <0>;
339 pinctrl-names = "default";
340 pinctrl-0 = <&hs_i2c4_bus>;
341 clocks = <&clock_peric0 PCLK_HSI2C4>;
342 clock-names = "hsi2c";
343 status = "disabled";
344 };
345
346 hsi2c_5: hsi2c@13670000 {
347 compatible = "samsung,exynos7-hsi2c";
348 reg = <0x13670000 0x1000>;
349 interrupts = <0 444 0>;
350 #address-cells = <1>;
351 #size-cells = <0>;
352 pinctrl-names = "default";
353 pinctrl-0 = <&hs_i2c5_bus>;
354 clocks = <&clock_peric0 PCLK_HSI2C5>;
355 clock-names = "hsi2c";
356 status = "disabled";
357 };
358
359 hsi2c_6: hsi2c@14e00000 {
360 compatible = "samsung,exynos7-hsi2c";
361 reg = <0x14e00000 0x1000>;
362 interrupts = <0 461 0>;
363 #address-cells = <1>;
364 #size-cells = <0>;
365 pinctrl-names = "default";
366 pinctrl-0 = <&hs_i2c6_bus>;
367 clocks = <&clock_peric1 PCLK_HSI2C6>;
368 clock-names = "hsi2c";
369 status = "disabled";
370 };
371
372 hsi2c_7: hsi2c@13e10000 {
373 compatible = "samsung,exynos7-hsi2c";
374 reg = <0x13e10000 0x1000>;
375 interrupts = <0 462 0>;
376 #address-cells = <1>;
377 #size-cells = <0>;
378 pinctrl-names = "default";
379 pinctrl-0 = <&hs_i2c7_bus>;
380 clocks = <&clock_peric1 PCLK_HSI2C7>;
381 clock-names = "hsi2c";
382 status = "disabled";
383 };
384
385 hsi2c_8: hsi2c@14e20000 {
386 compatible = "samsung,exynos7-hsi2c";
387 reg = <0x14e20000 0x1000>;
388 interrupts = <0 463 0>;
389 #address-cells = <1>;
390 #size-cells = <0>;
391 pinctrl-names = "default";
392 pinctrl-0 = <&hs_i2c8_bus>;
393 clocks = <&clock_peric1 PCLK_HSI2C8>;
394 clock-names = "hsi2c";
395 status = "disabled";
396 };
397
398 hsi2c_9: hsi2c@13680000 {
399 compatible = "samsung,exynos7-hsi2c";
400 reg = <0x13680000 0x1000>;
401 interrupts = <0 445 0>;
402 #address-cells = <1>;
403 #size-cells = <0>;
404 pinctrl-names = "default";
405 pinctrl-0 = <&hs_i2c9_bus>;
406 clocks = <&clock_peric0 PCLK_HSI2C9>;
407 clock-names = "hsi2c";
408 status = "disabled";
409 };
410
411 hsi2c_10: hsi2c@13690000 {
412 compatible = "samsung,exynos7-hsi2c";
413 reg = <0x13690000 0x1000>;
414 interrupts = <0 446 0>;
415 #address-cells = <1>;
416 #size-cells = <0>;
417 pinctrl-names = "default";
418 pinctrl-0 = <&hs_i2c10_bus>;
419 clocks = <&clock_peric0 PCLK_HSI2C10>;
420 clock-names = "hsi2c";
421 status = "disabled";
422 };
423
424 hsi2c_11: hsi2c@136a0000 {
425 compatible = "samsung,exynos7-hsi2c";
426 reg = <0x136a0000 0x1000>;
427 interrupts = <0 447 0>;
428 #address-cells = <1>;
429 #size-cells = <0>;
430 pinctrl-names = "default";
431 pinctrl-0 = <&hs_i2c11_bus>;
432 clocks = <&clock_peric0 PCLK_HSI2C11>;
433 clock-names = "hsi2c";
434 status = "disabled";
435 };
436
437 timer {
438 compatible = "arm,armv8-timer";
439 interrupts = <1 13 0xff01>,
440 <1 14 0xff01>,
441 <1 11 0xff01>,
442 <1 10 0xff01>;
443 };
444
445 pmu_system_controller: system-controller@105c0000 {
446 compatible = "samsung,exynos7-pmu", "syscon";
447 reg = <0x105c0000 0x5000>;
448 };
449
450 rtc: rtc@10590000 {
451 compatible = "samsung,s3c6410-rtc";
452 reg = <0x10590000 0x100>;
453 interrupts = <0 355 0>, <0 356 0>;
454 clocks = <&clock_ccore PCLK_RTC>;
455 clock-names = "rtc";
456 status = "disabled";
457 };
458
459 watchdog: watchdog@101d0000 {
460 compatible = "samsung,exynos7-wdt";
461 reg = <0x101d0000 0x100>;
462 interrupts = <0 110 0>;
463 clocks = <&clock_peris PCLK_WDT>;
464 clock-names = "watchdog";
465 samsung,syscon-phandle = <&pmu_system_controller>;
466 status = "disabled";
467 };
468
469 mmc_0: mmc@15740000 {
470 compatible = "samsung,exynos7-dw-mshc-smu";
471 interrupts = <0 201 0>;
472 #address-cells = <1>;
473 #size-cells = <0>;
474 reg = <0x15740000 0x2000>;
475 clocks = <&clock_fsys1 ACLK_MMC0>,
476 <&clock_top1 CLK_SCLK_MMC0>;
477 clock-names = "biu", "ciu";
478 fifo-depth = <0x40>;
479 status = "disabled";
480 };
481
482 mmc_1: mmc@15750000 {
483 compatible = "samsung,exynos7-dw-mshc";
484 interrupts = <0 202 0>;
485 #address-cells = <1>;
486 #size-cells = <0>;
487 reg = <0x15750000 0x2000>;
488 clocks = <&clock_fsys1 ACLK_MMC1>,
489 <&clock_top1 CLK_SCLK_MMC1>;
490 clock-names = "biu", "ciu";
491 fifo-depth = <0x40>;
492 status = "disabled";
493 };
494
495 mmc_2: mmc@15560000 {
496 compatible = "samsung,exynos7-dw-mshc-smu";
497 interrupts = <0 216 0>;
498 #address-cells = <1>;
499 #size-cells = <0>;
500 reg = <0x15560000 0x2000>;
501 clocks = <&clock_fsys0 ACLK_MMC2>,
502 <&clock_top1 CLK_SCLK_MMC2>;
503 clock-names = "biu", "ciu";
504 fifo-depth = <0x40>;
505 status = "disabled";
506 };
507
508 adc: adc@13620000 {
509 compatible = "samsung,exynos7-adc";
510 reg = <0x13620000 0x100>;
511 interrupts = <0 448 0>;
512 clocks = <&clock_peric0 PCLK_ADCIF>;
513 clock-names = "adc";
514 #io-channel-cells = <1>;
515 io-channel-ranges;
516 status = "disabled";
517 };
518
519 pwm: pwm@136c0000 {
520 compatible = "samsung,exynos4210-pwm";
521 reg = <0x136c0000 0x100>;
522 samsung,pwm-outputs = <0>, <1>, <2>, <3>;
523 #pwm-cells = <3>;
524 clocks = <&clock_peric0 PCLK_PWM>;
525 clock-names = "timers";
526 };
527 };
528};
529
530#include "exynos7-pinctrl.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
new file mode 100644
index 000000000000..4f2de3e789ee
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -0,0 +1,5 @@
1dtb-$(CONFIG_ARCH_FSL_LS2085A) += fsl-ls2085a-simu.dtb
2
3always := $(dtb-y)
4subdir-y := $(dts-dirs)
5clean-files := *.dtb
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts b/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts
new file mode 100644
index 000000000000..82e2a6fccc64
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts
@@ -0,0 +1,65 @@
1/*
2 * Device Tree file for Freescale LS2085a software Simulator model
3 *
4 * Copyright (C) 2014, Freescale Semiconductor
5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 *
8 * This file is dual-licensed: you can use it either under the terms
9 * of the GPL or the X11 license, at your option. Note that this dual
10 * licensing only applies to this file, and not this project as a
11 * whole.
12 *
13 * a) This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
17 *
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public
24 * License along with this library; if not, write to the Free
25 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
26 * MA 02110-1301 USA
27 *
28 * Or, alternatively,
29 *
30 * b) Permission is hereby granted, free of charge, to any person
31 * obtaining a copy of this software and associated documentation
32 * files (the "Software"), to deal in the Software without
33 * restriction, including without limitation the rights to use,
34 * copy, modify, merge, publish, distribute, sublicense, and/or
35 * sell copies of the Software, and to permit persons to whom the
36 * Software is furnished to do so, subject to the following
37 * conditions:
38 *
39 * The above copyright notice and this permission notice shall be
40 * included in all copies or substantial portions of the Software.
41 *
42 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
43 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
44 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
45 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
46 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
47 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
48 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
49 * OTHER DEALINGS IN THE SOFTWARE.
50 */
51
52/dts-v1/;
53
54/include/ "fsl-ls2085a.dtsi"
55
56/ {
57 model = "Freescale Layerscape 2085a software Simulator model";
58 compatible = "fsl,ls2085a-simu", "fsl,ls2085a";
59
60 ethernet@2210000 {
61 compatible = "smsc,lan91c111";
62 reg = <0x0 0x2210000 0x0 0x100>;
63 interrupts = <0 58 0x1>;
64 };
65};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi
new file mode 100644
index 000000000000..e281ceb338c3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi
@@ -0,0 +1,163 @@
1/*
2 * Device Tree Include file for Freescale Layerscape-2085A family SoC.
3 *
4 * Copyright (C) 2014, Freescale Semiconductor
5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 *
8 * This file is dual-licensed: you can use it either under the terms
9 * of the GPLv2 or the X11 license, at your option. Note that this dual
10 * licensing only applies to this file, and not this project as a
11 * whole.
12 *
13 * a) This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
17 *
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public
24 * License along with this library; if not, write to the Free
25 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
26 * MA 02110-1301 USA
27 *
28 * Or, alternatively,
29 *
30 * b) Permission is hereby granted, free of charge, to any person
31 * obtaining a copy of this software and associated documentation
32 * files (the "Software"), to deal in the Software without
33 * restriction, including without limitation the rights to use,
34 * copy, modify, merge, publish, distribute, sublicense, and/or
35 * sell copies of the Software, and to permit persons to whom the
36 * Software is furnished to do so, subject to the following
37 * conditions:
38 *
39 * The above copyright notice and this permission notice shall be
40 * included in all copies or substantial portions of the Software.
41 *
42 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
43 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
44 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
45 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
46 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
47 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
48 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
49 * OTHER DEALINGS IN THE SOFTWARE.
50 */
51
52/ {
53 compatible = "fsl,ls2085a";
54 interrupt-parent = <&gic>;
55 #address-cells = <2>;
56 #size-cells = <2>;
57
58 cpus {
59 #address-cells = <2>;
60 #size-cells = <0>;
61
62 /*
63 * We expect the enable-method for cpu's to be "psci", but this
64 * is dependent on the SoC FW, which will fill this in.
65 *
66 * Currently supported enable-method is psci v0.2
67 */
68
69 /* We have 4 clusters having 2 Cortex-A57 cores each */
70 cpu@0 {
71 device_type = "cpu";
72 compatible = "arm,cortex-a57";
73 reg = <0x0 0x0>;
74 };
75
76 cpu@1 {
77 device_type = "cpu";
78 compatible = "arm,cortex-a57";
79 reg = <0x0 0x1>;
80 };
81
82 cpu@100 {
83 device_type = "cpu";
84 compatible = "arm,cortex-a57";
85 reg = <0x0 0x100>;
86 };
87
88 cpu@101 {
89 device_type = "cpu";
90 compatible = "arm,cortex-a57";
91 reg = <0x0 0x101>;
92 };
93
94 cpu@200 {
95 device_type = "cpu";
96 compatible = "arm,cortex-a57";
97 reg = <0x0 0x200>;
98 };
99
100 cpu@201 {
101 device_type = "cpu";
102 compatible = "arm,cortex-a57";
103 reg = <0x0 0x201>;
104 };
105
106 cpu@300 {
107 device_type = "cpu";
108 compatible = "arm,cortex-a57";
109 reg = <0x0 0x300>;
110 };
111
112 cpu@301 {
113 device_type = "cpu";
114 compatible = "arm,cortex-a57";
115 reg = <0x0 0x301>;
116 };
117 };
118
119 memory@80000000 {
120 device_type = "memory";
121 reg = <0x00000000 0x80000000 0 0x80000000>;
122 /* DRAM space - 1, size : 2 GB DRAM */
123 };
124
125 gic: interrupt-controller@6000000 {
126 compatible = "arm,gic-v3";
127 reg = <0x0 0x06000000 0 0x10000>, /* GIC Dist */
128 <0x0 0x06100000 0 0x100000>; /* GICR (RD_base + SGI_base) */
129 #interrupt-cells = <3>;
130 interrupt-controller;
131 interrupts = <1 9 0x4>;
132 };
133
134 timer {
135 compatible = "arm,armv8-timer";
136 interrupts = <1 13 0x8>, /* Physical Secure PPI, active-low */
137 <1 14 0x8>, /* Physical Non-Secure PPI, active-low */
138 <1 11 0x8>, /* Virtual PPI, active-low */
139 <1 10 0x8>; /* Hypervisor PPI, active-low */
140 };
141
142 serial0: serial@21c0500 {
143 device_type = "serial";
144 compatible = "fsl,ns16550", "ns16550a";
145 reg = <0x0 0x21c0500 0x0 0x100>;
146 clock-frequency = <0>; /* Updated by bootloader */
147 interrupts = <0 32 0x1>; /* edge triggered */
148 };
149
150 serial1: serial@21c0600 {
151 device_type = "serial";
152 compatible = "fsl,ns16550", "ns16550a";
153 reg = <0x0 0x21c0600 0x0 0x100>;
154 clock-frequency = <0>; /* Updated by bootloader */
155 interrupts = <0 32 0x1>; /* edge triggered */
156 };
157
158 fsl_mc: fsl-mc@80c000000 {
159 compatible = "fsl,qoriq-mc";
160 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */
161 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */
162 };
163};
diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile
new file mode 100644
index 000000000000..3ce24622b231
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/Makefile
@@ -0,0 +1,5 @@
1dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-evb.dtb
2
3always := $(dtb-y)
4subdir-y := $(dts-dirs)
5clean-files := *.dtb
diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
new file mode 100644
index 000000000000..43d54017b779
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
@@ -0,0 +1,38 @@
1/*
2 * Copyright (c) 2014 MediaTek Inc.
3 * Author: Eddie Huang <eddie.huang@mediatek.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 */
14
15/dts-v1/;
16#include "mt8173.dtsi"
17
18/ {
19 model = "mediatek,mt8173-evb";
20
21 aliases {
22 serial0 = &uart0;
23 serial1 = &uart1;
24 serial2 = &uart2;
25 serial3 = &uart3;
26 };
27
28 memory@40000000 {
29 device_type = "memory";
30 reg = <0 0x40000000 0 0x80000000>;
31 };
32
33 chosen { };
34};
35
36&uart0 {
37 status = "okay";
38};
diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
new file mode 100644
index 000000000000..8554ec31dd9e
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
@@ -0,0 +1,168 @@
1/*
2 * Copyright (c) 2014 MediaTek Inc.
3 * Author: Eddie Huang <eddie.huang@mediatek.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <dt-bindings/interrupt-controller/irq.h>
15#include <dt-bindings/interrupt-controller/arm-gic.h>
16
17/ {
18 compatible = "mediatek,mt8173";
19 interrupt-parent = <&sysirq>;
20 #address-cells = <2>;
21 #size-cells = <2>;
22
23 cpus {
24 #address-cells = <1>;
25 #size-cells = <0>;
26
27 cpu-map {
28 cluster0 {
29 core0 {
30 cpu = <&cpu0>;
31 };
32 core1 {
33 cpu = <&cpu1>;
34 };
35 };
36
37 cluster1 {
38 core0 {
39 cpu = <&cpu2>;
40 };
41 core1 {
42 cpu = <&cpu3>;
43 };
44 };
45 };
46
47 cpu0: cpu@0 {
48 device_type = "cpu";
49 compatible = "arm,cortex-a53";
50 reg = <0x000>;
51 };
52
53 cpu1: cpu@1 {
54 device_type = "cpu";
55 compatible = "arm,cortex-a53";
56 reg = <0x001>;
57 enable-method = "psci";
58 };
59
60 cpu2: cpu@100 {
61 device_type = "cpu";
62 compatible = "arm,cortex-a57";
63 reg = <0x100>;
64 enable-method = "psci";
65 };
66
67 cpu3: cpu@101 {
68 device_type = "cpu";
69 compatible = "arm,cortex-a57";
70 reg = <0x101>;
71 enable-method = "psci";
72 };
73 };
74
75 psci {
76 compatible = "arm,psci";
77 method = "smc";
78 cpu_suspend = <0x84000001>;
79 cpu_off = <0x84000002>;
80 cpu_on = <0x84000003>;
81 };
82
83 uart_clk: dummy26m {
84 compatible = "fixed-clock";
85 clock-frequency = <26000000>;
86 #clock-cells = <0>;
87 };
88
89 timer {
90 compatible = "arm,armv8-timer";
91 interrupt-parent = <&gic>;
92 interrupts = <GIC_PPI 13
93 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
94 <GIC_PPI 14
95 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
96 <GIC_PPI 11
97 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
98 <GIC_PPI 10
99 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
100 };
101
102 soc {
103 #address-cells = <2>;
104 #size-cells = <2>;
105 compatible = "simple-bus";
106 ranges;
107
108 sysirq: intpol-controller@10200620 {
109 compatible = "mediatek,mt8173-sysirq",
110 "mediatek,mt6577-sysirq";
111 interrupt-controller;
112 #interrupt-cells = <3>;
113 interrupt-parent = <&gic>;
114 reg = <0 0x10200620 0 0x20>;
115 };
116
117 gic: interrupt-controller@10220000 {
118 compatible = "arm,gic-400";
119 #interrupt-cells = <3>;
120 interrupt-parent = <&gic>;
121 interrupt-controller;
122 reg = <0 0x10221000 0 0x1000>,
123 <0 0x10222000 0 0x2000>,
124 <0 0x10224000 0 0x2000>,
125 <0 0x10226000 0 0x2000>;
126 interrupts = <GIC_PPI 9
127 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
128 };
129
130 uart0: serial@11002000 {
131 compatible = "mediatek,mt8173-uart",
132 "mediatek,mt6577-uart";
133 reg = <0 0x11002000 0 0x400>;
134 interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_LOW>;
135 clocks = <&uart_clk>;
136 status = "disabled";
137 };
138
139 uart1: serial@11003000 {
140 compatible = "mediatek,mt8173-uart",
141 "mediatek,mt6577-uart";
142 reg = <0 0x11003000 0 0x400>;
143 interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_LOW>;
144 clocks = <&uart_clk>;
145 status = "disabled";
146 };
147
148 uart2: serial@11004000 {
149 compatible = "mediatek,mt8173-uart",
150 "mediatek,mt6577-uart";
151 reg = <0 0x11004000 0 0x400>;
152 interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_LOW>;
153 clocks = <&uart_clk>;
154 status = "disabled";
155 };
156
157 uart3: serial@11005000 {
158 compatible = "mediatek,mt8173-uart",
159 "mediatek,mt6577-uart";
160 reg = <0 0x11005000 0 0x400>;
161 interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_LOW>;
162 clocks = <&uart_clk>;
163 status = "disabled";
164 };
165 };
166
167};
168
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 5376d908eabe..be1f12a5a5f0 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -31,6 +31,8 @@ CONFIG_MODULES=y
31CONFIG_MODULE_UNLOAD=y 31CONFIG_MODULE_UNLOAD=y
32# CONFIG_BLK_DEV_BSG is not set 32# CONFIG_BLK_DEV_BSG is not set
33# CONFIG_IOSCHED_DEADLINE is not set 33# CONFIG_IOSCHED_DEADLINE is not set
34CONFIG_ARCH_FSL_LS2085A=y
35CONFIG_ARCH_MEDIATEK=y
34CONFIG_ARCH_THUNDER=y 36CONFIG_ARCH_THUNDER=y
35CONFIG_ARCH_VEXPRESS=y 37CONFIG_ARCH_VEXPRESS=y
36CONFIG_ARCH_XGENE=y 38CONFIG_ARCH_XGENE=y
@@ -45,6 +47,8 @@ CONFIG_CMA=y
45CONFIG_CMDLINE="console=ttyAMA0" 47CONFIG_CMDLINE="console=ttyAMA0"
46# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set 48# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
47CONFIG_COMPAT=y 49CONFIG_COMPAT=y
50CONFIG_CPU_IDLE=y
51CONFIG_ARM64_CPUIDLE=y
48CONFIG_NET=y 52CONFIG_NET=y
49CONFIG_PACKET=y 53CONFIG_PACKET=y
50CONFIG_UNIX=y 54CONFIG_UNIX=y
@@ -86,6 +90,7 @@ CONFIG_SERIO_AMBAKMI=y
86CONFIG_LEGACY_PTY_COUNT=16 90CONFIG_LEGACY_PTY_COUNT=16
87CONFIG_SERIAL_8250=y 91CONFIG_SERIAL_8250=y
88CONFIG_SERIAL_8250_CONSOLE=y 92CONFIG_SERIAL_8250_CONSOLE=y
93CONFIG_SERIAL_8250_MT6577=y
89CONFIG_SERIAL_AMBA_PL011=y 94CONFIG_SERIAL_AMBA_PL011=y
90CONFIG_SERIAL_AMBA_PL011_CONSOLE=y 95CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
91CONFIG_SERIAL_OF_PLATFORM=y 96CONFIG_SERIAL_OF_PLATFORM=y
diff --git a/arch/arm64/include/asm/bitrev.h b/arch/arm64/include/asm/bitrev.h
new file mode 100644
index 000000000000..a5a0c3660137
--- /dev/null
+++ b/arch/arm64/include/asm/bitrev.h
@@ -0,0 +1,19 @@
1#ifndef __ASM_BITREV_H
2#define __ASM_BITREV_H
3static __always_inline __attribute_const__ u32 __arch_bitrev32(u32 x)
4{
5 __asm__ ("rbit %w0, %w1" : "=r" (x) : "r" (x));
6 return x;
7}
8
9static __always_inline __attribute_const__ u16 __arch_bitrev16(u16 x)
10{
11 return __arch_bitrev32((u32)x) >> 16;
12}
13
14static __always_inline __attribute_const__ u8 __arch_bitrev8(u8 x)
15{
16 return __arch_bitrev32((u32)x) >> 24;
17}
18
19#endif
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 7ae31a2cc6c0..67d309cc3b6b 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages);
152int set_memory_rw(unsigned long addr, int numpages); 152int set_memory_rw(unsigned long addr, int numpages);
153int set_memory_x(unsigned long addr, int numpages); 153int set_memory_x(unsigned long addr, int numpages);
154int set_memory_nx(unsigned long addr, int numpages); 154int set_memory_nx(unsigned long addr, int numpages);
155
156#ifdef CONFIG_DEBUG_RODATA
157void mark_rodata_ro(void);
158#endif
159
155#endif 160#endif
diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h
index 4c631a0a3609..da2fc9e3cedd 100644
--- a/arch/arm64/include/asm/cachetype.h
+++ b/arch/arm64/include/asm/cachetype.h
@@ -39,24 +39,41 @@
39 39
40extern unsigned long __icache_flags; 40extern unsigned long __icache_flags;
41 41
42/*
43 * NumSets, bits[27:13] - (Number of sets in cache) - 1
44 * Associativity, bits[12:3] - (Associativity of cache) - 1
45 * LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2
46 */
47#define CCSIDR_EL1_WRITE_THROUGH BIT(31)
48#define CCSIDR_EL1_WRITE_BACK BIT(30)
49#define CCSIDR_EL1_READ_ALLOCATE BIT(29)
50#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28)
42#define CCSIDR_EL1_LINESIZE_MASK 0x7 51#define CCSIDR_EL1_LINESIZE_MASK 0x7
43#define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK) 52#define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK)
44 53#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3
54#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff
55#define CCSIDR_EL1_ASSOCIATIVITY(x) \
56 (((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK)
45#define CCSIDR_EL1_NUMSETS_SHIFT 13 57#define CCSIDR_EL1_NUMSETS_SHIFT 13
46#define CCSIDR_EL1_NUMSETS_MASK (0x7fff << CCSIDR_EL1_NUMSETS_SHIFT) 58#define CCSIDR_EL1_NUMSETS_MASK 0x7fff
47#define CCSIDR_EL1_NUMSETS(x) \ 59#define CCSIDR_EL1_NUMSETS(x) \
48 (((x) & CCSIDR_EL1_NUMSETS_MASK) >> CCSIDR_EL1_NUMSETS_SHIFT) 60 (((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK)
61
62#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x))
63#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1)
64#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1)
49 65
50extern u64 __attribute_const__ icache_get_ccsidr(void); 66extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr);
51 67
68/* Helpers for Level 1 Instruction cache csselr = 1L */
52static inline int icache_get_linesize(void) 69static inline int icache_get_linesize(void)
53{ 70{
54 return 16 << CCSIDR_EL1_LINESIZE(icache_get_ccsidr()); 71 return CACHE_LINESIZE(cache_get_ccsidr(1L));
55} 72}
56 73
57static inline int icache_get_numsets(void) 74static inline int icache_get_numsets(void)
58{ 75{
59 return 1 + CCSIDR_EL1_NUMSETS(icache_get_ccsidr()); 76 return CACHE_NUMSETS(cache_get_ccsidr(1L));
60} 77}
61 78
62/* 79/*
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index 3fb053fa6e98..7fbed6919b54 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -161,7 +161,6 @@ typedef struct compat_siginfo {
161 int si_code; 161 int si_code;
162 162
163 union { 163 union {
164 /* The padding is the same size as AArch64. */
165 int _pad[128/sizeof(int) - 3]; 164 int _pad[128/sizeof(int) - 3];
166 165
167 /* kill() */ 166 /* kill() */
diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h
index 6f8e2ef9094a..da301ee7395c 100644
--- a/arch/arm64/include/asm/cpu_ops.h
+++ b/arch/arm64/include/asm/cpu_ops.h
@@ -28,8 +28,6 @@ struct device_node;
28 * enable-method property. 28 * enable-method property.
29 * @cpu_init: Reads any data necessary for a specific enable-method from the 29 * @cpu_init: Reads any data necessary for a specific enable-method from the
30 * devicetree, for a given cpu node and proposed logical id. 30 * devicetree, for a given cpu node and proposed logical id.
31 * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
32 * devicetree, for a given cpu node and proposed logical id.
33 * @cpu_prepare: Early one-time preparation step for a cpu. If there is a 31 * @cpu_prepare: Early one-time preparation step for a cpu. If there is a
34 * mechanism for doing so, tests whether it is possible to boot 32 * mechanism for doing so, tests whether it is possible to boot
35 * the given CPU. 33 * the given CPU.
@@ -42,6 +40,8 @@ struct device_node;
42 * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the 40 * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
43 * cpu being killed. 41 * cpu being killed.
44 * @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu. 42 * @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu.
43 * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
44 * devicetree, for a given cpu node and proposed logical id.
45 * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing 45 * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing
46 * to wrong parameters or error conditions. Called from the 46 * to wrong parameters or error conditions. Called from the
47 * CPU being suspended. Must be called with IRQs disabled. 47 * CPU being suspended. Must be called with IRQs disabled.
@@ -49,7 +49,6 @@ struct device_node;
49struct cpu_operations { 49struct cpu_operations {
50 const char *name; 50 const char *name;
51 int (*cpu_init)(struct device_node *, unsigned int); 51 int (*cpu_init)(struct device_node *, unsigned int);
52 int (*cpu_init_idle)(struct device_node *, unsigned int);
53 int (*cpu_prepare)(unsigned int); 52 int (*cpu_prepare)(unsigned int);
54 int (*cpu_boot)(unsigned int); 53 int (*cpu_boot)(unsigned int);
55 void (*cpu_postboot)(void); 54 void (*cpu_postboot)(void);
@@ -58,7 +57,8 @@ struct cpu_operations {
58 void (*cpu_die)(unsigned int cpu); 57 void (*cpu_die)(unsigned int cpu);
59 int (*cpu_kill)(unsigned int cpu); 58 int (*cpu_kill)(unsigned int cpu);
60#endif 59#endif
61#ifdef CONFIG_ARM64_CPU_SUSPEND 60#ifdef CONFIG_CPU_IDLE
61 int (*cpu_init_idle)(struct device_node *, unsigned int);
62 int (*cpu_suspend)(unsigned long); 62 int (*cpu_suspend)(unsigned long);
63#endif 63#endif
64}; 64};
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 07547ccc1f2b..b6c16d5f622f 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -52,6 +52,8 @@ static inline void cpus_set_cap(unsigned int num)
52} 52}
53 53
54void check_local_cpu_errata(void); 54void check_local_cpu_errata(void);
55bool cpu_supports_mixed_endian_el0(void);
56bool system_supports_mixed_endian_el0(void);
55 57
56#endif /* __ASSEMBLY__ */ 58#endif /* __ASSEMBLY__ */
57 59
diff --git a/arch/arm64/include/asm/cpuidle.h b/arch/arm64/include/asm/cpuidle.h
index b52a9932e2b1..0710654631e7 100644
--- a/arch/arm64/include/asm/cpuidle.h
+++ b/arch/arm64/include/asm/cpuidle.h
@@ -3,11 +3,17 @@
3 3
4#ifdef CONFIG_CPU_IDLE 4#ifdef CONFIG_CPU_IDLE
5extern int cpu_init_idle(unsigned int cpu); 5extern int cpu_init_idle(unsigned int cpu);
6extern int cpu_suspend(unsigned long arg);
6#else 7#else
7static inline int cpu_init_idle(unsigned int cpu) 8static inline int cpu_init_idle(unsigned int cpu)
8{ 9{
9 return -EOPNOTSUPP; 10 return -EOPNOTSUPP;
10} 11}
12
13static inline int cpu_suspend(unsigned long arg)
14{
15 return -EOPNOTSUPP;
16}
11#endif 17#endif
12 18
13#endif 19#endif
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index 8adb986a3086..a84ec605bed8 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -72,6 +72,18 @@
72 72
73#define APM_CPU_PART_POTENZA 0x000 73#define APM_CPU_PART_POTENZA 0x000
74 74
75#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16
76#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)
77#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \
78 (((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT)
79#define ID_AA64MMFR0_BIGEND_SHIFT 8
80#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT)
81#define ID_AA64MMFR0_BIGEND(mmfr0) \
82 (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
83
84#define SCTLR_EL1_CP15BEN (0x1 << 5)
85#define SCTLR_EL1_SED (0x1 << 8)
86
75#ifndef __ASSEMBLY__ 87#ifndef __ASSEMBLY__
76 88
77/* 89/*
@@ -104,6 +116,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void)
104 return read_cpuid(CTR_EL0); 116 return read_cpuid(CTR_EL0);
105} 117}
106 118
119static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
120{
121 return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) ||
122 (ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1);
123}
107#endif /* __ASSEMBLY__ */ 124#endif /* __ASSEMBLY__ */
108 125
109#endif 126#endif
diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h
index 9ce3e680ae1c..6932bb57dba0 100644
--- a/arch/arm64/include/asm/dma-mapping.h
+++ b/arch/arm64/include/asm/dma-mapping.h
@@ -28,8 +28,6 @@
28 28
29#define DMA_ERROR_CODE (~(dma_addr_t)0) 29#define DMA_ERROR_CODE (~(dma_addr_t)0)
30extern struct dma_map_ops *dma_ops; 30extern struct dma_map_ops *dma_ops;
31extern struct dma_map_ops coherent_swiotlb_dma_ops;
32extern struct dma_map_ops noncoherent_swiotlb_dma_ops;
33 31
34static inline struct dma_map_ops *__generic_dma_ops(struct device *dev) 32static inline struct dma_map_ops *__generic_dma_ops(struct device *dev)
35{ 33{
@@ -47,23 +45,18 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
47 return __generic_dma_ops(dev); 45 return __generic_dma_ops(dev);
48} 46}
49 47
50static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops)
51{
52 dev->archdata.dma_ops = ops;
53}
54
55static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 48static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
56 struct iommu_ops *iommu, bool coherent) 49 struct iommu_ops *iommu, bool coherent)
57{ 50{
58 dev->archdata.dma_coherent = coherent; 51 dev->archdata.dma_coherent = coherent;
59 if (coherent)
60 set_dma_ops(dev, &coherent_swiotlb_dma_ops);
61} 52}
62#define arch_setup_dma_ops arch_setup_dma_ops 53#define arch_setup_dma_ops arch_setup_dma_ops
63 54
64/* do not use this function in a driver */ 55/* do not use this function in a driver */
65static inline bool is_device_dma_coherent(struct device *dev) 56static inline bool is_device_dma_coherent(struct device *dev)
66{ 57{
58 if (!dev)
59 return false;
67 return dev->archdata.dma_coherent; 60 return dev->archdata.dma_coherent;
68} 61}
69 62
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index a34fd3b12e2b..ef572206f1c3 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -6,29 +6,33 @@
6 6
7#ifdef CONFIG_EFI 7#ifdef CONFIG_EFI
8extern void efi_init(void); 8extern void efi_init(void);
9extern void efi_idmap_init(void);
10#else 9#else
11#define efi_init() 10#define efi_init()
12#define efi_idmap_init()
13#endif 11#endif
14 12
15#define efi_call_virt(f, ...) \ 13#define efi_call_virt(f, ...) \
16({ \ 14({ \
17 efi_##f##_t *__f = efi.systab->runtime->f; \ 15 efi_##f##_t *__f; \
18 efi_status_t __s; \ 16 efi_status_t __s; \
19 \ 17 \
20 kernel_neon_begin(); \ 18 kernel_neon_begin(); \
19 efi_virtmap_load(); \
20 __f = efi.systab->runtime->f; \
21 __s = __f(__VA_ARGS__); \ 21 __s = __f(__VA_ARGS__); \
22 efi_virtmap_unload(); \
22 kernel_neon_end(); \ 23 kernel_neon_end(); \
23 __s; \ 24 __s; \
24}) 25})
25 26
26#define __efi_call_virt(f, ...) \ 27#define __efi_call_virt(f, ...) \
27({ \ 28({ \
28 efi_##f##_t *__f = efi.systab->runtime->f; \ 29 efi_##f##_t *__f; \
29 \ 30 \
30 kernel_neon_begin(); \ 31 kernel_neon_begin(); \
32 efi_virtmap_load(); \
33 __f = efi.systab->runtime->f; \
31 __f(__VA_ARGS__); \ 34 __f(__VA_ARGS__); \
35 efi_virtmap_unload(); \
32 kernel_neon_end(); \ 36 kernel_neon_end(); \
33}) 37})
34 38
@@ -44,4 +48,22 @@ extern void efi_idmap_init(void);
44 48
45#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__) 49#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__)
46 50
51#define EFI_ALLOC_ALIGN SZ_64K
52
53/*
54 * On ARM systems, virtually remapped UEFI runtime services are set up in two
55 * distinct stages:
56 * - The stub retrieves the final version of the memory map from UEFI, populates
57 * the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime
58 * service to communicate the new mapping to the firmware (Note that the new
59 * mapping is not live at this time)
60 * - During an early initcall(), the EFI system table is permanently remapped
61 * and the virtual remapping of the UEFI Runtime Services regions is loaded
62 * into a private set of page tables. If this all succeeds, the Runtime
63 * Services are enabled and the EFI_RUNTIME_SERVICES bit set.
64 */
65
66void efi_virtmap_load(void);
67void efi_virtmap_unload(void);
68
47#endif /* _ASM_EFI_H */ 69#endif /* _ASM_EFI_H */
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 72674f4c3871..92bbae381598 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -18,40 +18,90 @@
18#ifndef __ASM_ESR_H 18#ifndef __ASM_ESR_H
19#define __ASM_ESR_H 19#define __ASM_ESR_H
20 20
21#define ESR_EL1_WRITE (1 << 6) 21#define ESR_ELx_EC_UNKNOWN (0x00)
22#define ESR_EL1_CM (1 << 8) 22#define ESR_ELx_EC_WFx (0x01)
23#define ESR_EL1_IL (1 << 25) 23/* Unallocated EC: 0x02 */
24#define ESR_ELx_EC_CP15_32 (0x03)
25#define ESR_ELx_EC_CP15_64 (0x04)
26#define ESR_ELx_EC_CP14_MR (0x05)
27#define ESR_ELx_EC_CP14_LS (0x06)
28#define ESR_ELx_EC_FP_ASIMD (0x07)
29#define ESR_ELx_EC_CP10_ID (0x08)
30/* Unallocated EC: 0x09 - 0x0B */
31#define ESR_ELx_EC_CP14_64 (0x0C)
32/* Unallocated EC: 0x0d */
33#define ESR_ELx_EC_ILL (0x0E)
34/* Unallocated EC: 0x0F - 0x10 */
35#define ESR_ELx_EC_SVC32 (0x11)
36#define ESR_ELx_EC_HVC32 (0x12)
37#define ESR_ELx_EC_SMC32 (0x13)
38/* Unallocated EC: 0x14 */
39#define ESR_ELx_EC_SVC64 (0x15)
40#define ESR_ELx_EC_HVC64 (0x16)
41#define ESR_ELx_EC_SMC64 (0x17)
42#define ESR_ELx_EC_SYS64 (0x18)
43/* Unallocated EC: 0x19 - 0x1E */
44#define ESR_ELx_EC_IMP_DEF (0x1f)
45#define ESR_ELx_EC_IABT_LOW (0x20)
46#define ESR_ELx_EC_IABT_CUR (0x21)
47#define ESR_ELx_EC_PC_ALIGN (0x22)
48/* Unallocated EC: 0x23 */
49#define ESR_ELx_EC_DABT_LOW (0x24)
50#define ESR_ELx_EC_DABT_CUR (0x25)
51#define ESR_ELx_EC_SP_ALIGN (0x26)
52/* Unallocated EC: 0x27 */
53#define ESR_ELx_EC_FP_EXC32 (0x28)
54/* Unallocated EC: 0x29 - 0x2B */
55#define ESR_ELx_EC_FP_EXC64 (0x2C)
56/* Unallocated EC: 0x2D - 0x2E */
57#define ESR_ELx_EC_SERROR (0x2F)
58#define ESR_ELx_EC_BREAKPT_LOW (0x30)
59#define ESR_ELx_EC_BREAKPT_CUR (0x31)
60#define ESR_ELx_EC_SOFTSTP_LOW (0x32)
61#define ESR_ELx_EC_SOFTSTP_CUR (0x33)
62#define ESR_ELx_EC_WATCHPT_LOW (0x34)
63#define ESR_ELx_EC_WATCHPT_CUR (0x35)
64/* Unallocated EC: 0x36 - 0x37 */
65#define ESR_ELx_EC_BKPT32 (0x38)
66/* Unallocated EC: 0x39 */
67#define ESR_ELx_EC_VECTOR32 (0x3A)
68/* Unallocted EC: 0x3B */
69#define ESR_ELx_EC_BRK64 (0x3C)
70/* Unallocated EC: 0x3D - 0x3F */
71#define ESR_ELx_EC_MAX (0x3F)
24 72
25#define ESR_EL1_EC_SHIFT (26) 73#define ESR_ELx_EC_SHIFT (26)
26#define ESR_EL1_EC_UNKNOWN (0x00) 74#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
27#define ESR_EL1_EC_WFI (0x01) 75
28#define ESR_EL1_EC_CP15_32 (0x03) 76#define ESR_ELx_IL (UL(1) << 25)
29#define ESR_EL1_EC_CP15_64 (0x04) 77#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)
30#define ESR_EL1_EC_CP14_MR (0x05) 78#define ESR_ELx_ISV (UL(1) << 24)
31#define ESR_EL1_EC_CP14_LS (0x06) 79#define ESR_ELx_SAS_SHIFT (22)
32#define ESR_EL1_EC_FP_ASIMD (0x07) 80#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)
33#define ESR_EL1_EC_CP10_ID (0x08) 81#define ESR_ELx_SSE (UL(1) << 21)
34#define ESR_EL1_EC_CP14_64 (0x0C) 82#define ESR_ELx_SRT_SHIFT (16)
35#define ESR_EL1_EC_ILL_ISS (0x0E) 83#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)
36#define ESR_EL1_EC_SVC32 (0x11) 84#define ESR_ELx_SF (UL(1) << 15)
37#define ESR_EL1_EC_SVC64 (0x15) 85#define ESR_ELx_AR (UL(1) << 14)
38#define ESR_EL1_EC_SYS64 (0x18) 86#define ESR_ELx_EA (UL(1) << 9)
39#define ESR_EL1_EC_IABT_EL0 (0x20) 87#define ESR_ELx_CM (UL(1) << 8)
40#define ESR_EL1_EC_IABT_EL1 (0x21) 88#define ESR_ELx_S1PTW (UL(1) << 7)
41#define ESR_EL1_EC_PC_ALIGN (0x22) 89#define ESR_ELx_WNR (UL(1) << 6)
42#define ESR_EL1_EC_DABT_EL0 (0x24) 90#define ESR_ELx_FSC (0x3F)
43#define ESR_EL1_EC_DABT_EL1 (0x25) 91#define ESR_ELx_FSC_TYPE (0x3C)
44#define ESR_EL1_EC_SP_ALIGN (0x26) 92#define ESR_ELx_FSC_EXTABT (0x10)
45#define ESR_EL1_EC_FP_EXC32 (0x28) 93#define ESR_ELx_FSC_FAULT (0x04)
46#define ESR_EL1_EC_FP_EXC64 (0x2C) 94#define ESR_ELx_FSC_PERM (0x0C)
47#define ESR_EL1_EC_SERROR (0x2F) 95#define ESR_ELx_CV (UL(1) << 24)
48#define ESR_EL1_EC_BREAKPT_EL0 (0x30) 96#define ESR_ELx_COND_SHIFT (20)
49#define ESR_EL1_EC_BREAKPT_EL1 (0x31) 97#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT)
50#define ESR_EL1_EC_SOFTSTP_EL0 (0x32) 98#define ESR_ELx_WFx_ISS_WFE (UL(1) << 0)
51#define ESR_EL1_EC_SOFTSTP_EL1 (0x33) 99#define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1)
52#define ESR_EL1_EC_WATCHPT_EL0 (0x34) 100
53#define ESR_EL1_EC_WATCHPT_EL1 (0x35) 101#ifndef __ASSEMBLY__
54#define ESR_EL1_EC_BKPT32 (0x38) 102#include <asm/types.h>
55#define ESR_EL1_EC_BRK64 (0x3C) 103
104const char *esr_get_class_string(u32 esr);
105#endif /* __ASSEMBLY */
56 106
57#endif /* __ASM_ESR_H */ 107#endif /* __ASM_ESR_H */
diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 9ef6eca905ca..defa0ff98250 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -49,6 +49,7 @@ enum fixed_addresses {
49 49
50 FIX_BTMAP_END = __end_of_permanent_fixed_addresses, 50 FIX_BTMAP_END = __end_of_permanent_fixed_addresses,
51 FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, 51 FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1,
52 FIX_TEXT_POKE0,
52 __end_of_fixed_addresses 53 __end_of_fixed_addresses
53}; 54};
54 55
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index 007618b8188c..a2daf1293028 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -76,7 +76,6 @@
76 fpsimd_restore_fpcr x\tmpnr, \state 76 fpsimd_restore_fpcr x\tmpnr, \state
77.endm 77.endm
78 78
79.altmacro
80.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2 79.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
81 mrs x\tmpnr1, fpsr 80 mrs x\tmpnr1, fpsr
82 str w\numnr, [\state, #8] 81 str w\numnr, [\state, #8]
@@ -86,11 +85,22 @@
86 add \state, \state, x\numnr, lsl #4 85 add \state, \state, x\numnr, lsl #4
87 sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1 86 sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
88 br x\tmpnr1 87 br x\tmpnr1
89 .irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 88 stp q30, q31, [\state, #-16 * 30 - 16]
90 .irp qb, %(qa + 1) 89 stp q28, q29, [\state, #-16 * 28 - 16]
91 stp q\qa, q\qb, [\state, # -16 * \qa - 16] 90 stp q26, q27, [\state, #-16 * 26 - 16]
92 .endr 91 stp q24, q25, [\state, #-16 * 24 - 16]
93 .endr 92 stp q22, q23, [\state, #-16 * 22 - 16]
93 stp q20, q21, [\state, #-16 * 20 - 16]
94 stp q18, q19, [\state, #-16 * 18 - 16]
95 stp q16, q17, [\state, #-16 * 16 - 16]
96 stp q14, q15, [\state, #-16 * 14 - 16]
97 stp q12, q13, [\state, #-16 * 12 - 16]
98 stp q10, q11, [\state, #-16 * 10 - 16]
99 stp q8, q9, [\state, #-16 * 8 - 16]
100 stp q6, q7, [\state, #-16 * 6 - 16]
101 stp q4, q5, [\state, #-16 * 4 - 16]
102 stp q2, q3, [\state, #-16 * 2 - 16]
103 stp q0, q1, [\state, #-16 * 0 - 16]
940: 1040:
95.endm 105.endm
96 106
@@ -103,10 +113,21 @@
103 add \state, \state, x\tmpnr2, lsl #4 113 add \state, \state, x\tmpnr2, lsl #4
104 sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1 114 sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
105 br x\tmpnr1 115 br x\tmpnr1
106 .irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 116 ldp q30, q31, [\state, #-16 * 30 - 16]
107 .irp qb, %(qa + 1) 117 ldp q28, q29, [\state, #-16 * 28 - 16]
108 ldp q\qa, q\qb, [\state, # -16 * \qa - 16] 118 ldp q26, q27, [\state, #-16 * 26 - 16]
109 .endr 119 ldp q24, q25, [\state, #-16 * 24 - 16]
110 .endr 120 ldp q22, q23, [\state, #-16 * 22 - 16]
121 ldp q20, q21, [\state, #-16 * 20 - 16]
122 ldp q18, q19, [\state, #-16 * 18 - 16]
123 ldp q16, q17, [\state, #-16 * 16 - 16]
124 ldp q14, q15, [\state, #-16 * 14 - 16]
125 ldp q12, q13, [\state, #-16 * 12 - 16]
126 ldp q10, q11, [\state, #-16 * 10 - 16]
127 ldp q8, q9, [\state, #-16 * 8 - 16]
128 ldp q6, q7, [\state, #-16 * 6 - 16]
129 ldp q4, q5, [\state, #-16 * 4 - 16]
130 ldp q2, q3, [\state, #-16 * 2 - 16]
131 ldp q0, q1, [\state, #-16 * 0 - 16]
1110: 1320:
112.endm 133.endm
diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index e8a3268a891c..6aae421f4d73 100644
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -20,7 +20,7 @@
20#include <linux/threads.h> 20#include <linux/threads.h>
21#include <asm/irq.h> 21#include <asm/irq.h>
22 22
23#define NR_IPI 6 23#define NR_IPI 5
24 24
25typedef struct { 25typedef struct {
26 unsigned int __softirq_pending; 26 unsigned int __softirq_pending;
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 949c406d4df4..540f7c0aea82 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -26,6 +26,7 @@
26 26
27#include <asm/byteorder.h> 27#include <asm/byteorder.h>
28#include <asm/barrier.h> 28#include <asm/barrier.h>
29#include <asm/memory.h>
29#include <asm/pgtable.h> 30#include <asm/pgtable.h>
30#include <asm/early_ioremap.h> 31#include <asm/early_ioremap.h>
31#include <asm/alternative.h> 32#include <asm/alternative.h>
@@ -145,8 +146,8 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
145 * I/O port access primitives. 146 * I/O port access primitives.
146 */ 147 */
147#define arch_has_dev_port() (1) 148#define arch_has_dev_port() (1)
148#define IO_SPACE_LIMIT (SZ_32M - 1) 149#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1)
149#define PCI_IOBASE ((void __iomem *)(MODULES_VADDR - SZ_32M)) 150#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
150 151
151/* 152/*
152 * String version of I/O memory access operations. 153 * String version of I/O memory access operations.
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 8afb863f5a9e..94674eb7e7bb 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -18,6 +18,7 @@
18#ifndef __ARM64_KVM_ARM_H__ 18#ifndef __ARM64_KVM_ARM_H__
19#define __ARM64_KVM_ARM_H__ 19#define __ARM64_KVM_ARM_H__
20 20
21#include <asm/esr.h>
21#include <asm/memory.h> 22#include <asm/memory.h>
22#include <asm/types.h> 23#include <asm/types.h>
23 24
@@ -184,77 +185,11 @@
184#define MDCR_EL2_TPMCR (1 << 5) 185#define MDCR_EL2_TPMCR (1 << 5)
185#define MDCR_EL2_HPMN_MASK (0x1F) 186#define MDCR_EL2_HPMN_MASK (0x1F)
186 187
187/* Exception Syndrome Register (ESR) bits */ 188/* For compatibility with fault code shared with 32-bit */
188#define ESR_EL2_EC_SHIFT (26) 189#define FSC_FAULT ESR_ELx_FSC_FAULT
189#define ESR_EL2_EC (UL(0x3f) << ESR_EL2_EC_SHIFT) 190#define FSC_PERM ESR_ELx_FSC_PERM
190#define ESR_EL2_IL (UL(1) << 25)
191#define ESR_EL2_ISS (ESR_EL2_IL - 1)
192#define ESR_EL2_ISV_SHIFT (24)
193#define ESR_EL2_ISV (UL(1) << ESR_EL2_ISV_SHIFT)
194#define ESR_EL2_SAS_SHIFT (22)
195#define ESR_EL2_SAS (UL(3) << ESR_EL2_SAS_SHIFT)
196#define ESR_EL2_SSE (1 << 21)
197#define ESR_EL2_SRT_SHIFT (16)
198#define ESR_EL2_SRT_MASK (0x1f << ESR_EL2_SRT_SHIFT)
199#define ESR_EL2_SF (1 << 15)
200#define ESR_EL2_AR (1 << 14)
201#define ESR_EL2_EA (1 << 9)
202#define ESR_EL2_CM (1 << 8)
203#define ESR_EL2_S1PTW (1 << 7)
204#define ESR_EL2_WNR (1 << 6)
205#define ESR_EL2_FSC (0x3f)
206#define ESR_EL2_FSC_TYPE (0x3c)
207
208#define ESR_EL2_CV_SHIFT (24)
209#define ESR_EL2_CV (UL(1) << ESR_EL2_CV_SHIFT)
210#define ESR_EL2_COND_SHIFT (20)
211#define ESR_EL2_COND (UL(0xf) << ESR_EL2_COND_SHIFT)
212
213
214#define FSC_FAULT (0x04)
215#define FSC_PERM (0x0c)
216 191
217/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 192/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
218#define HPFAR_MASK (~UL(0xf)) 193#define HPFAR_MASK (~UL(0xf))
219 194
220#define ESR_EL2_EC_UNKNOWN (0x00)
221#define ESR_EL2_EC_WFI (0x01)
222#define ESR_EL2_EC_CP15_32 (0x03)
223#define ESR_EL2_EC_CP15_64 (0x04)
224#define ESR_EL2_EC_CP14_MR (0x05)
225#define ESR_EL2_EC_CP14_LS (0x06)
226#define ESR_EL2_EC_FP_ASIMD (0x07)
227#define ESR_EL2_EC_CP10_ID (0x08)
228#define ESR_EL2_EC_CP14_64 (0x0C)
229#define ESR_EL2_EC_ILL_ISS (0x0E)
230#define ESR_EL2_EC_SVC32 (0x11)
231#define ESR_EL2_EC_HVC32 (0x12)
232#define ESR_EL2_EC_SMC32 (0x13)
233#define ESR_EL2_EC_SVC64 (0x15)
234#define ESR_EL2_EC_HVC64 (0x16)
235#define ESR_EL2_EC_SMC64 (0x17)
236#define ESR_EL2_EC_SYS64 (0x18)
237#define ESR_EL2_EC_IABT (0x20)
238#define ESR_EL2_EC_IABT_HYP (0x21)
239#define ESR_EL2_EC_PC_ALIGN (0x22)
240#define ESR_EL2_EC_DABT (0x24)
241#define ESR_EL2_EC_DABT_HYP (0x25)
242#define ESR_EL2_EC_SP_ALIGN (0x26)
243#define ESR_EL2_EC_FP_EXC32 (0x28)
244#define ESR_EL2_EC_FP_EXC64 (0x2C)
245#define ESR_EL2_EC_SERROR (0x2F)
246#define ESR_EL2_EC_BREAKPT (0x30)
247#define ESR_EL2_EC_BREAKPT_HYP (0x31)
248#define ESR_EL2_EC_SOFTSTP (0x32)
249#define ESR_EL2_EC_SOFTSTP_HYP (0x33)
250#define ESR_EL2_EC_WATCHPT (0x34)
251#define ESR_EL2_EC_WATCHPT_HYP (0x35)
252#define ESR_EL2_EC_BKPT32 (0x38)
253#define ESR_EL2_EC_VECTOR32 (0x3A)
254#define ESR_EL2_EC_BRK64 (0x3C)
255
256#define ESR_EL2_EC_xABT_xFSR_EXTABT 0x10
257
258#define ESR_EL2_EC_WFI_ISS_WFE (1 << 0)
259
260#endif /* __ARM64_KVM_ARM_H__ */ 195#endif /* __ARM64_KVM_ARM_H__ */
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 483842180f8f..4f7310fa77f0 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -126,6 +126,7 @@ extern char __kvm_hyp_vector[];
126 126
127extern void __kvm_flush_vm_context(void); 127extern void __kvm_flush_vm_context(void);
128extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); 128extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
129extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
129 130
130extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); 131extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
131 132
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 3cb4c856b10d..17e92f05b1fe 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -23,10 +23,13 @@
23#define __ARM64_KVM_EMULATE_H__ 23#define __ARM64_KVM_EMULATE_H__
24 24
25#include <linux/kvm_host.h> 25#include <linux/kvm_host.h>
26#include <asm/kvm_asm.h> 26
27#include <asm/esr.h>
27#include <asm/kvm_arm.h> 28#include <asm/kvm_arm.h>
29#include <asm/kvm_asm.h>
28#include <asm/kvm_mmio.h> 30#include <asm/kvm_mmio.h>
29#include <asm/ptrace.h> 31#include <asm/ptrace.h>
32#include <asm/cputype.h>
30 33
31unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num); 34unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num);
32unsigned long *vcpu_spsr32(const struct kvm_vcpu *vcpu); 35unsigned long *vcpu_spsr32(const struct kvm_vcpu *vcpu);
@@ -138,70 +141,75 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
138 return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8; 141 return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
139} 142}
140 143
144static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
145{
146 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_xVC_IMM_MASK;
147}
148
141static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu) 149static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
142{ 150{
143 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_ISV); 151 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
144} 152}
145 153
146static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) 154static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
147{ 155{
148 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_WNR); 156 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR);
149} 157}
150 158
151static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu) 159static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
152{ 160{
153 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SSE); 161 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE);
154} 162}
155 163
156static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) 164static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
157{ 165{
158 return (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SRT_MASK) >> ESR_EL2_SRT_SHIFT; 166 return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
159} 167}
160 168
161static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu) 169static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
162{ 170{
163 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EA); 171 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_EA);
164} 172}
165 173
166static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu) 174static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu)
167{ 175{
168 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_S1PTW); 176 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW);
169} 177}
170 178
171static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) 179static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
172{ 180{
173 return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SAS) >> ESR_EL2_SAS_SHIFT); 181 return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
174} 182}
175 183
176/* This one is not specific to Data Abort */ 184/* This one is not specific to Data Abort */
177static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu) 185static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
178{ 186{
179 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_IL); 187 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL);
180} 188}
181 189
182static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) 190static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
183{ 191{
184 return kvm_vcpu_get_hsr(vcpu) >> ESR_EL2_EC_SHIFT; 192 return kvm_vcpu_get_hsr(vcpu) >> ESR_ELx_EC_SHIFT;
185} 193}
186 194
187static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) 195static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
188{ 196{
189 return kvm_vcpu_trap_get_class(vcpu) == ESR_EL2_EC_IABT; 197 return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
190} 198}
191 199
192static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) 200static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
193{ 201{
194 return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC; 202 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC;
195} 203}
196 204
197static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu) 205static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
198{ 206{
199 return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC_TYPE; 207 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
200} 208}
201 209
202static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu) 210static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
203{ 211{
204 return vcpu_sys_reg(vcpu, MPIDR_EL1); 212 return vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
205} 213}
206 214
207static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) 215static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index acd101a9014d..8ac3c70fe3c6 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -59,6 +59,9 @@ struct kvm_arch {
59 /* VTTBR value associated with above pgd and vmid */ 59 /* VTTBR value associated with above pgd and vmid */
60 u64 vttbr; 60 u64 vttbr;
61 61
62 /* The maximum number of vCPUs depends on the used GIC model */
63 int max_vcpus;
64
62 /* Interrupt controller */ 65 /* Interrupt controller */
63 struct vgic_dist vgic; 66 struct vgic_dist vgic;
64 67
@@ -159,6 +162,7 @@ struct kvm_vm_stat {
159}; 162};
160 163
161struct kvm_vcpu_stat { 164struct kvm_vcpu_stat {
165 u32 halt_successful_poll;
162 u32 halt_wakeup; 166 u32 halt_wakeup;
163}; 167};
164 168
@@ -196,6 +200,7 @@ struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
196 200
197u64 kvm_call_hyp(void *hypfn, ...); 201u64 kvm_call_hyp(void *hypfn, ...);
198void force_vm_exit(const cpumask_t *mask); 202void force_vm_exit(const cpumask_t *mask);
203void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
199 204
200int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, 205int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
201 int exception_index); 206 int exception_index);
@@ -203,6 +208,8 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
203int kvm_perf_init(void); 208int kvm_perf_init(void);
204int kvm_perf_teardown(void); 209int kvm_perf_teardown(void);
205 210
211struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
212
206static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr, 213static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
207 phys_addr_t pgd_ptr, 214 phys_addr_t pgd_ptr,
208 unsigned long hyp_stack_ptr, 215 unsigned long hyp_stack_ptr,
diff --git a/arch/arm64/include/asm/kvm_mmio.h b/arch/arm64/include/asm/kvm_mmio.h
index fc2f689c0694..9f52beb7cb13 100644
--- a/arch/arm64/include/asm/kvm_mmio.h
+++ b/arch/arm64/include/asm/kvm_mmio.h
@@ -40,6 +40,7 @@ struct kvm_exit_mmio {
40 u8 data[8]; 40 u8 data[8];
41 u32 len; 41 u32 len;
42 bool is_write; 42 bool is_write;
43 void *private;
43}; 44};
44 45
45static inline void kvm_prepare_mmio(struct kvm_run *run, 46static inline void kvm_prepare_mmio(struct kvm_run *run,
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index adcf49547301..6458b5373142 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -118,6 +118,27 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
118 pmd_val(*pmd) |= PMD_S2_RDWR; 118 pmd_val(*pmd) |= PMD_S2_RDWR;
119} 119}
120 120
121static inline void kvm_set_s2pte_readonly(pte_t *pte)
122{
123 pte_val(*pte) = (pte_val(*pte) & ~PTE_S2_RDWR) | PTE_S2_RDONLY;
124}
125
126static inline bool kvm_s2pte_readonly(pte_t *pte)
127{
128 return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY;
129}
130
131static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
132{
133 pmd_val(*pmd) = (pmd_val(*pmd) & ~PMD_S2_RDWR) | PMD_S2_RDONLY;
134}
135
136static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
137{
138 return (pmd_val(*pmd) & PMD_S2_RDWR) == PMD_S2_RDONLY;
139}
140
141
121#define kvm_pgd_addr_end(addr, end) pgd_addr_end(addr, end) 142#define kvm_pgd_addr_end(addr, end) pgd_addr_end(addr, end)
122#define kvm_pud_addr_end(addr, end) pud_addr_end(addr, end) 143#define kvm_pud_addr_end(addr, end) pud_addr_end(addr, end)
123#define kvm_pmd_addr_end(addr, end) pmd_addr_end(addr, end) 144#define kvm_pmd_addr_end(addr, end) pmd_addr_end(addr, end)
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 6486b2bfd562..f800d45ea226 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -33,6 +33,12 @@
33#define UL(x) _AC(x, UL) 33#define UL(x) _AC(x, UL)
34 34
35/* 35/*
36 * Size of the PCI I/O space. This must remain a power of two so that
37 * IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses.
38 */
39#define PCI_IO_SIZE SZ_16M
40
41/*
36 * PAGE_OFFSET - the virtual address of the start of the kernel image (top 42 * PAGE_OFFSET - the virtual address of the start of the kernel image (top
37 * (VA_BITS - 1)) 43 * (VA_BITS - 1))
38 * VA_BITS - the maximum number of bits for virtual addresses. 44 * VA_BITS - the maximum number of bits for virtual addresses.
@@ -45,7 +51,9 @@
45#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) 51#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
46#define MODULES_END (PAGE_OFFSET) 52#define MODULES_END (PAGE_OFFSET)
47#define MODULES_VADDR (MODULES_END - SZ_64M) 53#define MODULES_VADDR (MODULES_END - SZ_64M)
48#define FIXADDR_TOP (MODULES_VADDR - SZ_2M - PAGE_SIZE) 54#define PCI_IO_END (MODULES_VADDR - SZ_2M)
55#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
56#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
49#define TASK_SIZE_64 (UL(1) << VA_BITS) 57#define TASK_SIZE_64 (UL(1) << VA_BITS)
50 58
51#ifdef CONFIG_COMPAT 59#ifdef CONFIG_COMPAT
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index c2f006c48bdb..3d311761e3c2 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -31,7 +31,8 @@ extern void paging_init(void);
31extern void setup_mm_for_reboot(void); 31extern void setup_mm_for_reboot(void);
32extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); 32extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
33extern void init_mem_pgprot(void); 33extern void init_mem_pgprot(void);
34/* create an identity mapping for memory (or io if map_io is true) */ 34extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
35extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io); 35 unsigned long virt, phys_addr_t size,
36 pgprot_t prot);
36 37
37#endif 38#endif
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 88174e0bfafe..5f930cc9ea83 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -119,6 +119,7 @@
119#define PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[2:1] */ 119#define PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[2:1] */
120#define PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */ 120#define PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */
121 121
122#define PMD_S2_RDONLY (_AT(pmdval_t, 1) << 6) /* HAP[2:1] */
122#define PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */ 123#define PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */
123 124
124/* 125/*
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 210d632aa5ad..16449c535e50 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -25,7 +25,6 @@
25 * Software defined PTE bits definition. 25 * Software defined PTE bits definition.
26 */ 26 */
27#define PTE_VALID (_AT(pteval_t, 1) << 0) 27#define PTE_VALID (_AT(pteval_t, 1) << 0)
28#define PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !pte_present() */
29#define PTE_DIRTY (_AT(pteval_t, 1) << 55) 28#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
30#define PTE_SPECIAL (_AT(pteval_t, 1) << 56) 29#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
31#define PTE_WRITE (_AT(pteval_t, 1) << 57) 30#define PTE_WRITE (_AT(pteval_t, 1) << 57)
@@ -46,7 +45,7 @@
46 45
47#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K)) 46#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
48 47
49#define FIRST_USER_ADDRESS 0 48#define FIRST_USER_ADDRESS 0UL
50 49
51#ifndef __ASSEMBLY__ 50#ifndef __ASSEMBLY__
52extern void __pte_error(const char *file, int line, unsigned long val); 51extern void __pte_error(const char *file, int line, unsigned long val);
@@ -264,6 +263,11 @@ static inline pmd_t pte_pmd(pte_t pte)
264 return __pmd(pte_val(pte)); 263 return __pmd(pte_val(pte));
265} 264}
266 265
266static inline pgprot_t mk_sect_prot(pgprot_t prot)
267{
268 return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
269}
270
267/* 271/*
268 * THP definitions. 272 * THP definitions.
269 */ 273 */
@@ -337,9 +341,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
337 341
338#ifdef CONFIG_ARM64_64K_PAGES 342#ifdef CONFIG_ARM64_64K_PAGES
339#define pud_sect(pud) (0) 343#define pud_sect(pud) (0)
344#define pud_table(pud) (1)
340#else 345#else
341#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ 346#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
342 PUD_TYPE_SECT) 347 PUD_TYPE_SECT)
348#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
349 PUD_TYPE_TABLE)
343#endif 350#endif
344 351
345static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 352static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
@@ -469,13 +476,12 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
469/* 476/*
470 * Encode and decode a swap entry: 477 * Encode and decode a swap entry:
471 * bits 0-1: present (must be zero) 478 * bits 0-1: present (must be zero)
472 * bit 2: PTE_FILE 479 * bits 2-7: swap type
473 * bits 3-8: swap type 480 * bits 8-57: swap offset
474 * bits 9-57: swap offset
475 */ 481 */
476#define __SWP_TYPE_SHIFT 3 482#define __SWP_TYPE_SHIFT 2
477#define __SWP_TYPE_BITS 6 483#define __SWP_TYPE_BITS 6
478#define __SWP_OFFSET_BITS 49 484#define __SWP_OFFSET_BITS 50
479#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) 485#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
480#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) 486#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
481#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) 487#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1)
@@ -493,18 +499,6 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
493 */ 499 */
494#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) 500#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
495 501
496/*
497 * Encode and decode a file entry:
498 * bits 0-1: present (must be zero)
499 * bit 2: PTE_FILE
500 * bits 3-57: file offset / PAGE_SIZE
501 */
502#define pte_file(pte) (pte_val(pte) & PTE_FILE)
503#define pte_to_pgoff(x) (pte_val(x) >> 3)
504#define pgoff_to_pte(x) __pte(((x) << 3) | PTE_FILE)
505
506#define PTE_FILE_MAX_BITS 55
507
508extern int kern_addr_valid(unsigned long addr); 502extern int kern_addr_valid(unsigned long addr);
509 503
510#include <asm-generic/pgtable.h> 504#include <asm-generic/pgtable.h>
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index 41ed9e13795e..d6dd9fdbc3be 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -58,6 +58,13 @@
58#define COMPAT_PSR_Z_BIT 0x40000000 58#define COMPAT_PSR_Z_BIT 0x40000000
59#define COMPAT_PSR_N_BIT 0x80000000 59#define COMPAT_PSR_N_BIT 0x80000000
60#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ 60#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
61
62#ifdef CONFIG_CPU_BIG_ENDIAN
63#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT
64#else
65#define COMPAT_PSR_ENDSTATE 0
66#endif
67
61/* 68/*
62 * These are 'magic' values for PTRACE_PEEKUSR that return info about where a 69 * These are 'magic' values for PTRACE_PEEKUSR that return info about where a
63 * process is located in memory. 70 * process is located in memory.
diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h
index 456d67c1f0fa..003802f58963 100644
--- a/arch/arm64/include/asm/suspend.h
+++ b/arch/arm64/include/asm/suspend.h
@@ -23,6 +23,4 @@ struct sleep_save_sp {
23 23
24extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); 24extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
25extern void cpu_resume(void); 25extern void cpu_resume(void);
26extern int cpu_suspend(unsigned long);
27
28#endif 26#endif
diff --git a/arch/arm64/include/asm/syscalls.h b/arch/arm64/include/asm/syscalls.h
deleted file mode 100644
index 48fe7c600e98..000000000000
--- a/arch/arm64/include/asm/syscalls.h
+++ /dev/null
@@ -1,30 +0,0 @@
1/*
2 * Copyright (C) 2012 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_SYSCALLS_H
17#define __ASM_SYSCALLS_H
18
19#include <linux/linkage.h>
20#include <linux/compiler.h>
21#include <linux/signal.h>
22
23/*
24 * System call wrappers implemented in kernel/entry.S.
25 */
26asmlinkage long sys_rt_sigreturn_wrapper(void);
27
28#include <asm-generic/syscalls.h>
29
30#endif /* __ASM_SYSCALLS_H */
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 459bf8e53208..702e1e6a0d80 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -48,7 +48,6 @@ struct thread_info {
48 mm_segment_t addr_limit; /* address limit */ 48 mm_segment_t addr_limit; /* address limit */
49 struct task_struct *task; /* main task structure */ 49 struct task_struct *task; /* main task structure */
50 struct exec_domain *exec_domain; /* execution domain */ 50 struct exec_domain *exec_domain; /* execution domain */
51 struct restart_block restart_block;
52 int preempt_count; /* 0 => preemptable, <0 => bug */ 51 int preempt_count; /* 0 => preemptable, <0 => bug */
53 int cpu; /* cpu */ 52 int cpu; /* cpu */
54}; 53};
@@ -60,9 +59,6 @@ struct thread_info {
60 .flags = 0, \ 59 .flags = 0, \
61 .preempt_count = INIT_PREEMPT_COUNT, \ 60 .preempt_count = INIT_PREEMPT_COUNT, \
62 .addr_limit = KERNEL_DS, \ 61 .addr_limit = KERNEL_DS, \
63 .restart_block = { \
64 .fn = do_no_restart_syscall, \
65 }, \
66} 62}
67 63
68#define init_thread_info (init_thread_union.thread_info) 64#define init_thread_info (init_thread_union.thread_info)
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 3bf8f4e99a51..07e1ba449bf1 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -63,7 +63,7 @@ static inline void set_fs(mm_segment_t fs)
63 current_thread_info()->addr_limit = fs; 63 current_thread_info()->addr_limit = fs;
64} 64}
65 65
66#define segment_eq(a,b) ((a) == (b)) 66#define segment_eq(a, b) ((a) == (b))
67 67
68/* 68/*
69 * Return 1 if addr < current->addr_limit, 0 otherwise. 69 * Return 1 if addr < current->addr_limit, 0 otherwise.
@@ -147,7 +147,7 @@ do { \
147 default: \ 147 default: \
148 BUILD_BUG(); \ 148 BUILD_BUG(); \
149 } \ 149 } \
150 (x) = (__typeof__(*(ptr)))__gu_val; \ 150 (x) = (__force __typeof__(*(ptr)))__gu_val; \
151} while (0) 151} while (0)
152 152
153#define __get_user(x, ptr) \ 153#define __get_user(x, ptr) \
diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h
index 23e9432ac112..3bc498c250dc 100644
--- a/arch/arm64/include/asm/unistd.h
+++ b/arch/arm64/include/asm/unistd.h
@@ -48,6 +48,9 @@
48#endif 48#endif
49 49
50#define __ARCH_WANT_SYS_CLONE 50#define __ARCH_WANT_SYS_CLONE
51
52#ifndef __COMPAT_SYSCALL_NR
51#include <uapi/asm/unistd.h> 53#include <uapi/asm/unistd.h>
54#endif
52 55
53#define NR_syscalls (__NR_syscalls) 56#define NR_syscalls (__NR_syscalls)
diff --git a/arch/arm64/include/uapi/asm/Kbuild b/arch/arm64/include/uapi/asm/Kbuild
index 942376d37d22..825b0fe51c2b 100644
--- a/arch/arm64/include/uapi/asm/Kbuild
+++ b/arch/arm64/include/uapi/asm/Kbuild
@@ -18,4 +18,5 @@ header-y += siginfo.h
18header-y += signal.h 18header-y += signal.h
19header-y += stat.h 19header-y += stat.h
20header-y += statfs.h 20header-y += statfs.h
21header-y += ucontext.h
21header-y += unistd.h 22header-y += unistd.h
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 8e38878c87c6..3ef77a466018 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -78,6 +78,13 @@ struct kvm_regs {
78#define KVM_VGIC_V2_DIST_SIZE 0x1000 78#define KVM_VGIC_V2_DIST_SIZE 0x1000
79#define KVM_VGIC_V2_CPU_SIZE 0x2000 79#define KVM_VGIC_V2_CPU_SIZE 0x2000
80 80
81/* Supported VGICv3 address types */
82#define KVM_VGIC_V3_ADDR_TYPE_DIST 2
83#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3
84
85#define KVM_VGIC_V3_DIST_SIZE SZ_64K
86#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K)
87
81#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ 88#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */
82#define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ 89#define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */
83#define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ 90#define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */
@@ -161,6 +168,8 @@ struct kvm_arch_memory_slot {
161#define KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0 168#define KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0
162#define KVM_DEV_ARM_VGIC_OFFSET_MASK (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) 169#define KVM_DEV_ARM_VGIC_OFFSET_MASK (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
163#define KVM_DEV_ARM_VGIC_GRP_NR_IRQS 3 170#define KVM_DEV_ARM_VGIC_GRP_NR_IRQS 3
171#define KVM_DEV_ARM_VGIC_GRP_CTRL 4
172#define KVM_DEV_ARM_VGIC_CTRL_INIT 0
164 173
165/* KVM_IRQ_LINE irq field index values */ 174/* KVM_IRQ_LINE irq field index values */
166#define KVM_ARM_IRQ_TYPE_SHIFT 24 175#define KVM_ARM_IRQ_TYPE_SHIFT 24
diff --git a/arch/arm64/include/asm/ucontext.h b/arch/arm64/include/uapi/asm/ucontext.h
index 42e04c877428..791de8e89e35 100644
--- a/arch/arm64/include/asm/ucontext.h
+++ b/arch/arm64/include/uapi/asm/ucontext.h
@@ -13,8 +13,10 @@
13 * You should have received a copy of the GNU General Public License 13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */ 15 */
16#ifndef __ASM_UCONTEXT_H 16#ifndef _UAPI__ASM_UCONTEXT_H
17#define __ASM_UCONTEXT_H 17#define _UAPI__ASM_UCONTEXT_H
18
19#include <linux/types.h>
18 20
19struct ucontext { 21struct ucontext {
20 unsigned long uc_flags; 22 unsigned long uc_flags;
@@ -27,4 +29,4 @@ struct ucontext {
27 struct sigcontext uc_mcontext; 29 struct sigcontext uc_mcontext;
28}; 30};
29 31
30#endif /* __ASM_UCONTEXT_H */ 32#endif /* _UAPI__ASM_UCONTEXT_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index eaa77ed7766a..bef04afd6031 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -16,10 +16,10 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
16 entry-fpsimd.o process.o ptrace.o setup.o signal.o \ 16 entry-fpsimd.o process.o ptrace.o setup.o signal.o \
17 sys.o stacktrace.o time.o traps.o io.o vdso.o \ 17 sys.o stacktrace.o time.o traps.o io.o vdso.o \
18 hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \ 18 hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \
19 cpuinfo.o cpu_errata.o alternative.o 19 cpuinfo.o cpu_errata.o alternative.o cacheinfo.o
20 20
21arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ 21arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
22 sys_compat.o \ 22 sys_compat.o entry32.o \
23 ../../arm/kernel/opcodes.o 23 ../../arm/kernel/opcodes.o
24arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o 24arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
25arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o 25arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
@@ -27,7 +27,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o
27arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o 27arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
28arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o 28arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
29arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o 29arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
30arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o 30arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o
31arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o 31arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o
32arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o 32arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
33arm64-obj-$(CONFIG_KGDB) += kgdb.o 33arm64-obj-$(CONFIG_KGDB) += kgdb.o
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index c363671d7509..7922c2e710ca 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -19,6 +19,7 @@
19#include <asm/system_misc.h> 19#include <asm/system_misc.h>
20#include <asm/traps.h> 20#include <asm/traps.h>
21#include <asm/uaccess.h> 21#include <asm/uaccess.h>
22#include <asm/cpufeature.h>
22 23
23#define CREATE_TRACE_POINTS 24#define CREATE_TRACE_POINTS
24#include "trace-events-emulation.h" 25#include "trace-events-emulation.h"
@@ -85,6 +86,57 @@ static void remove_emulation_hooks(struct insn_emulation_ops *ops)
85 pr_notice("Removed %s emulation handler\n", ops->name); 86 pr_notice("Removed %s emulation handler\n", ops->name);
86} 87}
87 88
89static void enable_insn_hw_mode(void *data)
90{
91 struct insn_emulation *insn = (struct insn_emulation *)data;
92 if (insn->ops->set_hw_mode)
93 insn->ops->set_hw_mode(true);
94}
95
96static void disable_insn_hw_mode(void *data)
97{
98 struct insn_emulation *insn = (struct insn_emulation *)data;
99 if (insn->ops->set_hw_mode)
100 insn->ops->set_hw_mode(false);
101}
102
103/* Run set_hw_mode(mode) on all active CPUs */
104static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
105{
106 if (!insn->ops->set_hw_mode)
107 return -EINVAL;
108 if (enable)
109 on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
110 else
111 on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
112 return 0;
113}
114
115/*
116 * Run set_hw_mode for all insns on a starting CPU.
117 * Returns:
118 * 0 - If all the hooks ran successfully.
119 * -EINVAL - At least one hook is not supported by the CPU.
120 */
121static int run_all_insn_set_hw_mode(unsigned long cpu)
122{
123 int rc = 0;
124 unsigned long flags;
125 struct insn_emulation *insn;
126
127 raw_spin_lock_irqsave(&insn_emulation_lock, flags);
128 list_for_each_entry(insn, &insn_emulation, node) {
129 bool enable = (insn->current_mode == INSN_HW);
130 if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
131 pr_warn("CPU[%ld] cannot support the emulation of %s",
132 cpu, insn->ops->name);
133 rc = -EINVAL;
134 }
135 }
136 raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
137 return rc;
138}
139
88static int update_insn_emulation_mode(struct insn_emulation *insn, 140static int update_insn_emulation_mode(struct insn_emulation *insn,
89 enum insn_emulation_mode prev) 141 enum insn_emulation_mode prev)
90{ 142{
@@ -97,10 +149,8 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
97 remove_emulation_hooks(insn->ops); 149 remove_emulation_hooks(insn->ops);
98 break; 150 break;
99 case INSN_HW: 151 case INSN_HW:
100 if (insn->ops->set_hw_mode) { 152 if (!run_all_cpu_set_hw_mode(insn, false))
101 insn->ops->set_hw_mode(false);
102 pr_notice("Disabled %s support\n", insn->ops->name); 153 pr_notice("Disabled %s support\n", insn->ops->name);
103 }
104 break; 154 break;
105 } 155 }
106 156
@@ -111,10 +161,9 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
111 register_emulation_hooks(insn->ops); 161 register_emulation_hooks(insn->ops);
112 break; 162 break;
113 case INSN_HW: 163 case INSN_HW:
114 if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) 164 ret = run_all_cpu_set_hw_mode(insn, true);
165 if (!ret)
115 pr_notice("Enabled %s support\n", insn->ops->name); 166 pr_notice("Enabled %s support\n", insn->ops->name);
116 else
117 ret = -EINVAL;
118 break; 167 break;
119 } 168 }
120 169
@@ -133,6 +182,8 @@ static void register_insn_emulation(struct insn_emulation_ops *ops)
133 switch (ops->status) { 182 switch (ops->status) {
134 case INSN_DEPRECATED: 183 case INSN_DEPRECATED:
135 insn->current_mode = INSN_EMULATE; 184 insn->current_mode = INSN_EMULATE;
185 /* Disable the HW mode if it was turned on at early boot time */
186 run_all_cpu_set_hw_mode(insn, false);
136 insn->max = INSN_HW; 187 insn->max = INSN_HW;
137 break; 188 break;
138 case INSN_OBSOLETE: 189 case INSN_OBSOLETE:
@@ -453,8 +504,6 @@ ret:
453 return 0; 504 return 0;
454} 505}
455 506
456#define SCTLR_EL1_CP15BEN (1 << 5)
457
458static inline void config_sctlr_el1(u32 clear, u32 set) 507static inline void config_sctlr_el1(u32 clear, u32 set)
459{ 508{
460 u32 val; 509 u32 val;
@@ -465,48 +514,13 @@ static inline void config_sctlr_el1(u32 clear, u32 set)
465 asm volatile("msr sctlr_el1, %0" : : "r" (val)); 514 asm volatile("msr sctlr_el1, %0" : : "r" (val));
466} 515}
467 516
468static void enable_cp15_ben(void *info)
469{
470 config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
471}
472
473static void disable_cp15_ben(void *info)
474{
475 config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
476}
477
478static int cpu_hotplug_notify(struct notifier_block *b,
479 unsigned long action, void *hcpu)
480{
481 switch (action) {
482 case CPU_STARTING:
483 case CPU_STARTING_FROZEN:
484 enable_cp15_ben(NULL);
485 return NOTIFY_DONE;
486 case CPU_DYING:
487 case CPU_DYING_FROZEN:
488 disable_cp15_ben(NULL);
489 return NOTIFY_DONE;
490 }
491
492 return NOTIFY_OK;
493}
494
495static struct notifier_block cpu_hotplug_notifier = {
496 .notifier_call = cpu_hotplug_notify,
497};
498
499static int cp15_barrier_set_hw_mode(bool enable) 517static int cp15_barrier_set_hw_mode(bool enable)
500{ 518{
501 if (enable) { 519 if (enable)
502 register_cpu_notifier(&cpu_hotplug_notifier); 520 config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
503 on_each_cpu(enable_cp15_ben, NULL, true); 521 else
504 } else { 522 config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
505 unregister_cpu_notifier(&cpu_hotplug_notifier); 523 return 0;
506 on_each_cpu(disable_cp15_ben, NULL, true);
507 }
508
509 return true;
510} 524}
511 525
512static struct undef_hook cp15_barrier_hooks[] = { 526static struct undef_hook cp15_barrier_hooks[] = {
@@ -534,6 +548,93 @@ static struct insn_emulation_ops cp15_barrier_ops = {
534 .set_hw_mode = cp15_barrier_set_hw_mode, 548 .set_hw_mode = cp15_barrier_set_hw_mode,
535}; 549};
536 550
551static int setend_set_hw_mode(bool enable)
552{
553 if (!cpu_supports_mixed_endian_el0())
554 return -EINVAL;
555
556 if (enable)
557 config_sctlr_el1(SCTLR_EL1_SED, 0);
558 else
559 config_sctlr_el1(0, SCTLR_EL1_SED);
560 return 0;
561}
562
563static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
564{
565 char *insn;
566
567 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
568
569 if (big_endian) {
570 insn = "setend be";
571 regs->pstate |= COMPAT_PSR_E_BIT;
572 } else {
573 insn = "setend le";
574 regs->pstate &= ~COMPAT_PSR_E_BIT;
575 }
576
577 trace_instruction_emulation(insn, regs->pc);
578 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
579 current->comm, (unsigned long)current->pid, regs->pc);
580
581 return 0;
582}
583
584static int a32_setend_handler(struct pt_regs *regs, u32 instr)
585{
586 int rc = compat_setend_handler(regs, (instr >> 9) & 1);
587 regs->pc += 4;
588 return rc;
589}
590
591static int t16_setend_handler(struct pt_regs *regs, u32 instr)
592{
593 int rc = compat_setend_handler(regs, (instr >> 3) & 1);
594 regs->pc += 2;
595 return rc;
596}
597
598static struct undef_hook setend_hooks[] = {
599 {
600 .instr_mask = 0xfffffdff,
601 .instr_val = 0xf1010000,
602 .pstate_mask = COMPAT_PSR_MODE_MASK,
603 .pstate_val = COMPAT_PSR_MODE_USR,
604 .fn = a32_setend_handler,
605 },
606 {
607 /* Thumb mode */
608 .instr_mask = 0x0000fff7,
609 .instr_val = 0x0000b650,
610 .pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK),
611 .pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR),
612 .fn = t16_setend_handler,
613 },
614 {}
615};
616
617static struct insn_emulation_ops setend_ops = {
618 .name = "setend",
619 .status = INSN_DEPRECATED,
620 .hooks = setend_hooks,
621 .set_hw_mode = setend_set_hw_mode,
622};
623
624static int insn_cpu_hotplug_notify(struct notifier_block *b,
625 unsigned long action, void *hcpu)
626{
627 int rc = 0;
628 if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING)
629 rc = run_all_insn_set_hw_mode((unsigned long)hcpu);
630
631 return notifier_from_errno(rc);
632}
633
634static struct notifier_block insn_cpu_hotplug_notifier = {
635 .notifier_call = insn_cpu_hotplug_notify,
636};
637
537/* 638/*
538 * Invoked as late_initcall, since not needed before init spawned. 639 * Invoked as late_initcall, since not needed before init spawned.
539 */ 640 */
@@ -545,6 +646,14 @@ static int __init armv8_deprecated_init(void)
545 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) 646 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
546 register_insn_emulation(&cp15_barrier_ops); 647 register_insn_emulation(&cp15_barrier_ops);
547 648
649 if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
650 if(system_supports_mixed_endian_el0())
651 register_insn_emulation(&setend_ops);
652 else
653 pr_info("setend instruction emulation is not supported on the system");
654 }
655
656 register_cpu_notifier(&insn_cpu_hotplug_notifier);
548 register_insn_emulation_sysctl(ctl_abi); 657 register_insn_emulation_sysctl(ctl_abi);
549 658
550 return 0; 659 return 0;
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 9a9fce090d58..f7fa65d4c352 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -140,6 +140,7 @@ int main(void)
140 DEFINE(VGIC_V2_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr)); 140 DEFINE(VGIC_V2_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr));
141 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr)); 141 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
142 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr)); 142 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
143 DEFINE(VGIC_V3_CPU_SRE, offsetof(struct vgic_cpu, vgic_v3.vgic_sre));
143 DEFINE(VGIC_V3_CPU_HCR, offsetof(struct vgic_cpu, vgic_v3.vgic_hcr)); 144 DEFINE(VGIC_V3_CPU_HCR, offsetof(struct vgic_cpu, vgic_v3.vgic_hcr));
144 DEFINE(VGIC_V3_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr)); 145 DEFINE(VGIC_V3_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr));
145 DEFINE(VGIC_V3_CPU_MISR, offsetof(struct vgic_cpu, vgic_v3.vgic_misr)); 146 DEFINE(VGIC_V3_CPU_MISR, offsetof(struct vgic_cpu, vgic_v3.vgic_misr));
@@ -152,7 +153,7 @@ int main(void)
152 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); 153 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
153 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); 154 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
154#endif 155#endif
155#ifdef CONFIG_ARM64_CPU_SUSPEND 156#ifdef CONFIG_CPU_PM
156 DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx)); 157 DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
157 DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); 158 DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
158 DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask)); 159 DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));
diff --git a/arch/arm64/kernel/cacheinfo.c b/arch/arm64/kernel/cacheinfo.c
new file mode 100644
index 000000000000..b8629d52fba9
--- /dev/null
+++ b/arch/arm64/kernel/cacheinfo.c
@@ -0,0 +1,128 @@
1/*
2 * ARM64 cacheinfo support
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 * All Rights Reserved
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
12 * kind, whether express or implied; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/bitops.h>
21#include <linux/cacheinfo.h>
22#include <linux/cpu.h>
23#include <linux/compiler.h>
24#include <linux/of.h>
25
26#include <asm/cachetype.h>
27#include <asm/processor.h>
28
29#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */
30/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */
31#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1))
32#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level))
33#define CLIDR_CTYPE(clidr, level) \
34 (((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level))
35
36static inline enum cache_type get_cache_type(int level)
37{
38 u64 clidr;
39
40 if (level > MAX_CACHE_LEVEL)
41 return CACHE_TYPE_NOCACHE;
42 asm volatile ("mrs %x0, clidr_el1" : "=r" (clidr));
43 return CLIDR_CTYPE(clidr, level);
44}
45
46/*
47 * Cache Size Selection Register(CSSELR) selects which Cache Size ID
48 * Register(CCSIDR) is accessible by specifying the required cache
49 * level and the cache type. We need to ensure that no one else changes
50 * CSSELR by calling this in non-preemtible context
51 */
52u64 __attribute_const__ cache_get_ccsidr(u64 csselr)
53{
54 u64 ccsidr;
55
56 WARN_ON(preemptible());
57
58 /* Put value into CSSELR */
59 asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
60 isb();
61 /* Read result out of CCSIDR */
62 asm volatile("mrs %x0, ccsidr_el1" : "=r" (ccsidr));
63
64 return ccsidr;
65}
66
67static void ci_leaf_init(struct cacheinfo *this_leaf,
68 enum cache_type type, unsigned int level)
69{
70 bool is_icache = type & CACHE_TYPE_INST;
71 u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache);
72
73 this_leaf->level = level;
74 this_leaf->type = type;
75 this_leaf->coherency_line_size = CACHE_LINESIZE(tmp);
76 this_leaf->number_of_sets = CACHE_NUMSETS(tmp);
77 this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp);
78 this_leaf->size = this_leaf->number_of_sets *
79 this_leaf->coherency_line_size * this_leaf->ways_of_associativity;
80 this_leaf->attributes =
81 ((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) |
82 ((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) |
83 ((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) |
84 ((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0);
85}
86
87static int __init_cache_level(unsigned int cpu)
88{
89 unsigned int ctype, level, leaves;
90 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
91
92 for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) {
93 ctype = get_cache_type(level);
94 if (ctype == CACHE_TYPE_NOCACHE) {
95 level--;
96 break;
97 }
98 /* Separate instruction and data caches */
99 leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1;
100 }
101
102 this_cpu_ci->num_levels = level;
103 this_cpu_ci->num_leaves = leaves;
104 return 0;
105}
106
107static int __populate_cache_leaves(unsigned int cpu)
108{
109 unsigned int level, idx;
110 enum cache_type type;
111 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
112 struct cacheinfo *this_leaf = this_cpu_ci->info_list;
113
114 for (idx = 0, level = 1; level <= this_cpu_ci->num_levels &&
115 idx < this_cpu_ci->num_leaves; idx++, level++) {
116 type = get_cache_type(level);
117 if (type == CACHE_TYPE_SEPARATE) {
118 ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level);
119 ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level);
120 } else {
121 ci_leaf_init(this_leaf++, type, level);
122 }
123 }
124 return 0;
125}
126
127DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level)
128DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves)
diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c
index 19d17f51db37..5c0896647fd1 100644
--- a/arch/arm64/kernel/cpuidle.c
+++ b/arch/arm64/kernel/cpuidle.c
@@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu)
29 of_node_put(cpu_node); 29 of_node_put(cpu_node);
30 return ret; 30 return ret;
31} 31}
32
33/**
34 * cpu_suspend() - function to enter a low-power idle state
35 * @arg: argument to pass to CPU suspend operations
36 *
37 * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
38 * operations back-end error code otherwise.
39 */
40int cpu_suspend(unsigned long arg)
41{
42 int cpu = smp_processor_id();
43
44 /*
45 * If cpu_ops have not been registered or suspend
46 * has not been initialized, cpu_suspend call fails early.
47 */
48 if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
49 return -EOPNOTSUPP;
50 return cpu_ops[cpu]->cpu_suspend(arg);
51}
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 07d435cf2eea..929855691dae 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -35,6 +35,7 @@
35 */ 35 */
36DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data); 36DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
37static struct cpuinfo_arm64 boot_cpu_data; 37static struct cpuinfo_arm64 boot_cpu_data;
38static bool mixed_endian_el0 = true;
38 39
39static char *icache_policy_str[] = { 40static char *icache_policy_str[] = {
40 [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", 41 [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN",
@@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
68 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); 69 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
69} 70}
70 71
72bool cpu_supports_mixed_endian_el0(void)
73{
74 return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
75}
76
77bool system_supports_mixed_endian_el0(void)
78{
79 return mixed_endian_el0;
80}
81
82static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info)
83{
84 mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0);
85}
86
87static void update_cpu_features(struct cpuinfo_arm64 *info)
88{
89 update_mixed_endian_el0_support(info);
90}
91
71static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu) 92static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu)
72{ 93{
73 if ((boot & mask) == (cur & mask)) 94 if ((boot & mask) == (cur & mask))
@@ -215,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
215 cpuinfo_detect_icache_policy(info); 236 cpuinfo_detect_icache_policy(info);
216 237
217 check_local_cpu_errata(); 238 check_local_cpu_errata();
239 update_cpu_features(info);
218} 240}
219 241
220void cpuinfo_store_cpu(void) 242void cpuinfo_store_cpu(void)
@@ -231,15 +253,3 @@ void __init cpuinfo_store_boot_cpu(void)
231 253
232 boot_cpu_data = *info; 254 boot_cpu_data = *info;
233} 255}
234
235u64 __attribute_const__ icache_get_ccsidr(void)
236{
237 u64 ccsidr;
238
239 WARN_ON(preemptible());
240
241 /* Select L1 I-cache and read its size ID register */
242 asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1"
243 : "=r"(ccsidr) : "r"(1L));
244 return ccsidr;
245}
diff --git a/arch/arm64/kernel/efi-stub.c b/arch/arm64/kernel/efi-stub.c
index d27dd982ff26..f5374065ad53 100644
--- a/arch/arm64/kernel/efi-stub.c
+++ b/arch/arm64/kernel/efi-stub.c
@@ -13,13 +13,13 @@
13#include <asm/efi.h> 13#include <asm/efi.h>
14#include <asm/sections.h> 14#include <asm/sections.h>
15 15
16efi_status_t handle_kernel_image(efi_system_table_t *sys_table, 16efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table,
17 unsigned long *image_addr, 17 unsigned long *image_addr,
18 unsigned long *image_size, 18 unsigned long *image_size,
19 unsigned long *reserve_addr, 19 unsigned long *reserve_addr,
20 unsigned long *reserve_size, 20 unsigned long *reserve_size,
21 unsigned long dram_base, 21 unsigned long dram_base,
22 efi_loaded_image_t *image) 22 efi_loaded_image_t *image)
23{ 23{
24 efi_status_t status; 24 efi_status_t status;
25 unsigned long kernel_size, kernel_memsize = 0; 25 unsigned long kernel_size, kernel_memsize = 0;
diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index 2bb4347d0edf..b42c7b480e1e 100644
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -11,27 +11,46 @@
11 * 11 *
12 */ 12 */
13 13
14#include <linux/atomic.h>
14#include <linux/dmi.h> 15#include <linux/dmi.h>
15#include <linux/efi.h> 16#include <linux/efi.h>
16#include <linux/export.h> 17#include <linux/export.h>
17#include <linux/memblock.h> 18#include <linux/memblock.h>
19#include <linux/mm_types.h>
18#include <linux/bootmem.h> 20#include <linux/bootmem.h>
19#include <linux/of.h> 21#include <linux/of.h>
20#include <linux/of_fdt.h> 22#include <linux/of_fdt.h>
23#include <linux/preempt.h>
24#include <linux/rbtree.h>
25#include <linux/rwsem.h>
21#include <linux/sched.h> 26#include <linux/sched.h>
22#include <linux/slab.h> 27#include <linux/slab.h>
28#include <linux/spinlock.h>
23 29
24#include <asm/cacheflush.h> 30#include <asm/cacheflush.h>
25#include <asm/efi.h> 31#include <asm/efi.h>
26#include <asm/tlbflush.h> 32#include <asm/tlbflush.h>
27#include <asm/mmu_context.h> 33#include <asm/mmu_context.h>
34#include <asm/mmu.h>
35#include <asm/pgtable.h>
28 36
29struct efi_memory_map memmap; 37struct efi_memory_map memmap;
30 38
31static efi_runtime_services_t *runtime;
32
33static u64 efi_system_table; 39static u64 efi_system_table;
34 40
41static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss;
42
43static struct mm_struct efi_mm = {
44 .mm_rb = RB_ROOT,
45 .pgd = efi_pgd,
46 .mm_users = ATOMIC_INIT(2),
47 .mm_count = ATOMIC_INIT(1),
48 .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem),
49 .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
50 .mmlist = LIST_HEAD_INIT(efi_mm.mmlist),
51 INIT_MM_CONTEXT(efi_mm)
52};
53
35static int uefi_debug __initdata; 54static int uefi_debug __initdata;
36static int __init uefi_debug_setup(char *str) 55static int __init uefi_debug_setup(char *str)
37{ 56{
@@ -48,30 +67,33 @@ static int __init is_normal_ram(efi_memory_desc_t *md)
48 return 0; 67 return 0;
49} 68}
50 69
51static void __init efi_setup_idmap(void) 70/*
71 * Translate a EFI virtual address into a physical address: this is necessary,
72 * as some data members of the EFI system table are virtually remapped after
73 * SetVirtualAddressMap() has been called.
74 */
75static phys_addr_t efi_to_phys(unsigned long addr)
52{ 76{
53 struct memblock_region *r;
54 efi_memory_desc_t *md; 77 efi_memory_desc_t *md;
55 u64 paddr, npages, size;
56 78
57 for_each_memblock(memory, r)
58 create_id_mapping(r->base, r->size, 0);
59
60 /* map runtime io spaces */
61 for_each_efi_memory_desc(&memmap, md) { 79 for_each_efi_memory_desc(&memmap, md) {
62 if (!(md->attribute & EFI_MEMORY_RUNTIME) || is_normal_ram(md)) 80 if (!(md->attribute & EFI_MEMORY_RUNTIME))
63 continue; 81 continue;
64 paddr = md->phys_addr; 82 if (md->virt_addr == 0)
65 npages = md->num_pages; 83 /* no virtual mapping has been installed by the stub */
66 memrange_efi_to_native(&paddr, &npages); 84 break;
67 size = npages << PAGE_SHIFT; 85 if (md->virt_addr <= addr &&
68 create_id_mapping(paddr, size, 1); 86 (addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT))
87 return md->phys_addr + addr - md->virt_addr;
69 } 88 }
89 return addr;
70} 90}
71 91
72static int __init uefi_init(void) 92static int __init uefi_init(void)
73{ 93{
74 efi_char16_t *c16; 94 efi_char16_t *c16;
95 void *config_tables;
96 u64 table_size;
75 char vendor[100] = "unknown"; 97 char vendor[100] = "unknown";
76 int i, retval; 98 int i, retval;
77 99
@@ -99,7 +121,7 @@ static int __init uefi_init(void)
99 efi.systab->hdr.revision & 0xffff); 121 efi.systab->hdr.revision & 0xffff);
100 122
101 /* Show what we know for posterity */ 123 /* Show what we know for posterity */
102 c16 = early_memremap(efi.systab->fw_vendor, 124 c16 = early_memremap(efi_to_phys(efi.systab->fw_vendor),
103 sizeof(vendor)); 125 sizeof(vendor));
104 if (c16) { 126 if (c16) {
105 for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i) 127 for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i)
@@ -112,8 +134,14 @@ static int __init uefi_init(void)
112 efi.systab->hdr.revision >> 16, 134 efi.systab->hdr.revision >> 16,
113 efi.systab->hdr.revision & 0xffff, vendor); 135 efi.systab->hdr.revision & 0xffff, vendor);
114 136
115 retval = efi_config_init(NULL); 137 table_size = sizeof(efi_config_table_64_t) * efi.systab->nr_tables;
138 config_tables = early_memremap(efi_to_phys(efi.systab->tables),
139 table_size);
116 140
141 retval = efi_config_parse_tables(config_tables, efi.systab->nr_tables,
142 sizeof(efi_config_table_64_t), NULL);
143
144 early_memunmap(config_tables, table_size);
117out: 145out:
118 early_memunmap(efi.systab, sizeof(efi_system_table_t)); 146 early_memunmap(efi.systab, sizeof(efi_system_table_t));
119 return retval; 147 return retval;
@@ -163,9 +191,7 @@ static __init void reserve_regions(void)
163 if (is_normal_ram(md)) 191 if (is_normal_ram(md))
164 early_init_dt_add_memory_arch(paddr, size); 192 early_init_dt_add_memory_arch(paddr, size);
165 193
166 if (is_reserve_region(md) || 194 if (is_reserve_region(md)) {
167 md->type == EFI_BOOT_SERVICES_CODE ||
168 md->type == EFI_BOOT_SERVICES_DATA) {
169 memblock_reserve(paddr, size); 195 memblock_reserve(paddr, size);
170 if (uefi_debug) 196 if (uefi_debug)
171 pr_cont("*"); 197 pr_cont("*");
@@ -178,123 +204,6 @@ static __init void reserve_regions(void)
178 set_bit(EFI_MEMMAP, &efi.flags); 204 set_bit(EFI_MEMMAP, &efi.flags);
179} 205}
180 206
181
182static u64 __init free_one_region(u64 start, u64 end)
183{
184 u64 size = end - start;
185
186 if (uefi_debug)
187 pr_info(" EFI freeing: 0x%012llx-0x%012llx\n", start, end - 1);
188
189 free_bootmem_late(start, size);
190 return size;
191}
192
193static u64 __init free_region(u64 start, u64 end)
194{
195 u64 map_start, map_end, total = 0;
196
197 if (end <= start)
198 return total;
199
200 map_start = (u64)memmap.phys_map;
201 map_end = PAGE_ALIGN(map_start + (memmap.map_end - memmap.map));
202 map_start &= PAGE_MASK;
203
204 if (start < map_end && end > map_start) {
205 /* region overlaps UEFI memmap */
206 if (start < map_start)
207 total += free_one_region(start, map_start);
208
209 if (map_end < end)
210 total += free_one_region(map_end, end);
211 } else
212 total += free_one_region(start, end);
213
214 return total;
215}
216
217static void __init free_boot_services(void)
218{
219 u64 total_freed = 0;
220 u64 keep_end, free_start, free_end;
221 efi_memory_desc_t *md;
222
223 /*
224 * If kernel uses larger pages than UEFI, we have to be careful
225 * not to inadvertantly free memory we want to keep if there is
226 * overlap at the kernel page size alignment. We do not want to
227 * free is_reserve_region() memory nor the UEFI memmap itself.
228 *
229 * The memory map is sorted, so we keep track of the end of
230 * any previous region we want to keep, remember any region
231 * we want to free and defer freeing it until we encounter
232 * the next region we want to keep. This way, before freeing
233 * it, we can clip it as needed to avoid freeing memory we
234 * want to keep for UEFI.
235 */
236
237 keep_end = 0;
238 free_start = 0;
239
240 for_each_efi_memory_desc(&memmap, md) {
241 u64 paddr, npages, size;
242
243 if (is_reserve_region(md)) {
244 /*
245 * We don't want to free any memory from this region.
246 */
247 if (free_start) {
248 /* adjust free_end then free region */
249 if (free_end > md->phys_addr)
250 free_end -= PAGE_SIZE;
251 total_freed += free_region(free_start, free_end);
252 free_start = 0;
253 }
254 keep_end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT);
255 continue;
256 }
257
258 if (md->type != EFI_BOOT_SERVICES_CODE &&
259 md->type != EFI_BOOT_SERVICES_DATA) {
260 /* no need to free this region */
261 continue;
262 }
263
264 /*
265 * We want to free memory from this region.
266 */
267 paddr = md->phys_addr;
268 npages = md->num_pages;
269 memrange_efi_to_native(&paddr, &npages);
270 size = npages << PAGE_SHIFT;
271
272 if (free_start) {
273 if (paddr <= free_end)
274 free_end = paddr + size;
275 else {
276 total_freed += free_region(free_start, free_end);
277 free_start = paddr;
278 free_end = paddr + size;
279 }
280 } else {
281 free_start = paddr;
282 free_end = paddr + size;
283 }
284 if (free_start < keep_end) {
285 free_start += PAGE_SIZE;
286 if (free_start >= free_end)
287 free_start = 0;
288 }
289 }
290 if (free_start)
291 total_freed += free_region(free_start, free_end);
292
293 if (total_freed)
294 pr_info("Freed 0x%llx bytes of EFI boot services memory",
295 total_freed);
296}
297
298void __init efi_init(void) 207void __init efi_init(void)
299{ 208{
300 struct efi_fdt_params params; 209 struct efi_fdt_params params;
@@ -317,159 +226,100 @@ void __init efi_init(void)
317 return; 226 return;
318 227
319 reserve_regions(); 228 reserve_regions();
229 early_memunmap(memmap.map, params.mmap_size);
320} 230}
321 231
322void __init efi_idmap_init(void) 232static bool __init efi_virtmap_init(void)
323{ 233{
324 if (!efi_enabled(EFI_BOOT)) 234 efi_memory_desc_t *md;
325 return;
326
327 /* boot time idmap_pg_dir is incomplete, so fill in missing parts */
328 efi_setup_idmap();
329 early_memunmap(memmap.map, memmap.map_end - memmap.map);
330}
331
332static int __init remap_region(efi_memory_desc_t *md, void **new)
333{
334 u64 paddr, vaddr, npages, size;
335
336 paddr = md->phys_addr;
337 npages = md->num_pages;
338 memrange_efi_to_native(&paddr, &npages);
339 size = npages << PAGE_SHIFT;
340 235
341 if (is_normal_ram(md)) 236 for_each_efi_memory_desc(&memmap, md) {
342 vaddr = (__force u64)ioremap_cache(paddr, size); 237 u64 paddr, npages, size;
343 else 238 pgprot_t prot;
344 vaddr = (__force u64)ioremap(paddr, size);
345 239
346 if (!vaddr) { 240 if (!(md->attribute & EFI_MEMORY_RUNTIME))
347 pr_err("Unable to remap 0x%llx pages @ %p\n", 241 continue;
348 npages, (void *)paddr); 242 if (md->virt_addr == 0)
349 return 0; 243 return false;
350 }
351 244
352 /* adjust for any rounding when EFI and system pagesize differs */ 245 paddr = md->phys_addr;
353 md->virt_addr = vaddr + (md->phys_addr - paddr); 246 npages = md->num_pages;
247 memrange_efi_to_native(&paddr, &npages);
248 size = npages << PAGE_SHIFT;
354 249
355 if (uefi_debug) 250 pr_info(" EFI remap 0x%016llx => %p\n",
356 pr_info(" EFI remap 0x%012llx => %p\n",
357 md->phys_addr, (void *)md->virt_addr); 251 md->phys_addr, (void *)md->virt_addr);
358 252
359 memcpy(*new, md, memmap.desc_size); 253 /*
360 *new += memmap.desc_size; 254 * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be
361 255 * executable, everything else can be mapped with the XN bits
362 return 1; 256 * set.
257 */
258 if (!is_normal_ram(md))
259 prot = __pgprot(PROT_DEVICE_nGnRE);
260 else if (md->type == EFI_RUNTIME_SERVICES_CODE)
261 prot = PAGE_KERNEL_EXEC;
262 else
263 prot = PAGE_KERNEL;
264
265 create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot);
266 }
267 return true;
363} 268}
364 269
365/* 270/*
366 * Switch UEFI from an identity map to a kernel virtual map 271 * Enable the UEFI Runtime Services if all prerequisites are in place, i.e.,
272 * non-early mapping of the UEFI system table and virtual mappings for all
273 * EFI_MEMORY_RUNTIME regions.
367 */ 274 */
368static int __init arm64_enter_virtual_mode(void) 275static int __init arm64_enable_runtime_services(void)
369{ 276{
370 efi_memory_desc_t *md;
371 phys_addr_t virtmap_phys;
372 void *virtmap, *virt_md;
373 efi_status_t status;
374 u64 mapsize; 277 u64 mapsize;
375 int count = 0;
376 unsigned long flags;
377 278
378 if (!efi_enabled(EFI_BOOT)) { 279 if (!efi_enabled(EFI_BOOT)) {
379 pr_info("EFI services will not be available.\n"); 280 pr_info("EFI services will not be available.\n");
380 return -1; 281 return -1;
381 } 282 }
382 283
383 mapsize = memmap.map_end - memmap.map;
384
385 if (efi_runtime_disabled()) { 284 if (efi_runtime_disabled()) {
386 pr_info("EFI runtime services will be disabled.\n"); 285 pr_info("EFI runtime services will be disabled.\n");
387 return -1; 286 return -1;
388 } 287 }
389 288
390 pr_info("Remapping and enabling EFI services.\n"); 289 pr_info("Remapping and enabling EFI services.\n");
391 /* replace early memmap mapping with permanent mapping */ 290
291 mapsize = memmap.map_end - memmap.map;
392 memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map, 292 memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map,
393 mapsize); 293 mapsize);
394 memmap.map_end = memmap.map + mapsize; 294 if (!memmap.map) {
395 295 pr_err("Failed to remap EFI memory map\n");
396 efi.memmap = &memmap;
397
398 /* Map the runtime regions */
399 virtmap = kmalloc(mapsize, GFP_KERNEL);
400 if (!virtmap) {
401 pr_err("Failed to allocate EFI virtual memmap\n");
402 return -1; 296 return -1;
403 } 297 }
404 virtmap_phys = virt_to_phys(virtmap); 298 memmap.map_end = memmap.map + mapsize;
405 virt_md = virtmap; 299 efi.memmap = &memmap;
406
407 for_each_efi_memory_desc(&memmap, md) {
408 if (!(md->attribute & EFI_MEMORY_RUNTIME))
409 continue;
410 if (!remap_region(md, &virt_md))
411 goto err_unmap;
412 ++count;
413 }
414 300
415 efi.systab = (__force void *)efi_lookup_mapped_addr(efi_system_table); 301 efi.systab = (__force void *)ioremap_cache(efi_system_table,
302 sizeof(efi_system_table_t));
416 if (!efi.systab) { 303 if (!efi.systab) {
417 /* 304 pr_err("Failed to remap EFI System Table\n");
418 * If we have no virtual mapping for the System Table at this 305 return -1;
419 * point, the memory map doesn't cover the physical offset where
420 * it resides. This means the System Table will be inaccessible
421 * to Runtime Services themselves once the virtual mapping is
422 * installed.
423 */
424 pr_err("Failed to remap EFI System Table -- buggy firmware?\n");
425 goto err_unmap;
426 } 306 }
427 set_bit(EFI_SYSTEM_TABLES, &efi.flags); 307 set_bit(EFI_SYSTEM_TABLES, &efi.flags);
428 308
429 local_irq_save(flags); 309 if (!efi_virtmap_init()) {
430 cpu_switch_mm(idmap_pg_dir, &init_mm); 310 pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n");
431
432 /* Call SetVirtualAddressMap with the physical address of the map */
433 runtime = efi.systab->runtime;
434 efi.set_virtual_address_map = runtime->set_virtual_address_map;
435
436 status = efi.set_virtual_address_map(count * memmap.desc_size,
437 memmap.desc_size,
438 memmap.desc_version,
439 (efi_memory_desc_t *)virtmap_phys);
440 cpu_set_reserved_ttbr0();
441 flush_tlb_all();
442 local_irq_restore(flags);
443
444 kfree(virtmap);
445
446 free_boot_services();
447
448 if (status != EFI_SUCCESS) {
449 pr_err("Failed to set EFI virtual address map! [%lx]\n",
450 status);
451 return -1; 311 return -1;
452 } 312 }
453 313
454 /* Set up runtime services function pointers */ 314 /* Set up runtime services function pointers */
455 runtime = efi.systab->runtime;
456 efi_native_runtime_setup(); 315 efi_native_runtime_setup();
457 set_bit(EFI_RUNTIME_SERVICES, &efi.flags); 316 set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
458 317
459 efi.runtime_version = efi.systab->hdr.revision; 318 efi.runtime_version = efi.systab->hdr.revision;
460 319
461 return 0; 320 return 0;
462
463err_unmap:
464 /* unmap all mappings that succeeded: there are 'count' of those */
465 for (virt_md = virtmap; count--; virt_md += memmap.desc_size) {
466 md = virt_md;
467 iounmap((__force void __iomem *)md->virt_addr);
468 }
469 kfree(virtmap);
470 return -1;
471} 321}
472early_initcall(arm64_enter_virtual_mode); 322early_initcall(arm64_enable_runtime_services);
473 323
474static int __init arm64_dmi_init(void) 324static int __init arm64_dmi_init(void)
475{ 325{
@@ -484,3 +334,23 @@ static int __init arm64_dmi_init(void)
484 return 0; 334 return 0;
485} 335}
486core_initcall(arm64_dmi_init); 336core_initcall(arm64_dmi_init);
337
338static void efi_set_pgd(struct mm_struct *mm)
339{
340 cpu_switch_mm(mm->pgd, mm);
341 flush_tlb_all();
342 if (icache_is_aivivt())
343 __flush_icache_all();
344}
345
346void efi_virtmap_load(void)
347{
348 preempt_disable();
349 efi_set_pgd(&efi_mm);
350}
351
352void efi_virtmap_unload(void)
353{
354 efi_set_pgd(current->active_mm);
355 preempt_enable();
356}
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index fd4fa374e5d2..cf21bb3bf752 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -269,18 +269,18 @@ ENDPROC(el1_error_invalid)
269el1_sync: 269el1_sync:
270 kernel_entry 1 270 kernel_entry 1
271 mrs x1, esr_el1 // read the syndrome register 271 mrs x1, esr_el1 // read the syndrome register
272 lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class 272 lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
273 cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1 273 cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
274 b.eq el1_da 274 b.eq el1_da
275 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap 275 cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
276 b.eq el1_undef 276 b.eq el1_undef
277 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception 277 cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
278 b.eq el1_sp_pc 278 b.eq el1_sp_pc
279 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception 279 cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
280 b.eq el1_sp_pc 280 b.eq el1_sp_pc
281 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1 281 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
282 b.eq el1_undef 282 b.eq el1_undef
283 cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1 283 cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
284 b.ge el1_dbg 284 b.ge el1_dbg
285 b el1_inv 285 b el1_inv
286el1_da: 286el1_da:
@@ -318,7 +318,7 @@ el1_dbg:
318 /* 318 /*
319 * Debug exception handling 319 * Debug exception handling
320 */ 320 */
321 cmp x24, #ESR_EL1_EC_BRK64 // if BRK64 321 cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
322 cinc x24, x24, eq // set bit '0' 322 cinc x24, x24, eq // set bit '0'
323 tbz x24, #0, el1_inv // EL1 only 323 tbz x24, #0, el1_inv // EL1 only
324 mrs x0, far_el1 324 mrs x0, far_el1
@@ -375,26 +375,26 @@ el1_preempt:
375el0_sync: 375el0_sync:
376 kernel_entry 0 376 kernel_entry 0
377 mrs x25, esr_el1 // read the syndrome register 377 mrs x25, esr_el1 // read the syndrome register
378 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class 378 lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
379 cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state 379 cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
380 b.eq el0_svc 380 b.eq el0_svc
381 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 381 cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
382 b.eq el0_da 382 b.eq el0_da
383 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 383 cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
384 b.eq el0_ia 384 b.eq el0_ia
385 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access 385 cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
386 b.eq el0_fpsimd_acc 386 b.eq el0_fpsimd_acc
387 cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception 387 cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
388 b.eq el0_fpsimd_exc 388 b.eq el0_fpsimd_exc
389 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap 389 cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
390 b.eq el0_undef 390 b.eq el0_undef
391 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception 391 cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
392 b.eq el0_sp_pc 392 b.eq el0_sp_pc
393 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception 393 cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
394 b.eq el0_sp_pc 394 b.eq el0_sp_pc
395 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 395 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
396 b.eq el0_undef 396 b.eq el0_undef
397 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 397 cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
398 b.ge el0_dbg 398 b.ge el0_dbg
399 b el0_inv 399 b el0_inv
400 400
@@ -403,37 +403,37 @@ el0_sync:
403el0_sync_compat: 403el0_sync_compat:
404 kernel_entry 0, 32 404 kernel_entry 0, 32
405 mrs x25, esr_el1 // read the syndrome register 405 mrs x25, esr_el1 // read the syndrome register
406 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class 406 lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
407 cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state 407 cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
408 b.eq el0_svc_compat 408 b.eq el0_svc_compat
409 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 409 cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
410 b.eq el0_da 410 b.eq el0_da
411 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 411 cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
412 b.eq el0_ia 412 b.eq el0_ia
413 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access 413 cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
414 b.eq el0_fpsimd_acc 414 b.eq el0_fpsimd_acc
415 cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception 415 cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
416 b.eq el0_fpsimd_exc 416 b.eq el0_fpsimd_exc
417 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 417 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
418 b.eq el0_undef 418 b.eq el0_undef
419 cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap 419 cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
420 b.eq el0_undef 420 b.eq el0_undef
421 cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap 421 cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
422 b.eq el0_undef 422 b.eq el0_undef
423 cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap 423 cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
424 b.eq el0_undef 424 b.eq el0_undef
425 cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap 425 cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
426 b.eq el0_undef 426 b.eq el0_undef
427 cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap 427 cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
428 b.eq el0_undef 428 b.eq el0_undef
429 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 429 cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
430 b.ge el0_dbg 430 b.ge el0_dbg
431 b el0_inv 431 b el0_inv
432el0_svc_compat: 432el0_svc_compat:
433 /* 433 /*
434 * AArch32 syscall handling 434 * AArch32 syscall handling
435 */ 435 */
436 adr stbl, compat_sys_call_table // load compat syscall table pointer 436 adrp stbl, compat_sys_call_table // load compat syscall table pointer
437 uxtw scno, w7 // syscall number in w7 (r7) 437 uxtw scno, w7 // syscall number in w7 (r7)
438 mov sc_nr, #__NR_compat_syscalls 438 mov sc_nr, #__NR_compat_syscalls
439 b el0_svc_naked 439 b el0_svc_naked
diff --git a/arch/arm64/kernel/sys32.S b/arch/arm64/kernel/entry32.S
index 423a5b3fc2be..9a8f6ae2530e 100644
--- a/arch/arm64/kernel/sys32.S
+++ b/arch/arm64/kernel/entry32.S
@@ -27,26 +27,26 @@
27 * System call wrappers for the AArch32 compatibility layer. 27 * System call wrappers for the AArch32 compatibility layer.
28 */ 28 */
29 29
30compat_sys_sigreturn_wrapper: 30ENTRY(compat_sys_sigreturn_wrapper)
31 mov x0, sp 31 mov x0, sp
32 mov x27, #0 // prevent syscall restart handling (why) 32 mov x27, #0 // prevent syscall restart handling (why)
33 b compat_sys_sigreturn 33 b compat_sys_sigreturn
34ENDPROC(compat_sys_sigreturn_wrapper) 34ENDPROC(compat_sys_sigreturn_wrapper)
35 35
36compat_sys_rt_sigreturn_wrapper: 36ENTRY(compat_sys_rt_sigreturn_wrapper)
37 mov x0, sp 37 mov x0, sp
38 mov x27, #0 // prevent syscall restart handling (why) 38 mov x27, #0 // prevent syscall restart handling (why)
39 b compat_sys_rt_sigreturn 39 b compat_sys_rt_sigreturn
40ENDPROC(compat_sys_rt_sigreturn_wrapper) 40ENDPROC(compat_sys_rt_sigreturn_wrapper)
41 41
42compat_sys_statfs64_wrapper: 42ENTRY(compat_sys_statfs64_wrapper)
43 mov w3, #84 43 mov w3, #84
44 cmp w1, #88 44 cmp w1, #88
45 csel w1, w3, w1, eq 45 csel w1, w3, w1, eq
46 b compat_sys_statfs64 46 b compat_sys_statfs64
47ENDPROC(compat_sys_statfs64_wrapper) 47ENDPROC(compat_sys_statfs64_wrapper)
48 48
49compat_sys_fstatfs64_wrapper: 49ENTRY(compat_sys_fstatfs64_wrapper)
50 mov w3, #84 50 mov w3, #84
51 cmp w1, #88 51 cmp w1, #88
52 csel w1, w3, w1, eq 52 csel w1, w3, w1, eq
@@ -58,33 +58,33 @@ ENDPROC(compat_sys_fstatfs64_wrapper)
58 * in registers or that take 32-bit parameters which require sign 58 * in registers or that take 32-bit parameters which require sign
59 * extension. 59 * extension.
60 */ 60 */
61compat_sys_pread64_wrapper: 61ENTRY(compat_sys_pread64_wrapper)
62 regs_to_64 x3, x4, x5 62 regs_to_64 x3, x4, x5
63 b sys_pread64 63 b sys_pread64
64ENDPROC(compat_sys_pread64_wrapper) 64ENDPROC(compat_sys_pread64_wrapper)
65 65
66compat_sys_pwrite64_wrapper: 66ENTRY(compat_sys_pwrite64_wrapper)
67 regs_to_64 x3, x4, x5 67 regs_to_64 x3, x4, x5
68 b sys_pwrite64 68 b sys_pwrite64
69ENDPROC(compat_sys_pwrite64_wrapper) 69ENDPROC(compat_sys_pwrite64_wrapper)
70 70
71compat_sys_truncate64_wrapper: 71ENTRY(compat_sys_truncate64_wrapper)
72 regs_to_64 x1, x2, x3 72 regs_to_64 x1, x2, x3
73 b sys_truncate 73 b sys_truncate
74ENDPROC(compat_sys_truncate64_wrapper) 74ENDPROC(compat_sys_truncate64_wrapper)
75 75
76compat_sys_ftruncate64_wrapper: 76ENTRY(compat_sys_ftruncate64_wrapper)
77 regs_to_64 x1, x2, x3 77 regs_to_64 x1, x2, x3
78 b sys_ftruncate 78 b sys_ftruncate
79ENDPROC(compat_sys_ftruncate64_wrapper) 79ENDPROC(compat_sys_ftruncate64_wrapper)
80 80
81compat_sys_readahead_wrapper: 81ENTRY(compat_sys_readahead_wrapper)
82 regs_to_64 x1, x2, x3 82 regs_to_64 x1, x2, x3
83 mov w2, w4 83 mov w2, w4
84 b sys_readahead 84 b sys_readahead
85ENDPROC(compat_sys_readahead_wrapper) 85ENDPROC(compat_sys_readahead_wrapper)
86 86
87compat_sys_fadvise64_64_wrapper: 87ENTRY(compat_sys_fadvise64_64_wrapper)
88 mov w6, w1 88 mov w6, w1
89 regs_to_64 x1, x2, x3 89 regs_to_64 x1, x2, x3
90 regs_to_64 x2, x4, x5 90 regs_to_64 x2, x4, x5
@@ -92,24 +92,14 @@ compat_sys_fadvise64_64_wrapper:
92 b sys_fadvise64_64 92 b sys_fadvise64_64
93ENDPROC(compat_sys_fadvise64_64_wrapper) 93ENDPROC(compat_sys_fadvise64_64_wrapper)
94 94
95compat_sys_sync_file_range2_wrapper: 95ENTRY(compat_sys_sync_file_range2_wrapper)
96 regs_to_64 x2, x2, x3 96 regs_to_64 x2, x2, x3
97 regs_to_64 x3, x4, x5 97 regs_to_64 x3, x4, x5
98 b sys_sync_file_range2 98 b sys_sync_file_range2
99ENDPROC(compat_sys_sync_file_range2_wrapper) 99ENDPROC(compat_sys_sync_file_range2_wrapper)
100 100
101compat_sys_fallocate_wrapper: 101ENTRY(compat_sys_fallocate_wrapper)
102 regs_to_64 x2, x2, x3 102 regs_to_64 x2, x2, x3
103 regs_to_64 x3, x4, x5 103 regs_to_64 x3, x4, x5
104 b sys_fallocate 104 b sys_fallocate
105ENDPROC(compat_sys_fallocate_wrapper) 105ENDPROC(compat_sys_fallocate_wrapper)
106
107#undef __SYSCALL
108#define __SYSCALL(x, y) .quad y // x
109
110/*
111 * The system calls table must be 4KB aligned.
112 */
113 .align 12
114ENTRY(compat_sys_call_table)
115#include <asm/unistd32.h>
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index df1cf15377b4..98bbe06e469c 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = {
894 .notifier_call = hw_breakpoint_reset_notify, 894 .notifier_call = hw_breakpoint_reset_notify,
895}; 895};
896 896
897#ifdef CONFIG_ARM64_CPU_SUSPEND 897#ifdef CONFIG_CPU_PM
898extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)); 898extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
899#else 899#else
900static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) 900static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
index 7e9327a0986d..27d4864577e5 100644
--- a/arch/arm64/kernel/insn.c
+++ b/arch/arm64/kernel/insn.c
@@ -17,14 +17,19 @@
17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */ 18 */
19#include <linux/bitops.h> 19#include <linux/bitops.h>
20#include <linux/bug.h>
20#include <linux/compiler.h> 21#include <linux/compiler.h>
21#include <linux/kernel.h> 22#include <linux/kernel.h>
23#include <linux/mm.h>
22#include <linux/smp.h> 24#include <linux/smp.h>
25#include <linux/spinlock.h>
23#include <linux/stop_machine.h> 26#include <linux/stop_machine.h>
27#include <linux/types.h>
24#include <linux/uaccess.h> 28#include <linux/uaccess.h>
25 29
26#include <asm/cacheflush.h> 30#include <asm/cacheflush.h>
27#include <asm/debug-monitors.h> 31#include <asm/debug-monitors.h>
32#include <asm/fixmap.h>
28#include <asm/insn.h> 33#include <asm/insn.h>
29 34
30#define AARCH64_INSN_SF_BIT BIT(31) 35#define AARCH64_INSN_SF_BIT BIT(31)
@@ -72,6 +77,29 @@ bool __kprobes aarch64_insn_is_nop(u32 insn)
72 } 77 }
73} 78}
74 79
80static DEFINE_SPINLOCK(patch_lock);
81
82static void __kprobes *patch_map(void *addr, int fixmap)
83{
84 unsigned long uintaddr = (uintptr_t) addr;
85 bool module = !core_kernel_text(uintaddr);
86 struct page *page;
87
88 if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
89 page = vmalloc_to_page(addr);
90 else
91 page = virt_to_page(addr);
92
93 BUG_ON(!page);
94 set_fixmap(fixmap, page_to_phys(page));
95
96 return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
97}
98
99static void __kprobes patch_unmap(int fixmap)
100{
101 clear_fixmap(fixmap);
102}
75/* 103/*
76 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always 104 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
77 * little-endian. 105 * little-endian.
@@ -88,10 +116,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
88 return ret; 116 return ret;
89} 117}
90 118
119static int __kprobes __aarch64_insn_write(void *addr, u32 insn)
120{
121 void *waddr = addr;
122 unsigned long flags = 0;
123 int ret;
124
125 spin_lock_irqsave(&patch_lock, flags);
126 waddr = patch_map(addr, FIX_TEXT_POKE0);
127
128 ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE);
129
130 patch_unmap(FIX_TEXT_POKE0);
131 spin_unlock_irqrestore(&patch_lock, flags);
132
133 return ret;
134}
135
91int __kprobes aarch64_insn_write(void *addr, u32 insn) 136int __kprobes aarch64_insn_write(void *addr, u32 insn)
92{ 137{
93 insn = cpu_to_le32(insn); 138 insn = cpu_to_le32(insn);
94 return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE); 139 return __aarch64_insn_write(addr, insn);
95} 140}
96 141
97static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) 142static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 9b6f71db2709..67bf4107f6ef 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -35,8 +35,8 @@
35void *module_alloc(unsigned long size) 35void *module_alloc(unsigned long size)
36{ 36{
37 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 37 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
38 GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, 38 GFP_KERNEL, PAGE_KERNEL_EXEC, 0,
39 __builtin_return_address(0)); 39 NUMA_NO_NODE, __builtin_return_address(0));
40} 40}
41 41
42enum aarch64_reloc_op { 42enum aarch64_reloc_op {
diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
index ce5836c14ec1..6f93c24ca801 100644
--- a/arch/arm64/kernel/pci.c
+++ b/arch/arm64/kernel/pci.c
@@ -46,25 +46,3 @@ int pcibios_add_device(struct pci_dev *dev)
46 46
47 return 0; 47 return 0;
48} 48}
49
50
51#ifdef CONFIG_PCI_DOMAINS_GENERIC
52static bool dt_domain_found = false;
53
54void pci_bus_assign_domain_nr(struct pci_bus *bus, struct device *parent)
55{
56 int domain = of_get_pci_domain_nr(parent->of_node);
57
58 if (domain >= 0) {
59 dt_domain_found = true;
60 } else if (dt_domain_found == true) {
61 dev_err(parent, "Node %s is missing \"linux,pci-domain\" property in DT\n",
62 parent->of_node->full_name);
63 return;
64 } else {
65 domain = pci_get_new_domain_nr();
66 }
67
68 bus->domain_nr = domain;
69}
70#endif
diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c
index f1dbca7d5c96..3425f311c49e 100644
--- a/arch/arm64/kernel/psci.c
+++ b/arch/arm64/kernel/psci.c
@@ -540,8 +540,6 @@ const struct cpu_operations cpu_psci_ops = {
540 .name = "psci", 540 .name = "psci",
541#ifdef CONFIG_CPU_IDLE 541#ifdef CONFIG_CPU_IDLE
542 .cpu_init_idle = cpu_psci_cpu_init_idle, 542 .cpu_init_idle = cpu_psci_cpu_init_idle,
543#endif
544#ifdef CONFIG_ARM64_CPU_SUSPEND
545 .cpu_suspend = cpu_psci_cpu_suspend, 543 .cpu_suspend = cpu_psci_cpu_suspend,
546#endif 544#endif
547#ifdef CONFIG_SMP 545#ifdef CONFIG_SMP
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 20fe2932ad0c..e8420f635bd4 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -40,6 +40,7 @@
40#include <linux/fs.h> 40#include <linux/fs.h>
41#include <linux/proc_fs.h> 41#include <linux/proc_fs.h>
42#include <linux/memblock.h> 42#include <linux/memblock.h>
43#include <linux/of_iommu.h>
43#include <linux/of_fdt.h> 44#include <linux/of_fdt.h>
44#include <linux/of_platform.h> 45#include <linux/of_platform.h>
45#include <linux/efi.h> 46#include <linux/efi.h>
@@ -322,25 +323,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys)
322 dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); 323 dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name());
323} 324}
324 325
325/*
326 * Limit the memory size that was specified via FDT.
327 */
328static int __init early_mem(char *p)
329{
330 phys_addr_t limit;
331
332 if (!p)
333 return 1;
334
335 limit = memparse(p, &p) & PAGE_MASK;
336 pr_notice("Memory limited to %lldMB\n", limit >> 20);
337
338 memblock_enforce_memory_limit(limit);
339
340 return 0;
341}
342early_param("mem", early_mem);
343
344static void __init request_standard_resources(void) 326static void __init request_standard_resources(void)
345{ 327{
346 struct memblock_region *region; 328 struct memblock_region *region;
@@ -401,7 +383,6 @@ void __init setup_arch(char **cmdline_p)
401 paging_init(); 383 paging_init();
402 request_standard_resources(); 384 request_standard_resources();
403 385
404 efi_idmap_init();
405 early_ioremap_reset(); 386 early_ioremap_reset();
406 387
407 unflatten_device_tree(); 388 unflatten_device_tree();
@@ -425,6 +406,7 @@ void __init setup_arch(char **cmdline_p)
425 406
426static int __init arm64_device_init(void) 407static int __init arm64_device_init(void)
427{ 408{
409 of_iommu_init();
428 of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); 410 of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
429 return 0; 411 return 0;
430} 412}
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 6fa792137eda..660ccf9f7524 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -131,7 +131,7 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
131 struct rt_sigframe __user *frame; 131 struct rt_sigframe __user *frame;
132 132
133 /* Always make any pending restarted system calls return -EINTR */ 133 /* Always make any pending restarted system calls return -EINTR */
134 current_thread_info()->restart_block.fn = do_no_restart_syscall; 134 current->restart_block.fn = do_no_restart_syscall;
135 135
136 /* 136 /*
137 * Since we stacked the signal on a 128-bit boundary, then 'sp' should 137 * Since we stacked the signal on a 128-bit boundary, then 'sp' should
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 5a1ba6e80d4e..c20a300e2213 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -347,7 +347,7 @@ asmlinkage int compat_sys_sigreturn(struct pt_regs *regs)
347 struct compat_sigframe __user *frame; 347 struct compat_sigframe __user *frame;
348 348
349 /* Always make any pending restarted system calls return -EINTR */ 349 /* Always make any pending restarted system calls return -EINTR */
350 current_thread_info()->restart_block.fn = do_no_restart_syscall; 350 current->restart_block.fn = do_no_restart_syscall;
351 351
352 /* 352 /*
353 * Since we stacked the signal on a 64-bit boundary, 353 * Since we stacked the signal on a 64-bit boundary,
@@ -381,7 +381,7 @@ asmlinkage int compat_sys_rt_sigreturn(struct pt_regs *regs)
381 struct compat_rt_sigframe __user *frame; 381 struct compat_rt_sigframe __user *frame;
382 382
383 /* Always make any pending restarted system calls return -EINTR */ 383 /* Always make any pending restarted system calls return -EINTR */
384 current_thread_info()->restart_block.fn = do_no_restart_syscall; 384 current->restart_block.fn = do_no_restart_syscall;
385 385
386 /* 386 /*
387 * Since we stacked the signal on a 64-bit boundary, 387 * Since we stacked the signal on a 64-bit boundary,
@@ -440,7 +440,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
440{ 440{
441 compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler); 441 compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
442 compat_ulong_t retcode; 442 compat_ulong_t retcode;
443 compat_ulong_t spsr = regs->pstate & ~PSR_f; 443 compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT);
444 int thumb; 444 int thumb;
445 445
446 /* Check if the handler is written for ARM or Thumb */ 446 /* Check if the handler is written for ARM or Thumb */
@@ -454,6 +454,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
454 /* The IT state must be cleared for both ARM and Thumb-2 */ 454 /* The IT state must be cleared for both ARM and Thumb-2 */
455 spsr &= ~COMPAT_PSR_IT_MASK; 455 spsr &= ~COMPAT_PSR_IT_MASK;
456 456
457 /* Restore the original endianness */
458 spsr |= COMPAT_PSR_ENDSTATE;
459
457 if (ka->sa.sa_flags & SA_RESTORER) { 460 if (ka->sa.sa_flags & SA_RESTORER) {
458 retcode = ptr_to_compat(ka->sa.sa_restorer); 461 retcode = ptr_to_compat(ka->sa.sa_restorer);
459 } else { 462 } else {
@@ -501,7 +504,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf,
501 504
502 __put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err); 505 __put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);
503 /* set the compat FSR WnR */ 506 /* set the compat FSR WnR */
504 __put_user_error(!!(current->thread.fault_code & ESR_EL1_WRITE) << 507 __put_user_error(!!(current->thread.fault_code & ESR_ELx_WNR) <<
505 FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err); 508 FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err);
506 __put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err); 509 __put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err);
507 __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err); 510 __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err);
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 7ae6ee085261..328b8ce4b007 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -65,7 +65,6 @@ struct secondary_data secondary_data;
65enum ipi_msg_type { 65enum ipi_msg_type {
66 IPI_RESCHEDULE, 66 IPI_RESCHEDULE,
67 IPI_CALL_FUNC, 67 IPI_CALL_FUNC,
68 IPI_CALL_FUNC_SINGLE,
69 IPI_CPU_STOP, 68 IPI_CPU_STOP,
70 IPI_TIMER, 69 IPI_TIMER,
71 IPI_IRQ_WORK, 70 IPI_IRQ_WORK,
@@ -483,7 +482,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
483#define S(x,s) [x] = s 482#define S(x,s) [x] = s
484 S(IPI_RESCHEDULE, "Rescheduling interrupts"), 483 S(IPI_RESCHEDULE, "Rescheduling interrupts"),
485 S(IPI_CALL_FUNC, "Function call interrupts"), 484 S(IPI_CALL_FUNC, "Function call interrupts"),
486 S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
487 S(IPI_CPU_STOP, "CPU stop interrupts"), 485 S(IPI_CPU_STOP, "CPU stop interrupts"),
488 S(IPI_TIMER, "Timer broadcast interrupts"), 486 S(IPI_TIMER, "Timer broadcast interrupts"),
489 S(IPI_IRQ_WORK, "IRQ work interrupts"), 487 S(IPI_IRQ_WORK, "IRQ work interrupts"),
@@ -527,7 +525,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
527 525
528void arch_send_call_function_single_ipi(int cpu) 526void arch_send_call_function_single_ipi(int cpu)
529{ 527{
530 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); 528 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
531} 529}
532 530
533#ifdef CONFIG_IRQ_WORK 531#ifdef CONFIG_IRQ_WORK
@@ -585,12 +583,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
585 irq_exit(); 583 irq_exit();
586 break; 584 break;
587 585
588 case IPI_CALL_FUNC_SINGLE:
589 irq_enter();
590 generic_smp_call_function_single_interrupt();
591 irq_exit();
592 break;
593
594 case IPI_CPU_STOP: 586 case IPI_CPU_STOP:
595 irq_enter(); 587 irq_enter();
596 ipi_cpu_stop(cpu); 588 ipi_cpu_stop(cpu);
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 2d6b6065fe7f..d7daf45ae7a2 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -1,7 +1,6 @@
1#include <linux/percpu.h> 1#include <linux/percpu.h>
2#include <linux/slab.h> 2#include <linux/slab.h>
3#include <asm/cacheflush.h> 3#include <asm/cacheflush.h>
4#include <asm/cpu_ops.h>
5#include <asm/debug-monitors.h> 4#include <asm/debug-monitors.h>
6#include <asm/pgtable.h> 5#include <asm/pgtable.h>
7#include <asm/memory.h> 6#include <asm/memory.h>
@@ -51,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
51 hw_breakpoint_restore = hw_bp_restore; 50 hw_breakpoint_restore = hw_bp_restore;
52} 51}
53 52
54/**
55 * cpu_suspend() - function to enter a low-power state
56 * @arg: argument to pass to CPU suspend operations
57 *
58 * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
59 * operations back-end error code otherwise.
60 */
61int cpu_suspend(unsigned long arg)
62{
63 int cpu = smp_processor_id();
64
65 /*
66 * If cpu_ops have not been registered or suspend
67 * has not been initialized, cpu_suspend call fails early.
68 */
69 if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
70 return -EOPNOTSUPP;
71 return cpu_ops[cpu]->cpu_suspend(arg);
72}
73
74/* 53/*
75 * __cpu_suspend 54 * __cpu_suspend
76 * 55 *
diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c
index 3fa98ff14f0e..75151aaf1a52 100644
--- a/arch/arm64/kernel/sys.c
+++ b/arch/arm64/kernel/sys.c
@@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
39/* 39/*
40 * Wrappers to pass the pt_regs argument. 40 * Wrappers to pass the pt_regs argument.
41 */ 41 */
42asmlinkage long sys_rt_sigreturn_wrapper(void);
42#define sys_rt_sigreturn sys_rt_sigreturn_wrapper 43#define sys_rt_sigreturn sys_rt_sigreturn_wrapper
43 44
44#include <asm/syscalls.h>
45
46#undef __SYSCALL 45#undef __SYSCALL
47#define __SYSCALL(nr, sym) [nr] = sym, 46#define __SYSCALL(nr, sym) [nr] = sym,
48 47
@@ -50,7 +49,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
50 * The sys_call_table array must be 4K aligned to be accessible from 49 * The sys_call_table array must be 4K aligned to be accessible from
51 * kernel/entry.S. 50 * kernel/entry.S.
52 */ 51 */
53void *sys_call_table[__NR_syscalls] __aligned(4096) = { 52void * const sys_call_table[__NR_syscalls] __aligned(4096) = {
54 [0 ... __NR_syscalls - 1] = sys_ni_syscall, 53 [0 ... __NR_syscalls - 1] = sys_ni_syscall,
55#include <asm/unistd.h> 54#include <asm/unistd.h>
56}; 55};
diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c
new file mode 100644
index 000000000000..2d5ab3c90b82
--- /dev/null
+++ b/arch/arm64/kernel/sys32.c
@@ -0,0 +1,51 @@
1/*
2 * arch/arm64/kernel/sys32.c
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 *
6 * This program is free software(void); you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http(void);//www.gnu.org/licenses/>.
17 */
18
19/*
20 * Needed to avoid conflicting __NR_* macros between uapi/asm/unistd.h and
21 * asm/unistd32.h.
22 */
23#define __COMPAT_SYSCALL_NR
24
25#include <linux/compiler.h>
26#include <linux/syscalls.h>
27
28asmlinkage long compat_sys_sigreturn_wrapper(void);
29asmlinkage long compat_sys_rt_sigreturn_wrapper(void);
30asmlinkage long compat_sys_statfs64_wrapper(void);
31asmlinkage long compat_sys_fstatfs64_wrapper(void);
32asmlinkage long compat_sys_pread64_wrapper(void);
33asmlinkage long compat_sys_pwrite64_wrapper(void);
34asmlinkage long compat_sys_truncate64_wrapper(void);
35asmlinkage long compat_sys_ftruncate64_wrapper(void);
36asmlinkage long compat_sys_readahead_wrapper(void);
37asmlinkage long compat_sys_fadvise64_64_wrapper(void);
38asmlinkage long compat_sys_sync_file_range2_wrapper(void);
39asmlinkage long compat_sys_fallocate_wrapper(void);
40
41#undef __SYSCALL
42#define __SYSCALL(nr, sym) [nr] = sym,
43
44/*
45 * The sys_call_table array must be 4K aligned to be accessible from
46 * kernel/entry.S.
47 */
48void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = {
49 [0 ... __NR_compat_syscalls - 1] = sys_ni_syscall,
50#include <asm/unistd32.h>
51};
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 0a801e3743d5..1ef2940df13c 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -33,6 +33,7 @@
33 33
34#include <asm/atomic.h> 34#include <asm/atomic.h>
35#include <asm/debug-monitors.h> 35#include <asm/debug-monitors.h>
36#include <asm/esr.h>
36#include <asm/traps.h> 37#include <asm/traps.h>
37#include <asm/stacktrace.h> 38#include <asm/stacktrace.h>
38#include <asm/exception.h> 39#include <asm/exception.h>
@@ -373,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs)
373 return sys_ni_syscall(); 374 return sys_ni_syscall();
374} 375}
375 376
377static const char *esr_class_str[] = {
378 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
379 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
380 [ESR_ELx_EC_WFx] = "WFI/WFE",
381 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
382 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
383 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
384 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
385 [ESR_ELx_EC_FP_ASIMD] = "ASIMD",
386 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
387 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
388 [ESR_ELx_EC_ILL] = "PSTATE.IL",
389 [ESR_ELx_EC_SVC32] = "SVC (AArch32)",
390 [ESR_ELx_EC_HVC32] = "HVC (AArch32)",
391 [ESR_ELx_EC_SMC32] = "SMC (AArch32)",
392 [ESR_ELx_EC_SVC64] = "SVC (AArch64)",
393 [ESR_ELx_EC_HVC64] = "HVC (AArch64)",
394 [ESR_ELx_EC_SMC64] = "SMC (AArch64)",
395 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
396 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
397 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
398 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
399 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
400 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
401 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
402 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
403 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
404 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
405 [ESR_ELx_EC_SERROR] = "SError",
406 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
407 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
408 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
409 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
410 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
411 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
412 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
413 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
414 [ESR_ELx_EC_BRK64] = "BRK (AArch64)",
415};
416
417const char *esr_get_class_string(u32 esr)
418{
419 return esr_class_str[esr >> ESR_ELx_EC_SHIFT];
420}
421
376/* 422/*
377 * bad_mode handles the impossible case in the exception vector. 423 * bad_mode handles the impossible case in the exception vector.
378 */ 424 */
@@ -382,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
382 void __user *pc = (void __user *)instruction_pointer(regs); 428 void __user *pc = (void __user *)instruction_pointer(regs);
383 console_verbose(); 429 console_verbose();
384 430
385 pr_crit("Bad mode in %s handler detected, code 0x%08x\n", 431 pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n",
386 handler[reason], esr); 432 handler[reason], esr, esr_get_class_string(esr));
387 __show_regs(regs); 433 __show_regs(regs);
388 434
389 info.si_signo = SIGILL; 435 info.si_signo = SIGILL;
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 9965ec87cbec..5d9d2dca530d 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -8,6 +8,7 @@
8#include <asm/thread_info.h> 8#include <asm/thread_info.h>
9#include <asm/memory.h> 9#include <asm/memory.h>
10#include <asm/page.h> 10#include <asm/page.h>
11#include <asm/pgtable.h>
11 12
12#include "image.h" 13#include "image.h"
13 14
@@ -49,6 +50,14 @@ PECOFF_FILE_ALIGNMENT = 0x200;
49#define PECOFF_EDATA_PADDING 50#define PECOFF_EDATA_PADDING
50#endif 51#endif
51 52
53#ifdef CONFIG_DEBUG_ALIGN_RODATA
54#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT);
55#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO
56#else
57#define ALIGN_DEBUG_RO
58#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min);
59#endif
60
52SECTIONS 61SECTIONS
53{ 62{
54 /* 63 /*
@@ -71,6 +80,7 @@ SECTIONS
71 _text = .; 80 _text = .;
72 HEAD_TEXT 81 HEAD_TEXT
73 } 82 }
83 ALIGN_DEBUG_RO
74 .text : { /* Real text segment */ 84 .text : { /* Real text segment */
75 _stext = .; /* Text and read-only data */ 85 _stext = .; /* Text and read-only data */
76 __exception_text_start = .; 86 __exception_text_start = .;
@@ -87,19 +97,22 @@ SECTIONS
87 *(.got) /* Global offset table */ 97 *(.got) /* Global offset table */
88 } 98 }
89 99
100 ALIGN_DEBUG_RO
90 RO_DATA(PAGE_SIZE) 101 RO_DATA(PAGE_SIZE)
91 EXCEPTION_TABLE(8) 102 EXCEPTION_TABLE(8)
92 NOTES 103 NOTES
104 ALIGN_DEBUG_RO
93 _etext = .; /* End of text and rodata section */ 105 _etext = .; /* End of text and rodata section */
94 106
95 . = ALIGN(PAGE_SIZE); 107 ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
96 __init_begin = .; 108 __init_begin = .;
97 109
98 INIT_TEXT_SECTION(8) 110 INIT_TEXT_SECTION(8)
99 .exit.text : { 111 .exit.text : {
100 ARM_EXIT_KEEP(EXIT_TEXT) 112 ARM_EXIT_KEEP(EXIT_TEXT)
101 } 113 }
102 . = ALIGN(16); 114
115 ALIGN_DEBUG_RO_MIN(16)
103 .init.data : { 116 .init.data : {
104 INIT_DATA 117 INIT_DATA
105 INIT_SETUP(16) 118 INIT_SETUP(16)
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 8ba85e9ea388..f5590c81d95f 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -22,10 +22,13 @@ config KVM
22 select PREEMPT_NOTIFIERS 22 select PREEMPT_NOTIFIERS
23 select ANON_INODES 23 select ANON_INODES
24 select HAVE_KVM_CPU_RELAX_INTERCEPT 24 select HAVE_KVM_CPU_RELAX_INTERCEPT
25 select HAVE_KVM_ARCH_TLB_FLUSH_ALL
25 select KVM_MMIO 26 select KVM_MMIO
26 select KVM_ARM_HOST 27 select KVM_ARM_HOST
27 select KVM_ARM_VGIC 28 select KVM_ARM_VGIC
28 select KVM_ARM_TIMER 29 select KVM_ARM_TIMER
30 select KVM_GENERIC_DIRTYLOG_READ_PROTECT
31 select SRCU
29 ---help--- 32 ---help---
30 Support hosting virtualized guest machines. 33 Support hosting virtualized guest machines.
31 34
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 32a096174b94..4e6e09ee4033 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -21,7 +21,9 @@ kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
21 21
22kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o 22kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
23kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o 23kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
24kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o
24kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o 25kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o
25kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o 26kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o
27kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3-emul.o
26kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o 28kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o
27kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o 29kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm64/kvm/emulate.c b/arch/arm64/kvm/emulate.c
index 124418d17049..f87d8fbaa48d 100644
--- a/arch/arm64/kvm/emulate.c
+++ b/arch/arm64/kvm/emulate.c
@@ -22,6 +22,7 @@
22 */ 22 */
23 23
24#include <linux/kvm_host.h> 24#include <linux/kvm_host.h>
25#include <asm/esr.h>
25#include <asm/kvm_emulate.h> 26#include <asm/kvm_emulate.h>
26 27
27/* 28/*
@@ -55,8 +56,8 @@ static int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
55{ 56{
56 u32 esr = kvm_vcpu_get_hsr(vcpu); 57 u32 esr = kvm_vcpu_get_hsr(vcpu);
57 58
58 if (esr & ESR_EL2_CV) 59 if (esr & ESR_ELx_CV)
59 return (esr & ESR_EL2_COND) >> ESR_EL2_COND_SHIFT; 60 return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
60 61
61 return -1; 62 return -1;
62} 63}
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 34b8bd0711e9..524fa25671fc 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -21,17 +21,25 @@
21 21
22#include <linux/kvm.h> 22#include <linux/kvm.h>
23#include <linux/kvm_host.h> 23#include <linux/kvm_host.h>
24#include <asm/kvm_emulate.h> 24
25#include <asm/esr.h>
25#include <asm/kvm_coproc.h> 26#include <asm/kvm_coproc.h>
27#include <asm/kvm_emulate.h>
26#include <asm/kvm_mmu.h> 28#include <asm/kvm_mmu.h>
27#include <asm/kvm_psci.h> 29#include <asm/kvm_psci.h>
28 30
31#define CREATE_TRACE_POINTS
32#include "trace.h"
33
29typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); 34typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *);
30 35
31static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) 36static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
32{ 37{
33 int ret; 38 int ret;
34 39
40 trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
41 kvm_vcpu_hvc_get_imm(vcpu));
42
35 ret = kvm_psci_call(vcpu); 43 ret = kvm_psci_call(vcpu);
36 if (ret < 0) { 44 if (ret < 0) {
37 kvm_inject_undefined(vcpu); 45 kvm_inject_undefined(vcpu);
@@ -61,10 +69,13 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
61 */ 69 */
62static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) 70static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
63{ 71{
64 if (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EC_WFI_ISS_WFE) 72 if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
73 trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
65 kvm_vcpu_on_spin(vcpu); 74 kvm_vcpu_on_spin(vcpu);
66 else 75 } else {
76 trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
67 kvm_vcpu_block(vcpu); 77 kvm_vcpu_block(vcpu);
78 }
68 79
69 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); 80 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
70 81
@@ -72,29 +83,30 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
72} 83}
73 84
74static exit_handle_fn arm_exit_handlers[] = { 85static exit_handle_fn arm_exit_handlers[] = {
75 [ESR_EL2_EC_WFI] = kvm_handle_wfx, 86 [ESR_ELx_EC_WFx] = kvm_handle_wfx,
76 [ESR_EL2_EC_CP15_32] = kvm_handle_cp15_32, 87 [ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32,
77 [ESR_EL2_EC_CP15_64] = kvm_handle_cp15_64, 88 [ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64,
78 [ESR_EL2_EC_CP14_MR] = kvm_handle_cp14_32, 89 [ESR_ELx_EC_CP14_MR] = kvm_handle_cp14_32,
79 [ESR_EL2_EC_CP14_LS] = kvm_handle_cp14_load_store, 90 [ESR_ELx_EC_CP14_LS] = kvm_handle_cp14_load_store,
80 [ESR_EL2_EC_CP14_64] = kvm_handle_cp14_64, 91 [ESR_ELx_EC_CP14_64] = kvm_handle_cp14_64,
81 [ESR_EL2_EC_HVC32] = handle_hvc, 92 [ESR_ELx_EC_HVC32] = handle_hvc,
82 [ESR_EL2_EC_SMC32] = handle_smc, 93 [ESR_ELx_EC_SMC32] = handle_smc,
83 [ESR_EL2_EC_HVC64] = handle_hvc, 94 [ESR_ELx_EC_HVC64] = handle_hvc,
84 [ESR_EL2_EC_SMC64] = handle_smc, 95 [ESR_ELx_EC_SMC64] = handle_smc,
85 [ESR_EL2_EC_SYS64] = kvm_handle_sys_reg, 96 [ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
86 [ESR_EL2_EC_IABT] = kvm_handle_guest_abort, 97 [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
87 [ESR_EL2_EC_DABT] = kvm_handle_guest_abort, 98 [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
88}; 99};
89 100
90static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) 101static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
91{ 102{
92 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); 103 u32 hsr = kvm_vcpu_get_hsr(vcpu);
104 u8 hsr_ec = hsr >> ESR_ELx_EC_SHIFT;
93 105
94 if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || 106 if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) ||
95 !arm_exit_handlers[hsr_ec]) { 107 !arm_exit_handlers[hsr_ec]) {
96 kvm_err("Unknown exception class: hsr: %#08x\n", 108 kvm_err("Unknown exception class: hsr: %#08x -- %s\n",
97 (unsigned int)kvm_vcpu_get_hsr(vcpu)); 109 hsr, esr_get_class_string(hsr));
98 BUG(); 110 BUG();
99 } 111 }
100 112
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index c3ca89c27c6b..5befd010e232 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -17,15 +17,16 @@
17 17
18#include <linux/linkage.h> 18#include <linux/linkage.h>
19 19
20#include <asm/assembler.h>
21#include <asm/memory.h>
22#include <asm/asm-offsets.h> 20#include <asm/asm-offsets.h>
21#include <asm/assembler.h>
23#include <asm/debug-monitors.h> 22#include <asm/debug-monitors.h>
23#include <asm/esr.h>
24#include <asm/fpsimdmacros.h> 24#include <asm/fpsimdmacros.h>
25#include <asm/kvm.h> 25#include <asm/kvm.h>
26#include <asm/kvm_asm.h>
27#include <asm/kvm_arm.h> 26#include <asm/kvm_arm.h>
27#include <asm/kvm_asm.h>
28#include <asm/kvm_mmu.h> 28#include <asm/kvm_mmu.h>
29#include <asm/memory.h>
29 30
30#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x) 31#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
31#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x) 32#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
@@ -1031,6 +1032,28 @@ ENTRY(__kvm_tlb_flush_vmid_ipa)
1031 ret 1032 ret
1032ENDPROC(__kvm_tlb_flush_vmid_ipa) 1033ENDPROC(__kvm_tlb_flush_vmid_ipa)
1033 1034
1035/**
1036 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
1037 * @struct kvm *kvm - pointer to kvm structure
1038 *
1039 * Invalidates all Stage 1 and 2 TLB entries for current VMID.
1040 */
1041ENTRY(__kvm_tlb_flush_vmid)
1042 dsb ishst
1043
1044 kern_hyp_va x0
1045 ldr x2, [x0, #KVM_VTTBR]
1046 msr vttbr_el2, x2
1047 isb
1048
1049 tlbi vmalls12e1is
1050 dsb ish
1051 isb
1052
1053 msr vttbr_el2, xzr
1054 ret
1055ENDPROC(__kvm_tlb_flush_vmid)
1056
1034ENTRY(__kvm_flush_vm_context) 1057ENTRY(__kvm_flush_vm_context)
1035 dsb ishst 1058 dsb ishst
1036 tlbi alle1is 1059 tlbi alle1is
@@ -1141,9 +1164,9 @@ el1_sync: // Guest trapped into EL2
1141 push x2, x3 1164 push x2, x3
1142 1165
1143 mrs x1, esr_el2 1166 mrs x1, esr_el2
1144 lsr x2, x1, #ESR_EL2_EC_SHIFT 1167 lsr x2, x1, #ESR_ELx_EC_SHIFT
1145 1168
1146 cmp x2, #ESR_EL2_EC_HVC64 1169 cmp x2, #ESR_ELx_EC_HVC64
1147 b.ne el1_trap 1170 b.ne el1_trap
1148 1171
1149 mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest 1172 mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
@@ -1178,13 +1201,13 @@ el1_trap:
1178 * x1: ESR 1201 * x1: ESR
1179 * x2: ESR_EC 1202 * x2: ESR_EC
1180 */ 1203 */
1181 cmp x2, #ESR_EL2_EC_DABT 1204 cmp x2, #ESR_ELx_EC_DABT_LOW
1182 mov x0, #ESR_EL2_EC_IABT 1205 mov x0, #ESR_ELx_EC_IABT_LOW
1183 ccmp x2, x0, #4, ne 1206 ccmp x2, x0, #4, ne
1184 b.ne 1f // Not an abort we care about 1207 b.ne 1f // Not an abort we care about
1185 1208
1186 /* This is an abort. Check for permission fault */ 1209 /* This is an abort. Check for permission fault */
1187 and x2, x1, #ESR_EL2_FSC_TYPE 1210 and x2, x1, #ESR_ELx_FSC_TYPE
1188 cmp x2, #FSC_PERM 1211 cmp x2, #FSC_PERM
1189 b.ne 1f // Not a permission fault 1212 b.ne 1f // Not a permission fault
1190 1213
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 81a02a8762b0..f02530e726f6 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -118,27 +118,27 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
118 * instruction set. Report an external synchronous abort. 118 * instruction set. Report an external synchronous abort.
119 */ 119 */
120 if (kvm_vcpu_trap_il_is32bit(vcpu)) 120 if (kvm_vcpu_trap_il_is32bit(vcpu))
121 esr |= ESR_EL1_IL; 121 esr |= ESR_ELx_IL;
122 122
123 /* 123 /*
124 * Here, the guest runs in AArch64 mode when in EL1. If we get 124 * Here, the guest runs in AArch64 mode when in EL1. If we get
125 * an AArch32 fault, it means we managed to trap an EL0 fault. 125 * an AArch32 fault, it means we managed to trap an EL0 fault.
126 */ 126 */
127 if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) 127 if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
128 esr |= (ESR_EL1_EC_IABT_EL0 << ESR_EL1_EC_SHIFT); 128 esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
129 else 129 else
130 esr |= (ESR_EL1_EC_IABT_EL1 << ESR_EL1_EC_SHIFT); 130 esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
131 131
132 if (!is_iabt) 132 if (!is_iabt)
133 esr |= ESR_EL1_EC_DABT_EL0; 133 esr |= ESR_ELx_EC_DABT_LOW;
134 134
135 vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_EL2_EC_xABT_xFSR_EXTABT; 135 vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_ELx_FSC_EXTABT;
136} 136}
137 137
138static void inject_undef64(struct kvm_vcpu *vcpu) 138static void inject_undef64(struct kvm_vcpu *vcpu)
139{ 139{
140 unsigned long cpsr = *vcpu_cpsr(vcpu); 140 unsigned long cpsr = *vcpu_cpsr(vcpu);
141 u32 esr = (ESR_EL1_EC_UNKNOWN << ESR_EL1_EC_SHIFT); 141 u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
142 142
143 *vcpu_spsr(vcpu) = cpsr; 143 *vcpu_spsr(vcpu) = cpsr;
144 *vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu); 144 *vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu);
@@ -151,7 +151,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
151 * set. 151 * set.
152 */ 152 */
153 if (kvm_vcpu_trap_il_is32bit(vcpu)) 153 if (kvm_vcpu_trap_il_is32bit(vcpu))
154 esr |= ESR_EL1_IL; 154 esr |= ESR_ELx_IL;
155 155
156 vcpu_sys_reg(vcpu, ESR_EL1) = esr; 156 vcpu_sys_reg(vcpu, ESR_EL1) = esr;
157} 157}
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index f31e8bb2bc5b..c370b4014799 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -20,17 +20,20 @@
20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 */ 21 */
22 22
23#include <linux/mm.h>
24#include <linux/kvm_host.h> 23#include <linux/kvm_host.h>
24#include <linux/mm.h>
25#include <linux/uaccess.h> 25#include <linux/uaccess.h>
26#include <asm/kvm_arm.h> 26
27#include <asm/kvm_host.h>
28#include <asm/kvm_emulate.h>
29#include <asm/kvm_coproc.h>
30#include <asm/kvm_mmu.h>
31#include <asm/cacheflush.h> 27#include <asm/cacheflush.h>
32#include <asm/cputype.h> 28#include <asm/cputype.h>
33#include <asm/debug-monitors.h> 29#include <asm/debug-monitors.h>
30#include <asm/esr.h>
31#include <asm/kvm_arm.h>
32#include <asm/kvm_coproc.h>
33#include <asm/kvm_emulate.h>
34#include <asm/kvm_host.h>
35#include <asm/kvm_mmu.h>
36
34#include <trace/events/kvm.h> 37#include <trace/events/kvm.h>
35 38
36#include "sys_regs.h" 39#include "sys_regs.h"
@@ -110,6 +113,27 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu,
110 return true; 113 return true;
111} 114}
112 115
116/*
117 * Trap handler for the GICv3 SGI generation system register.
118 * Forward the request to the VGIC emulation.
119 * The cp15_64 code makes sure this automatically works
120 * for both AArch64 and AArch32 accesses.
121 */
122static bool access_gic_sgi(struct kvm_vcpu *vcpu,
123 const struct sys_reg_params *p,
124 const struct sys_reg_desc *r)
125{
126 u64 val;
127
128 if (!p->is_write)
129 return read_from_write_only(vcpu, p);
130
131 val = *vcpu_reg(vcpu, p->Rt);
132 vgic_v3_dispatch_sgi(vcpu, val);
133
134 return true;
135}
136
113static bool trap_raz_wi(struct kvm_vcpu *vcpu, 137static bool trap_raz_wi(struct kvm_vcpu *vcpu,
114 const struct sys_reg_params *p, 138 const struct sys_reg_params *p,
115 const struct sys_reg_desc *r) 139 const struct sys_reg_desc *r)
@@ -197,10 +221,19 @@ static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
197 221
198static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) 222static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
199{ 223{
224 u64 mpidr;
225
200 /* 226 /*
201 * Simply map the vcpu_id into the Aff0 field of the MPIDR. 227 * Map the vcpu_id into the first three affinity level fields of
228 * the MPIDR. We limit the number of VCPUs in level 0 due to a
229 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
230 * of the GICv3 to be able to address each CPU directly when
231 * sending IPIs.
202 */ 232 */
203 vcpu_sys_reg(vcpu, MPIDR_EL1) = (1UL << 31) | (vcpu->vcpu_id & 0xff); 233 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
234 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
235 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
236 vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
204} 237}
205 238
206/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */ 239/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
@@ -370,6 +403,9 @@ static const struct sys_reg_desc sys_reg_descs[] = {
370 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000), 403 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
371 NULL, reset_val, VBAR_EL1, 0 }, 404 NULL, reset_val, VBAR_EL1, 0 },
372 405
406 /* ICC_SGI1R_EL1 */
407 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
408 access_gic_sgi },
373 /* ICC_SRE_EL1 */ 409 /* ICC_SRE_EL1 */
374 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101), 410 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
375 trap_raz_wi }, 411 trap_raz_wi },
@@ -602,6 +638,8 @@ static const struct sys_reg_desc cp14_64_regs[] = {
602 * register). 638 * register).
603 */ 639 */
604static const struct sys_reg_desc cp15_regs[] = { 640static const struct sys_reg_desc cp15_regs[] = {
641 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
642
605 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR }, 643 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
606 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, 644 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
607 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, 645 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
@@ -649,6 +687,7 @@ static const struct sys_reg_desc cp15_regs[] = {
649 687
650static const struct sys_reg_desc cp15_64_regs[] = { 688static const struct sys_reg_desc cp15_64_regs[] = {
651 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, 689 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
690 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
652 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 }, 691 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
653}; 692};
654 693
@@ -760,12 +799,12 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu,
760 int cp; 799 int cp;
761 800
762 switch(hsr_ec) { 801 switch(hsr_ec) {
763 case ESR_EL2_EC_CP15_32: 802 case ESR_ELx_EC_CP15_32:
764 case ESR_EL2_EC_CP15_64: 803 case ESR_ELx_EC_CP15_64:
765 cp = 15; 804 cp = 15;
766 break; 805 break;
767 case ESR_EL2_EC_CP14_MR: 806 case ESR_ELx_EC_CP14_MR:
768 case ESR_EL2_EC_CP14_64: 807 case ESR_ELx_EC_CP14_64:
769 cp = 14; 808 cp = 14;
770 break; 809 break;
771 default: 810 default:
diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
new file mode 100644
index 000000000000..157416e963f2
--- /dev/null
+++ b/arch/arm64/kvm/trace.h
@@ -0,0 +1,55 @@
1#if !defined(_TRACE_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
2#define _TRACE_ARM64_KVM_H
3
4#include <linux/tracepoint.h>
5
6#undef TRACE_SYSTEM
7#define TRACE_SYSTEM kvm
8
9TRACE_EVENT(kvm_wfx_arm64,
10 TP_PROTO(unsigned long vcpu_pc, bool is_wfe),
11 TP_ARGS(vcpu_pc, is_wfe),
12
13 TP_STRUCT__entry(
14 __field(unsigned long, vcpu_pc)
15 __field(bool, is_wfe)
16 ),
17
18 TP_fast_assign(
19 __entry->vcpu_pc = vcpu_pc;
20 __entry->is_wfe = is_wfe;
21 ),
22
23 TP_printk("guest executed wf%c at: 0x%08lx",
24 __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc)
25);
26
27TRACE_EVENT(kvm_hvc_arm64,
28 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
29 TP_ARGS(vcpu_pc, r0, imm),
30
31 TP_STRUCT__entry(
32 __field(unsigned long, vcpu_pc)
33 __field(unsigned long, r0)
34 __field(unsigned long, imm)
35 ),
36
37 TP_fast_assign(
38 __entry->vcpu_pc = vcpu_pc;
39 __entry->r0 = r0;
40 __entry->imm = imm;
41 ),
42
43 TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx)",
44 __entry->vcpu_pc, __entry->r0, __entry->imm)
45);
46
47#endif /* _TRACE_ARM64_KVM_H */
48
49#undef TRACE_INCLUDE_PATH
50#define TRACE_INCLUDE_PATH .
51#undef TRACE_INCLUDE_FILE
52#define TRACE_INCLUDE_FILE trace
53
54/* This part must be outside protection */
55#include <trace/define_trace.h>
diff --git a/arch/arm64/kvm/vgic-v3-switch.S b/arch/arm64/kvm/vgic-v3-switch.S
index d16046999e06..617a012a0107 100644
--- a/arch/arm64/kvm/vgic-v3-switch.S
+++ b/arch/arm64/kvm/vgic-v3-switch.S
@@ -148,17 +148,18 @@
148 * x0: Register pointing to VCPU struct 148 * x0: Register pointing to VCPU struct
149 */ 149 */
150.macro restore_vgic_v3_state 150.macro restore_vgic_v3_state
151 // Disable SRE_EL1 access. Necessary, otherwise
152 // ICH_VMCR_EL2.VFIQEn becomes one, and FIQ happens...
153 msr_s ICC_SRE_EL1, xzr
154 isb
155
156 // Compute the address of struct vgic_cpu 151 // Compute the address of struct vgic_cpu
157 add x3, x0, #VCPU_VGIC_CPU 152 add x3, x0, #VCPU_VGIC_CPU
158 153
159 // Restore all interesting registers 154 // Restore all interesting registers
160 ldr w4, [x3, #VGIC_V3_CPU_HCR] 155 ldr w4, [x3, #VGIC_V3_CPU_HCR]
161 ldr w5, [x3, #VGIC_V3_CPU_VMCR] 156 ldr w5, [x3, #VGIC_V3_CPU_VMCR]
157 ldr w25, [x3, #VGIC_V3_CPU_SRE]
158
159 msr_s ICC_SRE_EL1, x25
160
161 // make sure SRE is valid before writing the other registers
162 isb
162 163
163 msr_s ICH_HCR_EL2, x4 164 msr_s ICH_HCR_EL2, x4
164 msr_s ICH_VMCR_EL2, x5 165 msr_s ICH_VMCR_EL2, x5
@@ -244,9 +245,12 @@
244 dsb sy 245 dsb sy
245 246
246 // Prevent the guest from touching the GIC system registers 247 // Prevent the guest from touching the GIC system registers
248 // if SRE isn't enabled for GICv3 emulation
249 cbnz x25, 1f
247 mrs_s x5, ICC_SRE_EL2 250 mrs_s x5, ICC_SRE_EL2
248 and x5, x5, #~ICC_SRE_EL2_ENABLE 251 and x5, x5, #~ICC_SRE_EL2_ENABLE
249 msr_s ICC_SRE_EL2, x5 252 msr_s ICC_SRE_EL2, x5
2531:
250.endm 254.endm
251 255
252ENTRY(__save_vgic_v3_state) 256ENTRY(__save_vgic_v3_state)
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index d92094203913..0a24b9b8c698 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -134,16 +134,17 @@ static void __dma_free_coherent(struct device *dev, size_t size,
134 swiotlb_free_coherent(dev, size, vaddr, dma_handle); 134 swiotlb_free_coherent(dev, size, vaddr, dma_handle);
135} 135}
136 136
137static void *__dma_alloc_noncoherent(struct device *dev, size_t size, 137static void *__dma_alloc(struct device *dev, size_t size,
138 dma_addr_t *dma_handle, gfp_t flags, 138 dma_addr_t *dma_handle, gfp_t flags,
139 struct dma_attrs *attrs) 139 struct dma_attrs *attrs)
140{ 140{
141 struct page *page; 141 struct page *page;
142 void *ptr, *coherent_ptr; 142 void *ptr, *coherent_ptr;
143 bool coherent = is_device_dma_coherent(dev);
143 144
144 size = PAGE_ALIGN(size); 145 size = PAGE_ALIGN(size);
145 146
146 if (!(flags & __GFP_WAIT)) { 147 if (!coherent && !(flags & __GFP_WAIT)) {
147 struct page *page = NULL; 148 struct page *page = NULL;
148 void *addr = __alloc_from_pool(size, &page); 149 void *addr = __alloc_from_pool(size, &page);
149 150
@@ -151,13 +152,16 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
151 *dma_handle = phys_to_dma(dev, page_to_phys(page)); 152 *dma_handle = phys_to_dma(dev, page_to_phys(page));
152 153
153 return addr; 154 return addr;
154
155 } 155 }
156 156
157 ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs); 157 ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
158 if (!ptr) 158 if (!ptr)
159 goto no_mem; 159 goto no_mem;
160 160
161 /* no need for non-cacheable mapping if coherent */
162 if (coherent)
163 return ptr;
164
161 /* remove any dirty cache lines on the kernel alias */ 165 /* remove any dirty cache lines on the kernel alias */
162 __dma_flush_range(ptr, ptr + size); 166 __dma_flush_range(ptr, ptr + size);
163 167
@@ -179,15 +183,17 @@ no_mem:
179 return NULL; 183 return NULL;
180} 184}
181 185
182static void __dma_free_noncoherent(struct device *dev, size_t size, 186static void __dma_free(struct device *dev, size_t size,
183 void *vaddr, dma_addr_t dma_handle, 187 void *vaddr, dma_addr_t dma_handle,
184 struct dma_attrs *attrs) 188 struct dma_attrs *attrs)
185{ 189{
186 void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle)); 190 void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
187 191
188 if (__free_from_pool(vaddr, size)) 192 if (!is_device_dma_coherent(dev)) {
189 return; 193 if (__free_from_pool(vaddr, size))
190 vunmap(vaddr); 194 return;
195 vunmap(vaddr);
196 }
191 __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs); 197 __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
192} 198}
193 199
@@ -199,7 +205,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
199 dma_addr_t dev_addr; 205 dma_addr_t dev_addr;
200 206
201 dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs); 207 dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
202 __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); 208 if (!is_device_dma_coherent(dev))
209 __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
203 210
204 return dev_addr; 211 return dev_addr;
205} 212}
@@ -209,7 +216,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
209 size_t size, enum dma_data_direction dir, 216 size_t size, enum dma_data_direction dir,
210 struct dma_attrs *attrs) 217 struct dma_attrs *attrs)
211{ 218{
212 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); 219 if (!is_device_dma_coherent(dev))
220 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
213 swiotlb_unmap_page(dev, dev_addr, size, dir, attrs); 221 swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
214} 222}
215 223
@@ -221,9 +229,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
221 int i, ret; 229 int i, ret;
222 230
223 ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs); 231 ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
224 for_each_sg(sgl, sg, ret, i) 232 if (!is_device_dma_coherent(dev))
225 __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), 233 for_each_sg(sgl, sg, ret, i)
226 sg->length, dir); 234 __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
235 sg->length, dir);
227 236
228 return ret; 237 return ret;
229} 238}
@@ -236,9 +245,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
236 struct scatterlist *sg; 245 struct scatterlist *sg;
237 int i; 246 int i;
238 247
239 for_each_sg(sgl, sg, nelems, i) 248 if (!is_device_dma_coherent(dev))
240 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), 249 for_each_sg(sgl, sg, nelems, i)
241 sg->length, dir); 250 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
251 sg->length, dir);
242 swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs); 252 swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
243} 253}
244 254
@@ -246,7 +256,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev,
246 dma_addr_t dev_addr, size_t size, 256 dma_addr_t dev_addr, size_t size,
247 enum dma_data_direction dir) 257 enum dma_data_direction dir)
248{ 258{
249 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); 259 if (!is_device_dma_coherent(dev))
260 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
250 swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir); 261 swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
251} 262}
252 263
@@ -255,7 +266,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev,
255 enum dma_data_direction dir) 266 enum dma_data_direction dir)
256{ 267{
257 swiotlb_sync_single_for_device(dev, dev_addr, size, dir); 268 swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
258 __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); 269 if (!is_device_dma_coherent(dev))
270 __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
259} 271}
260 272
261static void __swiotlb_sync_sg_for_cpu(struct device *dev, 273static void __swiotlb_sync_sg_for_cpu(struct device *dev,
@@ -265,9 +277,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev,
265 struct scatterlist *sg; 277 struct scatterlist *sg;
266 int i; 278 int i;
267 279
268 for_each_sg(sgl, sg, nelems, i) 280 if (!is_device_dma_coherent(dev))
269 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), 281 for_each_sg(sgl, sg, nelems, i)
270 sg->length, dir); 282 __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
283 sg->length, dir);
271 swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir); 284 swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
272} 285}
273 286
@@ -279,9 +292,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
279 int i; 292 int i;
280 293
281 swiotlb_sync_sg_for_device(dev, sgl, nelems, dir); 294 swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
282 for_each_sg(sgl, sg, nelems, i) 295 if (!is_device_dma_coherent(dev))
283 __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), 296 for_each_sg(sgl, sg, nelems, i)
284 sg->length, dir); 297 __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
298 sg->length, dir);
285} 299}
286 300
287/* vma->vm_page_prot must be set appropriately before calling this function */ 301/* vma->vm_page_prot must be set appropriately before calling this function */
@@ -308,28 +322,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
308 return ret; 322 return ret;
309} 323}
310 324
311static int __swiotlb_mmap_noncoherent(struct device *dev, 325static int __swiotlb_mmap(struct device *dev,
312 struct vm_area_struct *vma, 326 struct vm_area_struct *vma,
313 void *cpu_addr, dma_addr_t dma_addr, size_t size, 327 void *cpu_addr, dma_addr_t dma_addr, size_t size,
314 struct dma_attrs *attrs) 328 struct dma_attrs *attrs)
315{
316 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
317 return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
318}
319
320static int __swiotlb_mmap_coherent(struct device *dev,
321 struct vm_area_struct *vma,
322 void *cpu_addr, dma_addr_t dma_addr, size_t size,
323 struct dma_attrs *attrs)
324{ 329{
325 /* Just use whatever page_prot attributes were specified */ 330 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
331 is_device_dma_coherent(dev));
326 return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); 332 return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
327} 333}
328 334
329struct dma_map_ops noncoherent_swiotlb_dma_ops = { 335static struct dma_map_ops swiotlb_dma_ops = {
330 .alloc = __dma_alloc_noncoherent, 336 .alloc = __dma_alloc,
331 .free = __dma_free_noncoherent, 337 .free = __dma_free,
332 .mmap = __swiotlb_mmap_noncoherent, 338 .mmap = __swiotlb_mmap,
333 .map_page = __swiotlb_map_page, 339 .map_page = __swiotlb_map_page,
334 .unmap_page = __swiotlb_unmap_page, 340 .unmap_page = __swiotlb_unmap_page,
335 .map_sg = __swiotlb_map_sg_attrs, 341 .map_sg = __swiotlb_map_sg_attrs,
@@ -341,24 +347,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = {
341 .dma_supported = swiotlb_dma_supported, 347 .dma_supported = swiotlb_dma_supported,
342 .mapping_error = swiotlb_dma_mapping_error, 348 .mapping_error = swiotlb_dma_mapping_error,
343}; 349};
344EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
345
346struct dma_map_ops coherent_swiotlb_dma_ops = {
347 .alloc = __dma_alloc_coherent,
348 .free = __dma_free_coherent,
349 .mmap = __swiotlb_mmap_coherent,
350 .map_page = swiotlb_map_page,
351 .unmap_page = swiotlb_unmap_page,
352 .map_sg = swiotlb_map_sg_attrs,
353 .unmap_sg = swiotlb_unmap_sg_attrs,
354 .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
355 .sync_single_for_device = swiotlb_sync_single_for_device,
356 .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
357 .sync_sg_for_device = swiotlb_sync_sg_for_device,
358 .dma_supported = swiotlb_dma_supported,
359 .mapping_error = swiotlb_dma_mapping_error,
360};
361EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
362 350
363extern int swiotlb_late_init_with_default_size(size_t default_size); 351extern int swiotlb_late_init_with_default_size(size_t default_size);
364 352
@@ -427,7 +415,7 @@ static int __init swiotlb_late_init(void)
427{ 415{
428 size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT); 416 size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
429 417
430 dma_ops = &noncoherent_swiotlb_dma_ops; 418 dma_ops = &swiotlb_dma_ops;
431 419
432 return swiotlb_late_init_with_default_size(swiotlb_size); 420 return swiotlb_late_init_with_default_size(swiotlb_size);
433} 421}
diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index d54dc9ac4b70..74c256744b25 100644
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -14,14 +14,18 @@
14 * of the License. 14 * of the License.
15 */ 15 */
16#include <linux/debugfs.h> 16#include <linux/debugfs.h>
17#include <linux/errno.h>
17#include <linux/fs.h> 18#include <linux/fs.h>
18#include <linux/io.h> 19#include <linux/io.h>
20#include <linux/init.h>
19#include <linux/mm.h> 21#include <linux/mm.h>
20#include <linux/sched.h> 22#include <linux/sched.h>
21#include <linux/seq_file.h> 23#include <linux/seq_file.h>
22 24
23#include <asm/fixmap.h> 25#include <asm/fixmap.h>
26#include <asm/memory.h>
24#include <asm/pgtable.h> 27#include <asm/pgtable.h>
28#include <asm/pgtable-hwdef.h>
25 29
26#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS) 30#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS)
27 31
@@ -37,10 +41,10 @@ enum address_markers_idx {
37 VMEMMAP_START_NR, 41 VMEMMAP_START_NR,
38 VMEMMAP_END_NR, 42 VMEMMAP_END_NR,
39#endif 43#endif
40 PCI_START_NR,
41 PCI_END_NR,
42 FIXADDR_START_NR, 44 FIXADDR_START_NR,
43 FIXADDR_END_NR, 45 FIXADDR_END_NR,
46 PCI_START_NR,
47 PCI_END_NR,
44 MODULES_START_NR, 48 MODULES_START_NR,
45 MODUELS_END_NR, 49 MODUELS_END_NR,
46 KERNEL_SPACE_NR, 50 KERNEL_SPACE_NR,
@@ -53,10 +57,10 @@ static struct addr_marker address_markers[] = {
53 { 0, "vmemmap start" }, 57 { 0, "vmemmap start" },
54 { 0, "vmemmap end" }, 58 { 0, "vmemmap end" },
55#endif 59#endif
56 { (unsigned long) PCI_IOBASE, "PCI I/O start" },
57 { (unsigned long) PCI_IOBASE + SZ_16M, "PCI I/O end" },
58 { FIXADDR_START, "Fixmap start" }, 60 { FIXADDR_START, "Fixmap start" },
59 { FIXADDR_TOP, "Fixmap end" }, 61 { FIXADDR_TOP, "Fixmap end" },
62 { PCI_IO_START, "PCI I/O start" },
63 { PCI_IO_END, "PCI I/O end" },
60 { MODULES_VADDR, "Modules start" }, 64 { MODULES_VADDR, "Modules start" },
61 { MODULES_END, "Modules end" }, 65 { MODULES_END, "Modules end" },
62 { PAGE_OFFSET, "Kernel Mapping" }, 66 { PAGE_OFFSET, "Kernel Mapping" },
@@ -246,10 +250,12 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
246 250
247 for (i = 0; i < PTRS_PER_PMD; i++, pmd++) { 251 for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
248 addr = start + i * PMD_SIZE; 252 addr = start + i * PMD_SIZE;
249 if (pmd_none(*pmd) || pmd_sect(*pmd) || pmd_bad(*pmd)) 253 if (pmd_none(*pmd) || pmd_sect(*pmd)) {
250 note_page(st, addr, 3, pmd_val(*pmd)); 254 note_page(st, addr, 3, pmd_val(*pmd));
251 else 255 } else {
256 BUG_ON(pmd_bad(*pmd));
252 walk_pte(st, pmd, addr); 257 walk_pte(st, pmd, addr);
258 }
253 } 259 }
254} 260}
255 261
@@ -261,10 +267,12 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
261 267
262 for (i = 0; i < PTRS_PER_PUD; i++, pud++) { 268 for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
263 addr = start + i * PUD_SIZE; 269 addr = start + i * PUD_SIZE;
264 if (pud_none(*pud) || pud_sect(*pud) || pud_bad(*pud)) 270 if (pud_none(*pud) || pud_sect(*pud)) {
265 note_page(st, addr, 2, pud_val(*pud)); 271 note_page(st, addr, 2, pud_val(*pud));
266 else 272 } else {
273 BUG_ON(pud_bad(*pud));
267 walk_pmd(st, pud, addr); 274 walk_pmd(st, pud, addr);
275 }
268 } 276 }
269} 277}
270 278
@@ -276,10 +284,12 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st
276 284
277 for (i = 0; i < PTRS_PER_PGD; i++, pgd++) { 285 for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
278 addr = start + i * PGDIR_SIZE; 286 addr = start + i * PGDIR_SIZE;
279 if (pgd_none(*pgd) || pgd_bad(*pgd)) 287 if (pgd_none(*pgd)) {
280 note_page(st, addr, 1, pgd_val(*pgd)); 288 note_page(st, addr, 1, pgd_val(*pgd));
281 else 289 } else {
290 BUG_ON(pgd_bad(*pgd));
282 walk_pud(st, pgd, addr); 291 walk_pud(st, pgd, addr);
292 }
283 } 293 }
284} 294}
285 295
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index c11cd27ca8f5..96da13167d4a 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
219 219
220 if (esr & ESR_LNX_EXEC) { 220 if (esr & ESR_LNX_EXEC) {
221 vm_flags = VM_EXEC; 221 vm_flags = VM_EXEC;
222 } else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) { 222 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
223 vm_flags = VM_WRITE; 223 vm_flags = VM_WRITE;
224 mm_flags |= FAULT_FLAG_WRITE; 224 mm_flags |= FAULT_FLAG_WRITE;
225 } 225 }
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 023747bf4dd7..2de9d2e59d96 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -38,12 +38,6 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
38} 38}
39#endif 39#endif
40 40
41struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
42 int write)
43{
44 return ERR_PTR(-EINVAL);
45}
46
47int pmd_huge(pmd_t pmd) 41int pmd_huge(pmd_t pmd)
48{ 42{
49 return !(pmd_val(pmd) & PMD_TABLE_BIT); 43 return !(pmd_val(pmd) & PMD_TABLE_BIT);
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index c95464a33f36..71145f952070 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -35,6 +35,7 @@
35#include <linux/efi.h> 35#include <linux/efi.h>
36 36
37#include <asm/fixmap.h> 37#include <asm/fixmap.h>
38#include <asm/memory.h>
38#include <asm/sections.h> 39#include <asm/sections.h>
39#include <asm/setup.h> 40#include <asm/setup.h>
40#include <asm/sizes.h> 41#include <asm/sizes.h>
@@ -136,10 +137,29 @@ static void arm64_memory_present(void)
136} 137}
137#endif 138#endif
138 139
140static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
141
142/*
143 * Limit the memory size that was specified via FDT.
144 */
145static int __init early_mem(char *p)
146{
147 if (!p)
148 return 1;
149
150 memory_limit = memparse(p, &p) & PAGE_MASK;
151 pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
152
153 return 0;
154}
155early_param("mem", early_mem);
156
139void __init arm64_memblock_init(void) 157void __init arm64_memblock_init(void)
140{ 158{
141 phys_addr_t dma_phys_limit = 0; 159 phys_addr_t dma_phys_limit = 0;
142 160
161 memblock_enforce_memory_limit(memory_limit);
162
143 /* 163 /*
144 * Register the kernel text, kernel data, initrd, and initial 164 * Register the kernel text, kernel data, initrd, and initial
145 * pagetables with memblock. 165 * pagetables with memblock.
@@ -277,8 +297,8 @@ void __init mem_init(void)
277 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" 297 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
278 " 0x%16lx - 0x%16lx (%6ld MB actual)\n" 298 " 0x%16lx - 0x%16lx (%6ld MB actual)\n"
279#endif 299#endif
280 " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
281 " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n" 300 " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n"
301 " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
282 " modules : 0x%16lx - 0x%16lx (%6ld MB)\n" 302 " modules : 0x%16lx - 0x%16lx (%6ld MB)\n"
283 " memory : 0x%16lx - 0x%16lx (%6ld MB)\n" 303 " memory : 0x%16lx - 0x%16lx (%6ld MB)\n"
284 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" 304 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
@@ -291,8 +311,8 @@ void __init mem_init(void)
291 MLM((unsigned long)virt_to_page(PAGE_OFFSET), 311 MLM((unsigned long)virt_to_page(PAGE_OFFSET),
292 (unsigned long)virt_to_page(high_memory)), 312 (unsigned long)virt_to_page(high_memory)),
293#endif 313#endif
294 MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M),
295 MLK(FIXADDR_START, FIXADDR_TOP), 314 MLK(FIXADDR_START, FIXADDR_TOP),
315 MLM(PCI_IO_START, PCI_IO_END),
296 MLM(MODULES_VADDR, MODULES_END), 316 MLM(MODULES_VADDR, MODULES_END),
297 MLM(PAGE_OFFSET, (unsigned long)high_memory), 317 MLM(PAGE_OFFSET, (unsigned long)high_memory),
298 MLK_ROUNDUP(__init_begin, __init_end), 318 MLK_ROUNDUP(__init_begin, __init_end),
@@ -325,6 +345,7 @@ void __init mem_init(void)
325 345
326void free_initmem(void) 346void free_initmem(void)
327{ 347{
348 fixup_init();
328 free_initmem_default(0); 349 free_initmem_default(0);
329 free_alternatives_memory(); 350 free_alternatives_memory();
330} 351}
diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
index cbb99c8f1e04..01e88c8bcab0 100644
--- a/arch/arm64/mm/ioremap.c
+++ b/arch/arm64/mm/ioremap.c
@@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
62 if (!area) 62 if (!area)
63 return NULL; 63 return NULL;
64 addr = (unsigned long)area->addr; 64 addr = (unsigned long)area->addr;
65 area->phys_addr = phys_addr;
65 66
66 err = ioremap_page_range(addr, addr + size, phys_addr, prot); 67 err = ioremap_page_range(addr, addr + size, phys_addr, prot);
67 if (err) { 68 if (err) {
diff --git a/arch/arm64/mm/mm.h b/arch/arm64/mm/mm.h
index 50c3351df9c7..ef47d99b5cbc 100644
--- a/arch/arm64/mm/mm.h
+++ b/arch/arm64/mm/mm.h
@@ -1 +1,3 @@
1extern void __init bootmem_init(void); 1extern void __init bootmem_init(void);
2
3void fixup_init(void);
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 6032f3e3056a..c6daaf6c6f97 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -26,6 +26,8 @@
26#include <linux/memblock.h> 26#include <linux/memblock.h>
27#include <linux/fs.h> 27#include <linux/fs.h>
28#include <linux/io.h> 28#include <linux/io.h>
29#include <linux/slab.h>
30#include <linux/stop_machine.h>
29 31
30#include <asm/cputype.h> 32#include <asm/cputype.h>
31#include <asm/fixmap.h> 33#include <asm/fixmap.h>
@@ -45,80 +47,6 @@
45struct page *empty_zero_page; 47struct page *empty_zero_page;
46EXPORT_SYMBOL(empty_zero_page); 48EXPORT_SYMBOL(empty_zero_page);
47 49
48struct cachepolicy {
49 const char policy[16];
50 u64 mair;
51 u64 tcr;
52};
53
54static struct cachepolicy cache_policies[] __initdata = {
55 {
56 .policy = "uncached",
57 .mair = 0x44, /* inner, outer non-cacheable */
58 .tcr = TCR_IRGN_NC | TCR_ORGN_NC,
59 }, {
60 .policy = "writethrough",
61 .mair = 0xaa, /* inner, outer write-through, read-allocate */
62 .tcr = TCR_IRGN_WT | TCR_ORGN_WT,
63 }, {
64 .policy = "writeback",
65 .mair = 0xee, /* inner, outer write-back, read-allocate */
66 .tcr = TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
67 }
68};
69
70/*
71 * These are useful for identifying cache coherency problems by allowing the
72 * cache or the cache and writebuffer to be turned off. It changes the Normal
73 * memory caching attributes in the MAIR_EL1 register.
74 */
75static int __init early_cachepolicy(char *p)
76{
77 int i;
78 u64 tmp;
79
80 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
81 int len = strlen(cache_policies[i].policy);
82
83 if (memcmp(p, cache_policies[i].policy, len) == 0)
84 break;
85 }
86 if (i == ARRAY_SIZE(cache_policies)) {
87 pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
88 return 0;
89 }
90
91 flush_cache_all();
92
93 /*
94 * Modify MT_NORMAL attributes in MAIR_EL1.
95 */
96 asm volatile(
97 " mrs %0, mair_el1\n"
98 " bfi %0, %1, %2, #8\n"
99 " msr mair_el1, %0\n"
100 " isb\n"
101 : "=&r" (tmp)
102 : "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
103
104 /*
105 * Modify TCR PTW cacheability attributes.
106 */
107 asm volatile(
108 " mrs %0, tcr_el1\n"
109 " bic %0, %0, %2\n"
110 " orr %0, %0, %1\n"
111 " msr tcr_el1, %0\n"
112 " isb\n"
113 : "=&r" (tmp)
114 : "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
115
116 flush_cache_all();
117
118 return 0;
119}
120early_param("cachepolicy", early_cachepolicy);
121
122pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 50pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
123 unsigned long size, pgprot_t vma_prot) 51 unsigned long size, pgprot_t vma_prot)
124{ 52{
@@ -133,19 +61,42 @@ EXPORT_SYMBOL(phys_mem_access_prot);
133static void __init *early_alloc(unsigned long sz) 61static void __init *early_alloc(unsigned long sz)
134{ 62{
135 void *ptr = __va(memblock_alloc(sz, sz)); 63 void *ptr = __va(memblock_alloc(sz, sz));
64 BUG_ON(!ptr);
136 memset(ptr, 0, sz); 65 memset(ptr, 0, sz);
137 return ptr; 66 return ptr;
138} 67}
139 68
140static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, 69/*
70 * remap a PMD into pages
71 */
72static void split_pmd(pmd_t *pmd, pte_t *pte)
73{
74 unsigned long pfn = pmd_pfn(*pmd);
75 int i = 0;
76
77 do {
78 /*
79 * Need to have the least restrictive permissions available
80 * permissions will be fixed up later
81 */
82 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
83 pfn++;
84 } while (pte++, i++, i < PTRS_PER_PTE);
85}
86
87static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
141 unsigned long end, unsigned long pfn, 88 unsigned long end, unsigned long pfn,
142 pgprot_t prot) 89 pgprot_t prot,
90 void *(*alloc)(unsigned long size))
143{ 91{
144 pte_t *pte; 92 pte_t *pte;
145 93
146 if (pmd_none(*pmd)) { 94 if (pmd_none(*pmd) || pmd_sect(*pmd)) {
147 pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t)); 95 pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
96 if (pmd_sect(*pmd))
97 split_pmd(pmd, pte);
148 __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); 98 __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
99 flush_tlb_all();
149 } 100 }
150 BUG_ON(pmd_bad(*pmd)); 101 BUG_ON(pmd_bad(*pmd));
151 102
@@ -156,30 +107,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
156 } while (pte++, addr += PAGE_SIZE, addr != end); 107 } while (pte++, addr += PAGE_SIZE, addr != end);
157} 108}
158 109
159static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, 110void split_pud(pud_t *old_pud, pmd_t *pmd)
160 unsigned long end, phys_addr_t phys, 111{
161 int map_io) 112 unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
113 pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
114 int i = 0;
115
116 do {
117 set_pmd(pmd, __pmd(addr | prot));
118 addr += PMD_SIZE;
119 } while (pmd++, i++, i < PTRS_PER_PMD);
120}
121
122static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
123 unsigned long addr, unsigned long end,
124 phys_addr_t phys, pgprot_t prot,
125 void *(*alloc)(unsigned long size))
162{ 126{
163 pmd_t *pmd; 127 pmd_t *pmd;
164 unsigned long next; 128 unsigned long next;
165 pmdval_t prot_sect;
166 pgprot_t prot_pte;
167
168 if (map_io) {
169 prot_sect = PROT_SECT_DEVICE_nGnRE;
170 prot_pte = __pgprot(PROT_DEVICE_nGnRE);
171 } else {
172 prot_sect = PROT_SECT_NORMAL_EXEC;
173 prot_pte = PAGE_KERNEL_EXEC;
174 }
175 129
176 /* 130 /*
177 * Check for initial section mappings in the pgd/pud and remove them. 131 * Check for initial section mappings in the pgd/pud and remove them.
178 */ 132 */
179 if (pud_none(*pud) || pud_bad(*pud)) { 133 if (pud_none(*pud) || pud_sect(*pud)) {
180 pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t)); 134 pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t));
181 pud_populate(&init_mm, pud, pmd); 135 if (pud_sect(*pud)) {
136 /*
137 * need to have the 1G of mappings continue to be
138 * present
139 */
140 split_pud(pud, pmd);
141 }
142 pud_populate(mm, pud, pmd);
143 flush_tlb_all();
182 } 144 }
145 BUG_ON(pud_bad(*pud));
183 146
184 pmd = pmd_offset(pud, addr); 147 pmd = pmd_offset(pud, addr);
185 do { 148 do {
@@ -187,31 +150,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
187 /* try section mapping first */ 150 /* try section mapping first */
188 if (((addr | next | phys) & ~SECTION_MASK) == 0) { 151 if (((addr | next | phys) & ~SECTION_MASK) == 0) {
189 pmd_t old_pmd =*pmd; 152 pmd_t old_pmd =*pmd;
190 set_pmd(pmd, __pmd(phys | prot_sect)); 153 set_pmd(pmd, __pmd(phys |
154 pgprot_val(mk_sect_prot(prot))));
191 /* 155 /*
192 * Check for previous table entries created during 156 * Check for previous table entries created during
193 * boot (__create_page_tables) and flush them. 157 * boot (__create_page_tables) and flush them.
194 */ 158 */
195 if (!pmd_none(old_pmd)) 159 if (!pmd_none(old_pmd)) {
196 flush_tlb_all(); 160 flush_tlb_all();
161 if (pmd_table(old_pmd)) {
162 phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0));
163 if (!WARN_ON_ONCE(slab_is_available()))
164 memblock_free(table, PAGE_SIZE);
165 }
166 }
197 } else { 167 } else {
198 alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 168 alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
199 prot_pte); 169 prot, alloc);
200 } 170 }
201 phys += next - addr; 171 phys += next - addr;
202 } while (pmd++, addr = next, addr != end); 172 } while (pmd++, addr = next, addr != end);
203} 173}
204 174
205static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, 175static inline bool use_1G_block(unsigned long addr, unsigned long next,
206 unsigned long end, phys_addr_t phys, 176 unsigned long phys)
207 int map_io) 177{
178 if (PAGE_SHIFT != 12)
179 return false;
180
181 if (((addr | next | phys) & ~PUD_MASK) != 0)
182 return false;
183
184 return true;
185}
186
187static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
188 unsigned long addr, unsigned long end,
189 phys_addr_t phys, pgprot_t prot,
190 void *(*alloc)(unsigned long size))
208{ 191{
209 pud_t *pud; 192 pud_t *pud;
210 unsigned long next; 193 unsigned long next;
211 194
212 if (pgd_none(*pgd)) { 195 if (pgd_none(*pgd)) {
213 pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t)); 196 pud = alloc(PTRS_PER_PUD * sizeof(pud_t));
214 pgd_populate(&init_mm, pgd, pud); 197 pgd_populate(mm, pgd, pud);
215 } 198 }
216 BUG_ON(pgd_bad(*pgd)); 199 BUG_ON(pgd_bad(*pgd));
217 200
@@ -222,10 +205,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
222 /* 205 /*
223 * For 4K granule only, attempt to put down a 1GB block 206 * For 4K granule only, attempt to put down a 1GB block
224 */ 207 */
225 if (!map_io && (PAGE_SHIFT == 12) && 208 if (use_1G_block(addr, next, phys)) {
226 ((addr | next | phys) & ~PUD_MASK) == 0) {
227 pud_t old_pud = *pud; 209 pud_t old_pud = *pud;
228 set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC)); 210 set_pud(pud, __pud(phys |
211 pgprot_val(mk_sect_prot(prot))));
229 212
230 /* 213 /*
231 * If we have an old value for a pud, it will 214 * If we have an old value for a pud, it will
@@ -235,12 +218,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
235 * Look up the old pmd table and free it. 218 * Look up the old pmd table and free it.
236 */ 219 */
237 if (!pud_none(old_pud)) { 220 if (!pud_none(old_pud)) {
238 phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
239 memblock_free(table, PAGE_SIZE);
240 flush_tlb_all(); 221 flush_tlb_all();
222 if (pud_table(old_pud)) {
223 phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
224 if (!WARN_ON_ONCE(slab_is_available()))
225 memblock_free(table, PAGE_SIZE);
226 }
241 } 227 }
242 } else { 228 } else {
243 alloc_init_pmd(pud, addr, next, phys, map_io); 229 alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc);
244 } 230 }
245 phys += next - addr; 231 phys += next - addr;
246 } while (pud++, addr = next, addr != end); 232 } while (pud++, addr = next, addr != end);
@@ -250,9 +236,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
250 * Create the page directory entries and any necessary page tables for the 236 * Create the page directory entries and any necessary page tables for the
251 * mapping specified by 'md'. 237 * mapping specified by 'md'.
252 */ 238 */
253static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys, 239static void __create_mapping(struct mm_struct *mm, pgd_t *pgd,
254 unsigned long virt, phys_addr_t size, 240 phys_addr_t phys, unsigned long virt,
255 int map_io) 241 phys_addr_t size, pgprot_t prot,
242 void *(*alloc)(unsigned long size))
256{ 243{
257 unsigned long addr, length, end, next; 244 unsigned long addr, length, end, next;
258 245
@@ -262,31 +249,95 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
262 end = addr + length; 249 end = addr + length;
263 do { 250 do {
264 next = pgd_addr_end(addr, end); 251 next = pgd_addr_end(addr, end);
265 alloc_init_pud(pgd, addr, next, phys, map_io); 252 alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc);
266 phys += next - addr; 253 phys += next - addr;
267 } while (pgd++, addr = next, addr != end); 254 } while (pgd++, addr = next, addr != end);
268} 255}
269 256
270static void __init create_mapping(phys_addr_t phys, unsigned long virt, 257static void *late_alloc(unsigned long size)
271 phys_addr_t size) 258{
259 void *ptr;
260
261 BUG_ON(size > PAGE_SIZE);
262 ptr = (void *)__get_free_page(PGALLOC_GFP);
263 BUG_ON(!ptr);
264 return ptr;
265}
266
267static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
268 phys_addr_t size, pgprot_t prot)
272{ 269{
273 if (virt < VMALLOC_START) { 270 if (virt < VMALLOC_START) {
274 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 271 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
275 &phys, virt); 272 &phys, virt);
276 return; 273 return;
277 } 274 }
278 __create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0); 275 __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
276 size, prot, early_alloc);
277}
278
279void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
280 unsigned long virt, phys_addr_t size,
281 pgprot_t prot)
282{
283 __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot,
284 late_alloc);
279} 285}
280 286
281void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io) 287static void create_mapping_late(phys_addr_t phys, unsigned long virt,
288 phys_addr_t size, pgprot_t prot)
282{ 289{
283 if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) { 290 if (virt < VMALLOC_START) {
284 pr_warn("BUG: not creating id mapping for %pa\n", &addr); 291 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
292 &phys, virt);
285 return; 293 return;
286 } 294 }
287 __create_mapping(&idmap_pg_dir[pgd_index(addr)], 295
288 addr, addr, size, map_io); 296 return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK),
297 phys, virt, size, prot, late_alloc);
298}
299
300#ifdef CONFIG_DEBUG_RODATA
301static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
302{
303 /*
304 * Set up the executable regions using the existing section mappings
305 * for now. This will get more fine grained later once all memory
306 * is mapped
307 */
308 unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
309 unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
310
311 if (end < kernel_x_start) {
312 create_mapping(start, __phys_to_virt(start),
313 end - start, PAGE_KERNEL);
314 } else if (start >= kernel_x_end) {
315 create_mapping(start, __phys_to_virt(start),
316 end - start, PAGE_KERNEL);
317 } else {
318 if (start < kernel_x_start)
319 create_mapping(start, __phys_to_virt(start),
320 kernel_x_start - start,
321 PAGE_KERNEL);
322 create_mapping(kernel_x_start,
323 __phys_to_virt(kernel_x_start),
324 kernel_x_end - kernel_x_start,
325 PAGE_KERNEL_EXEC);
326 if (kernel_x_end < end)
327 create_mapping(kernel_x_end,
328 __phys_to_virt(kernel_x_end),
329 end - kernel_x_end,
330 PAGE_KERNEL);
331 }
332
289} 333}
334#else
335static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
336{
337 create_mapping(start, __phys_to_virt(start), end - start,
338 PAGE_KERNEL_EXEC);
339}
340#endif
290 341
291static void __init map_mem(void) 342static void __init map_mem(void)
292{ 343{
@@ -332,14 +383,53 @@ static void __init map_mem(void)
332 memblock_set_current_limit(limit); 383 memblock_set_current_limit(limit);
333 } 384 }
334#endif 385#endif
335 386 __map_memblock(start, end);
336 create_mapping(start, __phys_to_virt(start), end - start);
337 } 387 }
338 388
339 /* Limit no longer required. */ 389 /* Limit no longer required. */
340 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 390 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
341} 391}
342 392
393void __init fixup_executable(void)
394{
395#ifdef CONFIG_DEBUG_RODATA
396 /* now that we are actually fully mapped, make the start/end more fine grained */
397 if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) {
398 unsigned long aligned_start = round_down(__pa(_stext),
399 SECTION_SIZE);
400
401 create_mapping(aligned_start, __phys_to_virt(aligned_start),
402 __pa(_stext) - aligned_start,
403 PAGE_KERNEL);
404 }
405
406 if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) {
407 unsigned long aligned_end = round_up(__pa(__init_end),
408 SECTION_SIZE);
409 create_mapping(__pa(__init_end), (unsigned long)__init_end,
410 aligned_end - __pa(__init_end),
411 PAGE_KERNEL);
412 }
413#endif
414}
415
416#ifdef CONFIG_DEBUG_RODATA
417void mark_rodata_ro(void)
418{
419 create_mapping_late(__pa(_stext), (unsigned long)_stext,
420 (unsigned long)_etext - (unsigned long)_stext,
421 PAGE_KERNEL_EXEC | PTE_RDONLY);
422
423}
424#endif
425
426void fixup_init(void)
427{
428 create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin,
429 (unsigned long)__init_end - (unsigned long)__init_begin,
430 PAGE_KERNEL);
431}
432
343/* 433/*
344 * paging_init() sets up the page tables, initialises the zone memory 434 * paging_init() sets up the page tables, initialises the zone memory
345 * maps and sets up the zero page. 435 * maps and sets up the zero page.
@@ -349,13 +439,7 @@ void __init paging_init(void)
349 void *zero_page; 439 void *zero_page;
350 440
351 map_mem(); 441 map_mem();
352 442 fixup_executable();
353 /*
354 * Finally flush the caches and tlb to ensure that we're in a
355 * consistent state.
356 */
357 flush_cache_all();
358 flush_tlb_all();
359 443
360 /* allocate the zero page. */ 444 /* allocate the zero page. */
361 zero_page = early_alloc(PAGE_SIZE); 445 zero_page = early_alloc(PAGE_SIZE);
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 4e778b13291b..28eebfb6af76 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -102,7 +102,7 @@ ENTRY(cpu_do_idle)
102 ret 102 ret
103ENDPROC(cpu_do_idle) 103ENDPROC(cpu_do_idle)
104 104
105#ifdef CONFIG_ARM64_CPU_SUSPEND 105#ifdef CONFIG_CPU_PM
106/** 106/**
107 * cpu_do_suspend - save CPU registers context 107 * cpu_do_suspend - save CPU registers context
108 * 108 *
@@ -244,14 +244,18 @@ ENTRY(__cpu_setup)
244ENDPROC(__cpu_setup) 244ENDPROC(__cpu_setup)
245 245
246 /* 246 /*
247 * We set the desired value explicitly, including those of the
248 * reserved bits. The values of bits EE & E0E were set early in
249 * el2_setup, which are left untouched below.
250 *
247 * n n T 251 * n n T
248 * U E WT T UD US IHBS 252 * U E WT T UD US IHBS
249 * CE0 XWHW CZ ME TEEA S 253 * CE0 XWHW CZ ME TEEA S
250 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM 254 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
251 * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved 255 * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
252 * .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings 256 * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
253 */ 257 */
254 .type crval, #object 258 .type crval, #object
255crval: 259crval:
256 .word 0x000802e2 // clear 260 .word 0xfcffffff // clear
257 .word 0x0405d11d // set 261 .word 0x34d5d91d // set
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-22 19:12:49 -0500