aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64')
-rw-r--r--arch/arm64/Kconfig57
-rw-r--r--arch/arm64/Kconfig.debug15
-rw-r--r--arch/arm64/Makefile4
-rw-r--r--arch/arm64/configs/defconfig7
-rw-r--r--arch/arm64/crypto/Makefile2
-rw-r--r--arch/arm64/include/asm/cacheflush.h11
-rw-r--r--arch/arm64/include/asm/cachetype.h16
-rw-r--r--arch/arm64/include/asm/cpu.h59
-rw-r--r--arch/arm64/include/asm/cputype.h35
-rw-r--r--arch/arm64/include/asm/fpsimdmacros.h17
-rw-r--r--arch/arm64/include/asm/memory.h6
-rw-r--r--arch/arm64/include/asm/page.h20
-rw-r--r--arch/arm64/include/asm/pgalloc.h24
-rw-r--r--arch/arm64/include/asm/pgtable-2level-hwdef.h43
-rw-r--r--arch/arm64/include/asm/pgtable-2level-types.h62
-rw-r--r--arch/arm64/include/asm/pgtable-3level-hwdef.h50
-rw-r--r--arch/arm64/include/asm/pgtable-hwdef.h42
-rw-r--r--arch/arm64/include/asm/pgtable-types.h (renamed from arch/arm64/include/asm/pgtable-3level-types.h)73
-rw-r--r--arch/arm64/include/asm/pgtable.h100
-rw-r--r--arch/arm64/include/asm/processor.h4
-rw-r--r--arch/arm64/include/asm/stackprotector.h38
-rw-r--r--arch/arm64/include/asm/syscall.h14
-rw-r--r--arch/arm64/include/asm/sysreg.h60
-rw-r--r--arch/arm64/include/asm/thread_info.h5
-rw-r--r--arch/arm64/include/asm/tlb.h11
-rw-r--r--arch/arm64/include/asm/tlbflush.h34
-rw-r--r--arch/arm64/include/asm/unistd.h17
-rw-r--r--arch/arm64/include/asm/unistd32.h1166
-rw-r--r--arch/arm64/kernel/Makefile3
-rw-r--r--arch/arm64/kernel/cpu_ops.c2
-rw-r--r--arch/arm64/kernel/cpuinfo.c192
-rw-r--r--arch/arm64/kernel/debug-monitors.c22
-rw-r--r--arch/arm64/kernel/entry-fpsimd.S2
-rw-r--r--arch/arm64/kernel/entry.S56
-rw-r--r--arch/arm64/kernel/head.S121
-rw-r--r--arch/arm64/kernel/hyp-stub.S1
-rw-r--r--arch/arm64/kernel/image.h62
-rw-r--r--arch/arm64/kernel/kuser32.S2
-rw-r--r--arch/arm64/kernel/process.c6
-rw-r--r--arch/arm64/kernel/psci.c8
-rw-r--r--arch/arm64/kernel/ptrace.c11
-rw-r--r--arch/arm64/kernel/setup.c47
-rw-r--r--arch/arm64/kernel/signal32.c2
-rw-r--r--arch/arm64/kernel/smp.c6
-rw-r--r--arch/arm64/kernel/suspend.c2
-rw-r--r--arch/arm64/kernel/sys_compat.c2
-rw-r--r--arch/arm64/kernel/topology.c47
-rw-r--r--arch/arm64/kernel/traps.c13
-rw-r--r--arch/arm64/kernel/vdso.c94
-rw-r--r--arch/arm64/kernel/vdso/Makefile6
-rw-r--r--arch/arm64/kernel/vdso/vdso.lds.S4
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S16
-rw-r--r--arch/arm64/mm/fault.c1
-rw-r--r--arch/arm64/mm/init.c34
-rw-r--r--arch/arm64/mm/ioremap.c30
-rw-r--r--arch/arm64/mm/mmu.c14
56 files changed, 1952 insertions, 846 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 839f48c26ef0..f3b584be76d7 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1,7 +1,6 @@
1config ARM64 1config ARM64
2 def_bool y 2 def_bool y
3 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE 3 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
4 select ARCH_HAS_OPP
5 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST 4 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
6 select ARCH_USE_CMPXCHG_LOCKREF 5 select ARCH_USE_CMPXCHG_LOCKREF
7 select ARCH_SUPPORTS_ATOMIC_RMW 6 select ARCH_SUPPORTS_ATOMIC_RMW
@@ -11,6 +10,8 @@ config ARM64
11 select ARM_AMBA 10 select ARM_AMBA
12 select ARM_ARCH_TIMER 11 select ARM_ARCH_TIMER
13 select ARM_GIC 12 select ARM_GIC
13 select AUDIT_ARCH_COMPAT_GENERIC
14 select ARM_GIC_V3
14 select BUILDTIME_EXTABLE_SORT 15 select BUILDTIME_EXTABLE_SORT
15 select CLONE_BACKWARDS 16 select CLONE_BACKWARDS
16 select COMMON_CLK 17 select COMMON_CLK
@@ -29,10 +30,12 @@ config ARM64
29 select GENERIC_STRNLEN_USER 30 select GENERIC_STRNLEN_USER
30 select GENERIC_TIME_VSYSCALL 31 select GENERIC_TIME_VSYSCALL
31 select HARDIRQS_SW_RESEND 32 select HARDIRQS_SW_RESEND
33 select HAVE_ARCH_AUDITSYSCALL
32 select HAVE_ARCH_JUMP_LABEL 34 select HAVE_ARCH_JUMP_LABEL
33 select HAVE_ARCH_KGDB 35 select HAVE_ARCH_KGDB
34 select HAVE_ARCH_TRACEHOOK 36 select HAVE_ARCH_TRACEHOOK
35 select HAVE_C_RECORDMCOUNT 37 select HAVE_C_RECORDMCOUNT
38 select HAVE_CC_STACKPROTECTOR
36 select HAVE_DEBUG_BUGVERBOSE 39 select HAVE_DEBUG_BUGVERBOSE
37 select HAVE_DEBUG_KMEMLEAK 40 select HAVE_DEBUG_KMEMLEAK
38 select HAVE_DMA_API_DEBUG 41 select HAVE_DMA_API_DEBUG
@@ -63,6 +66,7 @@ config ARM64
63 select RTC_LIB 66 select RTC_LIB
64 select SPARSE_IRQ 67 select SPARSE_IRQ
65 select SYSCTL_EXCEPTION_TRACE 68 select SYSCTL_EXCEPTION_TRACE
69 select HAVE_CONTEXT_TRACKING
66 help 70 help
67 ARM 64-bit (AArch64) Linux support. 71 ARM 64-bit (AArch64) Linux support.
68 72
@@ -155,14 +159,63 @@ endmenu
155 159
156menu "Kernel Features" 160menu "Kernel Features"
157 161
162choice
163 prompt "Page size"
164 default ARM64_4K_PAGES
165 help
166 Page size (translation granule) configuration.
167
168config ARM64_4K_PAGES
169 bool "4KB"
170 help
171 This feature enables 4KB pages support.
172
158config ARM64_64K_PAGES 173config ARM64_64K_PAGES
159 bool "Enable 64KB pages support" 174 bool "64KB"
160 help 175 help
161 This feature enables 64KB pages support (4KB by default) 176 This feature enables 64KB pages support (4KB by default)
162 allowing only two levels of page tables and faster TLB 177 allowing only two levels of page tables and faster TLB
163 look-up. AArch32 emulation is not available when this feature 178 look-up. AArch32 emulation is not available when this feature
164 is enabled. 179 is enabled.
165 180
181endchoice
182
183choice
184 prompt "Virtual address space size"
185 default ARM64_VA_BITS_39 if ARM64_4K_PAGES
186 default ARM64_VA_BITS_42 if ARM64_64K_PAGES
187 help
188 Allows choosing one of multiple possible virtual address
189 space sizes. The level of translation table is determined by
190 a combination of page size and virtual address space size.
191
192config ARM64_VA_BITS_39
193 bool "39-bit"
194 depends on ARM64_4K_PAGES
195
196config ARM64_VA_BITS_42
197 bool "42-bit"
198 depends on ARM64_64K_PAGES
199
200config ARM64_VA_BITS_48
201 bool "48-bit"
202 depends on BROKEN
203
204endchoice
205
206config ARM64_VA_BITS
207 int
208 default 39 if ARM64_VA_BITS_39
209 default 42 if ARM64_VA_BITS_42
210 default 48 if ARM64_VA_BITS_48
211
212config ARM64_PGTABLE_LEVELS
213 int
214 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42
215 default 3 if ARM64_64K_PAGES && ARM64_VA_BITS_48
216 default 3 if ARM64_4K_PAGES && ARM64_VA_BITS_39
217 default 4 if ARM64_4K_PAGES && ARM64_VA_BITS_48
218
166config CPU_BIG_ENDIAN 219config CPU_BIG_ENDIAN
167 bool "Build big-endian kernel" 220 bool "Build big-endian kernel"
168 help 221 help
diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug
index 1c1b75629842..4ee8e90b7a45 100644
--- a/arch/arm64/Kconfig.debug
+++ b/arch/arm64/Kconfig.debug
@@ -28,4 +28,19 @@ config PID_IN_CONTEXTIDR
28 instructions during context switch. Say Y here only if you are 28 instructions during context switch. Say Y here only if you are
29 planning to use hardware trace tools with this kernel. 29 planning to use hardware trace tools with this kernel.
30 30
31config ARM64_RANDOMIZE_TEXT_OFFSET
32 bool "Randomize TEXT_OFFSET at build time"
33 help
34 Say Y here if you want the image load offset (AKA TEXT_OFFSET)
35 of the kernel to be randomized at build-time. When selected,
36 this option will cause TEXT_OFFSET to be randomized upon any
37 build of the kernel, and the offset will be reflected in the
38 text_offset field of the resulting Image. This can be used to
39 fuzz-test bootloaders which respect text_offset.
40
41 This option is intended for bootloader and/or kernel testing
42 only. Bootloaders must make no assumptions regarding the value
43 of TEXT_OFFSET and platforms must not require a specific
44 value.
45
31endmenu 46endmenu
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 8185a913c5ed..e8d025c1459e 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -38,7 +38,11 @@ CHECKFLAGS += -D__aarch64__
38head-y := arch/arm64/kernel/head.o 38head-y := arch/arm64/kernel/head.o
39 39
40# The byte offset of the kernel image in RAM from the start of RAM. 40# The byte offset of the kernel image in RAM from the start of RAM.
41ifeq ($(CONFIG_ARM64_RANDOMIZE_TEXT_OFFSET), y)
42TEXT_OFFSET := $(shell awk 'BEGIN {srand(); printf "0x%04x0\n", int(65535 * rand())}')
43else
41TEXT_OFFSET := 0x00080000 44TEXT_OFFSET := 0x00080000
45endif
42 46
43export TEXT_OFFSET GZFLAGS 47export TEXT_OFFSET GZFLAGS
44 48
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 3421f316f5dc..1e52b741d806 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -52,8 +52,11 @@ CONFIG_IP_PNP_BOOTP=y
52# CONFIG_INET_LRO is not set 52# CONFIG_INET_LRO is not set
53# CONFIG_IPV6 is not set 53# CONFIG_IPV6 is not set
54# CONFIG_WIRELESS is not set 54# CONFIG_WIRELESS is not set
55CONFIG_NET_9P=y
56CONFIG_NET_9P_VIRTIO=y
55CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" 57CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
56CONFIG_DEVTMPFS=y 58CONFIG_DEVTMPFS=y
59CONFIG_DEVTMPFS_MOUNT=y
57CONFIG_DMA_CMA=y 60CONFIG_DMA_CMA=y
58CONFIG_BLK_DEV_LOOP=y 61CONFIG_BLK_DEV_LOOP=y
59CONFIG_VIRTIO_BLK=y 62CONFIG_VIRTIO_BLK=y
@@ -65,6 +68,7 @@ CONFIG_PATA_PLATFORM=y
65CONFIG_PATA_OF_PLATFORM=y 68CONFIG_PATA_OF_PLATFORM=y
66CONFIG_NETDEVICES=y 69CONFIG_NETDEVICES=y
67CONFIG_TUN=y 70CONFIG_TUN=y
71CONFIG_VIRTIO_NET=y
68CONFIG_SMC91X=y 72CONFIG_SMC91X=y
69CONFIG_SMSC911X=y 73CONFIG_SMSC911X=y
70# CONFIG_WLAN is not set 74# CONFIG_WLAN is not set
@@ -76,6 +80,7 @@ CONFIG_SERIAL_8250_CONSOLE=y
76CONFIG_SERIAL_AMBA_PL011=y 80CONFIG_SERIAL_AMBA_PL011=y
77CONFIG_SERIAL_AMBA_PL011_CONSOLE=y 81CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
78CONFIG_SERIAL_OF_PLATFORM=y 82CONFIG_SERIAL_OF_PLATFORM=y
83CONFIG_VIRTIO_CONSOLE=y
79# CONFIG_HW_RANDOM is not set 84# CONFIG_HW_RANDOM is not set
80# CONFIG_HWMON is not set 85# CONFIG_HWMON is not set
81CONFIG_REGULATOR=y 86CONFIG_REGULATOR=y
@@ -90,6 +95,7 @@ CONFIG_USB_ISP1760_HCD=y
90CONFIG_USB_STORAGE=y 95CONFIG_USB_STORAGE=y
91CONFIG_MMC=y 96CONFIG_MMC=y
92CONFIG_MMC_ARMMMCI=y 97CONFIG_MMC_ARMMMCI=y
98CONFIG_VIRTIO_BALLOON=y
93CONFIG_VIRTIO_MMIO=y 99CONFIG_VIRTIO_MMIO=y
94# CONFIG_IOMMU_SUPPORT is not set 100# CONFIG_IOMMU_SUPPORT is not set
95CONFIG_EXT2_FS=y 101CONFIG_EXT2_FS=y
@@ -107,6 +113,7 @@ CONFIG_HUGETLBFS=y
107# CONFIG_MISC_FILESYSTEMS is not set 113# CONFIG_MISC_FILESYSTEMS is not set
108CONFIG_NFS_FS=y 114CONFIG_NFS_FS=y
109CONFIG_ROOT_NFS=y 115CONFIG_ROOT_NFS=y
116CONFIG_9P_FS=y
110CONFIG_NLS_CODEPAGE_437=y 117CONFIG_NLS_CODEPAGE_437=y
111CONFIG_NLS_ISO8859_1=y 118CONFIG_NLS_ISO8859_1=y
112CONFIG_VIRTUALIZATION=y 119CONFIG_VIRTUALIZATION=y
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 2070a56ecc46..a3f935fde975 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -35,4 +35,4 @@ AFLAGS_aes-neon.o := -DINTERLEAVE=4
35CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS 35CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS
36 36
37$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE 37$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE
38 $(call if_changed_dep,cc_o_c) 38 $(call if_changed_rule,cc_o_c)
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index a5176cf32dad..f2defe1c380c 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -138,19 +138,10 @@ static inline void __flush_icache_all(void)
138#define flush_icache_page(vma,page) do { } while (0) 138#define flush_icache_page(vma,page) do { } while (0)
139 139
140/* 140/*
141 * flush_cache_vmap() is used when creating mappings (eg, via vmap, 141 * Not required on AArch64 (PIPT or VIPT non-aliasing D-cache).
142 * vmalloc, ioremap etc) in kernel space for pages. On non-VIPT
143 * caches, since the direct-mappings of these pages may contain cached
144 * data, we need to do a full cache flush to ensure that writebacks
145 * don't corrupt data placed into these pages via the new mappings.
146 */ 142 */
147static inline void flush_cache_vmap(unsigned long start, unsigned long end) 143static inline void flush_cache_vmap(unsigned long start, unsigned long end)
148{ 144{
149 /*
150 * set_pte_at() called from vmap_pte_range() does not
151 * have a DSB after cleaning the cache line.
152 */
153 dsb(ish);
154} 145}
155 146
156static inline void flush_cache_vunmap(unsigned long start, unsigned long end) 147static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h
index 4b23e758d5e0..7a2e0762cb40 100644
--- a/arch/arm64/include/asm/cachetype.h
+++ b/arch/arm64/include/asm/cachetype.h
@@ -30,10 +30,14 @@
30 30
31#ifndef __ASSEMBLY__ 31#ifndef __ASSEMBLY__
32 32
33static inline u32 icache_policy(void) 33#include <linux/bitops.h>
34{ 34
35 return (read_cpuid_cachetype() >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK; 35#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
36} 36
37#define ICACHEF_ALIASING BIT(0)
38#define ICACHEF_AIVIVT BIT(1)
39
40extern unsigned long __icache_flags;
37 41
38/* 42/*
39 * Whilst the D-side always behaves as PIPT on AArch64, aliasing is 43 * Whilst the D-side always behaves as PIPT on AArch64, aliasing is
@@ -41,12 +45,12 @@ static inline u32 icache_policy(void)
41 */ 45 */
42static inline int icache_is_aliasing(void) 46static inline int icache_is_aliasing(void)
43{ 47{
44 return icache_policy() != ICACHE_POLICY_PIPT; 48 return test_bit(ICACHEF_ALIASING, &__icache_flags);
45} 49}
46 50
47static inline int icache_is_aivivt(void) 51static inline int icache_is_aivivt(void)
48{ 52{
49 return icache_policy() == ICACHE_POLICY_AIVIVT; 53 return test_bit(ICACHEF_AIVIVT, &__icache_flags);
50} 54}
51 55
52static inline u32 cache_type_cwg(void) 56static inline u32 cache_type_cwg(void)
diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h
new file mode 100644
index 000000000000..056443086019
--- /dev/null
+++ b/arch/arm64/include/asm/cpu.h
@@ -0,0 +1,59 @@
1/*
2 * Copyright (C) 2014 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_CPU_H
17#define __ASM_CPU_H
18
19#include <linux/cpu.h>
20#include <linux/init.h>
21#include <linux/percpu.h>
22
23/*
24 * Records attributes of an individual CPU.
25 */
26struct cpuinfo_arm64 {
27 struct cpu cpu;
28 u32 reg_ctr;
29 u32 reg_cntfrq;
30 u32 reg_dczid;
31 u32 reg_midr;
32
33 u64 reg_id_aa64isar0;
34 u64 reg_id_aa64isar1;
35 u64 reg_id_aa64mmfr0;
36 u64 reg_id_aa64mmfr1;
37 u64 reg_id_aa64pfr0;
38 u64 reg_id_aa64pfr1;
39
40 u32 reg_id_isar0;
41 u32 reg_id_isar1;
42 u32 reg_id_isar2;
43 u32 reg_id_isar3;
44 u32 reg_id_isar4;
45 u32 reg_id_isar5;
46 u32 reg_id_mmfr0;
47 u32 reg_id_mmfr1;
48 u32 reg_id_mmfr2;
49 u32 reg_id_mmfr3;
50 u32 reg_id_pfr0;
51 u32 reg_id_pfr1;
52};
53
54DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data);
55
56void cpuinfo_store_cpu(void);
57void __init cpuinfo_store_boot_cpu(void);
58
59#endif /* __ASM_CPU_H */
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index 27f54a7cc81b..379d0b874328 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -18,6 +18,8 @@
18 18
19#define INVALID_HWID ULONG_MAX 19#define INVALID_HWID ULONG_MAX
20 20
21#define MPIDR_UP_BITMASK (0x1 << 30)
22#define MPIDR_MT_BITMASK (0x1 << 24)
21#define MPIDR_HWID_BITMASK 0xff00ffffff 23#define MPIDR_HWID_BITMASK 0xff00ffffff
22 24
23#define MPIDR_LEVEL_BITS_SHIFT 3 25#define MPIDR_LEVEL_BITS_SHIFT 3
@@ -36,15 +38,34 @@
36 __val; \ 38 __val; \
37}) 39})
38 40
41#define MIDR_REVISION_MASK 0xf
42#define MIDR_REVISION(midr) ((midr) & MIDR_REVISION_MASK)
43#define MIDR_PARTNUM_SHIFT 4
44#define MIDR_PARTNUM_MASK (0xfff << MIDR_PARTNUM_SHIFT)
45#define MIDR_PARTNUM(midr) \
46 (((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT)
47#define MIDR_ARCHITECTURE_SHIFT 16
48#define MIDR_ARCHITECTURE_MASK (0xf << MIDR_ARCHITECTURE_SHIFT)
49#define MIDR_ARCHITECTURE(midr) \
50 (((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT)
51#define MIDR_VARIANT_SHIFT 20
52#define MIDR_VARIANT_MASK (0xf << MIDR_VARIANT_SHIFT)
53#define MIDR_VARIANT(midr) \
54 (((midr) & MIDR_VARIANT_MASK) >> MIDR_VARIANT_SHIFT)
55#define MIDR_IMPLEMENTOR_SHIFT 24
56#define MIDR_IMPLEMENTOR_MASK (0xff << MIDR_IMPLEMENTOR_SHIFT)
57#define MIDR_IMPLEMENTOR(midr) \
58 (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT)
59
39#define ARM_CPU_IMP_ARM 0x41 60#define ARM_CPU_IMP_ARM 0x41
40#define ARM_CPU_IMP_APM 0x50 61#define ARM_CPU_IMP_APM 0x50
41 62
42#define ARM_CPU_PART_AEM_V8 0xD0F0 63#define ARM_CPU_PART_AEM_V8 0xD0F
43#define ARM_CPU_PART_FOUNDATION 0xD000 64#define ARM_CPU_PART_FOUNDATION 0xD00
44#define ARM_CPU_PART_CORTEX_A53 0xD030 65#define ARM_CPU_PART_CORTEX_A57 0xD07
45#define ARM_CPU_PART_CORTEX_A57 0xD070 66#define ARM_CPU_PART_CORTEX_A53 0xD03
46 67
47#define APM_CPU_PART_POTENZA 0x0000 68#define APM_CPU_PART_POTENZA 0x000
48 69
49#ifndef __ASSEMBLY__ 70#ifndef __ASSEMBLY__
50 71
@@ -65,12 +86,12 @@ static inline u64 __attribute_const__ read_cpuid_mpidr(void)
65 86
66static inline unsigned int __attribute_const__ read_cpuid_implementor(void) 87static inline unsigned int __attribute_const__ read_cpuid_implementor(void)
67{ 88{
68 return (read_cpuid_id() & 0xFF000000) >> 24; 89 return MIDR_IMPLEMENTOR(read_cpuid_id());
69} 90}
70 91
71static inline unsigned int __attribute_const__ read_cpuid_part_number(void) 92static inline unsigned int __attribute_const__ read_cpuid_part_number(void)
72{ 93{
73 return (read_cpuid_id() & 0xFFF0); 94 return MIDR_PARTNUM(read_cpuid_id());
74} 95}
75 96
76static inline u32 __attribute_const__ read_cpuid_cachetype(void) 97static inline u32 __attribute_const__ read_cpuid_cachetype(void)
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index 768414d55e64..007618b8188c 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -40,6 +40,19 @@
40 str w\tmpnr, [\state, #16 * 2 + 4] 40 str w\tmpnr, [\state, #16 * 2 + 4]
41.endm 41.endm
42 42
43.macro fpsimd_restore_fpcr state, tmp
44 /*
45 * Writes to fpcr may be self-synchronising, so avoid restoring
46 * the register if it hasn't changed.
47 */
48 mrs \tmp, fpcr
49 cmp \tmp, \state
50 b.eq 9999f
51 msr fpcr, \state
529999:
53.endm
54
55/* Clobbers \state */
43.macro fpsimd_restore state, tmpnr 56.macro fpsimd_restore state, tmpnr
44 ldp q0, q1, [\state, #16 * 0] 57 ldp q0, q1, [\state, #16 * 0]
45 ldp q2, q3, [\state, #16 * 2] 58 ldp q2, q3, [\state, #16 * 2]
@@ -60,7 +73,7 @@
60 ldr w\tmpnr, [\state, #16 * 2] 73 ldr w\tmpnr, [\state, #16 * 2]
61 msr fpsr, x\tmpnr 74 msr fpsr, x\tmpnr
62 ldr w\tmpnr, [\state, #16 * 2 + 4] 75 ldr w\tmpnr, [\state, #16 * 2 + 4]
63 msr fpcr, x\tmpnr 76 fpsimd_restore_fpcr x\tmpnr, \state
64.endm 77.endm
65 78
66.altmacro 79.altmacro
@@ -84,7 +97,7 @@
84.macro fpsimd_restore_partial state, tmpnr1, tmpnr2 97.macro fpsimd_restore_partial state, tmpnr1, tmpnr2
85 ldp w\tmpnr1, w\tmpnr2, [\state] 98 ldp w\tmpnr1, w\tmpnr2, [\state]
86 msr fpsr, x\tmpnr1 99 msr fpsr, x\tmpnr1
87 msr fpcr, x\tmpnr2 100 fpsimd_restore_fpcr x\tmpnr2, x\tmpnr1
88 adr x\tmpnr1, 0f 101 adr x\tmpnr1, 0f
89 ldr w\tmpnr2, [\state, #8] 102 ldr w\tmpnr2, [\state, #8]
90 add \state, \state, x\tmpnr2, lsl #4 103 add \state, \state, x\tmpnr2, lsl #4
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 902eb708804a..ccc7087d3c4e 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -41,11 +41,7 @@
41 * The module space lives between the addresses given by TASK_SIZE 41 * The module space lives between the addresses given by TASK_SIZE
42 * and PAGE_OFFSET - it must be within 128MB of the kernel text. 42 * and PAGE_OFFSET - it must be within 128MB of the kernel text.
43 */ 43 */
44#ifdef CONFIG_ARM64_64K_PAGES 44#define VA_BITS (CONFIG_ARM64_VA_BITS)
45#define VA_BITS (42)
46#else
47#define VA_BITS (39)
48#endif
49#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) 45#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
50#define MODULES_END (PAGE_OFFSET) 46#define MODULES_END (PAGE_OFFSET)
51#define MODULES_VADDR (MODULES_END - SZ_64M) 47#define MODULES_VADDR (MODULES_END - SZ_64M)
diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index 46bf66628b6a..7a3f462133b0 100644
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -31,14 +31,26 @@
31/* We do define AT_SYSINFO_EHDR but don't use the gate mechanism */ 31/* We do define AT_SYSINFO_EHDR but don't use the gate mechanism */
32#define __HAVE_ARCH_GATE_AREA 1 32#define __HAVE_ARCH_GATE_AREA 1
33 33
34#ifndef __ASSEMBLY__ 34/*
35 35 * The idmap and swapper page tables need some space reserved in the kernel
36 * image. Both require pgd, pud (4 levels only) and pmd tables to (section)
37 * map the kernel. With the 64K page configuration, swapper and idmap need to
38 * map to pte level. The swapper also maps the FDT (see __create_page_tables
39 * for more information).
40 */
36#ifdef CONFIG_ARM64_64K_PAGES 41#ifdef CONFIG_ARM64_64K_PAGES
37#include <asm/pgtable-2level-types.h> 42#define SWAPPER_PGTABLE_LEVELS (CONFIG_ARM64_PGTABLE_LEVELS)
38#else 43#else
39#include <asm/pgtable-3level-types.h> 44#define SWAPPER_PGTABLE_LEVELS (CONFIG_ARM64_PGTABLE_LEVELS - 1)
40#endif 45#endif
41 46
47#define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
48#define IDMAP_DIR_SIZE (SWAPPER_DIR_SIZE)
49
50#ifndef __ASSEMBLY__
51
52#include <asm/pgtable-types.h>
53
42extern void __cpu_clear_user_page(void *p, unsigned long user); 54extern void __cpu_clear_user_page(void *p, unsigned long user);
43extern void __cpu_copy_user_page(void *to, const void *from, 55extern void __cpu_copy_user_page(void *to, const void *from,
44 unsigned long user); 56 unsigned long user);
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 9bea6e74a001..d5bed02073d6 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -26,7 +26,7 @@
26 26
27#define check_pgt_cache() do { } while (0) 27#define check_pgt_cache() do { } while (0)
28 28
29#ifndef CONFIG_ARM64_64K_PAGES 29#if CONFIG_ARM64_PGTABLE_LEVELS > 2
30 30
31static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 31static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
32{ 32{
@@ -44,7 +44,27 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
44 set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE)); 44 set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
45} 45}
46 46
47#endif /* CONFIG_ARM64_64K_PAGES */ 47#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 2 */
48
49#if CONFIG_ARM64_PGTABLE_LEVELS > 3
50
51static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
52{
53 return (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
54}
55
56static inline void pud_free(struct mm_struct *mm, pud_t *pud)
57{
58 BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
59 free_page((unsigned long)pud);
60}
61
62static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
63{
64 set_pgd(pgd, __pgd(__pa(pud) | PUD_TYPE_TABLE));
65}
66
67#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 3 */
48 68
49extern pgd_t *pgd_alloc(struct mm_struct *mm); 69extern pgd_t *pgd_alloc(struct mm_struct *mm);
50extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 70extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
diff --git a/arch/arm64/include/asm/pgtable-2level-hwdef.h b/arch/arm64/include/asm/pgtable-2level-hwdef.h
deleted file mode 100644
index 2593b490c56a..000000000000
--- a/arch/arm64/include/asm/pgtable-2level-hwdef.h
+++ /dev/null
@@ -1,43 +0,0 @@
1/*
2 * Copyright (C) 2012 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_PGTABLE_2LEVEL_HWDEF_H
17#define __ASM_PGTABLE_2LEVEL_HWDEF_H
18
19/*
20 * With LPAE and 64KB pages, there are 2 levels of page tables. Each level has
21 * 8192 entries of 8 bytes each, occupying a 64KB page. Levels 0 and 1 are not
22 * used. The 2nd level table (PGD for Linux) can cover a range of 4TB, each
23 * entry representing 512MB. The user and kernel address spaces are limited to
24 * 4TB in the 64KB page configuration.
25 */
26#define PTRS_PER_PTE 8192
27#define PTRS_PER_PGD 8192
28
29/*
30 * PGDIR_SHIFT determines the size a top-level page table entry can map.
31 */
32#define PGDIR_SHIFT 29
33#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
34#define PGDIR_MASK (~(PGDIR_SIZE-1))
35
36/*
37 * section address mask and size definitions.
38 */
39#define SECTION_SHIFT 29
40#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT)
41#define SECTION_MASK (~(SECTION_SIZE-1))
42
43#endif
diff --git a/arch/arm64/include/asm/pgtable-2level-types.h b/arch/arm64/include/asm/pgtable-2level-types.h
deleted file mode 100644
index 5f101e63dfc1..000000000000
--- a/arch/arm64/include/asm/pgtable-2level-types.h
+++ /dev/null
@@ -1,62 +0,0 @@
1/*
2 * Copyright (C) 2012 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_PGTABLE_2LEVEL_TYPES_H
17#define __ASM_PGTABLE_2LEVEL_TYPES_H
18
19#include <asm/types.h>
20
21typedef u64 pteval_t;
22typedef u64 pgdval_t;
23typedef pgdval_t pmdval_t;
24
25#undef STRICT_MM_TYPECHECKS
26
27#ifdef STRICT_MM_TYPECHECKS
28
29/*
30 * These are used to make use of C type-checking..
31 */
32typedef struct { pteval_t pte; } pte_t;
33typedef struct { pgdval_t pgd; } pgd_t;
34typedef struct { pteval_t pgprot; } pgprot_t;
35
36#define pte_val(x) ((x).pte)
37#define pgd_val(x) ((x).pgd)
38#define pgprot_val(x) ((x).pgprot)
39
40#define __pte(x) ((pte_t) { (x) } )
41#define __pgd(x) ((pgd_t) { (x) } )
42#define __pgprot(x) ((pgprot_t) { (x) } )
43
44#else /* !STRICT_MM_TYPECHECKS */
45
46typedef pteval_t pte_t;
47typedef pgdval_t pgd_t;
48typedef pteval_t pgprot_t;
49
50#define pte_val(x) (x)
51#define pgd_val(x) (x)
52#define pgprot_val(x) (x)
53
54#define __pte(x) (x)
55#define __pgd(x) (x)
56#define __pgprot(x) (x)
57
58#endif /* STRICT_MM_TYPECHECKS */
59
60#include <asm-generic/pgtable-nopmd.h>
61
62#endif /* __ASM_PGTABLE_2LEVEL_TYPES_H */
diff --git a/arch/arm64/include/asm/pgtable-3level-hwdef.h b/arch/arm64/include/asm/pgtable-3level-hwdef.h
deleted file mode 100644
index 3dbf941d7767..000000000000
--- a/arch/arm64/include/asm/pgtable-3level-hwdef.h
+++ /dev/null
@@ -1,50 +0,0 @@
1/*
2 * Copyright (C) 2012 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16#ifndef __ASM_PGTABLE_3LEVEL_HWDEF_H
17#define __ASM_PGTABLE_3LEVEL_HWDEF_H
18
19/*
20 * With LPAE and 4KB pages, there are 3 levels of page tables. Each level has
21 * 512 entries of 8 bytes each, occupying a 4K page. The first level table
22 * covers a range of 512GB, each entry representing 1GB. The user and kernel
23 * address spaces are limited to 512GB each.
24 */
25#define PTRS_PER_PTE 512
26#define PTRS_PER_PMD 512
27#define PTRS_PER_PGD 512
28
29/*
30 * PGDIR_SHIFT determines the size a top-level page table entry can map.
31 */
32#define PGDIR_SHIFT 30
33#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
34#define PGDIR_MASK (~(PGDIR_SIZE-1))
35
36/*
37 * PMD_SHIFT determines the size a middle-level page table entry can map.
38 */
39#define PMD_SHIFT 21
40#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
41#define PMD_MASK (~(PMD_SIZE-1))
42
43/*
44 * section address mask and size definitions.
45 */
46#define SECTION_SHIFT 21
47#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT)
48#define SECTION_MASK (~(SECTION_SIZE-1))
49
50#endif
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 955e8c5f0afb..88174e0bfafe 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -16,18 +16,50 @@
16#ifndef __ASM_PGTABLE_HWDEF_H 16#ifndef __ASM_PGTABLE_HWDEF_H
17#define __ASM_PGTABLE_HWDEF_H 17#define __ASM_PGTABLE_HWDEF_H
18 18
19#ifdef CONFIG_ARM64_64K_PAGES 19#define PTRS_PER_PTE (1 << (PAGE_SHIFT - 3))
20#include <asm/pgtable-2level-hwdef.h> 20
21#else 21/*
22#include <asm/pgtable-3level-hwdef.h> 22 * PMD_SHIFT determines the size a level 2 page table entry can map.
23 */
24#if CONFIG_ARM64_PGTABLE_LEVELS > 2
25#define PMD_SHIFT ((PAGE_SHIFT - 3) * 2 + 3)
26#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
27#define PMD_MASK (~(PMD_SIZE-1))
28#define PTRS_PER_PMD PTRS_PER_PTE
29#endif
30
31/*
32 * PUD_SHIFT determines the size a level 1 page table entry can map.
33 */
34#if CONFIG_ARM64_PGTABLE_LEVELS > 3
35#define PUD_SHIFT ((PAGE_SHIFT - 3) * 3 + 3)
36#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT)
37#define PUD_MASK (~(PUD_SIZE-1))
38#define PTRS_PER_PUD PTRS_PER_PTE
23#endif 39#endif
24 40
25/* 41/*
42 * PGDIR_SHIFT determines the size a top-level page table entry can map
43 * (depending on the configuration, this level can be 0, 1 or 2).
44 */
45#define PGDIR_SHIFT ((PAGE_SHIFT - 3) * CONFIG_ARM64_PGTABLE_LEVELS + 3)
46#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
47#define PGDIR_MASK (~(PGDIR_SIZE-1))
48#define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT))
49
50/*
51 * Section address mask and size definitions.
52 */
53#define SECTION_SHIFT PMD_SHIFT
54#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT)
55#define SECTION_MASK (~(SECTION_SIZE-1))
56
57/*
26 * Hardware page table definitions. 58 * Hardware page table definitions.
27 * 59 *
28 * Level 1 descriptor (PUD). 60 * Level 1 descriptor (PUD).
29 */ 61 */
30 62#define PUD_TYPE_TABLE (_AT(pudval_t, 3) << 0)
31#define PUD_TABLE_BIT (_AT(pgdval_t, 1) << 1) 63#define PUD_TABLE_BIT (_AT(pgdval_t, 1) << 1)
32#define PUD_TYPE_MASK (_AT(pgdval_t, 3) << 0) 64#define PUD_TYPE_MASK (_AT(pgdval_t, 3) << 0)
33#define PUD_TYPE_SECT (_AT(pgdval_t, 1) << 0) 65#define PUD_TYPE_SECT (_AT(pgdval_t, 1) << 0)
diff --git a/arch/arm64/include/asm/pgtable-3level-types.h b/arch/arm64/include/asm/pgtable-types.h
index 4e94424938a4..ca9df80af896 100644
--- a/arch/arm64/include/asm/pgtable-3level-types.h
+++ b/arch/arm64/include/asm/pgtable-types.h
@@ -1,7 +1,10 @@
1/* 1/*
2 * Copyright (C) 2012 ARM Ltd. 2 * Page table types definitions.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * Copyright (C) 2014 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
6 *
7 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 8 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation. 9 * published by the Free Software Foundation.
7 * 10 *
@@ -13,13 +16,15 @@
13 * You should have received a copy of the GNU General Public License 16 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */ 18 */
16#ifndef __ASM_PGTABLE_3LEVEL_TYPES_H 19
17#define __ASM_PGTABLE_3LEVEL_TYPES_H 20#ifndef __ASM_PGTABLE_TYPES_H
21#define __ASM_PGTABLE_TYPES_H
18 22
19#include <asm/types.h> 23#include <asm/types.h>
20 24
21typedef u64 pteval_t; 25typedef u64 pteval_t;
22typedef u64 pmdval_t; 26typedef u64 pmdval_t;
27typedef u64 pudval_t;
23typedef u64 pgdval_t; 28typedef u64 pgdval_t;
24 29
25#undef STRICT_MM_TYPECHECKS 30#undef STRICT_MM_TYPECHECKS
@@ -30,39 +35,61 @@ typedef u64 pgdval_t;
30 * These are used to make use of C type-checking.. 35 * These are used to make use of C type-checking..
31 */ 36 */
32typedef struct { pteval_t pte; } pte_t; 37typedef struct { pteval_t pte; } pte_t;
38#define pte_val(x) ((x).pte)
39#define __pte(x) ((pte_t) { (x) } )
40
41#if CONFIG_ARM64_PGTABLE_LEVELS > 2
33typedef struct { pmdval_t pmd; } pmd_t; 42typedef struct { pmdval_t pmd; } pmd_t;
34typedef struct { pgdval_t pgd; } pgd_t; 43#define pmd_val(x) ((x).pmd)
35typedef struct { pteval_t pgprot; } pgprot_t; 44#define __pmd(x) ((pmd_t) { (x) } )
45#endif
36 46
37#define pte_val(x) ((x).pte) 47#if CONFIG_ARM64_PGTABLE_LEVELS > 3
38#define pmd_val(x) ((x).pmd) 48typedef struct { pudval_t pud; } pud_t;
39#define pgd_val(x) ((x).pgd) 49#define pud_val(x) ((x).pud)
40#define pgprot_val(x) ((x).pgprot) 50#define __pud(x) ((pud_t) { (x) } )
51#endif
41 52
42#define __pte(x) ((pte_t) { (x) } ) 53typedef struct { pgdval_t pgd; } pgd_t;
43#define __pmd(x) ((pmd_t) { (x) } ) 54#define pgd_val(x) ((x).pgd)
44#define __pgd(x) ((pgd_t) { (x) } ) 55#define __pgd(x) ((pgd_t) { (x) } )
45#define __pgprot(x) ((pgprot_t) { (x) } ) 56
57typedef struct { pteval_t pgprot; } pgprot_t;
58#define pgprot_val(x) ((x).pgprot)
59#define __pgprot(x) ((pgprot_t) { (x) } )
46 60
47#else /* !STRICT_MM_TYPECHECKS */ 61#else /* !STRICT_MM_TYPECHECKS */
48 62
49typedef pteval_t pte_t; 63typedef pteval_t pte_t;
50typedef pmdval_t pmd_t;
51typedef pgdval_t pgd_t;
52typedef pteval_t pgprot_t;
53
54#define pte_val(x) (x) 64#define pte_val(x) (x)
55#define pmd_val(x) (x)
56#define pgd_val(x) (x)
57#define pgprot_val(x) (x)
58
59#define __pte(x) (x) 65#define __pte(x) (x)
66
67#if CONFIG_ARM64_PGTABLE_LEVELS > 2
68typedef pmdval_t pmd_t;
69#define pmd_val(x) (x)
60#define __pmd(x) (x) 70#define __pmd(x) (x)
71#endif
72
73#if CONFIG_ARM64_PGTABLE_LEVELS > 3
74typedef pudval_t pud_t;
75#define pud_val(x) (x)
76#define __pud(x) (x)
77#endif
78
79typedef pgdval_t pgd_t;
80#define pgd_val(x) (x)
61#define __pgd(x) (x) 81#define __pgd(x) (x)
82
83typedef pteval_t pgprot_t;
84#define pgprot_val(x) (x)
62#define __pgprot(x) (x) 85#define __pgprot(x) (x)
63 86
64#endif /* STRICT_MM_TYPECHECKS */ 87#endif /* STRICT_MM_TYPECHECKS */
65 88
89#if CONFIG_ARM64_PGTABLE_LEVELS == 2
90#include <asm-generic/pgtable-nopmd.h>
91#elif CONFIG_ARM64_PGTABLE_LEVELS == 3
66#include <asm-generic/pgtable-nopud.h> 92#include <asm-generic/pgtable-nopud.h>
93#endif
67 94
68#endif /* __ASM_PGTABLE_3LEVEL_TYPES_H */ 95#endif /* __ASM_PGTABLE_TYPES_H */
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index e0ccceb317d9..ffe1ba0506d1 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -33,9 +33,16 @@
33 33
34/* 34/*
35 * VMALLOC and SPARSEMEM_VMEMMAP ranges. 35 * VMALLOC and SPARSEMEM_VMEMMAP ranges.
36 *
37 * VMEMAP_SIZE: allows the whole VA space to be covered by a struct page array
38 * (rounded up to PUD_SIZE).
39 * VMALLOC_START: beginning of the kernel VA space
40 * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space,
41 * fixed mappings and modules
36 */ 42 */
43#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE)
37#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS) 44#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS)
38#define VMALLOC_END (PAGE_OFFSET - UL(0x400000000) - SZ_64K) 45#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
39 46
40#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K)) 47#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
41 48
@@ -44,14 +51,9 @@
44#ifndef __ASSEMBLY__ 51#ifndef __ASSEMBLY__
45extern void __pte_error(const char *file, int line, unsigned long val); 52extern void __pte_error(const char *file, int line, unsigned long val);
46extern void __pmd_error(const char *file, int line, unsigned long val); 53extern void __pmd_error(const char *file, int line, unsigned long val);
54extern void __pud_error(const char *file, int line, unsigned long val);
47extern void __pgd_error(const char *file, int line, unsigned long val); 55extern void __pgd_error(const char *file, int line, unsigned long val);
48 56
49#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
50#ifndef CONFIG_ARM64_64K_PAGES
51#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
52#endif
53#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
54
55#ifdef CONFIG_SMP 57#ifdef CONFIG_SMP
56#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) 58#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
57#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) 59#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
@@ -112,6 +114,8 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
112extern struct page *empty_zero_page; 114extern struct page *empty_zero_page;
113#define ZERO_PAGE(vaddr) (empty_zero_page) 115#define ZERO_PAGE(vaddr) (empty_zero_page)
114 116
117#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
118
115#define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT) 119#define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT)
116 120
117#define pfn_pte(pfn,prot) (__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) 121#define pfn_pte(pfn,prot) (__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
@@ -119,6 +123,10 @@ extern struct page *empty_zero_page;
119#define pte_none(pte) (!pte_val(pte)) 123#define pte_none(pte) (!pte_val(pte))
120#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0)) 124#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
121#define pte_page(pte) (pfn_to_page(pte_pfn(pte))) 125#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
126
127/* Find an entry in the third-level page table. */
128#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
129
122#define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + pte_index(addr)) 130#define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + pte_index(addr))
123 131
124#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) 132#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
@@ -138,6 +146,8 @@ extern struct page *empty_zero_page;
138 146
139#define pte_valid_user(pte) \ 147#define pte_valid_user(pte) \
140 ((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) 148 ((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER))
149#define pte_valid_not_user(pte) \
150 ((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID)
141 151
142static inline pte_t pte_wrprotect(pte_t pte) 152static inline pte_t pte_wrprotect(pte_t pte)
143{ 153{
@@ -184,6 +194,15 @@ static inline pte_t pte_mkspecial(pte_t pte)
184static inline void set_pte(pte_t *ptep, pte_t pte) 194static inline void set_pte(pte_t *ptep, pte_t pte)
185{ 195{
186 *ptep = pte; 196 *ptep = pte;
197
198 /*
199 * Only if the new pte is valid and kernel, otherwise TLB maintenance
200 * or update_mmu_cache() have the necessary barriers.
201 */
202 if (pte_valid_not_user(pte)) {
203 dsb(ishst);
204 isb();
205 }
187} 206}
188 207
189extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); 208extern void __sync_icache_dcache(pte_t pteval, unsigned long addr);
@@ -303,6 +322,7 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
303{ 322{
304 *pmdp = pmd; 323 *pmdp = pmd;
305 dsb(ishst); 324 dsb(ishst);
325 isb();
306} 326}
307 327
308static inline void pmd_clear(pmd_t *pmdp) 328static inline void pmd_clear(pmd_t *pmdp)
@@ -323,7 +343,9 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
323 */ 343 */
324#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot) 344#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot)
325 345
326#ifndef CONFIG_ARM64_64K_PAGES 346#if CONFIG_ARM64_PGTABLE_LEVELS > 2
347
348#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
327 349
328#define pud_none(pud) (!pud_val(pud)) 350#define pud_none(pud) (!pud_val(pud))
329#define pud_bad(pud) (!(pud_val(pud) & 2)) 351#define pud_bad(pud) (!(pud_val(pud) & 2))
@@ -333,6 +355,7 @@ static inline void set_pud(pud_t *pudp, pud_t pud)
333{ 355{
334 *pudp = pud; 356 *pudp = pud;
335 dsb(ishst); 357 dsb(ishst);
358 isb();
336} 359}
337 360
338static inline void pud_clear(pud_t *pudp) 361static inline void pud_clear(pud_t *pudp)
@@ -345,7 +368,51 @@ static inline pmd_t *pud_page_vaddr(pud_t pud)
345 return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); 368 return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
346} 369}
347 370
348#endif /* CONFIG_ARM64_64K_PAGES */ 371/* Find an entry in the second-level page table. */
372#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
373
374static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
375{
376 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
377}
378
379#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 2 */
380
381#if CONFIG_ARM64_PGTABLE_LEVELS > 3
382
383#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
384
385#define pgd_none(pgd) (!pgd_val(pgd))
386#define pgd_bad(pgd) (!(pgd_val(pgd) & 2))
387#define pgd_present(pgd) (pgd_val(pgd))
388
389static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
390{
391 *pgdp = pgd;
392 dsb(ishst);
393}
394
395static inline void pgd_clear(pgd_t *pgdp)
396{
397 set_pgd(pgdp, __pgd(0));
398}
399
400static inline pud_t *pgd_page_vaddr(pgd_t pgd)
401{
402 return __va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
403}
404
405/* Find an entry in the frst-level page table. */
406#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
407
408static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr)
409{
410 return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr);
411}
412
413#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 3 */
414
415#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
349 416
350/* to find an entry in a page-table-directory */ 417/* to find an entry in a page-table-directory */
351#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 418#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
@@ -355,18 +422,6 @@ static inline pmd_t *pud_page_vaddr(pud_t pud)
355/* to find an entry in a kernel page-table-directory */ 422/* to find an entry in a kernel page-table-directory */
356#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) 423#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
357 424
358/* Find an entry in the second-level page table.. */
359#ifndef CONFIG_ARM64_64K_PAGES
360#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
361static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
362{
363 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
364}
365#endif
366
367/* Find an entry in the third-level page table.. */
368#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
369
370static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) 425static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
371{ 426{
372 const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | 427 const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
@@ -383,9 +438,6 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
383extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; 438extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
384extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; 439extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
385 440
386#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE)
387#define IDMAP_DIR_SIZE (2 * PAGE_SIZE)
388
389/* 441/*
390 * Encode and decode a swap entry: 442 * Encode and decode a swap entry:
391 * bits 0-1: present (must be zero) 443 * bits 0-1: present (must be zero)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 34de2a8f7d93..77712454486b 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -137,8 +137,8 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev,
137#define task_pt_regs(p) \ 137#define task_pt_regs(p) \
138 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) 138 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
139 139
140#define KSTK_EIP(tsk) task_pt_regs(tsk)->pc 140#define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)
141#define KSTK_ESP(tsk) task_pt_regs(tsk)->sp 141#define KSTK_ESP(tsk) ((unsigned long)task_pt_regs(tsk)->sp)
142 142
143/* 143/*
144 * Prefetching support 144 * Prefetching support
diff --git a/arch/arm64/include/asm/stackprotector.h b/arch/arm64/include/asm/stackprotector.h
new file mode 100644
index 000000000000..fe5e287dc56b
--- /dev/null
+++ b/arch/arm64/include/asm/stackprotector.h
@@ -0,0 +1,38 @@
1/*
2 * GCC stack protector support.
3 *
4 * Stack protector works by putting predefined pattern at the start of
5 * the stack frame and verifying that it hasn't been overwritten when
6 * returning from the function. The pattern is called stack canary
7 * and gcc expects it to be defined by a global variable called
8 * "__stack_chk_guard" on ARM. This unfortunately means that on SMP
9 * we cannot have a different canary value per task.
10 */
11
12#ifndef __ASM_STACKPROTECTOR_H
13#define __ASM_STACKPROTECTOR_H
14
15#include <linux/random.h>
16#include <linux/version.h>
17
18extern unsigned long __stack_chk_guard;
19
20/*
21 * Initialize the stackprotector canary value.
22 *
23 * NOTE: this must only be called from functions that never return,
24 * and it must always be inlined.
25 */
26static __always_inline void boot_init_stack_canary(void)
27{
28 unsigned long canary;
29
30 /* Try to get a semi random initial value. */
31 get_random_bytes(&canary, sizeof(canary));
32 canary ^= LINUX_VERSION_CODE;
33
34 current->stack_canary = canary;
35 __stack_chk_guard = current->stack_canary;
36}
37
38#endif /* _ASM_STACKPROTECTOR_H */
diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
index 383771eb0b87..709a574468f0 100644
--- a/arch/arm64/include/asm/syscall.h
+++ b/arch/arm64/include/asm/syscall.h
@@ -16,6 +16,8 @@
16#ifndef __ASM_SYSCALL_H 16#ifndef __ASM_SYSCALL_H
17#define __ASM_SYSCALL_H 17#define __ASM_SYSCALL_H
18 18
19#include <uapi/linux/audit.h>
20#include <linux/compat.h>
19#include <linux/err.h> 21#include <linux/err.h>
20 22
21extern const void *sys_call_table[]; 23extern const void *sys_call_table[];
@@ -105,4 +107,16 @@ static inline void syscall_set_arguments(struct task_struct *task,
105 memcpy(&regs->regs[i], args, n * sizeof(args[0])); 107 memcpy(&regs->regs[i], args, n * sizeof(args[0]));
106} 108}
107 109
110/*
111 * We don't care about endianness (__AUDIT_ARCH_LE bit) here because
112 * AArch64 has the same system calls both on little- and big- endian.
113 */
114static inline int syscall_get_arch(void)
115{
116 if (is_compat_task())
117 return AUDIT_ARCH_ARM;
118
119 return AUDIT_ARCH_AARCH64;
120}
121
108#endif /* __ASM_SYSCALL_H */ 122#endif /* __ASM_SYSCALL_H */
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
new file mode 100644
index 000000000000..5c89df0acbcb
--- /dev/null
+++ b/arch/arm64/include/asm/sysreg.h
@@ -0,0 +1,60 @@
1/*
2 * Macros for accessing system registers with older binutils.
3 *
4 * Copyright (C) 2014 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
6 *
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#ifndef __ASM_SYSREG_H
21#define __ASM_SYSREG_H
22
23#define sys_reg(op0, op1, crn, crm, op2) \
24 ((((op0)-2)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5))
25
26#ifdef __ASSEMBLY__
27
28 .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
29 .equ __reg_num_x\num, \num
30 .endr
31 .equ __reg_num_xzr, 31
32
33 .macro mrs_s, rt, sreg
34 .inst 0xd5300000|(\sreg)|(__reg_num_\rt)
35 .endm
36
37 .macro msr_s, sreg, rt
38 .inst 0xd5100000|(\sreg)|(__reg_num_\rt)
39 .endm
40
41#else
42
43asm(
44" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"
45" .equ __reg_num_x\\num, \\num\n"
46" .endr\n"
47" .equ __reg_num_xzr, 31\n"
48"\n"
49" .macro mrs_s, rt, sreg\n"
50" .inst 0xd5300000|(\\sreg)|(__reg_num_\\rt)\n"
51" .endm\n"
52"\n"
53" .macro msr_s, sreg, rt\n"
54" .inst 0xd5100000|(\\sreg)|(__reg_num_\\rt)\n"
55" .endm\n"
56);
57
58#endif
59
60#endif /* __ASM_SYSREG_H */
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index e40b6d06d515..45108d802f5e 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -103,6 +103,7 @@ static inline struct thread_info *current_thread_info(void)
103#define TIF_NEED_RESCHED 1 103#define TIF_NEED_RESCHED 1
104#define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ 104#define TIF_NOTIFY_RESUME 2 /* callback before returning to user */
105#define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */ 105#define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */
106#define TIF_NOHZ 7
106#define TIF_SYSCALL_TRACE 8 107#define TIF_SYSCALL_TRACE 8
107#define TIF_SYSCALL_AUDIT 9 108#define TIF_SYSCALL_AUDIT 9
108#define TIF_SYSCALL_TRACEPOINT 10 109#define TIF_SYSCALL_TRACEPOINT 10
@@ -118,6 +119,7 @@ static inline struct thread_info *current_thread_info(void)
118#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) 119#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
119#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) 120#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
120#define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE) 121#define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
122#define _TIF_NOHZ (1 << TIF_NOHZ)
121#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) 123#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
122#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) 124#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
123#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) 125#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
@@ -128,7 +130,8 @@ static inline struct thread_info *current_thread_info(void)
128 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE) 130 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE)
129 131
130#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ 132#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
131 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) 133 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
134 _TIF_NOHZ)
132 135
133#endif /* __KERNEL__ */ 136#endif /* __KERNEL__ */
134#endif /* __ASM_THREAD_INFO_H */ 137#endif /* __ASM_THREAD_INFO_H */
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 80e2c08900d6..62731ef9749a 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -91,7 +91,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
91 tlb_remove_page(tlb, pte); 91 tlb_remove_page(tlb, pte);
92} 92}
93 93
94#ifndef CONFIG_ARM64_64K_PAGES 94#if CONFIG_ARM64_PGTABLE_LEVELS > 2
95static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 95static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
96 unsigned long addr) 96 unsigned long addr)
97{ 97{
@@ -100,6 +100,15 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
100} 100}
101#endif 101#endif
102 102
103#if CONFIG_ARM64_PGTABLE_LEVELS > 3
104static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
105 unsigned long addr)
106{
107 tlb_add_flush(tlb, addr);
108 tlb_remove_page(tlb, virt_to_page(pudp));
109}
110#endif
111
103static inline void __tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, 112static inline void __tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp,
104 unsigned long address) 113 unsigned long address)
105{ 114{
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index b9349c4513ea..73f0ce570fb3 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -98,8 +98,8 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
98 dsb(ish); 98 dsb(ish);
99} 99}
100 100
101static inline void flush_tlb_range(struct vm_area_struct *vma, 101static inline void __flush_tlb_range(struct vm_area_struct *vma,
102 unsigned long start, unsigned long end) 102 unsigned long start, unsigned long end)
103{ 103{
104 unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48; 104 unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
105 unsigned long addr; 105 unsigned long addr;
@@ -112,7 +112,7 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
112 dsb(ish); 112 dsb(ish);
113} 113}
114 114
115static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) 115static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long end)
116{ 116{
117 unsigned long addr; 117 unsigned long addr;
118 start >>= 12; 118 start >>= 12;
@@ -122,6 +122,30 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
122 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) 122 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
123 asm("tlbi vaae1is, %0" : : "r"(addr)); 123 asm("tlbi vaae1is, %0" : : "r"(addr));
124 dsb(ish); 124 dsb(ish);
125 isb();
126}
127
128/*
129 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
130 * necessarily a performance improvement.
131 */
132#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT)
133
134static inline void flush_tlb_range(struct vm_area_struct *vma,
135 unsigned long start, unsigned long end)
136{
137 if ((end - start) <= MAX_TLB_RANGE)
138 __flush_tlb_range(vma, start, end);
139 else
140 flush_tlb_mm(vma->vm_mm);
141}
142
143static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
144{
145 if ((end - start) <= MAX_TLB_RANGE)
146 __flush_tlb_kernel_range(start, end);
147 else
148 flush_tlb_all();
125} 149}
126 150
127/* 151/*
@@ -131,8 +155,8 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,
131 unsigned long addr, pte_t *ptep) 155 unsigned long addr, pte_t *ptep)
132{ 156{
133 /* 157 /*
134 * set_pte() does not have a DSB, so make sure that the page table 158 * set_pte() does not have a DSB for user mappings, so make sure that
135 * write is visible. 159 * the page table write is visible.
136 */ 160 */
137 dsb(ishst); 161 dsb(ishst);
138} 162}
diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h
index e5f47df00c24..4bc95d27e063 100644
--- a/arch/arm64/include/asm/unistd.h
+++ b/arch/arm64/include/asm/unistd.h
@@ -26,7 +26,24 @@
26#define __ARCH_WANT_COMPAT_SYS_SENDFILE 26#define __ARCH_WANT_COMPAT_SYS_SENDFILE
27#define __ARCH_WANT_SYS_FORK 27#define __ARCH_WANT_SYS_FORK
28#define __ARCH_WANT_SYS_VFORK 28#define __ARCH_WANT_SYS_VFORK
29
30/*
31 * Compat syscall numbers used by the AArch64 kernel.
32 */
33#define __NR_compat_restart_syscall 0
34#define __NR_compat_sigreturn 119
35#define __NR_compat_rt_sigreturn 173
36
37/*
38 * The following SVCs are ARM private.
39 */
40#define __ARM_NR_COMPAT_BASE 0x0f0000
41#define __ARM_NR_compat_cacheflush (__ARM_NR_COMPAT_BASE+2)
42#define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE+5)
43
44#define __NR_compat_syscalls 383
29#endif 45#endif
46
30#define __ARCH_WANT_SYS_CLONE 47#define __ARCH_WANT_SYS_CLONE
31#include <uapi/asm/unistd.h> 48#include <uapi/asm/unistd.h>
32 49
diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h
index c8d8fc17bd5a..e242600c4046 100644
--- a/arch/arm64/include/asm/unistd32.h
+++ b/arch/arm64/include/asm/unistd32.h
@@ -21,403 +21,769 @@
21#define __SYSCALL(x, y) 21#define __SYSCALL(x, y)
22#endif 22#endif
23 23
24__SYSCALL(0, sys_restart_syscall) 24#define __NR_restart_syscall 0
25__SYSCALL(1, sys_exit) 25__SYSCALL(__NR_restart_syscall, sys_restart_syscall)
26__SYSCALL(2, sys_fork) 26#define __NR_exit 1
27__SYSCALL(3, sys_read) 27__SYSCALL(__NR_exit, sys_exit)
28__SYSCALL(4, sys_write) 28#define __NR_fork 2
29__SYSCALL(5, compat_sys_open) 29__SYSCALL(__NR_fork, sys_fork)
30__SYSCALL(6, sys_close) 30#define __NR_read 3
31__SYSCALL(7, sys_ni_syscall) /* 7 was sys_waitpid */ 31__SYSCALL(__NR_read, sys_read)
32__SYSCALL(8, sys_creat) 32#define __NR_write 4
33__SYSCALL(9, sys_link) 33__SYSCALL(__NR_write, sys_write)
34__SYSCALL(10, sys_unlink) 34#define __NR_open 5
35__SYSCALL(11, compat_sys_execve) 35__SYSCALL(__NR_open, compat_sys_open)
36__SYSCALL(12, sys_chdir) 36#define __NR_close 6
37__SYSCALL(13, sys_ni_syscall) /* 13 was sys_time */ 37__SYSCALL(__NR_close, sys_close)
38__SYSCALL(14, sys_mknod) 38 /* 7 was sys_waitpid */
39__SYSCALL(15, sys_chmod) 39__SYSCALL(7, sys_ni_syscall)
40__SYSCALL(16, sys_lchown16) 40#define __NR_creat 8
41__SYSCALL(17, sys_ni_syscall) /* 17 was sys_break */ 41__SYSCALL(__NR_creat, sys_creat)
42__SYSCALL(18, sys_ni_syscall) /* 18 was sys_stat */ 42#define __NR_link 9
43__SYSCALL(19, compat_sys_lseek) 43__SYSCALL(__NR_link, sys_link)
44__SYSCALL(20, sys_getpid) 44#define __NR_unlink 10
45__SYSCALL(21, compat_sys_mount) 45__SYSCALL(__NR_unlink, sys_unlink)
46__SYSCALL(22, sys_ni_syscall) /* 22 was sys_umount */ 46#define __NR_execve 11
47__SYSCALL(23, sys_setuid16) 47__SYSCALL(__NR_execve, compat_sys_execve)
48__SYSCALL(24, sys_getuid16) 48#define __NR_chdir 12
49__SYSCALL(25, sys_ni_syscall) /* 25 was sys_stime */ 49__SYSCALL(__NR_chdir, sys_chdir)
50__SYSCALL(26, compat_sys_ptrace) 50 /* 13 was sys_time */
51__SYSCALL(27, sys_ni_syscall) /* 27 was sys_alarm */ 51__SYSCALL(13, sys_ni_syscall)
52__SYSCALL(28, sys_ni_syscall) /* 28 was sys_fstat */ 52#define __NR_mknod 14
53__SYSCALL(29, sys_pause) 53__SYSCALL(__NR_mknod, sys_mknod)
54__SYSCALL(30, sys_ni_syscall) /* 30 was sys_utime */ 54#define __NR_chmod 15
55__SYSCALL(31, sys_ni_syscall) /* 31 was sys_stty */ 55__SYSCALL(__NR_chmod, sys_chmod)
56__SYSCALL(32, sys_ni_syscall) /* 32 was sys_gtty */ 56#define __NR_lchown 16
57__SYSCALL(33, sys_access) 57__SYSCALL(__NR_lchown, sys_lchown16)
58__SYSCALL(34, sys_nice) 58 /* 17 was sys_break */
59__SYSCALL(35, sys_ni_syscall) /* 35 was sys_ftime */ 59__SYSCALL(17, sys_ni_syscall)
60__SYSCALL(36, sys_sync) 60 /* 18 was sys_stat */
61__SYSCALL(37, sys_kill) 61__SYSCALL(18, sys_ni_syscall)
62__SYSCALL(38, sys_rename) 62#define __NR_lseek 19
63__SYSCALL(39, sys_mkdir) 63__SYSCALL(__NR_lseek, compat_sys_lseek)
64__SYSCALL(40, sys_rmdir) 64#define __NR_getpid 20
65__SYSCALL(41, sys_dup) 65__SYSCALL(__NR_getpid, sys_getpid)
66__SYSCALL(42, sys_pipe) 66#define __NR_mount 21
67__SYSCALL(43, compat_sys_times) 67__SYSCALL(__NR_mount, compat_sys_mount)
68__SYSCALL(44, sys_ni_syscall) /* 44 was sys_prof */ 68 /* 22 was sys_umount */
69__SYSCALL(45, sys_brk) 69__SYSCALL(22, sys_ni_syscall)
70__SYSCALL(46, sys_setgid16) 70#define __NR_setuid 23
71__SYSCALL(47, sys_getgid16) 71__SYSCALL(__NR_setuid, sys_setuid16)
72__SYSCALL(48, sys_ni_syscall) /* 48 was sys_signal */ 72#define __NR_getuid 24
73__SYSCALL(49, sys_geteuid16) 73__SYSCALL(__NR_getuid, sys_getuid16)
74__SYSCALL(50, sys_getegid16) 74 /* 25 was sys_stime */
75__SYSCALL(51, sys_acct) 75__SYSCALL(25, sys_ni_syscall)
76__SYSCALL(52, sys_umount) 76#define __NR_ptrace 26
77__SYSCALL(53, sys_ni_syscall) /* 53 was sys_lock */ 77__SYSCALL(__NR_ptrace, compat_sys_ptrace)
78__SYSCALL(54, compat_sys_ioctl) 78 /* 27 was sys_alarm */
79__SYSCALL(55, compat_sys_fcntl) 79__SYSCALL(27, sys_ni_syscall)
80__SYSCALL(56, sys_ni_syscall) /* 56 was sys_mpx */ 80 /* 28 was sys_fstat */
81__SYSCALL(57, sys_setpgid) 81__SYSCALL(28, sys_ni_syscall)
82__SYSCALL(58, sys_ni_syscall) /* 58 was sys_ulimit */ 82#define __NR_pause 29
83__SYSCALL(59, sys_ni_syscall) /* 59 was sys_olduname */ 83__SYSCALL(__NR_pause, sys_pause)
84__SYSCALL(60, sys_umask) 84 /* 30 was sys_utime */
85__SYSCALL(61, sys_chroot) 85__SYSCALL(30, sys_ni_syscall)
86__SYSCALL(62, compat_sys_ustat) 86 /* 31 was sys_stty */
87__SYSCALL(63, sys_dup2) 87__SYSCALL(31, sys_ni_syscall)
88__SYSCALL(64, sys_getppid) 88 /* 32 was sys_gtty */
89__SYSCALL(65, sys_getpgrp) 89__SYSCALL(32, sys_ni_syscall)
90__SYSCALL(66, sys_setsid) 90#define __NR_access 33
91__SYSCALL(67, compat_sys_sigaction) 91__SYSCALL(__NR_access, sys_access)
92__SYSCALL(68, sys_ni_syscall) /* 68 was sys_sgetmask */ 92#define __NR_nice 34
93__SYSCALL(69, sys_ni_syscall) /* 69 was sys_ssetmask */ 93__SYSCALL(__NR_nice, sys_nice)
94__SYSCALL(70, sys_setreuid16) 94 /* 35 was sys_ftime */
95__SYSCALL(71, sys_setregid16) 95__SYSCALL(35, sys_ni_syscall)
96__SYSCALL(72, sys_sigsuspend) 96#define __NR_sync 36
97__SYSCALL(73, compat_sys_sigpending) 97__SYSCALL(__NR_sync, sys_sync)
98__SYSCALL(74, sys_sethostname) 98#define __NR_kill 37
99__SYSCALL(75, compat_sys_setrlimit) 99__SYSCALL(__NR_kill, sys_kill)
100__SYSCALL(76, sys_ni_syscall) /* 76 was compat_sys_getrlimit */ 100#define __NR_rename 38
101__SYSCALL(77, compat_sys_getrusage) 101__SYSCALL(__NR_rename, sys_rename)
102__SYSCALL(78, compat_sys_gettimeofday) 102#define __NR_mkdir 39
103__SYSCALL(79, compat_sys_settimeofday) 103__SYSCALL(__NR_mkdir, sys_mkdir)
104__SYSCALL(80, sys_getgroups16) 104#define __NR_rmdir 40
105__SYSCALL(81, sys_setgroups16) 105__SYSCALL(__NR_rmdir, sys_rmdir)
106__SYSCALL(82, sys_ni_syscall) /* 82 was compat_sys_select */ 106#define __NR_dup 41
107__SYSCALL(83, sys_symlink) 107__SYSCALL(__NR_dup, sys_dup)
108__SYSCALL(84, sys_ni_syscall) /* 84 was sys_lstat */ 108#define __NR_pipe 42
109__SYSCALL(85, sys_readlink) 109__SYSCALL(__NR_pipe, sys_pipe)
110__SYSCALL(86, sys_uselib) 110#define __NR_times 43
111__SYSCALL(87, sys_swapon) 111__SYSCALL(__NR_times, compat_sys_times)
112__SYSCALL(88, sys_reboot) 112 /* 44 was sys_prof */
113__SYSCALL(89, sys_ni_syscall) /* 89 was sys_readdir */ 113__SYSCALL(44, sys_ni_syscall)
114__SYSCALL(90, sys_ni_syscall) /* 90 was sys_mmap */ 114#define __NR_brk 45
115__SYSCALL(91, sys_munmap) 115__SYSCALL(__NR_brk, sys_brk)
116__SYSCALL(92, compat_sys_truncate) 116#define __NR_setgid 46
117__SYSCALL(93, compat_sys_ftruncate) 117__SYSCALL(__NR_setgid, sys_setgid16)
118__SYSCALL(94, sys_fchmod) 118#define __NR_getgid 47
119__SYSCALL(95, sys_fchown16) 119__SYSCALL(__NR_getgid, sys_getgid16)
120__SYSCALL(96, sys_getpriority) 120 /* 48 was sys_signal */
121__SYSCALL(97, sys_setpriority) 121__SYSCALL(48, sys_ni_syscall)
122__SYSCALL(98, sys_ni_syscall) /* 98 was sys_profil */ 122#define __NR_geteuid 49
123__SYSCALL(99, compat_sys_statfs) 123__SYSCALL(__NR_geteuid, sys_geteuid16)
124__SYSCALL(100, compat_sys_fstatfs) 124#define __NR_getegid 50
125__SYSCALL(101, sys_ni_syscall) /* 101 was sys_ioperm */ 125__SYSCALL(__NR_getegid, sys_getegid16)
126__SYSCALL(102, sys_ni_syscall) /* 102 was sys_socketcall */ 126#define __NR_acct 51
127__SYSCALL(103, sys_syslog) 127__SYSCALL(__NR_acct, sys_acct)
128__SYSCALL(104, compat_sys_setitimer) 128#define __NR_umount2 52
129__SYSCALL(105, compat_sys_getitimer) 129__SYSCALL(__NR_umount2, sys_umount)
130__SYSCALL(106, compat_sys_newstat) 130 /* 53 was sys_lock */
131__SYSCALL(107, compat_sys_newlstat) 131__SYSCALL(53, sys_ni_syscall)
132__SYSCALL(108, compat_sys_newfstat) 132#define __NR_ioctl 54
133__SYSCALL(109, sys_ni_syscall) /* 109 was sys_uname */ 133__SYSCALL(__NR_ioctl, compat_sys_ioctl)
134__SYSCALL(110, sys_ni_syscall) /* 110 was sys_iopl */ 134#define __NR_fcntl 55
135__SYSCALL(111, sys_vhangup) 135__SYSCALL(__NR_fcntl, compat_sys_fcntl)
136__SYSCALL(112, sys_ni_syscall) /* 112 was sys_idle */ 136 /* 56 was sys_mpx */
137__SYSCALL(113, sys_ni_syscall) /* 113 was sys_syscall */ 137__SYSCALL(56, sys_ni_syscall)
138__SYSCALL(114, compat_sys_wait4) 138#define __NR_setpgid 57
139__SYSCALL(115, sys_swapoff) 139__SYSCALL(__NR_setpgid, sys_setpgid)
140__SYSCALL(116, compat_sys_sysinfo) 140 /* 58 was sys_ulimit */
141__SYSCALL(117, sys_ni_syscall) /* 117 was sys_ipc */ 141__SYSCALL(58, sys_ni_syscall)
142__SYSCALL(118, sys_fsync) 142 /* 59 was sys_olduname */
143__SYSCALL(119, compat_sys_sigreturn_wrapper) 143__SYSCALL(59, sys_ni_syscall)
144__SYSCALL(120, sys_clone) 144#define __NR_umask 60
145__SYSCALL(121, sys_setdomainname) 145__SYSCALL(__NR_umask, sys_umask)
146__SYSCALL(122, sys_newuname) 146#define __NR_chroot 61
147__SYSCALL(123, sys_ni_syscall) /* 123 was sys_modify_ldt */ 147__SYSCALL(__NR_chroot, sys_chroot)
148__SYSCALL(124, compat_sys_adjtimex) 148#define __NR_ustat 62
149__SYSCALL(125, sys_mprotect) 149__SYSCALL(__NR_ustat, compat_sys_ustat)
150__SYSCALL(126, compat_sys_sigprocmask) 150#define __NR_dup2 63
151__SYSCALL(127, sys_ni_syscall) /* 127 was sys_create_module */ 151__SYSCALL(__NR_dup2, sys_dup2)
152__SYSCALL(128, sys_init_module) 152#define __NR_getppid 64
153__SYSCALL(129, sys_delete_module) 153__SYSCALL(__NR_getppid, sys_getppid)
154__SYSCALL(130, sys_ni_syscall) /* 130 was sys_get_kernel_syms */ 154#define __NR_getpgrp 65
155__SYSCALL(131, sys_quotactl) 155__SYSCALL(__NR_getpgrp, sys_getpgrp)
156__SYSCALL(132, sys_getpgid) 156#define __NR_setsid 66
157__SYSCALL(133, sys_fchdir) 157__SYSCALL(__NR_setsid, sys_setsid)
158__SYSCALL(134, sys_bdflush) 158#define __NR_sigaction 67
159__SYSCALL(135, sys_sysfs) 159__SYSCALL(__NR_sigaction, compat_sys_sigaction)
160__SYSCALL(136, sys_personality) 160 /* 68 was sys_sgetmask */
161__SYSCALL(137, sys_ni_syscall) /* 137 was sys_afs_syscall */ 161__SYSCALL(68, sys_ni_syscall)
162__SYSCALL(138, sys_setfsuid16) 162 /* 69 was sys_ssetmask */
163__SYSCALL(139, sys_setfsgid16) 163__SYSCALL(69, sys_ni_syscall)
164__SYSCALL(140, sys_llseek) 164#define __NR_setreuid 70
165__SYSCALL(141, compat_sys_getdents) 165__SYSCALL(__NR_setreuid, sys_setreuid16)
166__SYSCALL(142, compat_sys_select) 166#define __NR_setregid 71
167__SYSCALL(143, sys_flock) 167__SYSCALL(__NR_setregid, sys_setregid16)
168__SYSCALL(144, sys_msync) 168#define __NR_sigsuspend 72
169__SYSCALL(145, compat_sys_readv) 169__SYSCALL(__NR_sigsuspend, sys_sigsuspend)
170__SYSCALL(146, compat_sys_writev) 170#define __NR_sigpending 73
171__SYSCALL(147, sys_getsid) 171__SYSCALL(__NR_sigpending, compat_sys_sigpending)
172__SYSCALL(148, sys_fdatasync) 172#define __NR_sethostname 74
173__SYSCALL(149, compat_sys_sysctl) 173__SYSCALL(__NR_sethostname, sys_sethostname)
174__SYSCALL(150, sys_mlock) 174#define __NR_setrlimit 75
175__SYSCALL(151, sys_munlock) 175__SYSCALL(__NR_setrlimit, compat_sys_setrlimit)
176__SYSCALL(152, sys_mlockall) 176 /* 76 was compat_sys_getrlimit */
177__SYSCALL(153, sys_munlockall) 177__SYSCALL(76, sys_ni_syscall)
178__SYSCALL(154, sys_sched_setparam) 178#define __NR_getrusage 77
179__SYSCALL(155, sys_sched_getparam) 179__SYSCALL(__NR_getrusage, compat_sys_getrusage)
180__SYSCALL(156, sys_sched_setscheduler) 180#define __NR_gettimeofday 78
181__SYSCALL(157, sys_sched_getscheduler) 181__SYSCALL(__NR_gettimeofday, compat_sys_gettimeofday)
182__SYSCALL(158, sys_sched_yield) 182#define __NR_settimeofday 79
183__SYSCALL(159, sys_sched_get_priority_max) 183__SYSCALL(__NR_settimeofday, compat_sys_settimeofday)
184__SYSCALL(160, sys_sched_get_priority_min) 184#define __NR_getgroups 80
185__SYSCALL(161, compat_sys_sched_rr_get_interval) 185__SYSCALL(__NR_getgroups, sys_getgroups16)
186__SYSCALL(162, compat_sys_nanosleep) 186#define __NR_setgroups 81
187__SYSCALL(163, sys_mremap) 187__SYSCALL(__NR_setgroups, sys_setgroups16)
188__SYSCALL(164, sys_setresuid16) 188 /* 82 was compat_sys_select */
189__SYSCALL(165, sys_getresuid16) 189__SYSCALL(82, sys_ni_syscall)
190__SYSCALL(166, sys_ni_syscall) /* 166 was sys_vm86 */ 190#define __NR_symlink 83
191__SYSCALL(167, sys_ni_syscall) /* 167 was sys_query_module */ 191__SYSCALL(__NR_symlink, sys_symlink)
192__SYSCALL(168, sys_poll) 192 /* 84 was sys_lstat */
193__SYSCALL(169, sys_ni_syscall) 193__SYSCALL(84, sys_ni_syscall)
194__SYSCALL(170, sys_setresgid16) 194#define __NR_readlink 85
195__SYSCALL(171, sys_getresgid16) 195__SYSCALL(__NR_readlink, sys_readlink)
196__SYSCALL(172, sys_prctl) 196#define __NR_uselib 86
197__SYSCALL(173, compat_sys_rt_sigreturn_wrapper) 197__SYSCALL(__NR_uselib, sys_uselib)
198__SYSCALL(174, compat_sys_rt_sigaction) 198#define __NR_swapon 87
199__SYSCALL(175, compat_sys_rt_sigprocmask) 199__SYSCALL(__NR_swapon, sys_swapon)
200__SYSCALL(176, compat_sys_rt_sigpending) 200#define __NR_reboot 88
201__SYSCALL(177, compat_sys_rt_sigtimedwait) 201__SYSCALL(__NR_reboot, sys_reboot)
202__SYSCALL(178, compat_sys_rt_sigqueueinfo) 202 /* 89 was sys_readdir */
203__SYSCALL(179, compat_sys_rt_sigsuspend) 203__SYSCALL(89, sys_ni_syscall)
204__SYSCALL(180, compat_sys_pread64_wrapper) 204 /* 90 was sys_mmap */
205__SYSCALL(181, compat_sys_pwrite64_wrapper) 205__SYSCALL(90, sys_ni_syscall)
206__SYSCALL(182, sys_chown16) 206#define __NR_munmap 91
207__SYSCALL(183, sys_getcwd) 207__SYSCALL(__NR_munmap, sys_munmap)
208__SYSCALL(184, sys_capget) 208#define __NR_truncate 92
209__SYSCALL(185, sys_capset) 209__SYSCALL(__NR_truncate, compat_sys_truncate)
210__SYSCALL(186, compat_sys_sigaltstack) 210#define __NR_ftruncate 93
211__SYSCALL(187, compat_sys_sendfile) 211__SYSCALL(__NR_ftruncate, compat_sys_ftruncate)
212__SYSCALL(188, sys_ni_syscall) /* 188 reserved */ 212#define __NR_fchmod 94
213__SYSCALL(189, sys_ni_syscall) /* 189 reserved */ 213__SYSCALL(__NR_fchmod, sys_fchmod)
214__SYSCALL(190, sys_vfork) 214#define __NR_fchown 95
215__SYSCALL(191, compat_sys_getrlimit) /* SuS compliant getrlimit */ 215__SYSCALL(__NR_fchown, sys_fchown16)
216__SYSCALL(192, sys_mmap_pgoff) 216#define __NR_getpriority 96
217__SYSCALL(193, compat_sys_truncate64_wrapper) 217__SYSCALL(__NR_getpriority, sys_getpriority)
218__SYSCALL(194, compat_sys_ftruncate64_wrapper) 218#define __NR_setpriority 97
219__SYSCALL(195, sys_stat64) 219__SYSCALL(__NR_setpriority, sys_setpriority)
220__SYSCALL(196, sys_lstat64) 220 /* 98 was sys_profil */
221__SYSCALL(197, sys_fstat64) 221__SYSCALL(98, sys_ni_syscall)
222__SYSCALL(198, sys_lchown) 222#define __NR_statfs 99
223__SYSCALL(199, sys_getuid) 223__SYSCALL(__NR_statfs, compat_sys_statfs)
224__SYSCALL(200, sys_getgid) 224#define __NR_fstatfs 100
225__SYSCALL(201, sys_geteuid) 225__SYSCALL(__NR_fstatfs, compat_sys_fstatfs)
226__SYSCALL(202, sys_getegid) 226 /* 101 was sys_ioperm */
227__SYSCALL(203, sys_setreuid) 227__SYSCALL(101, sys_ni_syscall)
228__SYSCALL(204, sys_setregid) 228 /* 102 was sys_socketcall */
229__SYSCALL(205, sys_getgroups) 229__SYSCALL(102, sys_ni_syscall)
230__SYSCALL(206, sys_setgroups) 230#define __NR_syslog 103
231__SYSCALL(207, sys_fchown) 231__SYSCALL(__NR_syslog, sys_syslog)
232__SYSCALL(208, sys_setresuid) 232#define __NR_setitimer 104
233__SYSCALL(209, sys_getresuid) 233__SYSCALL(__NR_setitimer, compat_sys_setitimer)
234__SYSCALL(210, sys_setresgid) 234#define __NR_getitimer 105
235__SYSCALL(211, sys_getresgid) 235__SYSCALL(__NR_getitimer, compat_sys_getitimer)
236__SYSCALL(212, sys_chown) 236#define __NR_stat 106
237__SYSCALL(213, sys_setuid) 237__SYSCALL(__NR_stat, compat_sys_newstat)
238__SYSCALL(214, sys_setgid) 238#define __NR_lstat 107
239__SYSCALL(215, sys_setfsuid) 239__SYSCALL(__NR_lstat, compat_sys_newlstat)
240__SYSCALL(216, sys_setfsgid) 240#define __NR_fstat 108
241__SYSCALL(217, compat_sys_getdents64) 241__SYSCALL(__NR_fstat, compat_sys_newfstat)
242__SYSCALL(218, sys_pivot_root) 242 /* 109 was sys_uname */
243__SYSCALL(219, sys_mincore) 243__SYSCALL(109, sys_ni_syscall)
244__SYSCALL(220, sys_madvise) 244 /* 110 was sys_iopl */
245__SYSCALL(221, compat_sys_fcntl64) 245__SYSCALL(110, sys_ni_syscall)
246__SYSCALL(222, sys_ni_syscall) /* 222 for tux */ 246#define __NR_vhangup 111
247__SYSCALL(223, sys_ni_syscall) /* 223 is unused */ 247__SYSCALL(__NR_vhangup, sys_vhangup)
248__SYSCALL(224, sys_gettid) 248 /* 112 was sys_idle */
249__SYSCALL(225, compat_sys_readahead_wrapper) 249__SYSCALL(112, sys_ni_syscall)
250__SYSCALL(226, sys_setxattr) 250 /* 113 was sys_syscall */
251__SYSCALL(227, sys_lsetxattr) 251__SYSCALL(113, sys_ni_syscall)
252__SYSCALL(228, sys_fsetxattr) 252#define __NR_wait4 114
253__SYSCALL(229, sys_getxattr) 253__SYSCALL(__NR_wait4, compat_sys_wait4)
254__SYSCALL(230, sys_lgetxattr) 254#define __NR_swapoff 115
255__SYSCALL(231, sys_fgetxattr) 255__SYSCALL(__NR_swapoff, sys_swapoff)
256__SYSCALL(232, sys_listxattr) 256#define __NR_sysinfo 116
257__SYSCALL(233, sys_llistxattr) 257__SYSCALL(__NR_sysinfo, compat_sys_sysinfo)
258__SYSCALL(234, sys_flistxattr) 258 /* 117 was sys_ipc */
259__SYSCALL(235, sys_removexattr) 259__SYSCALL(117, sys_ni_syscall)
260__SYSCALL(236, sys_lremovexattr) 260#define __NR_fsync 118
261__SYSCALL(237, sys_fremovexattr) 261__SYSCALL(__NR_fsync, sys_fsync)
262__SYSCALL(238, sys_tkill) 262#define __NR_sigreturn 119
263__SYSCALL(239, sys_sendfile64) 263__SYSCALL(__NR_sigreturn, compat_sys_sigreturn_wrapper)
264__SYSCALL(240, compat_sys_futex) 264#define __NR_clone 120
265__SYSCALL(241, compat_sys_sched_setaffinity) 265__SYSCALL(__NR_clone, sys_clone)
266__SYSCALL(242, compat_sys_sched_getaffinity) 266#define __NR_setdomainname 121
267__SYSCALL(243, compat_sys_io_setup) 267__SYSCALL(__NR_setdomainname, sys_setdomainname)
268__SYSCALL(244, sys_io_destroy) 268#define __NR_uname 122
269__SYSCALL(245, compat_sys_io_getevents) 269__SYSCALL(__NR_uname, sys_newuname)
270__SYSCALL(246, compat_sys_io_submit) 270 /* 123 was sys_modify_ldt */
271__SYSCALL(247, sys_io_cancel) 271__SYSCALL(123, sys_ni_syscall)
272__SYSCALL(248, sys_exit_group) 272#define __NR_adjtimex 124
273__SYSCALL(249, compat_sys_lookup_dcookie) 273__SYSCALL(__NR_adjtimex, compat_sys_adjtimex)
274__SYSCALL(250, sys_epoll_create) 274#define __NR_mprotect 125
275__SYSCALL(251, sys_epoll_ctl) 275__SYSCALL(__NR_mprotect, sys_mprotect)
276__SYSCALL(252, sys_epoll_wait) 276#define __NR_sigprocmask 126
277__SYSCALL(253, sys_remap_file_pages) 277__SYSCALL(__NR_sigprocmask, compat_sys_sigprocmask)
278__SYSCALL(254, sys_ni_syscall) /* 254 for set_thread_area */ 278 /* 127 was sys_create_module */
279__SYSCALL(255, sys_ni_syscall) /* 255 for get_thread_area */ 279__SYSCALL(127, sys_ni_syscall)
280__SYSCALL(256, sys_set_tid_address) 280#define __NR_init_module 128
281__SYSCALL(257, compat_sys_timer_create) 281__SYSCALL(__NR_init_module, sys_init_module)
282__SYSCALL(258, compat_sys_timer_settime) 282#define __NR_delete_module 129
283__SYSCALL(259, compat_sys_timer_gettime) 283__SYSCALL(__NR_delete_module, sys_delete_module)
284__SYSCALL(260, sys_timer_getoverrun) 284 /* 130 was sys_get_kernel_syms */
285__SYSCALL(261, sys_timer_delete) 285__SYSCALL(130, sys_ni_syscall)
286__SYSCALL(262, compat_sys_clock_settime) 286#define __NR_quotactl 131
287__SYSCALL(263, compat_sys_clock_gettime) 287__SYSCALL(__NR_quotactl, sys_quotactl)
288__SYSCALL(264, compat_sys_clock_getres) 288#define __NR_getpgid 132
289__SYSCALL(265, compat_sys_clock_nanosleep) 289__SYSCALL(__NR_getpgid, sys_getpgid)
290__SYSCALL(266, compat_sys_statfs64_wrapper) 290#define __NR_fchdir 133
291__SYSCALL(267, compat_sys_fstatfs64_wrapper) 291__SYSCALL(__NR_fchdir, sys_fchdir)
292__SYSCALL(268, sys_tgkill) 292#define __NR_bdflush 134
293__SYSCALL(269, compat_sys_utimes) 293__SYSCALL(__NR_bdflush, sys_bdflush)
294__SYSCALL(270, compat_sys_fadvise64_64_wrapper) 294#define __NR_sysfs 135
295__SYSCALL(271, sys_pciconfig_iobase) 295__SYSCALL(__NR_sysfs, sys_sysfs)
296__SYSCALL(272, sys_pciconfig_read) 296#define __NR_personality 136
297__SYSCALL(273, sys_pciconfig_write) 297__SYSCALL(__NR_personality, sys_personality)
298__SYSCALL(274, compat_sys_mq_open) 298 /* 137 was sys_afs_syscall */
299__SYSCALL(275, sys_mq_unlink) 299__SYSCALL(137, sys_ni_syscall)
300__SYSCALL(276, compat_sys_mq_timedsend) 300#define __NR_setfsuid 138
301__SYSCALL(277, compat_sys_mq_timedreceive) 301__SYSCALL(__NR_setfsuid, sys_setfsuid16)
302__SYSCALL(278, compat_sys_mq_notify) 302#define __NR_setfsgid 139
303__SYSCALL(279, compat_sys_mq_getsetattr) 303__SYSCALL(__NR_setfsgid, sys_setfsgid16)
304__SYSCALL(280, compat_sys_waitid) 304#define __NR__llseek 140
305__SYSCALL(281, sys_socket) 305__SYSCALL(__NR__llseek, sys_llseek)
306__SYSCALL(282, sys_bind) 306#define __NR_getdents 141
307__SYSCALL(283, sys_connect) 307__SYSCALL(__NR_getdents, compat_sys_getdents)
308__SYSCALL(284, sys_listen) 308#define __NR__newselect 142
309__SYSCALL(285, sys_accept) 309__SYSCALL(__NR__newselect, compat_sys_select)
310__SYSCALL(286, sys_getsockname) 310#define __NR_flock 143
311__SYSCALL(287, sys_getpeername) 311__SYSCALL(__NR_flock, sys_flock)
312__SYSCALL(288, sys_socketpair) 312#define __NR_msync 144
313__SYSCALL(289, sys_send) 313__SYSCALL(__NR_msync, sys_msync)
314__SYSCALL(290, sys_sendto) 314#define __NR_readv 145
315__SYSCALL(291, compat_sys_recv) 315__SYSCALL(__NR_readv, compat_sys_readv)
316__SYSCALL(292, compat_sys_recvfrom) 316#define __NR_writev 146
317__SYSCALL(293, sys_shutdown) 317__SYSCALL(__NR_writev, compat_sys_writev)
318__SYSCALL(294, compat_sys_setsockopt) 318#define __NR_getsid 147
319__SYSCALL(295, compat_sys_getsockopt) 319__SYSCALL(__NR_getsid, sys_getsid)
320__SYSCALL(296, compat_sys_sendmsg) 320#define __NR_fdatasync 148
321__SYSCALL(297, compat_sys_recvmsg) 321__SYSCALL(__NR_fdatasync, sys_fdatasync)
322__SYSCALL(298, sys_semop) 322#define __NR__sysctl 149
323__SYSCALL(299, sys_semget) 323__SYSCALL(__NR__sysctl, compat_sys_sysctl)
324__SYSCALL(300, compat_sys_semctl) 324#define __NR_mlock 150
325__SYSCALL(301, compat_sys_msgsnd) 325__SYSCALL(__NR_mlock, sys_mlock)
326__SYSCALL(302, compat_sys_msgrcv) 326#define __NR_munlock 151
327__SYSCALL(303, sys_msgget) 327__SYSCALL(__NR_munlock, sys_munlock)
328__SYSCALL(304, compat_sys_msgctl) 328#define __NR_mlockall 152
329__SYSCALL(305, compat_sys_shmat) 329__SYSCALL(__NR_mlockall, sys_mlockall)
330__SYSCALL(306, sys_shmdt) 330#define __NR_munlockall 153
331__SYSCALL(307, sys_shmget) 331__SYSCALL(__NR_munlockall, sys_munlockall)
332__SYSCALL(308, compat_sys_shmctl) 332#define __NR_sched_setparam 154
333__SYSCALL(309, sys_add_key) 333__SYSCALL(__NR_sched_setparam, sys_sched_setparam)
334__SYSCALL(310, sys_request_key) 334#define __NR_sched_getparam 155
335__SYSCALL(311, compat_sys_keyctl) 335__SYSCALL(__NR_sched_getparam, sys_sched_getparam)
336__SYSCALL(312, compat_sys_semtimedop) 336#define __NR_sched_setscheduler 156
337__SYSCALL(313, sys_ni_syscall) 337__SYSCALL(__NR_sched_setscheduler, sys_sched_setscheduler)
338__SYSCALL(314, sys_ioprio_set) 338#define __NR_sched_getscheduler 157
339__SYSCALL(315, sys_ioprio_get) 339__SYSCALL(__NR_sched_getscheduler, sys_sched_getscheduler)
340__SYSCALL(316, sys_inotify_init) 340#define __NR_sched_yield 158
341__SYSCALL(317, sys_inotify_add_watch) 341__SYSCALL(__NR_sched_yield, sys_sched_yield)
342__SYSCALL(318, sys_inotify_rm_watch) 342#define __NR_sched_get_priority_max 159
343__SYSCALL(319, compat_sys_mbind) 343__SYSCALL(__NR_sched_get_priority_max, sys_sched_get_priority_max)
344__SYSCALL(320, compat_sys_get_mempolicy) 344#define __NR_sched_get_priority_min 160
345__SYSCALL(321, compat_sys_set_mempolicy) 345__SYSCALL(__NR_sched_get_priority_min, sys_sched_get_priority_min)
346__SYSCALL(322, compat_sys_openat) 346#define __NR_sched_rr_get_interval 161
347__SYSCALL(323, sys_mkdirat) 347__SYSCALL(__NR_sched_rr_get_interval, compat_sys_sched_rr_get_interval)
348__SYSCALL(324, sys_mknodat) 348#define __NR_nanosleep 162
349__SYSCALL(325, sys_fchownat) 349__SYSCALL(__NR_nanosleep, compat_sys_nanosleep)
350__SYSCALL(326, compat_sys_futimesat) 350#define __NR_mremap 163
351__SYSCALL(327, sys_fstatat64) 351__SYSCALL(__NR_mremap, sys_mremap)
352__SYSCALL(328, sys_unlinkat) 352#define __NR_setresuid 164
353__SYSCALL(329, sys_renameat) 353__SYSCALL(__NR_setresuid, sys_setresuid16)
354__SYSCALL(330, sys_linkat) 354#define __NR_getresuid 165
355__SYSCALL(331, sys_symlinkat) 355__SYSCALL(__NR_getresuid, sys_getresuid16)
356__SYSCALL(332, sys_readlinkat) 356 /* 166 was sys_vm86 */
357__SYSCALL(333, sys_fchmodat) 357__SYSCALL(166, sys_ni_syscall)
358__SYSCALL(334, sys_faccessat) 358 /* 167 was sys_query_module */
359__SYSCALL(335, compat_sys_pselect6) 359__SYSCALL(167, sys_ni_syscall)
360__SYSCALL(336, compat_sys_ppoll) 360#define __NR_poll 168
361__SYSCALL(337, sys_unshare) 361__SYSCALL(__NR_poll, sys_poll)
362__SYSCALL(338, compat_sys_set_robust_list) 362#define __NR_nfsservctl 169
363__SYSCALL(339, compat_sys_get_robust_list) 363__SYSCALL(__NR_nfsservctl, sys_ni_syscall)
364__SYSCALL(340, sys_splice) 364#define __NR_setresgid 170
365__SYSCALL(341, compat_sys_sync_file_range2_wrapper) 365__SYSCALL(__NR_setresgid, sys_setresgid16)
366__SYSCALL(342, sys_tee) 366#define __NR_getresgid 171
367__SYSCALL(343, compat_sys_vmsplice) 367__SYSCALL(__NR_getresgid, sys_getresgid16)
368__SYSCALL(344, compat_sys_move_pages) 368#define __NR_prctl 172
369__SYSCALL(345, sys_getcpu) 369__SYSCALL(__NR_prctl, sys_prctl)
370__SYSCALL(346, compat_sys_epoll_pwait) 370#define __NR_rt_sigreturn 173
371__SYSCALL(347, compat_sys_kexec_load) 371__SYSCALL(__NR_rt_sigreturn, compat_sys_rt_sigreturn_wrapper)
372__SYSCALL(348, compat_sys_utimensat) 372#define __NR_rt_sigaction 174
373__SYSCALL(349, compat_sys_signalfd) 373__SYSCALL(__NR_rt_sigaction, compat_sys_rt_sigaction)
374__SYSCALL(350, sys_timerfd_create) 374#define __NR_rt_sigprocmask 175
375__SYSCALL(351, sys_eventfd) 375__SYSCALL(__NR_rt_sigprocmask, compat_sys_rt_sigprocmask)
376__SYSCALL(352, compat_sys_fallocate_wrapper) 376#define __NR_rt_sigpending 176
377__SYSCALL(353, compat_sys_timerfd_settime) 377__SYSCALL(__NR_rt_sigpending, compat_sys_rt_sigpending)
378__SYSCALL(354, compat_sys_timerfd_gettime) 378#define __NR_rt_sigtimedwait 177
379__SYSCALL(355, compat_sys_signalfd4) 379__SYSCALL(__NR_rt_sigtimedwait, compat_sys_rt_sigtimedwait)
380__SYSCALL(356, sys_eventfd2) 380#define __NR_rt_sigqueueinfo 178
381__SYSCALL(357, sys_epoll_create1) 381__SYSCALL(__NR_rt_sigqueueinfo, compat_sys_rt_sigqueueinfo)
382__SYSCALL(358, sys_dup3) 382#define __NR_rt_sigsuspend 179
383__SYSCALL(359, sys_pipe2) 383__SYSCALL(__NR_rt_sigsuspend, compat_sys_rt_sigsuspend)
384__SYSCALL(360, sys_inotify_init1) 384#define __NR_pread64 180
385__SYSCALL(361, compat_sys_preadv) 385__SYSCALL(__NR_pread64, compat_sys_pread64_wrapper)
386__SYSCALL(362, compat_sys_pwritev) 386#define __NR_pwrite64 181
387__SYSCALL(363, compat_sys_rt_tgsigqueueinfo) 387__SYSCALL(__NR_pwrite64, compat_sys_pwrite64_wrapper)
388__SYSCALL(364, sys_perf_event_open) 388#define __NR_chown 182
389__SYSCALL(365, compat_sys_recvmmsg) 389__SYSCALL(__NR_chown, sys_chown16)
390__SYSCALL(366, sys_accept4) 390#define __NR_getcwd 183
391__SYSCALL(367, sys_fanotify_init) 391__SYSCALL(__NR_getcwd, sys_getcwd)
392__SYSCALL(368, compat_sys_fanotify_mark) 392#define __NR_capget 184
393__SYSCALL(369, sys_prlimit64) 393__SYSCALL(__NR_capget, sys_capget)
394__SYSCALL(370, sys_name_to_handle_at) 394#define __NR_capset 185
395__SYSCALL(371, compat_sys_open_by_handle_at) 395__SYSCALL(__NR_capset, sys_capset)
396__SYSCALL(372, compat_sys_clock_adjtime) 396#define __NR_sigaltstack 186
397__SYSCALL(373, sys_syncfs) 397__SYSCALL(__NR_sigaltstack, compat_sys_sigaltstack)
398__SYSCALL(374, compat_sys_sendmmsg) 398#define __NR_sendfile 187
399__SYSCALL(375, sys_setns) 399__SYSCALL(__NR_sendfile, compat_sys_sendfile)
400__SYSCALL(376, compat_sys_process_vm_readv) 400 /* 188 reserved */
401__SYSCALL(377, compat_sys_process_vm_writev) 401__SYSCALL(188, sys_ni_syscall)
402__SYSCALL(378, sys_kcmp) 402 /* 189 reserved */
403__SYSCALL(379, sys_finit_module) 403__SYSCALL(189, sys_ni_syscall)
404__SYSCALL(380, sys_sched_setattr) 404#define __NR_vfork 190
405__SYSCALL(381, sys_sched_getattr) 405__SYSCALL(__NR_vfork, sys_vfork)
406__SYSCALL(382, sys_renameat2) 406#define __NR_ugetrlimit 191 /* SuS compliant getrlimit */
407 407__SYSCALL(__NR_ugetrlimit, compat_sys_getrlimit) /* SuS compliant getrlimit */
408#define __NR_compat_syscalls 383 408#define __NR_mmap2 192
409 409__SYSCALL(__NR_mmap2, sys_mmap_pgoff)
410/* 410#define __NR_truncate64 193
411 * Compat syscall numbers used by the AArch64 kernel. 411__SYSCALL(__NR_truncate64, compat_sys_truncate64_wrapper)
412 */ 412#define __NR_ftruncate64 194
413#define __NR_compat_restart_syscall 0 413__SYSCALL(__NR_ftruncate64, compat_sys_ftruncate64_wrapper)
414#define __NR_compat_sigreturn 119 414#define __NR_stat64 195
415#define __NR_compat_rt_sigreturn 173 415__SYSCALL(__NR_stat64, sys_stat64)
416 416#define __NR_lstat64 196
417 417__SYSCALL(__NR_lstat64, sys_lstat64)
418/* 418#define __NR_fstat64 197
419 * The following SVCs are ARM private. 419__SYSCALL(__NR_fstat64, sys_fstat64)
420 */ 420#define __NR_lchown32 198
421#define __ARM_NR_COMPAT_BASE 0x0f0000 421__SYSCALL(__NR_lchown32, sys_lchown)
422#define __ARM_NR_compat_cacheflush (__ARM_NR_COMPAT_BASE+2) 422#define __NR_getuid32 199
423#define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE+5) 423__SYSCALL(__NR_getuid32, sys_getuid)
424#define __NR_getgid32 200
425__SYSCALL(__NR_getgid32, sys_getgid)
426#define __NR_geteuid32 201
427__SYSCALL(__NR_geteuid32, sys_geteuid)
428#define __NR_getegid32 202
429__SYSCALL(__NR_getegid32, sys_getegid)
430#define __NR_setreuid32 203
431__SYSCALL(__NR_setreuid32, sys_setreuid)
432#define __NR_setregid32 204
433__SYSCALL(__NR_setregid32, sys_setregid)
434#define __NR_getgroups32 205
435__SYSCALL(__NR_getgroups32, sys_getgroups)
436#define __NR_setgroups32 206
437__SYSCALL(__NR_setgroups32, sys_setgroups)
438#define __NR_fchown32 207
439__SYSCALL(__NR_fchown32, sys_fchown)
440#define __NR_setresuid32 208
441__SYSCALL(__NR_setresuid32, sys_setresuid)
442#define __NR_getresuid32 209
443__SYSCALL(__NR_getresuid32, sys_getresuid)
444#define __NR_setresgid32 210
445__SYSCALL(__NR_setresgid32, sys_setresgid)
446#define __NR_getresgid32 211
447__SYSCALL(__NR_getresgid32, sys_getresgid)
448#define __NR_chown32 212
449__SYSCALL(__NR_chown32, sys_chown)
450#define __NR_setuid32 213
451__SYSCALL(__NR_setuid32, sys_setuid)
452#define __NR_setgid32 214
453__SYSCALL(__NR_setgid32, sys_setgid)
454#define __NR_setfsuid32 215
455__SYSCALL(__NR_setfsuid32, sys_setfsuid)
456#define __NR_setfsgid32 216
457__SYSCALL(__NR_setfsgid32, sys_setfsgid)
458#define __NR_getdents64 217
459__SYSCALL(__NR_getdents64, compat_sys_getdents64)
460#define __NR_pivot_root 218
461__SYSCALL(__NR_pivot_root, sys_pivot_root)
462#define __NR_mincore 219
463__SYSCALL(__NR_mincore, sys_mincore)
464#define __NR_madvise 220
465__SYSCALL(__NR_madvise, sys_madvise)
466#define __NR_fcntl64 221
467__SYSCALL(__NR_fcntl64, compat_sys_fcntl64)
468 /* 222 for tux */
469__SYSCALL(222, sys_ni_syscall)
470 /* 223 is unused */
471__SYSCALL(223, sys_ni_syscall)
472#define __NR_gettid 224
473__SYSCALL(__NR_gettid, sys_gettid)
474#define __NR_readahead 225
475__SYSCALL(__NR_readahead, compat_sys_readahead_wrapper)
476#define __NR_setxattr 226
477__SYSCALL(__NR_setxattr, sys_setxattr)
478#define __NR_lsetxattr 227
479__SYSCALL(__NR_lsetxattr, sys_lsetxattr)
480#define __NR_fsetxattr 228
481__SYSCALL(__NR_fsetxattr, sys_fsetxattr)
482#define __NR_getxattr 229
483__SYSCALL(__NR_getxattr, sys_getxattr)
484#define __NR_lgetxattr 230
485__SYSCALL(__NR_lgetxattr, sys_lgetxattr)
486#define __NR_fgetxattr 231
487__SYSCALL(__NR_fgetxattr, sys_fgetxattr)
488#define __NR_listxattr 232
489__SYSCALL(__NR_listxattr, sys_listxattr)
490#define __NR_llistxattr 233
491__SYSCALL(__NR_llistxattr, sys_llistxattr)
492#define __NR_flistxattr 234
493__SYSCALL(__NR_flistxattr, sys_flistxattr)
494#define __NR_removexattr 235
495__SYSCALL(__NR_removexattr, sys_removexattr)
496#define __NR_lremovexattr 236
497__SYSCALL(__NR_lremovexattr, sys_lremovexattr)
498#define __NR_fremovexattr 237
499__SYSCALL(__NR_fremovexattr, sys_fremovexattr)
500#define __NR_tkill 238
501__SYSCALL(__NR_tkill, sys_tkill)
502#define __NR_sendfile64 239
503__SYSCALL(__NR_sendfile64, sys_sendfile64)
504#define __NR_futex 240
505__SYSCALL(__NR_futex, compat_sys_futex)
506#define __NR_sched_setaffinity 241
507__SYSCALL(__NR_sched_setaffinity, compat_sys_sched_setaffinity)
508#define __NR_sched_getaffinity 242
509__SYSCALL(__NR_sched_getaffinity, compat_sys_sched_getaffinity)
510#define __NR_io_setup 243
511__SYSCALL(__NR_io_setup, compat_sys_io_setup)
512#define __NR_io_destroy 244
513__SYSCALL(__NR_io_destroy, sys_io_destroy)
514#define __NR_io_getevents 245
515__SYSCALL(__NR_io_getevents, compat_sys_io_getevents)
516#define __NR_io_submit 246
517__SYSCALL(__NR_io_submit, compat_sys_io_submit)
518#define __NR_io_cancel 247
519__SYSCALL(__NR_io_cancel, sys_io_cancel)
520#define __NR_exit_group 248
521__SYSCALL(__NR_exit_group, sys_exit_group)
522#define __NR_lookup_dcookie 249
523__SYSCALL(__NR_lookup_dcookie, compat_sys_lookup_dcookie)
524#define __NR_epoll_create 250
525__SYSCALL(__NR_epoll_create, sys_epoll_create)
526#define __NR_epoll_ctl 251
527__SYSCALL(__NR_epoll_ctl, sys_epoll_ctl)
528#define __NR_epoll_wait 252
529__SYSCALL(__NR_epoll_wait, sys_epoll_wait)
530#define __NR_remap_file_pages 253
531__SYSCALL(__NR_remap_file_pages, sys_remap_file_pages)
532 /* 254 for set_thread_area */
533__SYSCALL(254, sys_ni_syscall)
534 /* 255 for get_thread_area */
535__SYSCALL(255, sys_ni_syscall)
536#define __NR_set_tid_address 256
537__SYSCALL(__NR_set_tid_address, sys_set_tid_address)
538#define __NR_timer_create 257
539__SYSCALL(__NR_timer_create, compat_sys_timer_create)
540#define __NR_timer_settime 258
541__SYSCALL(__NR_timer_settime, compat_sys_timer_settime)
542#define __NR_timer_gettime 259
543__SYSCALL(__NR_timer_gettime, compat_sys_timer_gettime)
544#define __NR_timer_getoverrun 260
545__SYSCALL(__NR_timer_getoverrun, sys_timer_getoverrun)
546#define __NR_timer_delete 261
547__SYSCALL(__NR_timer_delete, sys_timer_delete)
548#define __NR_clock_settime 262
549__SYSCALL(__NR_clock_settime, compat_sys_clock_settime)
550#define __NR_clock_gettime 263
551__SYSCALL(__NR_clock_gettime, compat_sys_clock_gettime)
552#define __NR_clock_getres 264
553__SYSCALL(__NR_clock_getres, compat_sys_clock_getres)
554#define __NR_clock_nanosleep 265
555__SYSCALL(__NR_clock_nanosleep, compat_sys_clock_nanosleep)
556#define __NR_statfs64 266
557__SYSCALL(__NR_statfs64, compat_sys_statfs64_wrapper)
558#define __NR_fstatfs64 267
559__SYSCALL(__NR_fstatfs64, compat_sys_fstatfs64_wrapper)
560#define __NR_tgkill 268
561__SYSCALL(__NR_tgkill, sys_tgkill)
562#define __NR_utimes 269
563__SYSCALL(__NR_utimes, compat_sys_utimes)
564#define __NR_arm_fadvise64_64 270
565__SYSCALL(__NR_arm_fadvise64_64, compat_sys_fadvise64_64_wrapper)
566#define __NR_pciconfig_iobase 271
567__SYSCALL(__NR_pciconfig_iobase, sys_pciconfig_iobase)
568#define __NR_pciconfig_read 272
569__SYSCALL(__NR_pciconfig_read, sys_pciconfig_read)
570#define __NR_pciconfig_write 273
571__SYSCALL(__NR_pciconfig_write, sys_pciconfig_write)
572#define __NR_mq_open 274
573__SYSCALL(__NR_mq_open, compat_sys_mq_open)
574#define __NR_mq_unlink 275
575__SYSCALL(__NR_mq_unlink, sys_mq_unlink)
576#define __NR_mq_timedsend 276
577__SYSCALL(__NR_mq_timedsend, compat_sys_mq_timedsend)
578#define __NR_mq_timedreceive 277
579__SYSCALL(__NR_mq_timedreceive, compat_sys_mq_timedreceive)
580#define __NR_mq_notify 278
581__SYSCALL(__NR_mq_notify, compat_sys_mq_notify)
582#define __NR_mq_getsetattr 279
583__SYSCALL(__NR_mq_getsetattr, compat_sys_mq_getsetattr)
584#define __NR_waitid 280
585__SYSCALL(__NR_waitid, compat_sys_waitid)
586#define __NR_socket 281
587__SYSCALL(__NR_socket, sys_socket)
588#define __NR_bind 282
589__SYSCALL(__NR_bind, sys_bind)
590#define __NR_connect 283
591__SYSCALL(__NR_connect, sys_connect)
592#define __NR_listen 284
593__SYSCALL(__NR_listen, sys_listen)
594#define __NR_accept 285
595__SYSCALL(__NR_accept, sys_accept)
596#define __NR_getsockname 286
597__SYSCALL(__NR_getsockname, sys_getsockname)
598#define __NR_getpeername 287
599__SYSCALL(__NR_getpeername, sys_getpeername)
600#define __NR_socketpair 288
601__SYSCALL(__NR_socketpair, sys_socketpair)
602#define __NR_send 289
603__SYSCALL(__NR_send, sys_send)
604#define __NR_sendto 290
605__SYSCALL(__NR_sendto, sys_sendto)
606#define __NR_recv 291
607__SYSCALL(__NR_recv, compat_sys_recv)
608#define __NR_recvfrom 292
609__SYSCALL(__NR_recvfrom, compat_sys_recvfrom)
610#define __NR_shutdown 293
611__SYSCALL(__NR_shutdown, sys_shutdown)
612#define __NR_setsockopt 294
613__SYSCALL(__NR_setsockopt, compat_sys_setsockopt)
614#define __NR_getsockopt 295
615__SYSCALL(__NR_getsockopt, compat_sys_getsockopt)
616#define __NR_sendmsg 296
617__SYSCALL(__NR_sendmsg, compat_sys_sendmsg)
618#define __NR_recvmsg 297
619__SYSCALL(__NR_recvmsg, compat_sys_recvmsg)
620#define __NR_semop 298
621__SYSCALL(__NR_semop, sys_semop)
622#define __NR_semget 299
623__SYSCALL(__NR_semget, sys_semget)
624#define __NR_semctl 300
625__SYSCALL(__NR_semctl, compat_sys_semctl)
626#define __NR_msgsnd 301
627__SYSCALL(__NR_msgsnd, compat_sys_msgsnd)
628#define __NR_msgrcv 302
629__SYSCALL(__NR_msgrcv, compat_sys_msgrcv)
630#define __NR_msgget 303
631__SYSCALL(__NR_msgget, sys_msgget)
632#define __NR_msgctl 304
633__SYSCALL(__NR_msgctl, compat_sys_msgctl)
634#define __NR_shmat 305
635__SYSCALL(__NR_shmat, compat_sys_shmat)
636#define __NR_shmdt 306
637__SYSCALL(__NR_shmdt, sys_shmdt)
638#define __NR_shmget 307
639__SYSCALL(__NR_shmget, sys_shmget)
640#define __NR_shmctl 308
641__SYSCALL(__NR_shmctl, compat_sys_shmctl)
642#define __NR_add_key 309
643__SYSCALL(__NR_add_key, sys_add_key)
644#define __NR_request_key 310
645__SYSCALL(__NR_request_key, sys_request_key)
646#define __NR_keyctl 311
647__SYSCALL(__NR_keyctl, compat_sys_keyctl)
648#define __NR_semtimedop 312
649__SYSCALL(__NR_semtimedop, compat_sys_semtimedop)
650#define __NR_vserver 313
651__SYSCALL(__NR_vserver, sys_ni_syscall)
652#define __NR_ioprio_set 314
653__SYSCALL(__NR_ioprio_set, sys_ioprio_set)
654#define __NR_ioprio_get 315
655__SYSCALL(__NR_ioprio_get, sys_ioprio_get)
656#define __NR_inotify_init 316
657__SYSCALL(__NR_inotify_init, sys_inotify_init)
658#define __NR_inotify_add_watch 317
659__SYSCALL(__NR_inotify_add_watch, sys_inotify_add_watch)
660#define __NR_inotify_rm_watch 318
661__SYSCALL(__NR_inotify_rm_watch, sys_inotify_rm_watch)
662#define __NR_mbind 319
663__SYSCALL(__NR_mbind, compat_sys_mbind)
664#define __NR_get_mempolicy 320
665__SYSCALL(__NR_get_mempolicy, compat_sys_get_mempolicy)
666#define __NR_set_mempolicy 321
667__SYSCALL(__NR_set_mempolicy, compat_sys_set_mempolicy)
668#define __NR_openat 322
669__SYSCALL(__NR_openat, compat_sys_openat)
670#define __NR_mkdirat 323
671__SYSCALL(__NR_mkdirat, sys_mkdirat)
672#define __NR_mknodat 324
673__SYSCALL(__NR_mknodat, sys_mknodat)
674#define __NR_fchownat 325
675__SYSCALL(__NR_fchownat, sys_fchownat)
676#define __NR_futimesat 326
677__SYSCALL(__NR_futimesat, compat_sys_futimesat)
678#define __NR_fstatat64 327
679__SYSCALL(__NR_fstatat64, sys_fstatat64)
680#define __NR_unlinkat 328
681__SYSCALL(__NR_unlinkat, sys_unlinkat)
682#define __NR_renameat 329
683__SYSCALL(__NR_renameat, sys_renameat)
684#define __NR_linkat 330
685__SYSCALL(__NR_linkat, sys_linkat)
686#define __NR_symlinkat 331
687__SYSCALL(__NR_symlinkat, sys_symlinkat)
688#define __NR_readlinkat 332
689__SYSCALL(__NR_readlinkat, sys_readlinkat)
690#define __NR_fchmodat 333
691__SYSCALL(__NR_fchmodat, sys_fchmodat)
692#define __NR_faccessat 334
693__SYSCALL(__NR_faccessat, sys_faccessat)
694#define __NR_pselect6 335
695__SYSCALL(__NR_pselect6, compat_sys_pselect6)
696#define __NR_ppoll 336
697__SYSCALL(__NR_ppoll, compat_sys_ppoll)
698#define __NR_unshare 337
699__SYSCALL(__NR_unshare, sys_unshare)
700#define __NR_set_robust_list 338
701__SYSCALL(__NR_set_robust_list, compat_sys_set_robust_list)
702#define __NR_get_robust_list 339
703__SYSCALL(__NR_get_robust_list, compat_sys_get_robust_list)
704#define __NR_splice 340
705__SYSCALL(__NR_splice, sys_splice)
706#define __NR_sync_file_range2 341
707__SYSCALL(__NR_sync_file_range2, compat_sys_sync_file_range2_wrapper)
708#define __NR_tee 342
709__SYSCALL(__NR_tee, sys_tee)
710#define __NR_vmsplice 343
711__SYSCALL(__NR_vmsplice, compat_sys_vmsplice)
712#define __NR_move_pages 344
713__SYSCALL(__NR_move_pages, compat_sys_move_pages)
714#define __NR_getcpu 345
715__SYSCALL(__NR_getcpu, sys_getcpu)
716#define __NR_epoll_pwait 346
717__SYSCALL(__NR_epoll_pwait, compat_sys_epoll_pwait)
718#define __NR_kexec_load 347
719__SYSCALL(__NR_kexec_load, compat_sys_kexec_load)
720#define __NR_utimensat 348
721__SYSCALL(__NR_utimensat, compat_sys_utimensat)
722#define __NR_signalfd 349
723__SYSCALL(__NR_signalfd, compat_sys_signalfd)
724#define __NR_timerfd_create 350
725__SYSCALL(__NR_timerfd_create, sys_timerfd_create)
726#define __NR_eventfd 351
727__SYSCALL(__NR_eventfd, sys_eventfd)
728#define __NR_fallocate 352
729__SYSCALL(__NR_fallocate, compat_sys_fallocate_wrapper)
730#define __NR_timerfd_settime 353
731__SYSCALL(__NR_timerfd_settime, compat_sys_timerfd_settime)
732#define __NR_timerfd_gettime 354
733__SYSCALL(__NR_timerfd_gettime, compat_sys_timerfd_gettime)
734#define __NR_signalfd4 355
735__SYSCALL(__NR_signalfd4, compat_sys_signalfd4)
736#define __NR_eventfd2 356
737__SYSCALL(__NR_eventfd2, sys_eventfd2)
738#define __NR_epoll_create1 357
739__SYSCALL(__NR_epoll_create1, sys_epoll_create1)
740#define __NR_dup3 358
741__SYSCALL(__NR_dup3, sys_dup3)
742#define __NR_pipe2 359
743__SYSCALL(__NR_pipe2, sys_pipe2)
744#define __NR_inotify_init1 360
745__SYSCALL(__NR_inotify_init1, sys_inotify_init1)
746#define __NR_preadv 361
747__SYSCALL(__NR_preadv, compat_sys_preadv)
748#define __NR_pwritev 362
749__SYSCALL(__NR_pwritev, compat_sys_pwritev)
750#define __NR_rt_tgsigqueueinfo 363
751__SYSCALL(__NR_rt_tgsigqueueinfo, compat_sys_rt_tgsigqueueinfo)
752#define __NR_perf_event_open 364
753__SYSCALL(__NR_perf_event_open, sys_perf_event_open)
754#define __NR_recvmmsg 365
755__SYSCALL(__NR_recvmmsg, compat_sys_recvmmsg)
756#define __NR_accept4 366
757__SYSCALL(__NR_accept4, sys_accept4)
758#define __NR_fanotify_init 367
759__SYSCALL(__NR_fanotify_init, sys_fanotify_init)
760#define __NR_fanotify_mark 368
761__SYSCALL(__NR_fanotify_mark, compat_sys_fanotify_mark)
762#define __NR_prlimit64 369
763__SYSCALL(__NR_prlimit64, sys_prlimit64)
764#define __NR_name_to_handle_at 370
765__SYSCALL(__NR_name_to_handle_at, sys_name_to_handle_at)
766#define __NR_open_by_handle_at 371
767__SYSCALL(__NR_open_by_handle_at, compat_sys_open_by_handle_at)
768#define __NR_clock_adjtime 372
769__SYSCALL(__NR_clock_adjtime, compat_sys_clock_adjtime)
770#define __NR_syncfs 373
771__SYSCALL(__NR_syncfs, sys_syncfs)
772#define __NR_sendmmsg 374
773__SYSCALL(__NR_sendmmsg, compat_sys_sendmmsg)
774#define __NR_setns 375
775__SYSCALL(__NR_setns, sys_setns)
776#define __NR_process_vm_readv 376
777__SYSCALL(__NR_process_vm_readv, compat_sys_process_vm_readv)
778#define __NR_process_vm_writev 377
779__SYSCALL(__NR_process_vm_writev, compat_sys_process_vm_writev)
780#define __NR_kcmp 378
781__SYSCALL(__NR_kcmp, sys_kcmp)
782#define __NR_finit_module 379
783__SYSCALL(__NR_finit_module, sys_finit_module)
784#define __NR_sched_setattr 380
785__SYSCALL(__NR_sched_setattr, sys_sched_setattr)
786#define __NR_sched_getattr 381
787__SYSCALL(__NR_sched_getattr, sys_sched_getattr)
788#define __NR_renameat2 382
789__SYSCALL(__NR_renameat2, sys_renameat2)
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index cdaedad3afe5..27c72ef4fd7a 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -15,7 +15,8 @@ CFLAGS_REMOVE_return_address.o = -pg
15arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ 15arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
16 entry-fpsimd.o process.o ptrace.o setup.o signal.o \ 16 entry-fpsimd.o process.o ptrace.o setup.o signal.o \
17 sys.o stacktrace.o time.o traps.o io.o vdso.o \ 17 sys.o stacktrace.o time.o traps.o io.o vdso.o \
18 hyp-stub.o psci.o cpu_ops.o insn.o return_address.o 18 hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \
19 cpuinfo.o
19 20
20arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ 21arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
21 sys_compat.o 22 sys_compat.o
diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c
index d62d12fb36c8..cce952440c64 100644
--- a/arch/arm64/kernel/cpu_ops.c
+++ b/arch/arm64/kernel/cpu_ops.c
@@ -30,8 +30,8 @@ const struct cpu_operations *cpu_ops[NR_CPUS];
30static const struct cpu_operations *supported_cpu_ops[] __initconst = { 30static const struct cpu_operations *supported_cpu_ops[] __initconst = {
31#ifdef CONFIG_SMP 31#ifdef CONFIG_SMP
32 &smp_spin_table_ops, 32 &smp_spin_table_ops,
33 &cpu_psci_ops,
34#endif 33#endif
34 &cpu_psci_ops,
35 NULL, 35 NULL,
36}; 36};
37 37
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
new file mode 100644
index 000000000000..f798f66634af
--- /dev/null
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -0,0 +1,192 @@
1/*
2 * Record and handle CPU attributes.
3 *
4 * Copyright (C) 2014 ARM Ltd.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17#include <asm/arch_timer.h>
18#include <asm/cachetype.h>
19#include <asm/cpu.h>
20#include <asm/cputype.h>
21
22#include <linux/bitops.h>
23#include <linux/init.h>
24#include <linux/kernel.h>
25#include <linux/printk.h>
26#include <linux/smp.h>
27
28/*
29 * In case the boot CPU is hotpluggable, we record its initial state and
30 * current state separately. Certain system registers may contain different
31 * values depending on configuration at or after reset.
32 */
33DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
34static struct cpuinfo_arm64 boot_cpu_data;
35
36static char *icache_policy_str[] = {
37 [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN",
38 [ICACHE_POLICY_AIVIVT] = "AIVIVT",
39 [ICACHE_POLICY_VIPT] = "VIPT",
40 [ICACHE_POLICY_PIPT] = "PIPT",
41};
42
43unsigned long __icache_flags;
44
45static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
46{
47 unsigned int cpu = smp_processor_id();
48 u32 l1ip = CTR_L1IP(info->reg_ctr);
49
50 if (l1ip != ICACHE_POLICY_PIPT)
51 set_bit(ICACHEF_ALIASING, &__icache_flags);
52 if (l1ip == ICACHE_POLICY_AIVIVT);
53 set_bit(ICACHEF_AIVIVT, &__icache_flags);
54
55 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
56}
57
58static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu)
59{
60 if ((boot & mask) == (cur & mask))
61 return 0;
62
63 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016lx, CPU%d: %#016lx\n",
64 name, (unsigned long)boot, cpu, (unsigned long)cur);
65
66 return 1;
67}
68
69#define CHECK_MASK(field, mask, boot, cur, cpu) \
70 check_reg_mask(#field, mask, (boot)->reg_ ## field, (cur)->reg_ ## field, cpu)
71
72#define CHECK(field, boot, cur, cpu) \
73 CHECK_MASK(field, ~0ULL, boot, cur, cpu)
74
75/*
76 * Verify that CPUs don't have unexpected differences that will cause problems.
77 */
78static void cpuinfo_sanity_check(struct cpuinfo_arm64 *cur)
79{
80 unsigned int cpu = smp_processor_id();
81 struct cpuinfo_arm64 *boot = &boot_cpu_data;
82 unsigned int diff = 0;
83
84 /*
85 * The kernel can handle differing I-cache policies, but otherwise
86 * caches should look identical. Userspace JITs will make use of
87 * *minLine.
88 */
89 diff |= CHECK_MASK(ctr, 0xffff3fff, boot, cur, cpu);
90
91 /*
92 * Userspace may perform DC ZVA instructions. Mismatched block sizes
93 * could result in too much or too little memory being zeroed if a
94 * process is preempted and migrated between CPUs.
95 */
96 diff |= CHECK(dczid, boot, cur, cpu);
97
98 /* If different, timekeeping will be broken (especially with KVM) */
99 diff |= CHECK(cntfrq, boot, cur, cpu);
100
101 /*
102 * Even in big.LITTLE, processors should be identical instruction-set
103 * wise.
104 */
105 diff |= CHECK(id_aa64isar0, boot, cur, cpu);
106 diff |= CHECK(id_aa64isar1, boot, cur, cpu);
107
108 /*
109 * Differing PARange support is fine as long as all peripherals and
110 * memory are mapped within the minimum PARange of all CPUs.
111 * Linux should not care about secure memory.
112 * ID_AA64MMFR1 is currently RES0.
113 */
114 diff |= CHECK_MASK(id_aa64mmfr0, 0xffffffffffff0ff0, boot, cur, cpu);
115 diff |= CHECK(id_aa64mmfr1, boot, cur, cpu);
116
117 /*
118 * EL3 is not our concern.
119 * ID_AA64PFR1 is currently RES0.
120 */
121 diff |= CHECK_MASK(id_aa64pfr0, 0xffffffffffff0fff, boot, cur, cpu);
122 diff |= CHECK(id_aa64pfr1, boot, cur, cpu);
123
124 /*
125 * If we have AArch32, we care about 32-bit features for compat. These
126 * registers should be RES0 otherwise.
127 */
128 diff |= CHECK(id_isar0, boot, cur, cpu);
129 diff |= CHECK(id_isar1, boot, cur, cpu);
130 diff |= CHECK(id_isar2, boot, cur, cpu);
131 diff |= CHECK(id_isar3, boot, cur, cpu);
132 diff |= CHECK(id_isar4, boot, cur, cpu);
133 diff |= CHECK(id_isar5, boot, cur, cpu);
134 diff |= CHECK(id_mmfr0, boot, cur, cpu);
135 diff |= CHECK(id_mmfr1, boot, cur, cpu);
136 diff |= CHECK(id_mmfr2, boot, cur, cpu);
137 diff |= CHECK(id_mmfr3, boot, cur, cpu);
138 diff |= CHECK(id_pfr0, boot, cur, cpu);
139 diff |= CHECK(id_pfr1, boot, cur, cpu);
140
141 /*
142 * Mismatched CPU features are a recipe for disaster. Don't even
143 * pretend to support them.
144 */
145 WARN_TAINT_ONCE(diff, TAINT_CPU_OUT_OF_SPEC,
146 "Unsupported CPU feature variation.");
147}
148
149static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
150{
151 info->reg_cntfrq = arch_timer_get_cntfrq();
152 info->reg_ctr = read_cpuid_cachetype();
153 info->reg_dczid = read_cpuid(DCZID_EL0);
154 info->reg_midr = read_cpuid_id();
155
156 info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1);
157 info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1);
158 info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
159 info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
160 info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1);
161 info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);
162
163 info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1);
164 info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1);
165 info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1);
166 info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1);
167 info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1);
168 info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1);
169 info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1);
170 info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1);
171 info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1);
172 info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1);
173 info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1);
174 info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1);
175
176 cpuinfo_detect_icache_policy(info);
177}
178
179void cpuinfo_store_cpu(void)
180{
181 struct cpuinfo_arm64 *info = this_cpu_ptr(&cpu_data);
182 __cpuinfo_store_cpu(info);
183 cpuinfo_sanity_check(info);
184}
185
186void __init cpuinfo_store_boot_cpu(void)
187{
188 struct cpuinfo_arm64 *info = &per_cpu(cpu_data, 0);
189 __cpuinfo_store_cpu(info);
190
191 boot_cpu_data = *info;
192}
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index a7fb874b595e..fe5b94078d82 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -315,20 +315,20 @@ static int brk_handler(unsigned long addr, unsigned int esr,
315{ 315{
316 siginfo_t info; 316 siginfo_t info;
317 317
318 if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED) 318 if (user_mode(regs)) {
319 return 0; 319 info = (siginfo_t) {
320 .si_signo = SIGTRAP,
321 .si_errno = 0,
322 .si_code = TRAP_BRKPT,
323 .si_addr = (void __user *)instruction_pointer(regs),
324 };
320 325
321 if (!user_mode(regs)) 326 force_sig_info(SIGTRAP, &info, current);
327 } else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) {
328 pr_warning("Unexpected kernel BRK exception at EL1\n");
322 return -EFAULT; 329 return -EFAULT;
330 }
323 331
324 info = (siginfo_t) {
325 .si_signo = SIGTRAP,
326 .si_errno = 0,
327 .si_code = TRAP_BRKPT,
328 .si_addr = (void __user *)instruction_pointer(regs),
329 };
330
331 force_sig_info(SIGTRAP, &info, current);
332 return 0; 332 return 0;
333} 333}
334 334
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index d358ccacfc00..c44a82f146b1 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -52,7 +52,7 @@ ENDPROC(fpsimd_load_state)
52ENTRY(fpsimd_save_partial_state) 52ENTRY(fpsimd_save_partial_state)
53 fpsimd_save_partial x0, 1, 8, 9 53 fpsimd_save_partial x0, 1, 8, 9
54 ret 54 ret
55ENDPROC(fpsimd_load_partial_state) 55ENDPROC(fpsimd_save_partial_state)
56 56
57/* 57/*
58 * Load the bottom n FP registers. 58 * Load the bottom n FP registers.
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 9ce04ba6bcb0..f0b5e5120a87 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -27,7 +27,32 @@
27#include <asm/esr.h> 27#include <asm/esr.h>
28#include <asm/thread_info.h> 28#include <asm/thread_info.h>
29#include <asm/unistd.h> 29#include <asm/unistd.h>
30#include <asm/unistd32.h> 30
31/*
32 * Context tracking subsystem. Used to instrument transitions
33 * between user and kernel mode.
34 */
35 .macro ct_user_exit, syscall = 0
36#ifdef CONFIG_CONTEXT_TRACKING
37 bl context_tracking_user_exit
38 .if \syscall == 1
39 /*
40 * Save/restore needed during syscalls. Restore syscall arguments from
41 * the values already saved on stack during kernel_entry.
42 */
43 ldp x0, x1, [sp]
44 ldp x2, x3, [sp, #S_X2]
45 ldp x4, x5, [sp, #S_X4]
46 ldp x6, x7, [sp, #S_X6]
47 .endif
48#endif
49 .endm
50
51 .macro ct_user_enter
52#ifdef CONFIG_CONTEXT_TRACKING
53 bl context_tracking_user_enter
54#endif
55 .endm
31 56
32/* 57/*
33 * Bad Abort numbers 58 * Bad Abort numbers
@@ -91,6 +116,7 @@
91 .macro kernel_exit, el, ret = 0 116 .macro kernel_exit, el, ret = 0
92 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR 117 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
93 .if \el == 0 118 .if \el == 0
119 ct_user_enter
94 ldr x23, [sp, #S_SP] // load return stack pointer 120 ldr x23, [sp, #S_SP] // load return stack pointer
95 .endif 121 .endif
96 .if \ret 122 .if \ret
@@ -353,7 +379,6 @@ el0_sync:
353 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class 379 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
354 cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state 380 cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state
355 b.eq el0_svc 381 b.eq el0_svc
356 adr lr, ret_to_user
357 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 382 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
358 b.eq el0_da 383 b.eq el0_da
359 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 384 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
@@ -382,7 +407,6 @@ el0_sync_compat:
382 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class 407 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
383 cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state 408 cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state
384 b.eq el0_svc_compat 409 b.eq el0_svc_compat
385 adr lr, ret_to_user
386 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 410 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
387 b.eq el0_da 411 b.eq el0_da
388 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 412 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
@@ -425,48 +449,59 @@ el0_da:
425 /* 449 /*
426 * Data abort handling 450 * Data abort handling
427 */ 451 */
428 mrs x0, far_el1 452 mrs x26, far_el1
429 bic x0, x0, #(0xff << 56)
430 // enable interrupts before calling the main handler 453 // enable interrupts before calling the main handler
431 enable_dbg_and_irq 454 enable_dbg_and_irq
455 ct_user_exit
456 bic x0, x26, #(0xff << 56)
432 mov x1, x25 457 mov x1, x25
433 mov x2, sp 458 mov x2, sp
459 adr lr, ret_to_user
434 b do_mem_abort 460 b do_mem_abort
435el0_ia: 461el0_ia:
436 /* 462 /*
437 * Instruction abort handling 463 * Instruction abort handling
438 */ 464 */
439 mrs x0, far_el1 465 mrs x26, far_el1
440 // enable interrupts before calling the main handler 466 // enable interrupts before calling the main handler
441 enable_dbg_and_irq 467 enable_dbg_and_irq
468 ct_user_exit
469 mov x0, x26
442 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts 470 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts
443 mov x2, sp 471 mov x2, sp
472 adr lr, ret_to_user
444 b do_mem_abort 473 b do_mem_abort
445el0_fpsimd_acc: 474el0_fpsimd_acc:
446 /* 475 /*
447 * Floating Point or Advanced SIMD access 476 * Floating Point or Advanced SIMD access
448 */ 477 */
449 enable_dbg 478 enable_dbg
479 ct_user_exit
450 mov x0, x25 480 mov x0, x25
451 mov x1, sp 481 mov x1, sp
482 adr lr, ret_to_user
452 b do_fpsimd_acc 483 b do_fpsimd_acc
453el0_fpsimd_exc: 484el0_fpsimd_exc:
454 /* 485 /*
455 * Floating Point or Advanced SIMD exception 486 * Floating Point or Advanced SIMD exception
456 */ 487 */
457 enable_dbg 488 enable_dbg
489 ct_user_exit
458 mov x0, x25 490 mov x0, x25
459 mov x1, sp 491 mov x1, sp
492 adr lr, ret_to_user
460 b do_fpsimd_exc 493 b do_fpsimd_exc
461el0_sp_pc: 494el0_sp_pc:
462 /* 495 /*
463 * Stack or PC alignment exception handling 496 * Stack or PC alignment exception handling
464 */ 497 */
465 mrs x0, far_el1 498 mrs x26, far_el1
466 // enable interrupts before calling the main handler 499 // enable interrupts before calling the main handler
467 enable_dbg_and_irq 500 enable_dbg_and_irq
501 mov x0, x26
468 mov x1, x25 502 mov x1, x25
469 mov x2, sp 503 mov x2, sp
504 adr lr, ret_to_user
470 b do_sp_pc_abort 505 b do_sp_pc_abort
471el0_undef: 506el0_undef:
472 /* 507 /*
@@ -474,7 +509,9 @@ el0_undef:
474 */ 509 */
475 // enable interrupts before calling the main handler 510 // enable interrupts before calling the main handler
476 enable_dbg_and_irq 511 enable_dbg_and_irq
512 ct_user_exit
477 mov x0, sp 513 mov x0, sp
514 adr lr, ret_to_user
478 b do_undefinstr 515 b do_undefinstr
479el0_dbg: 516el0_dbg:
480 /* 517 /*
@@ -486,12 +523,15 @@ el0_dbg:
486 mov x2, sp 523 mov x2, sp
487 bl do_debug_exception 524 bl do_debug_exception
488 enable_dbg 525 enable_dbg
526 ct_user_exit
489 b ret_to_user 527 b ret_to_user
490el0_inv: 528el0_inv:
491 enable_dbg 529 enable_dbg
530 ct_user_exit
492 mov x0, sp 531 mov x0, sp
493 mov x1, #BAD_SYNC 532 mov x1, #BAD_SYNC
494 mrs x2, esr_el1 533 mrs x2, esr_el1
534 adr lr, ret_to_user
495 b bad_mode 535 b bad_mode
496ENDPROC(el0_sync) 536ENDPROC(el0_sync)
497 537
@@ -504,6 +544,7 @@ el0_irq_naked:
504 bl trace_hardirqs_off 544 bl trace_hardirqs_off
505#endif 545#endif
506 546
547 ct_user_exit
507 irq_handler 548 irq_handler
508 549
509#ifdef CONFIG_TRACE_IRQFLAGS 550#ifdef CONFIG_TRACE_IRQFLAGS
@@ -608,6 +649,7 @@ el0_svc:
608el0_svc_naked: // compat entry point 649el0_svc_naked: // compat entry point
609 stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number 650 stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
610 enable_dbg_and_irq 651 enable_dbg_and_irq
652 ct_user_exit 1
611 653
612 ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks 654 ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks
613 tst x16, #_TIF_SYSCALL_WORK 655 tst x16, #_TIF_SYSCALL_WORK
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index a2c1195abb7f..144f10567f82 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -22,6 +22,7 @@
22 22
23#include <linux/linkage.h> 23#include <linux/linkage.h>
24#include <linux/init.h> 24#include <linux/init.h>
25#include <linux/irqchip/arm-gic-v3.h>
25 26
26#include <asm/assembler.h> 27#include <asm/assembler.h>
27#include <asm/ptrace.h> 28#include <asm/ptrace.h>
@@ -35,37 +36,31 @@
35#include <asm/page.h> 36#include <asm/page.h>
36#include <asm/virt.h> 37#include <asm/virt.h>
37 38
38/*
39 * swapper_pg_dir is the virtual address of the initial page table. We place
40 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has
41 * 2 pages and is placed below swapper_pg_dir.
42 */
43#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 39#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET)
44 40
45#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000 41#if (TEXT_OFFSET & 0xf) != 0
46#error KERNEL_RAM_VADDR must start at 0xXXX80000 42#error TEXT_OFFSET must be at least 16B aligned
43#elif (PAGE_OFFSET & 0xfffff) != 0
44#error PAGE_OFFSET must be at least 2MB aligned
45#elif TEXT_OFFSET > 0xfffff
46#error TEXT_OFFSET must be less than 2MB
47#endif 47#endif
48 48
49#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE) 49 .macro pgtbl, ttb0, ttb1, virt_to_phys
50#define IDMAP_DIR_SIZE (2 * PAGE_SIZE) 50 ldr \ttb1, =swapper_pg_dir
51 51 ldr \ttb0, =idmap_pg_dir
52 .globl swapper_pg_dir 52 add \ttb1, \ttb1, \virt_to_phys
53 .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE 53 add \ttb0, \ttb0, \virt_to_phys
54
55 .globl idmap_pg_dir
56 .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE
57
58 .macro pgtbl, ttb0, ttb1, phys
59 add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE
60 sub \ttb0, \ttb1, #IDMAP_DIR_SIZE
61 .endm 54 .endm
62 55
63#ifdef CONFIG_ARM64_64K_PAGES 56#ifdef CONFIG_ARM64_64K_PAGES
64#define BLOCK_SHIFT PAGE_SHIFT 57#define BLOCK_SHIFT PAGE_SHIFT
65#define BLOCK_SIZE PAGE_SIZE 58#define BLOCK_SIZE PAGE_SIZE
59#define TABLE_SHIFT PMD_SHIFT
66#else 60#else
67#define BLOCK_SHIFT SECTION_SHIFT 61#define BLOCK_SHIFT SECTION_SHIFT
68#define BLOCK_SIZE SECTION_SIZE 62#define BLOCK_SIZE SECTION_SIZE
63#define TABLE_SHIFT PUD_SHIFT
69#endif 64#endif
70 65
71#define KERNEL_START KERNEL_RAM_VADDR 66#define KERNEL_START KERNEL_RAM_VADDR
@@ -120,9 +115,9 @@ efi_head:
120 b stext // branch to kernel start, magic 115 b stext // branch to kernel start, magic
121 .long 0 // reserved 116 .long 0 // reserved
122#endif 117#endif
123 .quad TEXT_OFFSET // Image load offset from start of RAM 118 .quad _kernel_offset_le // Image load offset from start of RAM, little-endian
124 .quad 0 // reserved 119 .quad _kernel_size_le // Effective size of kernel image, little-endian
125 .quad 0 // reserved 120 .quad _kernel_flags_le // Informative flags, little-endian
126 .quad 0 // reserved 121 .quad 0 // reserved
127 .quad 0 // reserved 122 .quad 0 // reserved
128 .quad 0 // reserved 123 .quad 0 // reserved
@@ -295,6 +290,23 @@ CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1
295 msr cnthctl_el2, x0 290 msr cnthctl_el2, x0
296 msr cntvoff_el2, xzr // Clear virtual offset 291 msr cntvoff_el2, xzr // Clear virtual offset
297 292
293#ifdef CONFIG_ARM_GIC_V3
294 /* GICv3 system register access */
295 mrs x0, id_aa64pfr0_el1
296 ubfx x0, x0, #24, #4
297 cmp x0, #1
298 b.ne 3f
299
300 mrs_s x0, ICC_SRE_EL2
301 orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1
302 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1
303 msr_s ICC_SRE_EL2, x0
304 isb // Make sure SRE is now set
305 msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
306
3073:
308#endif
309
298 /* Populate ID registers. */ 310 /* Populate ID registers. */
299 mrs x0, midr_el1 311 mrs x0, midr_el1
300 mrs x1, mpidr_el1 312 mrs x1, mpidr_el1
@@ -413,7 +425,7 @@ ENTRY(secondary_startup)
413 mov x23, x0 // x23=current cpu_table 425 mov x23, x0 // x23=current cpu_table
414 cbz x23, __error_p // invalid processor (x23=0)? 426 cbz x23, __error_p // invalid processor (x23=0)?
415 427
416 pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1 428 pgtbl x25, x26, x28 // x25=TTBR0, x26=TTBR1
417 ldr x12, [x23, #CPU_INFO_SETUP] 429 ldr x12, [x23, #CPU_INFO_SETUP]
418 add x12, x12, x28 // __virt_to_phys 430 add x12, x12, x28 // __virt_to_phys
419 blr x12 // initialise processor 431 blr x12 // initialise processor
@@ -455,8 +467,13 @@ ENDPROC(__enable_mmu)
455 * x27 = *virtual* address to jump to upon completion 467 * x27 = *virtual* address to jump to upon completion
456 * 468 *
457 * other registers depend on the function called upon completion 469 * other registers depend on the function called upon completion
470 *
471 * We align the entire function to the smallest power of two larger than it to
472 * ensure it fits within a single block map entry. Otherwise were PHYS_OFFSET
473 * close to the end of a 512MB or 1GB block we might require an additional
474 * table to map the entire function.
458 */ 475 */
459 .align 6 476 .align 4
460__turn_mmu_on: 477__turn_mmu_on:
461 msr sctlr_el1, x0 478 msr sctlr_el1, x0
462 isb 479 isb
@@ -479,17 +496,38 @@ ENDPROC(__calc_phys_offset)
479 .quad PAGE_OFFSET 496 .quad PAGE_OFFSET
480 497
481/* 498/*
482 * Macro to populate the PGD for the corresponding block entry in the next 499 * Macro to create a table entry to the next page.
483 * level (tbl) for the given virtual address. 500 *
501 * tbl: page table address
502 * virt: virtual address
503 * shift: #imm page table shift
504 * ptrs: #imm pointers per table page
505 *
506 * Preserves: virt
507 * Corrupts: tmp1, tmp2
508 * Returns: tbl -> next level table page address
509 */
510 .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
511 lsr \tmp1, \virt, #\shift
512 and \tmp1, \tmp1, #\ptrs - 1 // table index
513 add \tmp2, \tbl, #PAGE_SIZE
514 orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type
515 str \tmp2, [\tbl, \tmp1, lsl #3]
516 add \tbl, \tbl, #PAGE_SIZE // next level table page
517 .endm
518
519/*
520 * Macro to populate the PGD (and possibily PUD) for the corresponding
521 * block entry in the next level (tbl) for the given virtual address.
484 * 522 *
485 * Preserves: pgd, tbl, virt 523 * Preserves: tbl, next, virt
486 * Corrupts: tmp1, tmp2 524 * Corrupts: tmp1, tmp2
487 */ 525 */
488 .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2 526 .macro create_pgd_entry, tbl, virt, tmp1, tmp2
489 lsr \tmp1, \virt, #PGDIR_SHIFT 527 create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
490 and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index 528#if SWAPPER_PGTABLE_LEVELS == 3
491 orr \tmp2, \tbl, #3 // PGD entry table type 529 create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
492 str \tmp2, [\pgd, \tmp1, lsl #3] 530#endif
493 .endm 531 .endm
494 532
495/* 533/*
@@ -522,7 +560,7 @@ ENDPROC(__calc_phys_offset)
522 * - pgd entry for fixed mappings (TTBR1) 560 * - pgd entry for fixed mappings (TTBR1)
523 */ 561 */
524__create_page_tables: 562__create_page_tables:
525 pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses 563 pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses
526 mov x27, lr 564 mov x27, lr
527 565
528 /* 566 /*
@@ -550,10 +588,10 @@ __create_page_tables:
550 /* 588 /*
551 * Create the identity mapping. 589 * Create the identity mapping.
552 */ 590 */
553 add x0, x25, #PAGE_SIZE // section table address 591 mov x0, x25 // idmap_pg_dir
554 ldr x3, =KERNEL_START 592 ldr x3, =KERNEL_START
555 add x3, x3, x28 // __pa(KERNEL_START) 593 add x3, x3, x28 // __pa(KERNEL_START)
556 create_pgd_entry x25, x0, x3, x5, x6 594 create_pgd_entry x0, x3, x5, x6
557 ldr x6, =KERNEL_END 595 ldr x6, =KERNEL_END
558 mov x5, x3 // __pa(KERNEL_START) 596 mov x5, x3 // __pa(KERNEL_START)
559 add x6, x6, x28 // __pa(KERNEL_END) 597 add x6, x6, x28 // __pa(KERNEL_END)
@@ -562,9 +600,9 @@ __create_page_tables:
562 /* 600 /*
563 * Map the kernel image (starting with PHYS_OFFSET). 601 * Map the kernel image (starting with PHYS_OFFSET).
564 */ 602 */
565 add x0, x26, #PAGE_SIZE // section table address 603 mov x0, x26 // swapper_pg_dir
566 mov x5, #PAGE_OFFSET 604 mov x5, #PAGE_OFFSET
567 create_pgd_entry x26, x0, x5, x3, x6 605 create_pgd_entry x0, x5, x3, x6
568 ldr x6, =KERNEL_END 606 ldr x6, =KERNEL_END
569 mov x3, x24 // phys offset 607 mov x3, x24 // phys offset
570 create_block_map x0, x7, x3, x5, x6 608 create_block_map x0, x7, x3, x5, x6
@@ -586,13 +624,6 @@ __create_page_tables:
586 create_block_map x0, x7, x3, x5, x6 624 create_block_map x0, x7, x3, x5, x6
5871: 6251:
588 /* 626 /*
589 * Create the pgd entry for the fixed mappings.
590 */
591 ldr x5, =FIXADDR_TOP // Fixed mapping virtual address
592 add x0, x26, #2 * PAGE_SIZE // section table address
593 create_pgd_entry x26, x0, x5, x6, x7
594
595 /*
596 * Since the page tables have been populated with non-cacheable 627 * Since the page tables have been populated with non-cacheable
597 * accesses (MMU disabled), invalidate the idmap and swapper page 628 * accesses (MMU disabled), invalidate the idmap and swapper page
598 * tables again to remove any speculatively loaded cache lines. 629 * tables again to remove any speculatively loaded cache lines.
@@ -611,7 +642,7 @@ ENDPROC(__create_page_tables)
611__switch_data: 642__switch_data:
612 .quad __mmap_switched 643 .quad __mmap_switched
613 .quad __bss_start // x6 644 .quad __bss_start // x6
614 .quad _end // x7 645 .quad __bss_stop // x7
615 .quad processor_id // x4 646 .quad processor_id // x4
616 .quad __fdt_pointer // x5 647 .quad __fdt_pointer // x5
617 .quad memstart_addr // x6 648 .quad memstart_addr // x6
diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S
index 0959611d9ff1..a272f335c289 100644
--- a/arch/arm64/kernel/hyp-stub.S
+++ b/arch/arm64/kernel/hyp-stub.S
@@ -19,6 +19,7 @@
19 19
20#include <linux/init.h> 20#include <linux/init.h>
21#include <linux/linkage.h> 21#include <linux/linkage.h>
22#include <linux/irqchip/arm-gic-v3.h>
22 23
23#include <asm/assembler.h> 24#include <asm/assembler.h>
24#include <asm/ptrace.h> 25#include <asm/ptrace.h>
diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
new file mode 100644
index 000000000000..8fae0756e175
--- /dev/null
+++ b/arch/arm64/kernel/image.h
@@ -0,0 +1,62 @@
1/*
2 * Linker script macros to generate Image header fields.
3 *
4 * Copyright (C) 2014 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18#ifndef __ASM_IMAGE_H
19#define __ASM_IMAGE_H
20
21#ifndef LINKER_SCRIPT
22#error This file should only be included in vmlinux.lds.S
23#endif
24
25/*
26 * There aren't any ELF relocations we can use to endian-swap values known only
27 * at link time (e.g. the subtraction of two symbol addresses), so we must get
28 * the linker to endian-swap certain values before emitting them.
29 */
30#ifdef CONFIG_CPU_BIG_ENDIAN
31#define DATA_LE64(data) \
32 ((((data) & 0x00000000000000ff) << 56) | \
33 (((data) & 0x000000000000ff00) << 40) | \
34 (((data) & 0x0000000000ff0000) << 24) | \
35 (((data) & 0x00000000ff000000) << 8) | \
36 (((data) & 0x000000ff00000000) >> 8) | \
37 (((data) & 0x0000ff0000000000) >> 24) | \
38 (((data) & 0x00ff000000000000) >> 40) | \
39 (((data) & 0xff00000000000000) >> 56))
40#else
41#define DATA_LE64(data) ((data) & 0xffffffffffffffff)
42#endif
43
44#ifdef CONFIG_CPU_BIG_ENDIAN
45#define __HEAD_FLAG_BE 1
46#else
47#define __HEAD_FLAG_BE 0
48#endif
49
50#define __HEAD_FLAGS (__HEAD_FLAG_BE << 0)
51
52/*
53 * These will output as part of the Image header, which should be little-endian
54 * regardless of the endianness of the kernel. While constant values could be
55 * endian swapped in head.S, all are done here for consistency.
56 */
57#define HEAD_SYMBOLS \
58 _kernel_size_le = DATA_LE64(_end - _text); \
59 _kernel_offset_le = DATA_LE64(TEXT_OFFSET); \
60 _kernel_flags_le = DATA_LE64(__HEAD_FLAGS);
61
62#endif /* __ASM_IMAGE_H */
diff --git a/arch/arm64/kernel/kuser32.S b/arch/arm64/kernel/kuser32.S
index 7787208e8cc6..997e6b27ff6a 100644
--- a/arch/arm64/kernel/kuser32.S
+++ b/arch/arm64/kernel/kuser32.S
@@ -28,7 +28,7 @@
28 * See Documentation/arm/kernel_user_helpers.txt for formal definitions. 28 * See Documentation/arm/kernel_user_helpers.txt for formal definitions.
29 */ 29 */
30 30
31#include <asm/unistd32.h> 31#include <asm/unistd.h>
32 32
33 .align 5 33 .align 5
34 .globl __kuser_helper_start 34 .globl __kuser_helper_start
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 43b7c34f92cb..1309d64aa926 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -51,6 +51,12 @@
51#include <asm/processor.h> 51#include <asm/processor.h>
52#include <asm/stacktrace.h> 52#include <asm/stacktrace.h>
53 53
54#ifdef CONFIG_CC_STACKPROTECTOR
55#include <linux/stackprotector.h>
56unsigned long __stack_chk_guard __read_mostly;
57EXPORT_SYMBOL(__stack_chk_guard);
58#endif
59
54static void setup_restart(void) 60static void setup_restart(void)
55{ 61{
56 /* 62 /*
diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c
index 9e9798f91172..553954771a67 100644
--- a/arch/arm64/kernel/psci.c
+++ b/arch/arm64/kernel/psci.c
@@ -235,7 +235,7 @@ static void psci_sys_poweroff(void)
235 * PSCI Function IDs for v0.2+ are well defined so use 235 * PSCI Function IDs for v0.2+ are well defined so use
236 * standard values. 236 * standard values.
237 */ 237 */
238static int psci_0_2_init(struct device_node *np) 238static int __init psci_0_2_init(struct device_node *np)
239{ 239{
240 int err, ver; 240 int err, ver;
241 241
@@ -296,7 +296,7 @@ out_put_node:
296/* 296/*
297 * PSCI < v0.2 get PSCI Function IDs via DT. 297 * PSCI < v0.2 get PSCI Function IDs via DT.
298 */ 298 */
299static int psci_0_1_init(struct device_node *np) 299static int __init psci_0_1_init(struct device_node *np)
300{ 300{
301 u32 id; 301 u32 id;
302 int err; 302 int err;
@@ -434,9 +434,11 @@ static int cpu_psci_cpu_kill(unsigned int cpu)
434 return 0; 434 return 0;
435} 435}
436#endif 436#endif
437#endif
437 438
438const struct cpu_operations cpu_psci_ops = { 439const struct cpu_operations cpu_psci_ops = {
439 .name = "psci", 440 .name = "psci",
441#ifdef CONFIG_SMP
440 .cpu_init = cpu_psci_cpu_init, 442 .cpu_init = cpu_psci_cpu_init,
441 .cpu_prepare = cpu_psci_cpu_prepare, 443 .cpu_prepare = cpu_psci_cpu_prepare,
442 .cpu_boot = cpu_psci_cpu_boot, 444 .cpu_boot = cpu_psci_cpu_boot,
@@ -445,6 +447,6 @@ const struct cpu_operations cpu_psci_ops = {
445 .cpu_die = cpu_psci_cpu_die, 447 .cpu_die = cpu_psci_cpu_die,
446 .cpu_kill = cpu_psci_cpu_kill, 448 .cpu_kill = cpu_psci_cpu_kill,
447#endif 449#endif
450#endif
448}; 451};
449 452
450#endif
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 9fde010c945f..0310811bd77d 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -19,6 +19,7 @@
19 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 */ 20 */
21 21
22#include <linux/audit.h>
22#include <linux/compat.h> 23#include <linux/compat.h>
23#include <linux/kernel.h> 24#include <linux/kernel.h>
24#include <linux/sched.h> 25#include <linux/sched.h>
@@ -39,6 +40,7 @@
39#include <asm/compat.h> 40#include <asm/compat.h>
40#include <asm/debug-monitors.h> 41#include <asm/debug-monitors.h>
41#include <asm/pgtable.h> 42#include <asm/pgtable.h>
43#include <asm/syscall.h>
42#include <asm/traps.h> 44#include <asm/traps.h>
43#include <asm/system_misc.h> 45#include <asm/system_misc.h>
44 46
@@ -1113,11 +1115,20 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs)
1113 if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) 1115 if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
1114 trace_sys_enter(regs, regs->syscallno); 1116 trace_sys_enter(regs, regs->syscallno);
1115 1117
1118#ifdef CONFIG_AUDITSYSCALL
1119 audit_syscall_entry(syscall_get_arch(), regs->syscallno,
1120 regs->orig_x0, regs->regs[1], regs->regs[2], regs->regs[3]);
1121#endif
1122
1116 return regs->syscallno; 1123 return regs->syscallno;
1117} 1124}
1118 1125
1119asmlinkage void syscall_trace_exit(struct pt_regs *regs) 1126asmlinkage void syscall_trace_exit(struct pt_regs *regs)
1120{ 1127{
1128#ifdef CONFIG_AUDITSYSCALL
1129 audit_syscall_exit(regs);
1130#endif
1131
1121 if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) 1132 if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
1122 trace_sys_exit(regs, regs_return_value(regs)); 1133 trace_sys_exit(regs, regs_return_value(regs));
1123 1134
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 46d1125571f6..f6f0ccf35ae6 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -45,6 +45,7 @@
45#include <linux/efi.h> 45#include <linux/efi.h>
46 46
47#include <asm/fixmap.h> 47#include <asm/fixmap.h>
48#include <asm/cpu.h>
48#include <asm/cputype.h> 49#include <asm/cputype.h>
49#include <asm/elf.h> 50#include <asm/elf.h>
50#include <asm/cputable.h> 51#include <asm/cputable.h>
@@ -77,7 +78,6 @@ unsigned int compat_elf_hwcap2 __read_mostly;
77#endif 78#endif
78 79
79static const char *cpu_name; 80static const char *cpu_name;
80static const char *machine_name;
81phys_addr_t __fdt_pointer __initdata; 81phys_addr_t __fdt_pointer __initdata;
82 82
83/* 83/*
@@ -219,6 +219,8 @@ static void __init setup_processor(void)
219 sprintf(init_utsname()->machine, ELF_PLATFORM); 219 sprintf(init_utsname()->machine, ELF_PLATFORM);
220 elf_hwcap = 0; 220 elf_hwcap = 0;
221 221
222 cpuinfo_store_boot_cpu();
223
222 /* 224 /*
223 * Check for sane CTR_EL0.CWG value. 225 * Check for sane CTR_EL0.CWG value.
224 */ 226 */
@@ -307,8 +309,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys)
307 while (true) 309 while (true)
308 cpu_relax(); 310 cpu_relax();
309 } 311 }
310
311 machine_name = of_flat_dt_get_machine_name();
312} 312}
313 313
314/* 314/*
@@ -417,14 +417,12 @@ static int __init arm64_device_init(void)
417} 417}
418arch_initcall_sync(arm64_device_init); 418arch_initcall_sync(arm64_device_init);
419 419
420static DEFINE_PER_CPU(struct cpu, cpu_data);
421
422static int __init topology_init(void) 420static int __init topology_init(void)
423{ 421{
424 int i; 422 int i;
425 423
426 for_each_possible_cpu(i) { 424 for_each_possible_cpu(i) {
427 struct cpu *cpu = &per_cpu(cpu_data, i); 425 struct cpu *cpu = &per_cpu(cpu_data.cpu, i);
428 cpu->hotpluggable = 1; 426 cpu->hotpluggable = 1;
429 register_cpu(cpu, i); 427 register_cpu(cpu, i);
430 } 428 }
@@ -449,10 +447,21 @@ static int c_show(struct seq_file *m, void *v)
449{ 447{
450 int i; 448 int i;
451 449
452 seq_printf(m, "Processor\t: %s rev %d (%s)\n", 450 /*
453 cpu_name, read_cpuid_id() & 15, ELF_PLATFORM); 451 * Dump out the common processor features in a single line. Userspace
452 * should read the hwcaps with getauxval(AT_HWCAP) rather than
453 * attempting to parse this.
454 */
455 seq_puts(m, "features\t:");
456 for (i = 0; hwcap_str[i]; i++)
457 if (elf_hwcap & (1 << i))
458 seq_printf(m, " %s", hwcap_str[i]);
459 seq_puts(m, "\n\n");
454 460
455 for_each_online_cpu(i) { 461 for_each_online_cpu(i) {
462 struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i);
463 u32 midr = cpuinfo->reg_midr;
464
456 /* 465 /*
457 * glibc reads /proc/cpuinfo to determine the number of 466 * glibc reads /proc/cpuinfo to determine the number of
458 * online processors, looking for lines beginning with 467 * online processors, looking for lines beginning with
@@ -461,25 +470,13 @@ static int c_show(struct seq_file *m, void *v)
461#ifdef CONFIG_SMP 470#ifdef CONFIG_SMP
462 seq_printf(m, "processor\t: %d\n", i); 471 seq_printf(m, "processor\t: %d\n", i);
463#endif 472#endif
473 seq_printf(m, "implementer\t: 0x%02x\n",
474 MIDR_IMPLEMENTOR(midr));
475 seq_printf(m, "variant\t\t: 0x%x\n", MIDR_VARIANT(midr));
476 seq_printf(m, "partnum\t\t: 0x%03x\n", MIDR_PARTNUM(midr));
477 seq_printf(m, "revision\t: 0x%x\n\n", MIDR_REVISION(midr));
464 } 478 }
465 479
466 /* dump out the processor features */
467 seq_puts(m, "Features\t: ");
468
469 for (i = 0; hwcap_str[i]; i++)
470 if (elf_hwcap & (1 << i))
471 seq_printf(m, "%s ", hwcap_str[i]);
472
473 seq_printf(m, "\nCPU implementer\t: 0x%02x\n", read_cpuid_id() >> 24);
474 seq_printf(m, "CPU architecture: AArch64\n");
475 seq_printf(m, "CPU variant\t: 0x%x\n", (read_cpuid_id() >> 20) & 15);
476 seq_printf(m, "CPU part\t: 0x%03x\n", (read_cpuid_id() >> 4) & 0xfff);
477 seq_printf(m, "CPU revision\t: %d\n", read_cpuid_id() & 15);
478
479 seq_puts(m, "\n");
480
481 seq_printf(m, "Hardware\t: %s\n", machine_name);
482
483 return 0; 480 return 0;
484} 481}
485 482
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 3491c638f172..c5ee208321c3 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -27,7 +27,7 @@
27#include <asm/fpsimd.h> 27#include <asm/fpsimd.h>
28#include <asm/signal32.h> 28#include <asm/signal32.h>
29#include <asm/uaccess.h> 29#include <asm/uaccess.h>
30#include <asm/unistd32.h> 30#include <asm/unistd.h>
31 31
32struct compat_sigcontext { 32struct compat_sigcontext {
33 /* We always set these two fields to 0 */ 33 /* We always set these two fields to 0 */
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 40f38f46c8e0..3e2f5ebbf63e 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -39,6 +39,7 @@
39 39
40#include <asm/atomic.h> 40#include <asm/atomic.h>
41#include <asm/cacheflush.h> 41#include <asm/cacheflush.h>
42#include <asm/cpu.h>
42#include <asm/cputype.h> 43#include <asm/cputype.h>
43#include <asm/cpu_ops.h> 44#include <asm/cpu_ops.h>
44#include <asm/mmu_context.h> 45#include <asm/mmu_context.h>
@@ -155,6 +156,11 @@ asmlinkage void secondary_start_kernel(void)
155 cpu_ops[cpu]->cpu_postboot(); 156 cpu_ops[cpu]->cpu_postboot();
156 157
157 /* 158 /*
159 * Log the CPU info before it is marked online and might get read.
160 */
161 cpuinfo_store_cpu();
162
163 /*
158 * Enable GIC and timers. 164 * Enable GIC and timers.
159 */ 165 */
160 notify_cpu_starting(cpu); 166 notify_cpu_starting(cpu);
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 1fa9ce4afd8f..55a99b9a97e0 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -119,7 +119,7 @@ int cpu_suspend(unsigned long arg)
119extern struct sleep_save_sp sleep_save_sp; 119extern struct sleep_save_sp sleep_save_sp;
120extern phys_addr_t sleep_idmap_phys; 120extern phys_addr_t sleep_idmap_phys;
121 121
122static int cpu_suspend_init(void) 122static int __init cpu_suspend_init(void)
123{ 123{
124 void *ctx_ptr; 124 void *ctx_ptr;
125 125
diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
index 26e9c4eeaba8..de2b0226e06d 100644
--- a/arch/arm64/kernel/sys_compat.c
+++ b/arch/arm64/kernel/sys_compat.c
@@ -26,7 +26,7 @@
26#include <linux/uaccess.h> 26#include <linux/uaccess.h>
27 27
28#include <asm/cacheflush.h> 28#include <asm/cacheflush.h>
29#include <asm/unistd32.h> 29#include <asm/unistd.h>
30 30
31static inline void 31static inline void
32do_compat_cache_op(unsigned long start, unsigned long end, int flags) 32do_compat_cache_op(unsigned long start, unsigned long end, int flags)
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 43514f905916..b6ee26b0939a 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -20,6 +20,7 @@
20#include <linux/of.h> 20#include <linux/of.h>
21#include <linux/sched.h> 21#include <linux/sched.h>
22 22
23#include <asm/cputype.h>
23#include <asm/topology.h> 24#include <asm/topology.h>
24 25
25static int __init get_cpu_for_node(struct device_node *node) 26static int __init get_cpu_for_node(struct device_node *node)
@@ -188,13 +189,9 @@ static int __init parse_dt_topology(void)
188 * Check that all cores are in the topology; the SMP code will 189 * Check that all cores are in the topology; the SMP code will
189 * only mark cores described in the DT as possible. 190 * only mark cores described in the DT as possible.
190 */ 191 */
191 for_each_possible_cpu(cpu) { 192 for_each_possible_cpu(cpu)
192 if (cpu_topology[cpu].cluster_id == -1) { 193 if (cpu_topology[cpu].cluster_id == -1)
193 pr_err("CPU%d: No topology information specified\n",
194 cpu);
195 ret = -EINVAL; 194 ret = -EINVAL;
196 }
197 }
198 195
199out_map: 196out_map:
200 of_node_put(map); 197 of_node_put(map);
@@ -219,14 +216,6 @@ static void update_siblings_masks(unsigned int cpuid)
219 struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; 216 struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
220 int cpu; 217 int cpu;
221 218
222 if (cpuid_topo->cluster_id == -1) {
223 /*
224 * DT does not contain topology information for this cpu.
225 */
226 pr_debug("CPU%u: No topology information configured\n", cpuid);
227 return;
228 }
229
230 /* update core and thread sibling masks */ 219 /* update core and thread sibling masks */
231 for_each_possible_cpu(cpu) { 220 for_each_possible_cpu(cpu) {
232 cpu_topo = &cpu_topology[cpu]; 221 cpu_topo = &cpu_topology[cpu];
@@ -249,6 +238,36 @@ static void update_siblings_masks(unsigned int cpuid)
249 238
250void store_cpu_topology(unsigned int cpuid) 239void store_cpu_topology(unsigned int cpuid)
251{ 240{
241 struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
242 u64 mpidr;
243
244 if (cpuid_topo->cluster_id != -1)
245 goto topology_populated;
246
247 mpidr = read_cpuid_mpidr();
248
249 /* Uniprocessor systems can rely on default topology values */
250 if (mpidr & MPIDR_UP_BITMASK)
251 return;
252
253 /* Create cpu topology mapping based on MPIDR. */
254 if (mpidr & MPIDR_MT_BITMASK) {
255 /* Multiprocessor system : Multi-threads per core */
256 cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
257 cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
258 cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
259 } else {
260 /* Multiprocessor system : Single-thread per core */
261 cpuid_topo->thread_id = -1;
262 cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
263 cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
264 }
265
266 pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
267 cpuid, cpuid_topo->cluster_id, cpuid_topo->core_id,
268 cpuid_topo->thread_id, mpidr);
269
270topology_populated:
252 update_siblings_masks(cpuid); 271 update_siblings_masks(cpuid);
253} 272}
254 273
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index c43cfa9b8304..02cd3f023e9a 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -156,7 +156,7 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
156 frame.pc = thread_saved_pc(tsk); 156 frame.pc = thread_saved_pc(tsk);
157 } 157 }
158 158
159 printk("Call trace:\n"); 159 pr_emerg("Call trace:\n");
160 while (1) { 160 while (1) {
161 unsigned long where = frame.pc; 161 unsigned long where = frame.pc;
162 int ret; 162 int ret;
@@ -331,17 +331,22 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
331 331
332void __pte_error(const char *file, int line, unsigned long val) 332void __pte_error(const char *file, int line, unsigned long val)
333{ 333{
334 printk("%s:%d: bad pte %016lx.\n", file, line, val); 334 pr_crit("%s:%d: bad pte %016lx.\n", file, line, val);
335} 335}
336 336
337void __pmd_error(const char *file, int line, unsigned long val) 337void __pmd_error(const char *file, int line, unsigned long val)
338{ 338{
339 printk("%s:%d: bad pmd %016lx.\n", file, line, val); 339 pr_crit("%s:%d: bad pmd %016lx.\n", file, line, val);
340}
341
342void __pud_error(const char *file, int line, unsigned long val)
343{
344 pr_crit("%s:%d: bad pud %016lx.\n", file, line, val);
340} 345}
341 346
342void __pgd_error(const char *file, int line, unsigned long val) 347void __pgd_error(const char *file, int line, unsigned long val)
343{ 348{
344 printk("%s:%d: bad pgd %016lx.\n", file, line, val); 349 pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val);
345} 350}
346 351
347void __init trap_init(void) 352void __init trap_init(void)
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 50384fec56c4..24f2e8c62479 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -88,22 +88,29 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp)
88{ 88{
89 struct mm_struct *mm = current->mm; 89 struct mm_struct *mm = current->mm;
90 unsigned long addr = AARCH32_VECTORS_BASE; 90 unsigned long addr = AARCH32_VECTORS_BASE;
91 int ret; 91 static struct vm_special_mapping spec = {
92 .name = "[vectors]",
93 .pages = vectors_page,
94
95 };
96 void *ret;
92 97
93 down_write(&mm->mmap_sem); 98 down_write(&mm->mmap_sem);
94 current->mm->context.vdso = (void *)addr; 99 current->mm->context.vdso = (void *)addr;
95 100
96 /* Map vectors page at the high address. */ 101 /* Map vectors page at the high address. */
97 ret = install_special_mapping(mm, addr, PAGE_SIZE, 102 ret = _install_special_mapping(mm, addr, PAGE_SIZE,
98 VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC, 103 VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC,
99 vectors_page); 104 &spec);
100 105
101 up_write(&mm->mmap_sem); 106 up_write(&mm->mmap_sem);
102 107
103 return ret; 108 return PTR_ERR_OR_ZERO(ret);
104} 109}
105#endif /* CONFIG_COMPAT */ 110#endif /* CONFIG_COMPAT */
106 111
112static struct vm_special_mapping vdso_spec[2];
113
107static int __init vdso_init(void) 114static int __init vdso_init(void)
108{ 115{
109 int i; 116 int i;
@@ -114,8 +121,8 @@ static int __init vdso_init(void)
114 } 121 }
115 122
116 vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT; 123 vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT;
117 pr_info("vdso: %ld pages (%ld code, %ld data) at base %p\n", 124 pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
118 vdso_pages + 1, vdso_pages, 1L, &vdso_start); 125 vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data);
119 126
120 /* Allocate the vDSO pagelist, plus a page for the data. */ 127 /* Allocate the vDSO pagelist, plus a page for the data. */
121 vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), 128 vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *),
@@ -123,12 +130,23 @@ static int __init vdso_init(void)
123 if (vdso_pagelist == NULL) 130 if (vdso_pagelist == NULL)
124 return -ENOMEM; 131 return -ENOMEM;
125 132
133 /* Grab the vDSO data page. */
134 vdso_pagelist[0] = virt_to_page(vdso_data);
135
126 /* Grab the vDSO code pages. */ 136 /* Grab the vDSO code pages. */
127 for (i = 0; i < vdso_pages; i++) 137 for (i = 0; i < vdso_pages; i++)
128 vdso_pagelist[i] = virt_to_page(&vdso_start + i * PAGE_SIZE); 138 vdso_pagelist[i + 1] = virt_to_page(&vdso_start + i * PAGE_SIZE);
129 139
130 /* Grab the vDSO data page. */ 140 /* Populate the special mapping structures */
131 vdso_pagelist[i] = virt_to_page(vdso_data); 141 vdso_spec[0] = (struct vm_special_mapping) {
142 .name = "[vvar]",
143 .pages = vdso_pagelist,
144 };
145
146 vdso_spec[1] = (struct vm_special_mapping) {
147 .name = "[vdso]",
148 .pages = &vdso_pagelist[1],
149 };
132 150
133 return 0; 151 return 0;
134} 152}
@@ -138,52 +156,42 @@ int arch_setup_additional_pages(struct linux_binprm *bprm,
138 int uses_interp) 156 int uses_interp)
139{ 157{
140 struct mm_struct *mm = current->mm; 158 struct mm_struct *mm = current->mm;
141 unsigned long vdso_base, vdso_mapping_len; 159 unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
142 int ret; 160 void *ret;
143 161
162 vdso_text_len = vdso_pages << PAGE_SHIFT;
144 /* Be sure to map the data page */ 163 /* Be sure to map the data page */
145 vdso_mapping_len = (vdso_pages + 1) << PAGE_SHIFT; 164 vdso_mapping_len = vdso_text_len + PAGE_SIZE;
146 165
147 down_write(&mm->mmap_sem); 166 down_write(&mm->mmap_sem);
148 vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); 167 vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
149 if (IS_ERR_VALUE(vdso_base)) { 168 if (IS_ERR_VALUE(vdso_base)) {
150 ret = vdso_base; 169 ret = ERR_PTR(vdso_base);
151 goto up_fail; 170 goto up_fail;
152 } 171 }
153 mm->context.vdso = (void *)vdso_base; 172 ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE,
154 173 VM_READ|VM_MAYREAD,
155 ret = install_special_mapping(mm, vdso_base, vdso_mapping_len, 174 &vdso_spec[0]);
156 VM_READ|VM_EXEC| 175 if (IS_ERR(ret))
157 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
158 vdso_pagelist);
159 if (ret) {
160 mm->context.vdso = NULL;
161 goto up_fail; 176 goto up_fail;
162 }
163 177
164up_fail: 178 vdso_base += PAGE_SIZE;
165 up_write(&mm->mmap_sem); 179 mm->context.vdso = (void *)vdso_base;
180 ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
181 VM_READ|VM_EXEC|
182 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
183 &vdso_spec[1]);
184 if (IS_ERR(ret))
185 goto up_fail;
166 186
167 return ret;
168}
169 187
170const char *arch_vma_name(struct vm_area_struct *vma) 188 up_write(&mm->mmap_sem);
171{ 189 return 0;
172 /*
173 * We can re-use the vdso pointer in mm_context_t for identifying
174 * the vectors page for compat applications. The vDSO will always
175 * sit above TASK_UNMAPPED_BASE and so we don't need to worry about
176 * it conflicting with the vectors base.
177 */
178 if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso) {
179#ifdef CONFIG_COMPAT
180 if (vma->vm_start == AARCH32_VECTORS_BASE)
181 return "[vectors]";
182#endif
183 return "[vdso]";
184 }
185 190
186 return NULL; 191up_fail:
192 mm->context.vdso = NULL;
193 up_write(&mm->mmap_sem);
194 return PTR_ERR(ret);
187} 195}
188 196
189/* 197/*
diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index 6d20b7d162d8..ff3bddea482d 100644
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -43,13 +43,13 @@ $(obj)/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE
43 $(call if_changed,vdsosym) 43 $(call if_changed,vdsosym)
44 44
45# Assembly rules for the .S files 45# Assembly rules for the .S files
46$(obj-vdso): %.o: %.S 46$(obj-vdso): %.o: %.S FORCE
47 $(call if_changed_dep,vdsoas) 47 $(call if_changed_dep,vdsoas)
48 48
49# Actual build commands 49# Actual build commands
50quiet_cmd_vdsold = VDSOL $@ 50quiet_cmd_vdsold = VDSOL $@
51 cmd_vdsold = $(CC) $(c_flags) -Wl,-n -Wl,-T $^ -o $@ 51 cmd_vdsold = $(CC) $(c_flags) -Wl,-n -Wl,-T $^ -o $@
52quiet_cmd_vdsoas = VDSOA $@ 52quiet_cmd_vdsoas = VDSOA $@
53 cmd_vdsoas = $(CC) $(a_flags) -c -o $@ $< 53 cmd_vdsoas = $(CC) $(a_flags) -c -o $@ $<
54 54
55# Install commands for the unstripped file 55# Install commands for the unstripped file
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index 8154b8d1c826..beca249bc2f3 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -28,6 +28,7 @@ OUTPUT_ARCH(aarch64)
28 28
29SECTIONS 29SECTIONS
30{ 30{
31 PROVIDE(_vdso_data = . - PAGE_SIZE);
31 . = VDSO_LBASE + SIZEOF_HEADERS; 32 . = VDSO_LBASE + SIZEOF_HEADERS;
32 33
33 .hash : { *(.hash) } :text 34 .hash : { *(.hash) } :text
@@ -57,9 +58,6 @@ SECTIONS
57 _end = .; 58 _end = .;
58 PROVIDE(end = .); 59 PROVIDE(end = .);
59 60
60 . = ALIGN(PAGE_SIZE);
61 PROVIDE(_vdso_data = .);
62
63 /DISCARD/ : { 61 /DISCARD/ : {
64 *(.note.GNU-stack) 62 *(.note.GNU-stack)
65 *(.data .data.* .gnu.linkonce.d.* .sdata*) 63 *(.data .data.* .gnu.linkonce.d.* .sdata*)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index f1e6d5c032e1..97f0c0429dfa 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -9,6 +9,8 @@
9#include <asm/memory.h> 9#include <asm/memory.h>
10#include <asm/page.h> 10#include <asm/page.h>
11 11
12#include "image.h"
13
12#define ARM_EXIT_KEEP(x) 14#define ARM_EXIT_KEEP(x)
13#define ARM_EXIT_DISCARD(x) x 15#define ARM_EXIT_DISCARD(x) x
14 16
@@ -104,9 +106,18 @@ SECTIONS
104 _edata = .; 106 _edata = .;
105 107
106 BSS_SECTION(0, 0, 0) 108 BSS_SECTION(0, 0, 0)
109
110 . = ALIGN(PAGE_SIZE);
111 idmap_pg_dir = .;
112 . += IDMAP_DIR_SIZE;
113 swapper_pg_dir = .;
114 . += SWAPPER_DIR_SIZE;
115
107 _end = .; 116 _end = .;
108 117
109 STABS_DEBUG 118 STABS_DEBUG
119
120 HEAD_SYMBOLS
110} 121}
111 122
112/* 123/*
@@ -114,3 +125,8 @@ SECTIONS
114 */ 125 */
115ASSERT(((__hyp_idmap_text_start + PAGE_SIZE) > __hyp_idmap_text_end), 126ASSERT(((__hyp_idmap_text_start + PAGE_SIZE) > __hyp_idmap_text_end),
116 "HYP init code too big") 127 "HYP init code too big")
128
129/*
130 * If padding is applied before .head.text, virt<->phys conversions will fail.
131 */
132ASSERT(_text == (PAGE_OFFSET + TEXT_OFFSET), "HEAD is misaligned")
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index bcc965e2cce1..41cb6d3d6075 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -62,6 +62,7 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
62 break; 62 break;
63 63
64 pud = pud_offset(pgd, addr); 64 pud = pud_offset(pgd, addr);
65 printk(", *pud=%016llx", pud_val(*pud));
65 if (pud_none(*pud) || pud_bad(*pud)) 66 if (pud_none(*pud) || pud_bad(*pud))
66 break; 67 break;
67 68
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index e90c5426fe14..5b4526ee3a01 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -33,6 +33,7 @@
33#include <linux/dma-mapping.h> 33#include <linux/dma-mapping.h>
34#include <linux/dma-contiguous.h> 34#include <linux/dma-contiguous.h>
35 35
36#include <asm/fixmap.h>
36#include <asm/sections.h> 37#include <asm/sections.h>
37#include <asm/setup.h> 38#include <asm/setup.h>
38#include <asm/sizes.h> 39#include <asm/sizes.h>
@@ -137,20 +138,16 @@ void __init arm64_memblock_init(void)
137{ 138{
138 phys_addr_t dma_phys_limit = 0; 139 phys_addr_t dma_phys_limit = 0;
139 140
140 /* Register the kernel text, kernel data and initrd with memblock */ 141 /*
142 * Register the kernel text, kernel data, initrd, and initial
143 * pagetables with memblock.
144 */
141 memblock_reserve(__pa(_text), _end - _text); 145 memblock_reserve(__pa(_text), _end - _text);
142#ifdef CONFIG_BLK_DEV_INITRD 146#ifdef CONFIG_BLK_DEV_INITRD
143 if (initrd_start) 147 if (initrd_start)
144 memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start); 148 memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start);
145#endif 149#endif
146 150
147 /*
148 * Reserve the page tables. These are already in use,
149 * and can only be in node 0.
150 */
151 memblock_reserve(__pa(swapper_pg_dir), SWAPPER_DIR_SIZE);
152 memblock_reserve(__pa(idmap_pg_dir), IDMAP_DIR_SIZE);
153
154 early_init_fdt_scan_reserved_mem(); 151 early_init_fdt_scan_reserved_mem();
155 152
156 /* 4GB maximum for 32-bit only capable devices */ 153 /* 4GB maximum for 32-bit only capable devices */
@@ -269,26 +266,33 @@ void __init mem_init(void)
269 266
270#define MLK(b, t) b, t, ((t) - (b)) >> 10 267#define MLK(b, t) b, t, ((t) - (b)) >> 10
271#define MLM(b, t) b, t, ((t) - (b)) >> 20 268#define MLM(b, t) b, t, ((t) - (b)) >> 20
269#define MLG(b, t) b, t, ((t) - (b)) >> 30
272#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) 270#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
273 271
274 pr_notice("Virtual kernel memory layout:\n" 272 pr_notice("Virtual kernel memory layout:\n"
275 " vmalloc : 0x%16lx - 0x%16lx (%6ld MB)\n" 273 " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n"
276#ifdef CONFIG_SPARSEMEM_VMEMMAP 274#ifdef CONFIG_SPARSEMEM_VMEMMAP
277 " vmemmap : 0x%16lx - 0x%16lx (%6ld MB)\n" 275 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
276 " 0x%16lx - 0x%16lx (%6ld MB actual)\n"
278#endif 277#endif
278 " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
279 " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n"
279 " modules : 0x%16lx - 0x%16lx (%6ld MB)\n" 280 " modules : 0x%16lx - 0x%16lx (%6ld MB)\n"
280 " memory : 0x%16lx - 0x%16lx (%6ld MB)\n" 281 " memory : 0x%16lx - 0x%16lx (%6ld MB)\n"
281 " .init : 0x%p" " - 0x%p" " (%6ld kB)\n" 282 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
282 " .text : 0x%p" " - 0x%p" " (%6ld kB)\n" 283 " .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
283 " .data : 0x%p" " - 0x%p" " (%6ld kB)\n", 284 " .data : 0x%p" " - 0x%p" " (%6ld KB)\n",
284 MLM(VMALLOC_START, VMALLOC_END), 285 MLG(VMALLOC_START, VMALLOC_END),
285#ifdef CONFIG_SPARSEMEM_VMEMMAP 286#ifdef CONFIG_SPARSEMEM_VMEMMAP
287 MLG((unsigned long)vmemmap,
288 (unsigned long)vmemmap + VMEMMAP_SIZE),
286 MLM((unsigned long)virt_to_page(PAGE_OFFSET), 289 MLM((unsigned long)virt_to_page(PAGE_OFFSET),
287 (unsigned long)virt_to_page(high_memory)), 290 (unsigned long)virt_to_page(high_memory)),
288#endif 291#endif
292 MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M),
293 MLK(FIXADDR_START, FIXADDR_TOP),
289 MLM(MODULES_VADDR, MODULES_END), 294 MLM(MODULES_VADDR, MODULES_END),
290 MLM(PAGE_OFFSET, (unsigned long)high_memory), 295 MLM(PAGE_OFFSET, (unsigned long)high_memory),
291
292 MLK_ROUNDUP(__init_begin, __init_end), 296 MLK_ROUNDUP(__init_begin, __init_end),
293 MLK_ROUNDUP(_text, _etext), 297 MLK_ROUNDUP(_text, _etext),
294 MLK_ROUNDUP(_sdata, _edata)); 298 MLK_ROUNDUP(_sdata, _edata));
diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
index 7ec328392ae0..fa324bd5a5c4 100644
--- a/arch/arm64/mm/ioremap.c
+++ b/arch/arm64/mm/ioremap.c
@@ -103,19 +103,28 @@ void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size)
103} 103}
104EXPORT_SYMBOL(ioremap_cache); 104EXPORT_SYMBOL(ioremap_cache);
105 105
106#ifndef CONFIG_ARM64_64K_PAGES
107static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 106static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
107#if CONFIG_ARM64_PGTABLE_LEVELS > 2
108static pte_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss;
109#endif
110#if CONFIG_ARM64_PGTABLE_LEVELS > 3
111static pte_t bm_pud[PTRS_PER_PUD] __page_aligned_bss;
108#endif 112#endif
109 113
110static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) 114static inline pud_t * __init early_ioremap_pud(unsigned long addr)
111{ 115{
112 pgd_t *pgd; 116 pgd_t *pgd;
113 pud_t *pud;
114 117
115 pgd = pgd_offset_k(addr); 118 pgd = pgd_offset_k(addr);
116 BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 119 BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));
117 120
118 pud = pud_offset(pgd, addr); 121 return pud_offset(pgd, addr);
122}
123
124static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
125{
126 pud_t *pud = early_ioremap_pud(addr);
127
119 BUG_ON(pud_none(*pud) || pud_bad(*pud)); 128 BUG_ON(pud_none(*pud) || pud_bad(*pud));
120 129
121 return pmd_offset(pud, addr); 130 return pmd_offset(pud, addr);
@@ -132,13 +141,18 @@ static inline pte_t * __init early_ioremap_pte(unsigned long addr)
132 141
133void __init early_ioremap_init(void) 142void __init early_ioremap_init(void)
134{ 143{
144 pgd_t *pgd;
145 pud_t *pud;
135 pmd_t *pmd; 146 pmd_t *pmd;
147 unsigned long addr = fix_to_virt(FIX_BTMAP_BEGIN);
136 148
137 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); 149 pgd = pgd_offset_k(addr);
138#ifndef CONFIG_ARM64_64K_PAGES 150 pgd_populate(&init_mm, pgd, bm_pud);
139 /* need to populate pmd for 4k pagesize only */ 151 pud = pud_offset(pgd, addr);
152 pud_populate(&init_mm, pud, bm_pmd);
153 pmd = pmd_offset(pud, addr);
140 pmd_populate_kernel(&init_mm, pmd, bm_pte); 154 pmd_populate_kernel(&init_mm, pmd, bm_pte);
141#endif 155
142 /* 156 /*
143 * The boot-ioremap range spans multiple pmds, for which 157 * The boot-ioremap range spans multiple pmds, for which
144 * we are not prepared: 158 * we are not prepared:
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index c43f1dd19489..c55567283cde 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -32,6 +32,7 @@
32#include <asm/setup.h> 32#include <asm/setup.h>
33#include <asm/sizes.h> 33#include <asm/sizes.h>
34#include <asm/tlb.h> 34#include <asm/tlb.h>
35#include <asm/memblock.h>
35#include <asm/mmu_context.h> 36#include <asm/mmu_context.h>
36 37
37#include "mm.h" 38#include "mm.h"
@@ -204,9 +205,16 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
204 unsigned long end, unsigned long phys, 205 unsigned long end, unsigned long phys,
205 int map_io) 206 int map_io)
206{ 207{
207 pud_t *pud = pud_offset(pgd, addr); 208 pud_t *pud;
208 unsigned long next; 209 unsigned long next;
209 210
211 if (pgd_none(*pgd)) {
212 pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
213 pgd_populate(&init_mm, pgd, pud);
214 }
215 BUG_ON(pgd_bad(*pgd));
216
217 pud = pud_offset(pgd, addr);
210 do { 218 do {
211 next = pud_addr_end(addr, end); 219 next = pud_addr_end(addr, end);
212 220
@@ -290,10 +298,10 @@ static void __init map_mem(void)
290 * memory addressable from the initial direct kernel mapping. 298 * memory addressable from the initial direct kernel mapping.
291 * 299 *
292 * The initial direct kernel mapping, located at swapper_pg_dir, 300 * The initial direct kernel mapping, located at swapper_pg_dir,
293 * gives us PGDIR_SIZE memory starting from PHYS_OFFSET (which must be 301 * gives us PUD_SIZE memory starting from PHYS_OFFSET (which must be
294 * aligned to 2MB as per Documentation/arm64/booting.txt). 302 * aligned to 2MB as per Documentation/arm64/booting.txt).
295 */ 303 */
296 limit = PHYS_OFFSET + PGDIR_SIZE; 304 limit = PHYS_OFFSET + PUD_SIZE;
297 memblock_set_current_limit(limit); 305 memblock_set_current_limit(limit);
298 306
299 /* map all the memory banks */ 307 /* map all the memory banks */
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-11 11:38:57 -0400