aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm')
-rw-r--r--arch/arm/Kconfig26
-rw-r--r--arch/arm/Makefile10
-rw-r--r--arch/arm/boot/Makefile2
-rw-r--r--arch/arm/boot/compressed/head.S52
-rw-r--r--arch/arm/boot/dts/am4372.dtsi11
-rw-r--r--arch/arm/boot/dts/am437x-gp-evm.dts1
-rw-r--r--arch/arm/boot/dts/am437x-sk-evm.dts1
-rw-r--r--arch/arm/boot/dts/am43x-epos-evm.dts1
-rw-r--r--arch/arm/boot/dts/am57xx-beagle-x15.dts3
-rw-r--r--arch/arm/boot/dts/at91sam9260.dtsi2
-rw-r--r--arch/arm/boot/dts/at91sam9263.dtsi2
-rw-r--r--arch/arm/boot/dts/at91sam9g45.dtsi2
-rw-r--r--arch/arm/boot/dts/at91sam9x5_macb0.dtsi2
-rw-r--r--arch/arm/boot/dts/at91sam9x5_macb1.dtsi2
-rw-r--r--arch/arm/boot/dts/dra7-evm.dts2
-rw-r--r--arch/arm/boot/dts/dra7.dtsi43
-rw-r--r--arch/arm/boot/dts/dra72-evm.dts1
-rw-r--r--arch/arm/boot/dts/dra72x.dtsi3
-rw-r--r--arch/arm/boot/dts/dra74x.dtsi5
-rw-r--r--arch/arm/boot/dts/exynos3250.dtsi4
-rw-r--r--arch/arm/boot/dts/exynos4.dtsi4
-rw-r--r--arch/arm/boot/dts/exynos5250.dtsi4
-rw-r--r--arch/arm/boot/dts/exynos5420.dtsi4
-rw-r--r--arch/arm/boot/dts/omap4-duovero.dtsi2
-rw-r--r--arch/arm/boot/dts/omap4-panda-common.dtsi8
-rw-r--r--arch/arm/boot/dts/omap4-sdp.dts8
-rw-r--r--arch/arm/boot/dts/omap4-var-som-om44.dtsi2
-rw-r--r--arch/arm/boot/dts/omap4.dtsi18
-rw-r--r--arch/arm/boot/dts/omap5-cm-t54.dts1
-rw-r--r--arch/arm/boot/dts/omap5-uevm.dts2
-rw-r--r--arch/arm/boot/dts/omap5.dtsi26
-rw-r--r--arch/arm/boot/dts/sama5d3_emac.dtsi2
-rw-r--r--arch/arm/boot/dts/stih416.dtsi4
-rw-r--r--arch/arm/boot/dts/tegra114.dtsi16
-rw-r--r--arch/arm/boot/dts/tegra124.dtsi16
-rw-r--r--arch/arm/boot/dts/tegra20.dtsi15
-rw-r--r--arch/arm/boot/dts/tegra30.dtsi16
-rw-r--r--arch/arm/common/bL_switcher.c16
-rw-r--r--arch/arm/configs/badge4_defconfig1
-rw-r--r--arch/arm/crypto/Kconfig130
-rw-r--r--arch/arm/crypto/Makefile27
-rw-r--r--arch/arm/crypto/aes-ce-core.S518
-rw-r--r--arch/arm/crypto/aes-ce-glue.c524
-rw-r--r--arch/arm/crypto/aesbs-glue.c9
-rw-r--r--arch/arm/crypto/ghash-ce-core.S94
-rw-r--r--arch/arm/crypto/ghash-ce-glue.c320
-rw-r--r--arch/arm/crypto/sha1-ce-core.S125
-rw-r--r--arch/arm/crypto/sha1-ce-glue.c96
-rw-r--r--arch/arm/crypto/sha1.h (renamed from arch/arm/include/asm/crypto/sha1.h)3
-rw-r--r--arch/arm/crypto/sha1_glue.c112
-rw-r--r--arch/arm/crypto/sha1_neon_glue.c137
-rw-r--r--arch/arm/crypto/sha2-ce-core.S125
-rw-r--r--arch/arm/crypto/sha2-ce-glue.c114
-rw-r--r--arch/arm/crypto/sha256-armv4.pl716
-rw-r--r--arch/arm/crypto/sha256-core.S_shipped2808
-rw-r--r--arch/arm/crypto/sha256_glue.c128
-rw-r--r--arch/arm/crypto/sha256_glue.h14
-rw-r--r--arch/arm/crypto/sha256_neon_glue.c101
-rw-r--r--arch/arm/include/asm/Kbuild1
-rw-r--r--arch/arm/include/asm/assembler.h3
-rw-r--r--arch/arm/include/asm/auxvec.h1
-rw-r--r--arch/arm/include/asm/cpuidle.h23
-rw-r--r--arch/arm/include/asm/cputype.h16
-rw-r--r--arch/arm/include/asm/elf.h15
-rw-r--r--arch/arm/include/asm/futex.h2
-rw-r--r--arch/arm/include/asm/jump_label.h5
-rw-r--r--arch/arm/include/asm/kvm_arm.h1
-rw-r--r--arch/arm/include/asm/kvm_host.h15
-rw-r--r--arch/arm/include/asm/kvm_mmio.h22
-rw-r--r--arch/arm/include/asm/mach/time.h3
-rw-r--r--arch/arm/include/asm/mmu.h3
-rw-r--r--arch/arm/include/asm/pmu.h1
-rw-r--r--arch/arm/include/asm/smp_plat.h1
-rw-r--r--arch/arm/include/asm/thread_info.h3
-rw-r--r--arch/arm/include/asm/uaccess.h10
-rw-r--r--arch/arm/include/asm/unified.h8
-rw-r--r--arch/arm/include/asm/vdso.h32
-rw-r--r--arch/arm/include/asm/vdso_datapage.h60
-rw-r--r--arch/arm/include/asm/word-at-a-time.h2
-rw-r--r--arch/arm/include/uapi/asm/Kbuild1
-rw-r--r--arch/arm/include/uapi/asm/auxvec.h7
-rw-r--r--arch/arm/include/uapi/asm/kvm.h3
-rw-r--r--arch/arm/kernel/Makefile6
-rw-r--r--arch/arm/kernel/arthur.c94
-rw-r--r--arch/arm/kernel/asm-offsets.c10
-rw-r--r--arch/arm/kernel/bios32.c10
-rw-r--r--arch/arm/kernel/cpuidle.c133
-rw-r--r--arch/arm/kernel/entry-armv.S2
-rw-r--r--arch/arm/kernel/head.S14
-rw-r--r--arch/arm/kernel/hibernate.c6
-rw-r--r--arch/arm/kernel/hw_breakpoint.c2
-rw-r--r--arch/arm/kernel/machine_kexec.c3
-rw-r--r--arch/arm/kernel/module.c38
-rw-r--r--arch/arm/kernel/perf_event.c21
-rw-r--r--arch/arm/kernel/perf_event_cpu.c71
-rw-r--r--arch/arm/kernel/perf_event_v7.c525
-rw-r--r--arch/arm/kernel/process.c159
-rw-r--r--arch/arm/kernel/psci-call.S31
-rw-r--r--arch/arm/kernel/psci.c39
-rw-r--r--arch/arm/kernel/reboot.c155
-rw-r--r--arch/arm/kernel/reboot.h7
-rw-r--r--arch/arm/kernel/return_address.c4
-rw-r--r--arch/arm/kernel/setup.c44
-rw-r--r--arch/arm/kernel/signal.c13
-rw-r--r--arch/arm/kernel/sleep.S15
-rw-r--r--arch/arm/kernel/smp.c5
-rw-r--r--arch/arm/kernel/swp_emulate.c2
-rw-r--r--arch/arm/kernel/time.c6
-rw-r--r--arch/arm/kernel/traps.c6
-rw-r--r--arch/arm/kernel/vdso.c337
-rw-r--r--arch/arm/kernel/vmlinux.lds.S7
-rw-r--r--arch/arm/kvm/Kconfig30
-rw-r--r--arch/arm/kvm/Makefile12
-rw-r--r--arch/arm/kvm/arm.c45
-rw-r--r--arch/arm/kvm/guest.c18
-rw-r--r--arch/arm/kvm/interrupts_head.S8
-rw-r--r--arch/arm/kvm/mmio.c64
-rw-r--r--arch/arm/kvm/mmu.c134
-rw-r--r--arch/arm/kvm/trace.h48
-rw-r--r--arch/arm/lib/clear_user.S2
-rw-r--r--arch/arm/lib/copy_to_user.S2
-rw-r--r--arch/arm/lib/csumpartialcopyuser.S2
-rw-r--r--arch/arm/lib/delay.c6
-rw-r--r--arch/arm/mach-davinci/cpuidle.c1
-rw-r--r--arch/arm/mach-dove/pcie.c12
-rw-r--r--arch/arm/mach-exynos/exynos.c15
-rw-r--r--arch/arm/mach-exynos/sleep.S31
-rw-r--r--arch/arm/mach-exynos/suspend.c135
-rw-r--r--arch/arm/mach-imx/Kconfig1
-rw-r--r--arch/arm/mach-imx/cpuidle-imx6q.c1
-rw-r--r--arch/arm/mach-imx/cpuidle-imx6sl.c1
-rw-r--r--arch/arm/mach-imx/cpuidle-imx6sx.c1
-rw-r--r--arch/arm/mach-mv78xx0/pcie.c12
-rw-r--r--arch/arm/mach-omap2/cpuidle44xx.c11
-rw-r--r--arch/arm/mach-omap2/hsmmc.c33
-rw-r--r--arch/arm/mach-omap2/omap-wakeupgen.c128
-rw-r--r--arch/arm/mach-omap2/omap-wakeupgen.h1
-rw-r--r--arch/arm/mach-omap2/omap4-common.c27
-rw-r--r--arch/arm/mach-orion5x/pci.c32
-rw-r--r--arch/arm/mach-pxa/raumfeld.c4
-rw-r--r--arch/arm/mach-s3c64xx/cpuidle.c2
-rw-r--r--arch/arm/mach-s5pv210/sleep.S2
-rw-r--r--arch/arm/mach-shmobile/intc-sh73a0.c7
-rw-r--r--arch/arm/mach-shmobile/setup-r8a7779.c7
-rw-r--r--arch/arm/mach-tegra/cpuidle-tegra114.c6
-rw-r--r--arch/arm/mach-tegra/cpuidle-tegra20.c11
-rw-r--r--arch/arm/mach-tegra/cpuidle-tegra30.c11
-rw-r--r--arch/arm/mach-tegra/iomap.h15
-rw-r--r--arch/arm/mach-tegra/irq.c209
-rw-r--r--arch/arm/mach-tegra/irq.h6
-rw-r--r--arch/arm/mach-tegra/tegra.c1
-rw-r--r--arch/arm/mach-ux500/cpu.c2
-rw-r--r--arch/arm/mach-vexpress/Kconfig1
-rw-r--r--arch/arm/mach-zynq/common.c2
-rw-r--r--arch/arm/mm/Kconfig16
-rw-r--r--arch/arm/mm/alignment.c6
-rw-r--r--arch/arm/mm/cache-l2x0.c7
-rw-r--r--arch/arm/mm/cache-v7.S38
-rw-r--r--arch/arm/mm/dma-mapping.c123
-rw-r--r--arch/arm/mm/init.c52
-rw-r--r--arch/arm/mm/mmap.c16
-rw-r--r--arch/arm/mm/proc-arm1020.S4
-rw-r--r--arch/arm/mm/proc-arm1020e.S4
-rw-r--r--arch/arm/mm/proc-arm1022.S4
-rw-r--r--arch/arm/mm/proc-arm1026.S4
-rw-r--r--arch/arm/mm/proc-arm720.S4
-rw-r--r--arch/arm/mm/proc-arm740.S4
-rw-r--r--arch/arm/mm/proc-arm7tdmi.S4
-rw-r--r--arch/arm/mm/proc-arm920.S4
-rw-r--r--arch/arm/mm/proc-arm922.S4
-rw-r--r--arch/arm/mm/proc-arm925.S4
-rw-r--r--arch/arm/mm/proc-arm926.S4
-rw-r--r--arch/arm/mm/proc-arm940.S30
-rw-r--r--arch/arm/mm/proc-arm946.S26
-rw-r--r--arch/arm/mm/proc-arm9tdmi.S4
-rw-r--r--arch/arm/mm/proc-fa526.S4
-rw-r--r--arch/arm/mm/proc-feroceon.S5
-rw-r--r--arch/arm/mm/proc-macros.S28
-rw-r--r--arch/arm/mm/proc-mohawk.S4
-rw-r--r--arch/arm/mm/proc-sa110.S4
-rw-r--r--arch/arm/mm/proc-sa1100.S4
-rw-r--r--arch/arm/mm/proc-v6.S4
-rw-r--r--arch/arm/mm/proc-v7-2level.S12
-rw-r--r--arch/arm/mm/proc-v7.S56
-rw-r--r--arch/arm/mm/proc-v7m.S4
-rw-r--r--arch/arm/mm/proc-xsc3.S4
-rw-r--r--arch/arm/mm/proc-xscale.S4
-rw-r--r--arch/arm/nwfpe/entry.S2
-rw-r--r--arch/arm/plat-omap/counter_32k.c20
-rw-r--r--arch/arm/vdso/.gitignore1
-rw-r--r--arch/arm/vdso/Makefile74
-rw-r--r--arch/arm/vdso/datapage.S15
-rw-r--r--arch/arm/vdso/vdso.S35
-rw-r--r--arch/arm/vdso/vdso.lds.S87
-rw-r--r--arch/arm/vdso/vdsomunge.c201
-rw-r--r--arch/arm/vdso/vgettimeofday.c282
196 files changed, 9210 insertions, 1545 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index cf4c0c99aa25..392e7ae69452 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1,8 +1,8 @@
1config ARM 1config ARM
2 bool 2 bool
3 default y 3 default y
4 select ARCH_BINFMT_ELF_RANDOMIZE_PIE
5 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE 4 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
5 select ARCH_HAS_ELF_RANDOMIZE
6 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST 6 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
7 select ARCH_HAVE_CUSTOM_GPIO_H 7 select ARCH_HAVE_CUSTOM_GPIO_H
8 select ARCH_HAS_GCOV_PROFILE_ALL 8 select ARCH_HAS_GCOV_PROFILE_ALL
@@ -21,6 +21,7 @@ config ARM
21 select GENERIC_IDLE_POLL_SETUP 21 select GENERIC_IDLE_POLL_SETUP
22 select GENERIC_IRQ_PROBE 22 select GENERIC_IRQ_PROBE
23 select GENERIC_IRQ_SHOW 23 select GENERIC_IRQ_SHOW
24 select GENERIC_IRQ_SHOW_LEVEL
24 select GENERIC_PCI_IOMAP 25 select GENERIC_PCI_IOMAP
25 select GENERIC_SCHED_CLOCK 26 select GENERIC_SCHED_CLOCK
26 select GENERIC_SMP_IDLE_THREAD 27 select GENERIC_SMP_IDLE_THREAD
@@ -286,6 +287,11 @@ config GENERIC_BUG
286 def_bool y 287 def_bool y
287 depends on BUG 288 depends on BUG
288 289
290config PGTABLE_LEVELS
291 int
292 default 3 if ARM_LPAE
293 default 2
294
289source "init/Kconfig" 295source "init/Kconfig"
290 296
291source "kernel/Kconfig.freezer" 297source "kernel/Kconfig.freezer"
@@ -1058,7 +1064,7 @@ config ARM_ERRATA_430973
1058 depends on CPU_V7 1064 depends on CPU_V7
1059 help 1065 help
1060 This option enables the workaround for the 430973 Cortex-A8 1066 This option enables the workaround for the 430973 Cortex-A8
1061 (r1p0..r1p2) erratum. If a code sequence containing an ARM/Thumb 1067 r1p* erratum. If a code sequence containing an ARM/Thumb
1062 interworking branch is replaced with another code sequence at the 1068 interworking branch is replaced with another code sequence at the
1063 same virtual address, whether due to self-modifying code or virtual 1069 same virtual address, whether due to self-modifying code or virtual
1064 to physical address re-mapping, Cortex-A8 does not recover from the 1070 to physical address re-mapping, Cortex-A8 does not recover from the
@@ -1127,6 +1133,7 @@ config ARM_ERRATA_742231
1127config ARM_ERRATA_643719 1133config ARM_ERRATA_643719
1128 bool "ARM errata: LoUIS bit field in CLIDR register is incorrect" 1134 bool "ARM errata: LoUIS bit field in CLIDR register is incorrect"
1129 depends on CPU_V7 && SMP 1135 depends on CPU_V7 && SMP
1136 default y
1130 help 1137 help
1131 This option enables the workaround for the 643719 Cortex-A9 (prior to 1138 This option enables the workaround for the 643719 Cortex-A9 (prior to
1132 r1p0) erratum. On affected cores the LoUIS bit field of the CLIDR 1139 r1p0) erratum. On affected cores the LoUIS bit field of the CLIDR
@@ -1344,7 +1351,7 @@ config SMP
1344 If you don't know what to do here, say N. 1351 If you don't know what to do here, say N.
1345 1352
1346config SMP_ON_UP 1353config SMP_ON_UP
1347 bool "Allow booting SMP kernel on uniprocessor systems (EXPERIMENTAL)" 1354 bool "Allow booting SMP kernel on uniprocessor systems"
1348 depends on SMP && !XIP_KERNEL && MMU 1355 depends on SMP && !XIP_KERNEL && MMU
1349 default y 1356 default y
1350 help 1357 help
@@ -2126,16 +2133,6 @@ menu "Userspace binary formats"
2126 2133
2127source "fs/Kconfig.binfmt" 2134source "fs/Kconfig.binfmt"
2128 2135
2129config ARTHUR
2130 tristate "RISC OS personality"
2131 depends on !AEABI
2132 help
2133 Say Y here to include the kernel code necessary if you want to run
2134 Acorn RISC OS/Arthur binaries under Linux. This code is still very
2135 experimental; if this sounds frightening, say N and sleep in peace.
2136 You can also say M here to compile this support as a module (which
2137 will be called arthur).
2138
2139endmenu 2136endmenu
2140 2137
2141menu "Power management options" 2138menu "Power management options"
@@ -2168,6 +2165,9 @@ source "arch/arm/Kconfig.debug"
2168source "security/Kconfig" 2165source "security/Kconfig"
2169 2166
2170source "crypto/Kconfig" 2167source "crypto/Kconfig"
2168if CRYPTO
2169source "arch/arm/crypto/Kconfig"
2170endif
2171 2171
2172source "lib/Kconfig" 2172source "lib/Kconfig"
2173 2173
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index eb7bb511f853..5575d9fa8806 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -13,7 +13,7 @@
13# Ensure linker flags are correct 13# Ensure linker flags are correct
14LDFLAGS := 14LDFLAGS :=
15 15
16LDFLAGS_vmlinux :=-p --no-undefined -X 16LDFLAGS_vmlinux :=-p --no-undefined -X --pic-veneer
17ifeq ($(CONFIG_CPU_ENDIAN_BE8),y) 17ifeq ($(CONFIG_CPU_ENDIAN_BE8),y)
18LDFLAGS_vmlinux += --be8 18LDFLAGS_vmlinux += --be8
19LDFLAGS_MODULE += --be8 19LDFLAGS_MODULE += --be8
@@ -264,6 +264,7 @@ core-$(CONFIG_FPE_FASTFPE) += $(FASTFPE_OBJ)
264core-$(CONFIG_VFP) += arch/arm/vfp/ 264core-$(CONFIG_VFP) += arch/arm/vfp/
265core-$(CONFIG_XEN) += arch/arm/xen/ 265core-$(CONFIG_XEN) += arch/arm/xen/
266core-$(CONFIG_KVM_ARM_HOST) += arch/arm/kvm/ 266core-$(CONFIG_KVM_ARM_HOST) += arch/arm/kvm/
267core-$(CONFIG_VDSO) += arch/arm/vdso/
267 268
268# If we have a machine-specific directory, then include it in the build. 269# If we have a machine-specific directory, then include it in the build.
269core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/ 270core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/
@@ -321,6 +322,12 @@ dtbs: prepare scripts
321dtbs_install: 322dtbs_install:
322 $(Q)$(MAKE) $(dtbinst)=$(boot)/dts 323 $(Q)$(MAKE) $(dtbinst)=$(boot)/dts
323 324
325PHONY += vdso_install
326vdso_install:
327ifeq ($(CONFIG_VDSO),y)
328 $(Q)$(MAKE) $(build)=arch/arm/vdso $@
329endif
330
324# We use MRPROPER_FILES and CLEAN_FILES now 331# We use MRPROPER_FILES and CLEAN_FILES now
325archclean: 332archclean:
326 $(Q)$(MAKE) $(clean)=$(boot) 333 $(Q)$(MAKE) $(clean)=$(boot)
@@ -345,4 +352,5 @@ define archhelp
345 echo ' Install using (your) ~/bin/$(INSTALLKERNEL) or' 352 echo ' Install using (your) ~/bin/$(INSTALLKERNEL) or'
346 echo ' (distribution) /sbin/$(INSTALLKERNEL) or' 353 echo ' (distribution) /sbin/$(INSTALLKERNEL) or'
347 echo ' install to $$(INSTALL_PATH) and run lilo' 354 echo ' install to $$(INSTALL_PATH) and run lilo'
355 echo ' vdso_install - Install unstripped vdso.so to $$(INSTALL_MOD_PATH)/vdso'
348endef 356endef
diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile
index ec2f8065f955..9eca7aee927f 100644
--- a/arch/arm/boot/Makefile
+++ b/arch/arm/boot/Makefile
@@ -12,7 +12,7 @@
12# 12#
13 13
14ifneq ($(MACHINE),) 14ifneq ($(MACHINE),)
15include $(srctree)/$(MACHINE)/Makefile.boot 15include $(MACHINE)/Makefile.boot
16endif 16endif
17 17
18# Note: the following conditions must always be true: 18# Note: the following conditions must always be true:
diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
index c41a793b519c..2c45b5709fa4 100644
--- a/arch/arm/boot/compressed/head.S
+++ b/arch/arm/boot/compressed/head.S
@@ -10,8 +10,11 @@
10 */ 10 */
11#include <linux/linkage.h> 11#include <linux/linkage.h>
12#include <asm/assembler.h> 12#include <asm/assembler.h>
13#include <asm/v7m.h>
14
15 AR_CLASS( .arch armv7-a )
16 M_CLASS( .arch armv7-m )
13 17
14 .arch armv7-a
15/* 18/*
16 * Debugging stuff 19 * Debugging stuff
17 * 20 *
@@ -114,7 +117,12 @@
114 * sort out different calling conventions 117 * sort out different calling conventions
115 */ 118 */
116 .align 119 .align
117 .arm @ Always enter in ARM state 120 /*
121 * Always enter in ARM state for CPUs that support the ARM ISA.
122 * As of today (2014) that's exactly the members of the A and R
123 * classes.
124 */
125 AR_CLASS( .arm )
118start: 126start:
119 .type start,#function 127 .type start,#function
120 .rept 7 128 .rept 7
@@ -132,14 +140,15 @@ start:
132 140
133 THUMB( .thumb ) 141 THUMB( .thumb )
1341: 1421:
135 ARM_BE8( setend be ) @ go BE8 if compiled for BE8 143 ARM_BE8( setend be ) @ go BE8 if compiled for BE8
136 mrs r9, cpsr 144 AR_CLASS( mrs r9, cpsr )
137#ifdef CONFIG_ARM_VIRT_EXT 145#ifdef CONFIG_ARM_VIRT_EXT
138 bl __hyp_stub_install @ get into SVC mode, reversibly 146 bl __hyp_stub_install @ get into SVC mode, reversibly
139#endif 147#endif
140 mov r7, r1 @ save architecture ID 148 mov r7, r1 @ save architecture ID
141 mov r8, r2 @ save atags pointer 149 mov r8, r2 @ save atags pointer
142 150
151#ifndef CONFIG_CPU_V7M
143 /* 152 /*
144 * Booting from Angel - need to enter SVC mode and disable 153 * Booting from Angel - need to enter SVC mode and disable
145 * FIQs/IRQs (numeric definitions from angel arm.h source). 154 * FIQs/IRQs (numeric definitions from angel arm.h source).
@@ -155,6 +164,7 @@ not_angel:
155 safe_svcmode_maskall r0 164 safe_svcmode_maskall r0
156 msr spsr_cxsf, r9 @ Save the CPU boot mode in 165 msr spsr_cxsf, r9 @ Save the CPU boot mode in
157 @ SPSR 166 @ SPSR
167#endif
158 /* 168 /*
159 * Note that some cache flushing and other stuff may 169 * Note that some cache flushing and other stuff may
160 * be needed here - is there an Angel SWI call for this? 170 * be needed here - is there an Angel SWI call for this?
@@ -168,9 +178,26 @@ not_angel:
168 .text 178 .text
169 179
170#ifdef CONFIG_AUTO_ZRELADDR 180#ifdef CONFIG_AUTO_ZRELADDR
171 @ determine final kernel image address 181 /*
182 * Find the start of physical memory. As we are executing
183 * without the MMU on, we are in the physical address space.
184 * We just need to get rid of any offset by aligning the
185 * address.
186 *
187 * This alignment is a balance between the requirements of
188 * different platforms - we have chosen 128MB to allow
189 * platforms which align the start of their physical memory
190 * to 128MB to use this feature, while allowing the zImage
191 * to be placed within the first 128MB of memory on other
192 * platforms. Increasing the alignment means we place
193 * stricter alignment requirements on the start of physical
194 * memory, but relaxing it means that we break people who
195 * are already placing their zImage in (eg) the top 64MB
196 * of this range.
197 */
172 mov r4, pc 198 mov r4, pc
173 and r4, r4, #0xf8000000 199 and r4, r4, #0xf8000000
200 /* Determine final kernel image address. */
174 add r4, r4, #TEXT_OFFSET 201 add r4, r4, #TEXT_OFFSET
175#else 202#else
176 ldr r4, =zreladdr 203 ldr r4, =zreladdr
@@ -810,6 +837,16 @@ __common_mmu_cache_on:
810call_cache_fn: adr r12, proc_types 837call_cache_fn: adr r12, proc_types
811#ifdef CONFIG_CPU_CP15 838#ifdef CONFIG_CPU_CP15
812 mrc p15, 0, r9, c0, c0 @ get processor ID 839 mrc p15, 0, r9, c0, c0 @ get processor ID
840#elif defined(CONFIG_CPU_V7M)
841 /*
842 * On v7-M the processor id is located in the V7M_SCB_CPUID
843 * register, but as cache handling is IMPLEMENTATION DEFINED on
844 * v7-M (if existant at all) we just return early here.
845 * If V7M_SCB_CPUID were used the cpu ID functions (i.e.
846 * __armv7_mmu_cache_{on,off,flush}) would be selected which
847 * use cp15 registers that are not implemented on v7-M.
848 */
849 bx lr
813#else 850#else
814 ldr r9, =CONFIG_PROCESSOR_ID 851 ldr r9, =CONFIG_PROCESSOR_ID
815#endif 852#endif
@@ -1310,8 +1347,9 @@ __hyp_reentry_vectors:
1310 1347
1311__enter_kernel: 1348__enter_kernel:
1312 mov r0, #0 @ must be 0 1349 mov r0, #0 @ must be 0
1313 ARM( mov pc, r4 ) @ call kernel 1350 ARM( mov pc, r4 ) @ call kernel
1314 THUMB( bx r4 ) @ entry point is always ARM 1351 M_CLASS( add r4, r4, #1 ) @ enter in Thumb mode for M class
1352 THUMB( bx r4 ) @ entry point is always ARM for A/R classes
1315 1353
1316reloc_code_end: 1354reloc_code_end:
1317 1355
diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi
index 1943fc333e7c..8a099bc10c1e 100644
--- a/arch/arm/boot/dts/am4372.dtsi
+++ b/arch/arm/boot/dts/am4372.dtsi
@@ -15,7 +15,7 @@
15 15
16/ { 16/ {
17 compatible = "ti,am4372", "ti,am43"; 17 compatible = "ti,am4372", "ti,am43";
18 interrupt-parent = <&gic>; 18 interrupt-parent = <&wakeupgen>;
19 19
20 20
21 aliases { 21 aliases {
@@ -48,6 +48,15 @@
48 #interrupt-cells = <3>; 48 #interrupt-cells = <3>;
49 reg = <0x48241000 0x1000>, 49 reg = <0x48241000 0x1000>,
50 <0x48240100 0x0100>; 50 <0x48240100 0x0100>;
51 interrupt-parent = <&gic>;
52 };
53
54 wakeupgen: interrupt-controller@48281000 {
55 compatible = "ti,omap4-wugen-mpu";
56 interrupt-controller;
57 #interrupt-cells = <3>;
58 reg = <0x48281000 0x1000>;
59 interrupt-parent = <&gic>;
51 }; 60 };
52 61
53 l2-cache-controller@48242000 { 62 l2-cache-controller@48242000 {
diff --git a/arch/arm/boot/dts/am437x-gp-evm.dts b/arch/arm/boot/dts/am437x-gp-evm.dts
index f84d9715a4a9..26956cb50835 100644
--- a/arch/arm/boot/dts/am437x-gp-evm.dts
+++ b/arch/arm/boot/dts/am437x-gp-evm.dts
@@ -352,7 +352,6 @@
352 reg = <0x24>; 352 reg = <0x24>;
353 compatible = "ti,tps65218"; 353 compatible = "ti,tps65218";
354 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */ 354 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */
355 interrupt-parent = <&gic>;
356 interrupt-controller; 355 interrupt-controller;
357 #interrupt-cells = <2>; 356 #interrupt-cells = <2>;
358 357
diff --git a/arch/arm/boot/dts/am437x-sk-evm.dts b/arch/arm/boot/dts/am437x-sk-evm.dts
index 832d24318f62..8ae29c955c11 100644
--- a/arch/arm/boot/dts/am437x-sk-evm.dts
+++ b/arch/arm/boot/dts/am437x-sk-evm.dts
@@ -392,7 +392,6 @@
392 tps@24 { 392 tps@24 {
393 compatible = "ti,tps65218"; 393 compatible = "ti,tps65218";
394 reg = <0x24>; 394 reg = <0x24>;
395 interrupt-parent = <&gic>;
396 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; 395 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
397 interrupt-controller; 396 interrupt-controller;
398 #interrupt-cells = <2>; 397 #interrupt-cells = <2>;
diff --git a/arch/arm/boot/dts/am43x-epos-evm.dts b/arch/arm/boot/dts/am43x-epos-evm.dts
index 257c099c347e..1d7109196872 100644
--- a/arch/arm/boot/dts/am43x-epos-evm.dts
+++ b/arch/arm/boot/dts/am43x-epos-evm.dts
@@ -369,7 +369,6 @@
369 reg = <0x24>; 369 reg = <0x24>;
370 compatible = "ti,tps65218"; 370 compatible = "ti,tps65218";
371 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */ 371 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */
372 interrupt-parent = <&gic>;
373 interrupt-controller; 372 interrupt-controller;
374 #interrupt-cells = <2>; 373 #interrupt-cells = <2>;
375 374
diff --git a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts
index 6463f9ef2b54..bd48dba16748 100644
--- a/arch/arm/boot/dts/am57xx-beagle-x15.dts
+++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts
@@ -454,7 +454,6 @@
454 mcp_rtc: rtc@6f { 454 mcp_rtc: rtc@6f {
455 compatible = "microchip,mcp7941x"; 455 compatible = "microchip,mcp7941x";
456 reg = <0x6f>; 456 reg = <0x6f>;
457 interrupt-parent = <&gic>;
458 interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_LOW>; /* IRQ_SYS_1N */ 457 interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_LOW>; /* IRQ_SYS_1N */
459 458
460 pinctrl-names = "default"; 459 pinctrl-names = "default";
@@ -477,7 +476,7 @@
477 476
478&uart3 { 477&uart3 {
479 status = "okay"; 478 status = "okay";
480 interrupts-extended = <&gic GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>, 479 interrupts-extended = <&crossbar_mpu GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
481 <&dra7_pmx_core 0x248>; 480 <&dra7_pmx_core 0x248>;
482 481
483 pinctrl-names = "default"; 482 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/at91sam9260.dtsi b/arch/arm/boot/dts/at91sam9260.dtsi
index e7f0a4ae271c..62d25b14deb8 100644
--- a/arch/arm/boot/dts/at91sam9260.dtsi
+++ b/arch/arm/boot/dts/at91sam9260.dtsi
@@ -842,7 +842,7 @@
842 }; 842 };
843 843
844 macb0: ethernet@fffc4000 { 844 macb0: ethernet@fffc4000 {
845 compatible = "cdns,at32ap7000-macb", "cdns,macb"; 845 compatible = "cdns,at91sam9260-macb", "cdns,macb";
846 reg = <0xfffc4000 0x100>; 846 reg = <0xfffc4000 0x100>;
847 interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>; 847 interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>;
848 pinctrl-names = "default"; 848 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/at91sam9263.dtsi b/arch/arm/boot/dts/at91sam9263.dtsi
index fce301c4e9d6..e4f61a979a57 100644
--- a/arch/arm/boot/dts/at91sam9263.dtsi
+++ b/arch/arm/boot/dts/at91sam9263.dtsi
@@ -845,7 +845,7 @@
845 }; 845 };
846 846
847 macb0: ethernet@fffbc000 { 847 macb0: ethernet@fffbc000 {
848 compatible = "cdns,at32ap7000-macb", "cdns,macb"; 848 compatible = "cdns,at91sam9260-macb", "cdns,macb";
849 reg = <0xfffbc000 0x100>; 849 reg = <0xfffbc000 0x100>;
850 interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>; 850 interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>;
851 pinctrl-names = "default"; 851 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/at91sam9g45.dtsi b/arch/arm/boot/dts/at91sam9g45.dtsi
index 488af63d5174..8ec05b11298a 100644
--- a/arch/arm/boot/dts/at91sam9g45.dtsi
+++ b/arch/arm/boot/dts/at91sam9g45.dtsi
@@ -956,7 +956,7 @@
956 }; 956 };
957 957
958 macb0: ethernet@fffbc000 { 958 macb0: ethernet@fffbc000 {
959 compatible = "cdns,at32ap7000-macb", "cdns,macb"; 959 compatible = "cdns,at91sam9260-macb", "cdns,macb";
960 reg = <0xfffbc000 0x100>; 960 reg = <0xfffbc000 0x100>;
961 interrupts = <25 IRQ_TYPE_LEVEL_HIGH 3>; 961 interrupts = <25 IRQ_TYPE_LEVEL_HIGH 3>;
962 pinctrl-names = "default"; 962 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/at91sam9x5_macb0.dtsi b/arch/arm/boot/dts/at91sam9x5_macb0.dtsi
index 57e89d1d0325..73d7e30965ba 100644
--- a/arch/arm/boot/dts/at91sam9x5_macb0.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5_macb0.dtsi
@@ -53,7 +53,7 @@
53 }; 53 };
54 54
55 macb0: ethernet@f802c000 { 55 macb0: ethernet@f802c000 {
56 compatible = "cdns,at32ap7000-macb", "cdns,macb"; 56 compatible = "cdns,at91sam9260-macb", "cdns,macb";
57 reg = <0xf802c000 0x100>; 57 reg = <0xf802c000 0x100>;
58 interrupts = <24 IRQ_TYPE_LEVEL_HIGH 3>; 58 interrupts = <24 IRQ_TYPE_LEVEL_HIGH 3>;
59 pinctrl-names = "default"; 59 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/at91sam9x5_macb1.dtsi b/arch/arm/boot/dts/at91sam9x5_macb1.dtsi
index 663676c02861..d81980c40c7d 100644
--- a/arch/arm/boot/dts/at91sam9x5_macb1.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5_macb1.dtsi
@@ -41,7 +41,7 @@
41 }; 41 };
42 42
43 macb1: ethernet@f8030000 { 43 macb1: ethernet@f8030000 {
44 compatible = "cdns,at32ap7000-macb", "cdns,macb"; 44 compatible = "cdns,at91sam9260-macb", "cdns,macb";
45 reg = <0xf8030000 0x100>; 45 reg = <0xf8030000 0x100>;
46 interrupts = <27 IRQ_TYPE_LEVEL_HIGH 3>; 46 interrupts = <27 IRQ_TYPE_LEVEL_HIGH 3>;
47 pinctrl-names = "default"; 47 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/dra7-evm.dts b/arch/arm/boot/dts/dra7-evm.dts
index 7563d7ce01bb..b1bd06c6c2a8 100644
--- a/arch/arm/boot/dts/dra7-evm.dts
+++ b/arch/arm/boot/dts/dra7-evm.dts
@@ -444,7 +444,7 @@
444 status = "okay"; 444 status = "okay";
445 pinctrl-names = "default"; 445 pinctrl-names = "default";
446 pinctrl-0 = <&uart1_pins>; 446 pinctrl-0 = <&uart1_pins>;
447 interrupts-extended = <&gic GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>, 447 interrupts-extended = <&crossbar_mpu GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
448 <&dra7_pmx_core 0x3e0>; 448 <&dra7_pmx_core 0x3e0>;
449}; 449};
450 450
diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
index c4659a979c41..a0afce7ad482 100644
--- a/arch/arm/boot/dts/dra7.dtsi
+++ b/arch/arm/boot/dts/dra7.dtsi
@@ -13,14 +13,13 @@
13#include "skeleton.dtsi" 13#include "skeleton.dtsi"
14 14
15#define MAX_SOURCES 400 15#define MAX_SOURCES 400
16#define DIRECT_IRQ(irq) (MAX_SOURCES + irq)
17 16
18/ { 17/ {
19 #address-cells = <1>; 18 #address-cells = <1>;
20 #size-cells = <1>; 19 #size-cells = <1>;
21 20
22 compatible = "ti,dra7xx"; 21 compatible = "ti,dra7xx";
23 interrupt-parent = <&gic>; 22 interrupt-parent = <&crossbar_mpu>;
24 23
25 aliases { 24 aliases {
26 i2c0 = &i2c1; 25 i2c0 = &i2c1;
@@ -50,18 +49,27 @@
50 <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>, 49 <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
51 <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>, 50 <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
52 <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>; 51 <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
52 interrupt-parent = <&gic>;
53 }; 53 };
54 54
55 gic: interrupt-controller@48211000 { 55 gic: interrupt-controller@48211000 {
56 compatible = "arm,cortex-a15-gic"; 56 compatible = "arm,cortex-a15-gic";
57 interrupt-controller; 57 interrupt-controller;
58 #interrupt-cells = <3>; 58 #interrupt-cells = <3>;
59 arm,routable-irqs = <192>;
60 reg = <0x48211000 0x1000>, 59 reg = <0x48211000 0x1000>,
61 <0x48212000 0x1000>, 60 <0x48212000 0x1000>,
62 <0x48214000 0x2000>, 61 <0x48214000 0x2000>,
63 <0x48216000 0x2000>; 62 <0x48216000 0x2000>;
64 interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>; 63 interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
64 interrupt-parent = <&gic>;
65 };
66
67 wakeupgen: interrupt-controller@48281000 {
68 compatible = "ti,omap5-wugen-mpu", "ti,omap4-wugen-mpu";
69 interrupt-controller;
70 #interrupt-cells = <3>;
71 reg = <0x48281000 0x1000>;
72 interrupt-parent = <&gic>;
65 }; 73 };
66 74
67 /* 75 /*
@@ -91,8 +99,8 @@
91 ti,hwmods = "l3_main_1", "l3_main_2"; 99 ti,hwmods = "l3_main_1", "l3_main_2";
92 reg = <0x44000000 0x1000000>, 100 reg = <0x44000000 0x1000000>,
93 <0x45000000 0x1000>; 101 <0x45000000 0x1000>;
94 interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, 102 interrupts-extended = <&crossbar_mpu GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
95 <GIC_SPI DIRECT_IRQ(10) IRQ_TYPE_LEVEL_HIGH>; 103 <&wakeupgen GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
96 104
97 prm: prm@4ae06000 { 105 prm: prm@4ae06000 {
98 compatible = "ti,dra7-prm"; 106 compatible = "ti,dra7-prm";
@@ -344,7 +352,7 @@
344 uart1: serial@4806a000 { 352 uart1: serial@4806a000 {
345 compatible = "ti,omap4-uart"; 353 compatible = "ti,omap4-uart";
346 reg = <0x4806a000 0x100>; 354 reg = <0x4806a000 0x100>;
347 interrupts-extended = <&gic GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>; 355 interrupts-extended = <&crossbar_mpu GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
348 ti,hwmods = "uart1"; 356 ti,hwmods = "uart1";
349 clock-frequency = <48000000>; 357 clock-frequency = <48000000>;
350 status = "disabled"; 358 status = "disabled";
@@ -355,7 +363,7 @@
355 uart2: serial@4806c000 { 363 uart2: serial@4806c000 {
356 compatible = "ti,omap4-uart"; 364 compatible = "ti,omap4-uart";
357 reg = <0x4806c000 0x100>; 365 reg = <0x4806c000 0x100>;
358 interrupts-extended = <&gic GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>; 366 interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
359 ti,hwmods = "uart2"; 367 ti,hwmods = "uart2";
360 clock-frequency = <48000000>; 368 clock-frequency = <48000000>;
361 status = "disabled"; 369 status = "disabled";
@@ -366,7 +374,7 @@
366 uart3: serial@48020000 { 374 uart3: serial@48020000 {
367 compatible = "ti,omap4-uart"; 375 compatible = "ti,omap4-uart";
368 reg = <0x48020000 0x100>; 376 reg = <0x48020000 0x100>;
369 interrupts-extended = <&gic GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>; 377 interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
370 ti,hwmods = "uart3"; 378 ti,hwmods = "uart3";
371 clock-frequency = <48000000>; 379 clock-frequency = <48000000>;
372 status = "disabled"; 380 status = "disabled";
@@ -377,7 +385,7 @@
377 uart4: serial@4806e000 { 385 uart4: serial@4806e000 {
378 compatible = "ti,omap4-uart"; 386 compatible = "ti,omap4-uart";
379 reg = <0x4806e000 0x100>; 387 reg = <0x4806e000 0x100>;
380 interrupts-extended = <&gic GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>; 388 interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
381 ti,hwmods = "uart4"; 389 ti,hwmods = "uart4";
382 clock-frequency = <48000000>; 390 clock-frequency = <48000000>;
383 status = "disabled"; 391 status = "disabled";
@@ -388,7 +396,7 @@
388 uart5: serial@48066000 { 396 uart5: serial@48066000 {
389 compatible = "ti,omap4-uart"; 397 compatible = "ti,omap4-uart";
390 reg = <0x48066000 0x100>; 398 reg = <0x48066000 0x100>;
391 interrupts-extended = <&gic GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>; 399 interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
392 ti,hwmods = "uart5"; 400 ti,hwmods = "uart5";
393 clock-frequency = <48000000>; 401 clock-frequency = <48000000>;
394 status = "disabled"; 402 status = "disabled";
@@ -399,7 +407,7 @@
399 uart6: serial@48068000 { 407 uart6: serial@48068000 {
400 compatible = "ti,omap4-uart"; 408 compatible = "ti,omap4-uart";
401 reg = <0x48068000 0x100>; 409 reg = <0x48068000 0x100>;
402 interrupts-extended = <&gic GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>; 410 interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
403 ti,hwmods = "uart6"; 411 ti,hwmods = "uart6";
404 clock-frequency = <48000000>; 412 clock-frequency = <48000000>;
405 status = "disabled"; 413 status = "disabled";
@@ -410,7 +418,7 @@
410 uart7: serial@48420000 { 418 uart7: serial@48420000 {
411 compatible = "ti,omap4-uart"; 419 compatible = "ti,omap4-uart";
412 reg = <0x48420000 0x100>; 420 reg = <0x48420000 0x100>;
413 interrupts-extended = <&gic GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>; 421 interrupts = <GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>;
414 ti,hwmods = "uart7"; 422 ti,hwmods = "uart7";
415 clock-frequency = <48000000>; 423 clock-frequency = <48000000>;
416 status = "disabled"; 424 status = "disabled";
@@ -419,7 +427,7 @@
419 uart8: serial@48422000 { 427 uart8: serial@48422000 {
420 compatible = "ti,omap4-uart"; 428 compatible = "ti,omap4-uart";
421 reg = <0x48422000 0x100>; 429 reg = <0x48422000 0x100>;
422 interrupts-extended = <&gic GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>; 430 interrupts = <GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>;
423 ti,hwmods = "uart8"; 431 ti,hwmods = "uart8";
424 clock-frequency = <48000000>; 432 clock-frequency = <48000000>;
425 status = "disabled"; 433 status = "disabled";
@@ -428,7 +436,7 @@
428 uart9: serial@48424000 { 436 uart9: serial@48424000 {
429 compatible = "ti,omap4-uart"; 437 compatible = "ti,omap4-uart";
430 reg = <0x48424000 0x100>; 438 reg = <0x48424000 0x100>;
431 interrupts-extended = <&gic GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>; 439 interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>;
432 ti,hwmods = "uart9"; 440 ti,hwmods = "uart9";
433 clock-frequency = <48000000>; 441 clock-frequency = <48000000>;
434 status = "disabled"; 442 status = "disabled";
@@ -437,7 +445,7 @@
437 uart10: serial@4ae2b000 { 445 uart10: serial@4ae2b000 {
438 compatible = "ti,omap4-uart"; 446 compatible = "ti,omap4-uart";
439 reg = <0x4ae2b000 0x100>; 447 reg = <0x4ae2b000 0x100>;
440 interrupts-extended = <&gic GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>; 448 interrupts = <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
441 ti,hwmods = "uart10"; 449 ti,hwmods = "uart10";
442 clock-frequency = <48000000>; 450 clock-frequency = <48000000>;
443 status = "disabled"; 451 status = "disabled";
@@ -1335,9 +1343,12 @@
1335 status = "disabled"; 1343 status = "disabled";
1336 }; 1344 };
1337 1345
1338 crossbar_mpu: crossbar@4a020000 { 1346 crossbar_mpu: crossbar@4a002a48 {
1339 compatible = "ti,irq-crossbar"; 1347 compatible = "ti,irq-crossbar";
1340 reg = <0x4a002a48 0x130>; 1348 reg = <0x4a002a48 0x130>;
1349 interrupt-controller;
1350 interrupt-parent = <&wakeupgen>;
1351 #interrupt-cells = <3>;
1341 ti,max-irqs = <160>; 1352 ti,max-irqs = <160>;
1342 ti,max-crossbar-sources = <MAX_SOURCES>; 1353 ti,max-crossbar-sources = <MAX_SOURCES>;
1343 ti,reg-size = <2>; 1354 ti,reg-size = <2>;
diff --git a/arch/arm/boot/dts/dra72-evm.dts b/arch/arm/boot/dts/dra72-evm.dts
index 40ed539ce474..daf28110d487 100644
--- a/arch/arm/boot/dts/dra72-evm.dts
+++ b/arch/arm/boot/dts/dra72-evm.dts
@@ -158,7 +158,6 @@
158 pinctrl-0 = <&tps65917_pins_default>; 158 pinctrl-0 = <&tps65917_pins_default>;
159 159
160 interrupts = <GIC_SPI 2 IRQ_TYPE_NONE>; /* IRQ_SYS_1N */ 160 interrupts = <GIC_SPI 2 IRQ_TYPE_NONE>; /* IRQ_SYS_1N */
161 interrupt-parent = <&gic>;
162 interrupt-controller; 161 interrupt-controller;
163 #interrupt-cells = <2>; 162 #interrupt-cells = <2>;
164 163
diff --git a/arch/arm/boot/dts/dra72x.dtsi b/arch/arm/boot/dts/dra72x.dtsi
index e5a3d23a3df1..f7fb0d0ef25a 100644
--- a/arch/arm/boot/dts/dra72x.dtsi
+++ b/arch/arm/boot/dts/dra72x.dtsi
@@ -25,6 +25,7 @@
25 25
26 pmu { 26 pmu {
27 compatible = "arm,cortex-a15-pmu"; 27 compatible = "arm,cortex-a15-pmu";
28 interrupts = <GIC_SPI DIRECT_IRQ(131) IRQ_TYPE_LEVEL_HIGH>; 28 interrupt-parent = <&wakeupgen>;
29 interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
29 }; 30 };
30}; 31};
diff --git a/arch/arm/boot/dts/dra74x.dtsi b/arch/arm/boot/dts/dra74x.dtsi
index 10173fab1a15..00eeed789b4b 100644
--- a/arch/arm/boot/dts/dra74x.dtsi
+++ b/arch/arm/boot/dts/dra74x.dtsi
@@ -41,8 +41,9 @@
41 41
42 pmu { 42 pmu {
43 compatible = "arm,cortex-a15-pmu"; 43 compatible = "arm,cortex-a15-pmu";
44 interrupts = <GIC_SPI DIRECT_IRQ(131) IRQ_TYPE_LEVEL_HIGH>, 44 interrupt-parent = <&wakeupgen>;
45 <GIC_SPI DIRECT_IRQ(132) IRQ_TYPE_LEVEL_HIGH>; 45 interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>,
46 <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>;
46 }; 47 };
47 48
48 ocp { 49 ocp {
diff --git a/arch/arm/boot/dts/exynos3250.dtsi b/arch/arm/boot/dts/exynos3250.dtsi
index ac6b0ae42caf..14ab515aa83c 100644
--- a/arch/arm/boot/dts/exynos3250.dtsi
+++ b/arch/arm/boot/dts/exynos3250.dtsi
@@ -131,6 +131,9 @@
131 pmu_system_controller: system-controller@10020000 { 131 pmu_system_controller: system-controller@10020000 {
132 compatible = "samsung,exynos3250-pmu", "syscon"; 132 compatible = "samsung,exynos3250-pmu", "syscon";
133 reg = <0x10020000 0x4000>; 133 reg = <0x10020000 0x4000>;
134 interrupt-controller;
135 #interrupt-cells = <3>;
136 interrupt-parent = <&gic>;
134 }; 137 };
135 138
136 mipi_phy: video-phy@10020710 { 139 mipi_phy: video-phy@10020710 {
@@ -185,6 +188,7 @@
185 compatible = "samsung,exynos3250-rtc"; 188 compatible = "samsung,exynos3250-rtc";
186 reg = <0x10070000 0x100>; 189 reg = <0x10070000 0x100>;
187 interrupts = <0 73 0>, <0 74 0>; 190 interrupts = <0 73 0>, <0 74 0>;
191 interrupt-parent = <&pmu_system_controller>;
188 status = "disabled"; 192 status = "disabled";
189 }; 193 };
190 194
diff --git a/arch/arm/boot/dts/exynos4.dtsi b/arch/arm/boot/dts/exynos4.dtsi
index 77ea547768f4..e20cdc24c3bb 100644
--- a/arch/arm/boot/dts/exynos4.dtsi
+++ b/arch/arm/boot/dts/exynos4.dtsi
@@ -154,6 +154,9 @@
154 pmu_system_controller: system-controller@10020000 { 154 pmu_system_controller: system-controller@10020000 {
155 compatible = "samsung,exynos4210-pmu", "syscon"; 155 compatible = "samsung,exynos4210-pmu", "syscon";
156 reg = <0x10020000 0x4000>; 156 reg = <0x10020000 0x4000>;
157 interrupt-controller;
158 #interrupt-cells = <3>;
159 interrupt-parent = <&gic>;
157 }; 160 };
158 161
159 dsi_0: dsi@11C80000 { 162 dsi_0: dsi@11C80000 {
@@ -266,6 +269,7 @@
266 rtc@10070000 { 269 rtc@10070000 {
267 compatible = "samsung,s3c6410-rtc"; 270 compatible = "samsung,s3c6410-rtc";
268 reg = <0x10070000 0x100>; 271 reg = <0x10070000 0x100>;
272 interrupt-parent = <&pmu_system_controller>;
269 interrupts = <0 44 0>, <0 45 0>; 273 interrupts = <0 44 0>, <0 45 0>;
270 clocks = <&clock CLK_RTC>; 274 clocks = <&clock CLK_RTC>;
271 clock-names = "rtc"; 275 clock-names = "rtc";
diff --git a/arch/arm/boot/dts/exynos5250.dtsi b/arch/arm/boot/dts/exynos5250.dtsi
index adbde1adad95..77f656eb8e6b 100644
--- a/arch/arm/boot/dts/exynos5250.dtsi
+++ b/arch/arm/boot/dts/exynos5250.dtsi
@@ -205,6 +205,9 @@
205 clock-names = "clkout16"; 205 clock-names = "clkout16";
206 clocks = <&clock CLK_FIN_PLL>; 206 clocks = <&clock CLK_FIN_PLL>;
207 #clock-cells = <1>; 207 #clock-cells = <1>;
208 interrupt-controller;
209 #interrupt-cells = <3>;
210 interrupt-parent = <&gic>;
208 }; 211 };
209 212
210 sysreg_system_controller: syscon@10050000 { 213 sysreg_system_controller: syscon@10050000 {
@@ -241,6 +244,7 @@
241 rtc: rtc@101E0000 { 244 rtc: rtc@101E0000 {
242 clocks = <&clock CLK_RTC>; 245 clocks = <&clock CLK_RTC>;
243 clock-names = "rtc"; 246 clock-names = "rtc";
247 interrupt-parent = <&pmu_system_controller>;
244 status = "disabled"; 248 status = "disabled";
245 }; 249 };
246 250
diff --git a/arch/arm/boot/dts/exynos5420.dtsi b/arch/arm/boot/dts/exynos5420.dtsi
index c0e98cf3514f..b3d2d53820e3 100644
--- a/arch/arm/boot/dts/exynos5420.dtsi
+++ b/arch/arm/boot/dts/exynos5420.dtsi
@@ -327,6 +327,7 @@
327 rtc: rtc@101E0000 { 327 rtc: rtc@101E0000 {
328 clocks = <&clock CLK_RTC>; 328 clocks = <&clock CLK_RTC>;
329 clock-names = "rtc"; 329 clock-names = "rtc";
330 interrupt-parent = <&pmu_system_controller>;
330 status = "disabled"; 331 status = "disabled";
331 }; 332 };
332 333
@@ -770,6 +771,9 @@
770 clock-names = "clkout16"; 771 clock-names = "clkout16";
771 clocks = <&clock CLK_FIN_PLL>; 772 clocks = <&clock CLK_FIN_PLL>;
772 #clock-cells = <1>; 773 #clock-cells = <1>;
774 interrupt-controller;
775 #interrupt-cells = <3>;
776 interrupt-parent = <&gic>;
773 }; 777 };
774 778
775 sysreg_system_controller: syscon@10050000 { 779 sysreg_system_controller: syscon@10050000 {
diff --git a/arch/arm/boot/dts/omap4-duovero.dtsi b/arch/arm/boot/dts/omap4-duovero.dtsi
index e860ccd9d09c..f2a94fa62552 100644
--- a/arch/arm/boot/dts/omap4-duovero.dtsi
+++ b/arch/arm/boot/dts/omap4-duovero.dtsi
@@ -173,14 +173,12 @@
173 twl: twl@48 { 173 twl: twl@48 {
174 reg = <0x48>; 174 reg = <0x48>;
175 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */ 175 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */
176 interrupt-parent = <&gic>;
177 }; 176 };
178 177
179 twl6040: twl@4b { 178 twl6040: twl@4b {
180 compatible = "ti,twl6040"; 179 compatible = "ti,twl6040";
181 reg = <0x4b>; 180 reg = <0x4b>;
182 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */ 181 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */
183 interrupt-parent = <&gic>;
184 ti,audpwron-gpio = <&gpio6 0 GPIO_ACTIVE_HIGH>; /* gpio_160 */ 182 ti,audpwron-gpio = <&gpio6 0 GPIO_ACTIVE_HIGH>; /* gpio_160 */
185 183
186 vio-supply = <&v1v8>; 184 vio-supply = <&v1v8>;
diff --git a/arch/arm/boot/dts/omap4-panda-common.dtsi b/arch/arm/boot/dts/omap4-panda-common.dtsi
index 150513506c19..7c15fb2e2fe4 100644
--- a/arch/arm/boot/dts/omap4-panda-common.dtsi
+++ b/arch/arm/boot/dts/omap4-panda-common.dtsi
@@ -372,7 +372,6 @@
372 reg = <0x48>; 372 reg = <0x48>;
373 /* IRQ# = 7 */ 373 /* IRQ# = 7 */
374 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */ 374 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */
375 interrupt-parent = <&gic>;
376 }; 375 };
377 376
378 twl6040: twl@4b { 377 twl6040: twl@4b {
@@ -384,7 +383,6 @@
384 383
385 /* IRQ# = 119 */ 384 /* IRQ# = 119 */
386 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */ 385 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */
387 interrupt-parent = <&gic>;
388 ti,audpwron-gpio = <&gpio4 31 GPIO_ACTIVE_HIGH>; /* gpio line 127 */ 386 ti,audpwron-gpio = <&gpio4 31 GPIO_ACTIVE_HIGH>; /* gpio line 127 */
389 387
390 vio-supply = <&v1v8>; 388 vio-supply = <&v1v8>;
@@ -479,17 +477,17 @@
479}; 477};
480 478
481&uart2 { 479&uart2 {
482 interrupts-extended = <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH 480 interrupts-extended = <&wakeupgen GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH
483 &omap4_pmx_core OMAP4_UART2_RX>; 481 &omap4_pmx_core OMAP4_UART2_RX>;
484}; 482};
485 483
486&uart3 { 484&uart3 {
487 interrupts-extended = <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH 485 interrupts-extended = <&wakeupgen GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH
488 &omap4_pmx_core OMAP4_UART3_RX>; 486 &omap4_pmx_core OMAP4_UART3_RX>;
489}; 487};
490 488
491&uart4 { 489&uart4 {
492 interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH 490 interrupts-extended = <&wakeupgen GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH
493 &omap4_pmx_core OMAP4_UART4_RX>; 491 &omap4_pmx_core OMAP4_UART4_RX>;
494}; 492};
495 493
diff --git a/arch/arm/boot/dts/omap4-sdp.dts b/arch/arm/boot/dts/omap4-sdp.dts
index 3e1da43068f6..8aca8dae968a 100644
--- a/arch/arm/boot/dts/omap4-sdp.dts
+++ b/arch/arm/boot/dts/omap4-sdp.dts
@@ -363,7 +363,6 @@
363 reg = <0x48>; 363 reg = <0x48>;
364 /* SPI = 0, IRQ# = 7, 4 = active high level-sensitive */ 364 /* SPI = 0, IRQ# = 7, 4 = active high level-sensitive */
365 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */ 365 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */
366 interrupt-parent = <&gic>;
367 }; 366 };
368 367
369 twl6040: twl@4b { 368 twl6040: twl@4b {
@@ -375,7 +374,6 @@
375 374
376 /* SPI = 0, IRQ# = 119, 4 = active high level-sensitive */ 375 /* SPI = 0, IRQ# = 119, 4 = active high level-sensitive */
377 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */ 376 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */
378 interrupt-parent = <&gic>;
379 ti,audpwron-gpio = <&gpio4 31 0>; /* gpio line 127 */ 377 ti,audpwron-gpio = <&gpio4 31 0>; /* gpio line 127 */
380 378
381 vio-supply = <&v1v8>; 379 vio-supply = <&v1v8>;
@@ -570,21 +568,21 @@
570}; 568};
571 569
572&uart2 { 570&uart2 {
573 interrupts-extended = <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH 571 interrupts-extended = <&wakeupgen GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH
574 &omap4_pmx_core OMAP4_UART2_RX>; 572 &omap4_pmx_core OMAP4_UART2_RX>;
575 pinctrl-names = "default"; 573 pinctrl-names = "default";
576 pinctrl-0 = <&uart2_pins>; 574 pinctrl-0 = <&uart2_pins>;
577}; 575};
578 576
579&uart3 { 577&uart3 {
580 interrupts-extended = <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH 578 interrupts-extended = <&wakeupgen GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH
581 &omap4_pmx_core OMAP4_UART3_RX>; 579 &omap4_pmx_core OMAP4_UART3_RX>;
582 pinctrl-names = "default"; 580 pinctrl-names = "default";
583 pinctrl-0 = <&uart3_pins>; 581 pinctrl-0 = <&uart3_pins>;
584}; 582};
585 583
586&uart4 { 584&uart4 {
587 interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH 585 interrupts-extended = <&wakeupgen GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH
588 &omap4_pmx_core OMAP4_UART4_RX>; 586 &omap4_pmx_core OMAP4_UART4_RX>;
589 pinctrl-names = "default"; 587 pinctrl-names = "default";
590 pinctrl-0 = <&uart4_pins>; 588 pinctrl-0 = <&uart4_pins>;
diff --git a/arch/arm/boot/dts/omap4-var-som-om44.dtsi b/arch/arm/boot/dts/omap4-var-som-om44.dtsi
index 062701e1a898..a4f1ba2e1903 100644
--- a/arch/arm/boot/dts/omap4-var-som-om44.dtsi
+++ b/arch/arm/boot/dts/omap4-var-som-om44.dtsi
@@ -185,7 +185,6 @@
185 reg = <0x48>; 185 reg = <0x48>;
186 /* SPI = 0, IRQ# = 7, 4 = active high level-sensitive */ 186 /* SPI = 0, IRQ# = 7, 4 = active high level-sensitive */
187 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */ 187 interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */
188 interrupt-parent = <&gic>;
189 }; 188 };
190 189
191 twl6040: twl@4b { 190 twl6040: twl@4b {
@@ -197,7 +196,6 @@
197 196
198 /* SPI = 0, IRQ# = 119, 4 = active high level-sensitive */ 197 /* SPI = 0, IRQ# = 119, 4 = active high level-sensitive */
199 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */ 198 interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_2N cascaded to gic */
200 interrupt-parent = <&gic>;
201 ti,audpwron-gpio = <&gpio6 22 0>; /* gpio 182 */ 199 ti,audpwron-gpio = <&gpio6 22 0>; /* gpio 182 */
202 200
203 vio-supply = <&v1v8>; 201 vio-supply = <&v1v8>;
diff --git a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi
index 87401d9f4d8b..f2091d1c9c36 100644
--- a/arch/arm/boot/dts/omap4.dtsi
+++ b/arch/arm/boot/dts/omap4.dtsi
@@ -14,7 +14,7 @@
14 14
15/ { 15/ {
16 compatible = "ti,omap4430", "ti,omap4"; 16 compatible = "ti,omap4430", "ti,omap4";
17 interrupt-parent = <&gic>; 17 interrupt-parent = <&wakeupgen>;
18 18
19 aliases { 19 aliases {
20 i2c0 = &i2c1; 20 i2c0 = &i2c1;
@@ -56,6 +56,7 @@
56 #interrupt-cells = <3>; 56 #interrupt-cells = <3>;
57 reg = <0x48241000 0x1000>, 57 reg = <0x48241000 0x1000>,
58 <0x48240100 0x0100>; 58 <0x48240100 0x0100>;
59 interrupt-parent = <&gic>;
59 }; 60 };
60 61
61 L2: l2-cache-controller@48242000 { 62 L2: l2-cache-controller@48242000 {
@@ -70,6 +71,15 @@
70 clocks = <&mpu_periphclk>; 71 clocks = <&mpu_periphclk>;
71 reg = <0x48240600 0x20>; 72 reg = <0x48240600 0x20>;
72 interrupts = <GIC_PPI 13 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_HIGH)>; 73 interrupts = <GIC_PPI 13 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_HIGH)>;
74 interrupt-parent = <&gic>;
75 };
76
77 wakeupgen: interrupt-controller@48281000 {
78 compatible = "ti,omap4-wugen-mpu";
79 interrupt-controller;
80 #interrupt-cells = <3>;
81 reg = <0x48281000 0x1000>;
82 interrupt-parent = <&gic>;
73 }; 83 };
74 84
75 /* 85 /*
@@ -319,7 +329,7 @@
319 uart2: serial@4806c000 { 329 uart2: serial@4806c000 {
320 compatible = "ti,omap4-uart"; 330 compatible = "ti,omap4-uart";
321 reg = <0x4806c000 0x100>; 331 reg = <0x4806c000 0x100>;
322 interrupts-extended = <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>; 332 interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
323 ti,hwmods = "uart2"; 333 ti,hwmods = "uart2";
324 clock-frequency = <48000000>; 334 clock-frequency = <48000000>;
325 }; 335 };
@@ -327,7 +337,7 @@
327 uart3: serial@48020000 { 337 uart3: serial@48020000 {
328 compatible = "ti,omap4-uart"; 338 compatible = "ti,omap4-uart";
329 reg = <0x48020000 0x100>; 339 reg = <0x48020000 0x100>;
330 interrupts-extended = <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>; 340 interrupts = <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
331 ti,hwmods = "uart3"; 341 ti,hwmods = "uart3";
332 clock-frequency = <48000000>; 342 clock-frequency = <48000000>;
333 }; 343 };
@@ -335,7 +345,7 @@
335 uart4: serial@4806e000 { 345 uart4: serial@4806e000 {
336 compatible = "ti,omap4-uart"; 346 compatible = "ti,omap4-uart";
337 reg = <0x4806e000 0x100>; 347 reg = <0x4806e000 0x100>;
338 interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>; 348 interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
339 ti,hwmods = "uart4"; 349 ti,hwmods = "uart4";
340 clock-frequency = <48000000>; 350 clock-frequency = <48000000>;
341 }; 351 };
diff --git a/arch/arm/boot/dts/omap5-cm-t54.dts b/arch/arm/boot/dts/omap5-cm-t54.dts
index b54b271e153b..61ad2ea34720 100644
--- a/arch/arm/boot/dts/omap5-cm-t54.dts
+++ b/arch/arm/boot/dts/omap5-cm-t54.dts
@@ -412,7 +412,6 @@
412 palmas: palmas@48 { 412 palmas: palmas@48 {
413 compatible = "ti,palmas"; 413 compatible = "ti,palmas";
414 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* IRQ_SYS_1N */ 414 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* IRQ_SYS_1N */
415 interrupt-parent = <&gic>;
416 reg = <0x48>; 415 reg = <0x48>;
417 interrupt-controller; 416 interrupt-controller;
418 #interrupt-cells = <2>; 417 #interrupt-cells = <2>;
diff --git a/arch/arm/boot/dts/omap5-uevm.dts b/arch/arm/boot/dts/omap5-uevm.dts
index 159720d6c956..74777a6e200a 100644
--- a/arch/arm/boot/dts/omap5-uevm.dts
+++ b/arch/arm/boot/dts/omap5-uevm.dts
@@ -311,7 +311,6 @@
311 palmas: palmas@48 { 311 palmas: palmas@48 {
312 compatible = "ti,palmas"; 312 compatible = "ti,palmas";
313 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* IRQ_SYS_1N */ 313 interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* IRQ_SYS_1N */
314 interrupt-parent = <&gic>;
315 reg = <0x48>; 314 reg = <0x48>;
316 interrupt-controller; 315 interrupt-controller;
317 #interrupt-cells = <2>; 316 #interrupt-cells = <2>;
@@ -521,7 +520,6 @@
521 pinctrl-0 = <&twl6040_pins>; 520 pinctrl-0 = <&twl6040_pins>;
522 521
523 interrupts = <GIC_SPI 119 IRQ_TYPE_NONE>; /* IRQ_SYS_2N cascaded to gic */ 522 interrupts = <GIC_SPI 119 IRQ_TYPE_NONE>; /* IRQ_SYS_2N cascaded to gic */
524 interrupt-parent = <&gic>;
525 ti,audpwron-gpio = <&gpio5 13 0>; /* gpio line 141 */ 523 ti,audpwron-gpio = <&gpio5 13 0>; /* gpio line 141 */
526 524
527 vio-supply = <&smps7_reg>; 525 vio-supply = <&smps7_reg>;
diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi
index 4a485b63a141..77b5f70d0ebc 100644
--- a/arch/arm/boot/dts/omap5.dtsi
+++ b/arch/arm/boot/dts/omap5.dtsi
@@ -18,7 +18,7 @@
18 #size-cells = <1>; 18 #size-cells = <1>;
19 19
20 compatible = "ti,omap5"; 20 compatible = "ti,omap5";
21 interrupt-parent = <&gic>; 21 interrupt-parent = <&wakeupgen>;
22 22
23 aliases { 23 aliases {
24 i2c0 = &i2c1; 24 i2c0 = &i2c1;
@@ -79,6 +79,7 @@
79 <GIC_PPI 14 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_LOW)>, 79 <GIC_PPI 14 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_LOW)>,
80 <GIC_PPI 11 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_LOW)>, 80 <GIC_PPI 11 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_LOW)>,
81 <GIC_PPI 10 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_LOW)>; 81 <GIC_PPI 10 (GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_LOW)>;
82 interrupt-parent = <&gic>;
82 }; 83 };
83 84
84 pmu { 85 pmu {
@@ -95,6 +96,15 @@
95 <0x48212000 0x1000>, 96 <0x48212000 0x1000>,
96 <0x48214000 0x2000>, 97 <0x48214000 0x2000>,
97 <0x48216000 0x2000>; 98 <0x48216000 0x2000>;
99 interrupt-parent = <&gic>;
100 };
101
102 wakeupgen: interrupt-controller@48281000 {
103 compatible = "ti,omap5-wugen-mpu", "ti,omap4-wugen-mpu";
104 interrupt-controller;
105 #interrupt-cells = <3>;
106 reg = <0x48281000 0x1000>;
107 interrupt-parent = <&gic>;
98 }; 108 };
99 109
100 /* 110 /*
@@ -458,7 +468,7 @@
458 uart1: serial@4806a000 { 468 uart1: serial@4806a000 {
459 compatible = "ti,omap4-uart"; 469 compatible = "ti,omap4-uart";
460 reg = <0x4806a000 0x100>; 470 reg = <0x4806a000 0x100>;
461 interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>; 471 interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
462 ti,hwmods = "uart1"; 472 ti,hwmods = "uart1";
463 clock-frequency = <48000000>; 473 clock-frequency = <48000000>;
464 }; 474 };
@@ -466,7 +476,7 @@
466 uart2: serial@4806c000 { 476 uart2: serial@4806c000 {
467 compatible = "ti,omap4-uart"; 477 compatible = "ti,omap4-uart";
468 reg = <0x4806c000 0x100>; 478 reg = <0x4806c000 0x100>;
469 interrupts-extended = <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>; 479 interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
470 ti,hwmods = "uart2"; 480 ti,hwmods = "uart2";
471 clock-frequency = <48000000>; 481 clock-frequency = <48000000>;
472 }; 482 };
@@ -474,7 +484,7 @@
474 uart3: serial@48020000 { 484 uart3: serial@48020000 {
475 compatible = "ti,omap4-uart"; 485 compatible = "ti,omap4-uart";
476 reg = <0x48020000 0x100>; 486 reg = <0x48020000 0x100>;
477 interrupts-extended = <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>; 487 interrupts = <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
478 ti,hwmods = "uart3"; 488 ti,hwmods = "uart3";
479 clock-frequency = <48000000>; 489 clock-frequency = <48000000>;
480 }; 490 };
@@ -482,7 +492,7 @@
482 uart4: serial@4806e000 { 492 uart4: serial@4806e000 {
483 compatible = "ti,omap4-uart"; 493 compatible = "ti,omap4-uart";
484 reg = <0x4806e000 0x100>; 494 reg = <0x4806e000 0x100>;
485 interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>; 495 interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
486 ti,hwmods = "uart4"; 496 ti,hwmods = "uart4";
487 clock-frequency = <48000000>; 497 clock-frequency = <48000000>;
488 }; 498 };
@@ -490,7 +500,7 @@
490 uart5: serial@48066000 { 500 uart5: serial@48066000 {
491 compatible = "ti,omap4-uart"; 501 compatible = "ti,omap4-uart";
492 reg = <0x48066000 0x100>; 502 reg = <0x48066000 0x100>;
493 interrupts-extended = <&gic GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>; 503 interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
494 ti,hwmods = "uart5"; 504 ti,hwmods = "uart5";
495 clock-frequency = <48000000>; 505 clock-frequency = <48000000>;
496 }; 506 };
@@ -498,7 +508,7 @@
498 uart6: serial@48068000 { 508 uart6: serial@48068000 {
499 compatible = "ti,omap4-uart"; 509 compatible = "ti,omap4-uart";
500 reg = <0x48068000 0x100>; 510 reg = <0x48068000 0x100>;
501 interrupts-extended = <&gic GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>; 511 interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
502 ti,hwmods = "uart6"; 512 ti,hwmods = "uart6";
503 clock-frequency = <48000000>; 513 clock-frequency = <48000000>;
504 }; 514 };
@@ -883,14 +893,12 @@
883 usbhsohci: ohci@4a064800 { 893 usbhsohci: ohci@4a064800 {
884 compatible = "ti,ohci-omap3"; 894 compatible = "ti,ohci-omap3";
885 reg = <0x4a064800 0x400>; 895 reg = <0x4a064800 0x400>;
886 interrupt-parent = <&gic>;
887 interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>; 896 interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
888 }; 897 };
889 898
890 usbhsehci: ehci@4a064c00 { 899 usbhsehci: ehci@4a064c00 {
891 compatible = "ti,ehci-omap"; 900 compatible = "ti,ehci-omap";
892 reg = <0x4a064c00 0x400>; 901 reg = <0x4a064c00 0x400>;
893 interrupt-parent = <&gic>;
894 interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>; 902 interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
895 }; 903 };
896 }; 904 };
diff --git a/arch/arm/boot/dts/sama5d3_emac.dtsi b/arch/arm/boot/dts/sama5d3_emac.dtsi
index fe2af9276312..b4544cf11bad 100644
--- a/arch/arm/boot/dts/sama5d3_emac.dtsi
+++ b/arch/arm/boot/dts/sama5d3_emac.dtsi
@@ -41,7 +41,7 @@
41 }; 41 };
42 42
43 macb1: ethernet@f802c000 { 43 macb1: ethernet@f802c000 {
44 compatible = "cdns,at32ap7000-macb", "cdns,macb"; 44 compatible = "cdns,at91sam9260-macb", "cdns,macb";
45 reg = <0xf802c000 0x100>; 45 reg = <0xf802c000 0x100>;
46 interrupts = <35 IRQ_TYPE_LEVEL_HIGH 3>; 46 interrupts = <35 IRQ_TYPE_LEVEL_HIGH 3>;
47 pinctrl-names = "default"; 47 pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/stih416.dtsi b/arch/arm/boot/dts/stih416.dtsi
index ea28ebadab1a..eeb7afecbbe6 100644
--- a/arch/arm/boot/dts/stih416.dtsi
+++ b/arch/arm/boot/dts/stih416.dtsi
@@ -10,7 +10,7 @@
10#include "stih416-clock.dtsi" 10#include "stih416-clock.dtsi"
11#include "stih416-pinctrl.dtsi" 11#include "stih416-pinctrl.dtsi"
12 12
13#include <dt-bindings/phy/phy-miphy365x.h> 13#include <dt-bindings/phy/phy.h>
14#include <dt-bindings/interrupt-controller/arm-gic.h> 14#include <dt-bindings/interrupt-controller/arm-gic.h>
15#include <dt-bindings/reset-controller/stih416-resets.h> 15#include <dt-bindings/reset-controller/stih416-resets.h>
16/ { 16/ {
@@ -306,7 +306,7 @@
306 reg = <0xfe380000 0x1000>; 306 reg = <0xfe380000 0x1000>;
307 interrupts = <GIC_SPI 157 IRQ_TYPE_NONE>; 307 interrupts = <GIC_SPI 157 IRQ_TYPE_NONE>;
308 interrupt-names = "hostc"; 308 interrupt-names = "hostc";
309 phys = <&phy_port0 MIPHY_TYPE_SATA>; 309 phys = <&phy_port0 PHY_TYPE_SATA>;
310 phy-names = "sata-phy"; 310 phy-names = "sata-phy";
311 resets = <&powerdown STIH416_SATA0_POWERDOWN>, 311 resets = <&powerdown STIH416_SATA0_POWERDOWN>,
312 <&softreset STIH416_SATA0_SOFTRESET>; 312 <&softreset STIH416_SATA0_SOFTRESET>;
diff --git a/arch/arm/boot/dts/tegra114.dtsi b/arch/arm/boot/dts/tegra114.dtsi
index 4296b5398bf5..f58a3d9d5f13 100644
--- a/arch/arm/boot/dts/tegra114.dtsi
+++ b/arch/arm/boot/dts/tegra114.dtsi
@@ -8,7 +8,7 @@
8 8
9/ { 9/ {
10 compatible = "nvidia,tegra114"; 10 compatible = "nvidia,tegra114";
11 interrupt-parent = <&gic>; 11 interrupt-parent = <&lic>;
12 12
13 host1x@50000000 { 13 host1x@50000000 {
14 compatible = "nvidia,tegra114-host1x", "simple-bus"; 14 compatible = "nvidia,tegra114-host1x", "simple-bus";
@@ -134,6 +134,19 @@
134 <0x50046000 0x2000>; 134 <0x50046000 0x2000>;
135 interrupts = <GIC_PPI 9 135 interrupts = <GIC_PPI 9
136 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>; 136 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
137 interrupt-parent = <&gic>;
138 };
139
140 lic: interrupt-controller@60004000 {
141 compatible = "nvidia,tegra114-ictlr", "nvidia,tegra30-ictlr";
142 reg = <0x60004000 0x100>,
143 <0x60004100 0x50>,
144 <0x60004200 0x50>,
145 <0x60004300 0x50>,
146 <0x60004400 0x50>;
147 interrupt-controller;
148 #interrupt-cells = <3>;
149 interrupt-parent = <&gic>;
137 }; 150 };
138 151
139 timer@60005000 { 152 timer@60005000 {
@@ -766,5 +779,6 @@
766 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, 779 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
767 <GIC_PPI 10 780 <GIC_PPI 10
768 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; 781 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
782 interrupt-parent = <&gic>;
769 }; 783 };
770}; 784};
diff --git a/arch/arm/boot/dts/tegra124.dtsi b/arch/arm/boot/dts/tegra124.dtsi
index 4be06c6ea0c8..db85695aa7aa 100644
--- a/arch/arm/boot/dts/tegra124.dtsi
+++ b/arch/arm/boot/dts/tegra124.dtsi
@@ -10,7 +10,7 @@
10 10
11/ { 11/ {
12 compatible = "nvidia,tegra124"; 12 compatible = "nvidia,tegra124";
13 interrupt-parent = <&gic>; 13 interrupt-parent = <&lic>;
14 #address-cells = <2>; 14 #address-cells = <2>;
15 #size-cells = <2>; 15 #size-cells = <2>;
16 16
@@ -173,6 +173,7 @@
173 <0x0 0x50046000 0x0 0x2000>; 173 <0x0 0x50046000 0x0 0x2000>;
174 interrupts = <GIC_PPI 9 174 interrupts = <GIC_PPI 9
175 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>; 175 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
176 interrupt-parent = <&gic>;
176 }; 177 };
177 178
178 gpu@0,57000000 { 179 gpu@0,57000000 {
@@ -190,6 +191,18 @@
190 status = "disabled"; 191 status = "disabled";
191 }; 192 };
192 193
194 lic: interrupt-controller@60004000 {
195 compatible = "nvidia,tegra124-ictlr", "nvidia,tegra30-ictlr";
196 reg = <0x0 0x60004000 0x0 0x100>,
197 <0x0 0x60004100 0x0 0x100>,
198 <0x0 0x60004200 0x0 0x100>,
199 <0x0 0x60004300 0x0 0x100>,
200 <0x0 0x60004400 0x0 0x100>;
201 interrupt-controller;
202 #interrupt-cells = <3>;
203 interrupt-parent = <&gic>;
204 };
205
193 timer@0,60005000 { 206 timer@0,60005000 {
194 compatible = "nvidia,tegra124-timer", "nvidia,tegra20-timer"; 207 compatible = "nvidia,tegra124-timer", "nvidia,tegra20-timer";
195 reg = <0x0 0x60005000 0x0 0x400>; 208 reg = <0x0 0x60005000 0x0 0x400>;
@@ -955,5 +968,6 @@
955 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, 968 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
956 <GIC_PPI 10 969 <GIC_PPI 10
957 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; 970 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
971 interrupt-parent = <&gic>;
958 }; 972 };
959}; 973};
diff --git a/arch/arm/boot/dts/tegra20.dtsi b/arch/arm/boot/dts/tegra20.dtsi
index e5527f742696..adf6b048d0bb 100644
--- a/arch/arm/boot/dts/tegra20.dtsi
+++ b/arch/arm/boot/dts/tegra20.dtsi
@@ -7,7 +7,7 @@
7 7
8/ { 8/ {
9 compatible = "nvidia,tegra20"; 9 compatible = "nvidia,tegra20";
10 interrupt-parent = <&intc>; 10 interrupt-parent = <&lic>;
11 11
12 host1x@50000000 { 12 host1x@50000000 {
13 compatible = "nvidia,tegra20-host1x", "simple-bus"; 13 compatible = "nvidia,tegra20-host1x", "simple-bus";
@@ -142,6 +142,7 @@
142 142
143 timer@50040600 { 143 timer@50040600 {
144 compatible = "arm,cortex-a9-twd-timer"; 144 compatible = "arm,cortex-a9-twd-timer";
145 interrupt-parent = <&intc>;
145 reg = <0x50040600 0x20>; 146 reg = <0x50040600 0x20>;
146 interrupts = <GIC_PPI 13 147 interrupts = <GIC_PPI 13
147 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>; 148 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
@@ -154,6 +155,7 @@
154 0x50040100 0x0100>; 155 0x50040100 0x0100>;
155 interrupt-controller; 156 interrupt-controller;
156 #interrupt-cells = <3>; 157 #interrupt-cells = <3>;
158 interrupt-parent = <&intc>;
157 }; 159 };
158 160
159 cache-controller@50043000 { 161 cache-controller@50043000 {
@@ -165,6 +167,17 @@
165 cache-level = <2>; 167 cache-level = <2>;
166 }; 168 };
167 169
170 lic: interrupt-controller@60004000 {
171 compatible = "nvidia,tegra20-ictlr";
172 reg = <0x60004000 0x100>,
173 <0x60004100 0x50>,
174 <0x60004200 0x50>,
175 <0x60004300 0x50>;
176 interrupt-controller;
177 #interrupt-cells = <3>;
178 interrupt-parent = <&intc>;
179 };
180
168 timer@60005000 { 181 timer@60005000 {
169 compatible = "nvidia,tegra20-timer"; 182 compatible = "nvidia,tegra20-timer";
170 reg = <0x60005000 0x60>; 183 reg = <0x60005000 0x60>;
diff --git a/arch/arm/boot/dts/tegra30.dtsi b/arch/arm/boot/dts/tegra30.dtsi
index db4810df142c..60e205a0f63d 100644
--- a/arch/arm/boot/dts/tegra30.dtsi
+++ b/arch/arm/boot/dts/tegra30.dtsi
@@ -8,7 +8,7 @@
8 8
9/ { 9/ {
10 compatible = "nvidia,tegra30"; 10 compatible = "nvidia,tegra30";
11 interrupt-parent = <&intc>; 11 interrupt-parent = <&lic>;
12 12
13 pcie-controller@00003000 { 13 pcie-controller@00003000 {
14 compatible = "nvidia,tegra30-pcie"; 14 compatible = "nvidia,tegra30-pcie";
@@ -228,6 +228,7 @@
228 timer@50040600 { 228 timer@50040600 {
229 compatible = "arm,cortex-a9-twd-timer"; 229 compatible = "arm,cortex-a9-twd-timer";
230 reg = <0x50040600 0x20>; 230 reg = <0x50040600 0x20>;
231 interrupt-parent = <&intc>;
231 interrupts = <GIC_PPI 13 232 interrupts = <GIC_PPI 13
232 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>; 233 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
233 clocks = <&tegra_car TEGRA30_CLK_TWD>; 234 clocks = <&tegra_car TEGRA30_CLK_TWD>;
@@ -239,6 +240,7 @@
239 0x50040100 0x0100>; 240 0x50040100 0x0100>;
240 interrupt-controller; 241 interrupt-controller;
241 #interrupt-cells = <3>; 242 #interrupt-cells = <3>;
243 interrupt-parent = <&intc>;
242 }; 244 };
243 245
244 cache-controller@50043000 { 246 cache-controller@50043000 {
@@ -250,6 +252,18 @@
250 cache-level = <2>; 252 cache-level = <2>;
251 }; 253 };
252 254
255 lic: interrupt-controller@60004000 {
256 compatible = "nvidia,tegra30-ictlr";
257 reg = <0x60004000 0x100>,
258 <0x60004100 0x50>,
259 <0x60004200 0x50>,
260 <0x60004300 0x50>,
261 <0x60004400 0x50>;
262 interrupt-controller;
263 #interrupt-cells = <3>;
264 interrupt-parent = <&intc>;
265 };
266
253 timer@60005000 { 267 timer@60005000 {
254 compatible = "nvidia,tegra30-timer", "nvidia,tegra20-timer"; 268 compatible = "nvidia,tegra30-timer", "nvidia,tegra20-timer";
255 reg = <0x60005000 0x400>; 269 reg = <0x60005000 0x400>;
diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c
index 6eaddc47c43d..37dc0fe1093f 100644
--- a/arch/arm/common/bL_switcher.c
+++ b/arch/arm/common/bL_switcher.c
@@ -151,8 +151,6 @@ static int bL_switch_to(unsigned int new_cluster_id)
151 unsigned int mpidr, this_cpu, that_cpu; 151 unsigned int mpidr, this_cpu, that_cpu;
152 unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster; 152 unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster;
153 struct completion inbound_alive; 153 struct completion inbound_alive;
154 struct tick_device *tdev;
155 enum clock_event_mode tdev_mode;
156 long volatile *handshake_ptr; 154 long volatile *handshake_ptr;
157 int ipi_nr, ret; 155 int ipi_nr, ret;
158 156
@@ -219,13 +217,7 @@ static int bL_switch_to(unsigned int new_cluster_id)
219 /* redirect GIC's SGIs to our counterpart */ 217 /* redirect GIC's SGIs to our counterpart */
220 gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]); 218 gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]);
221 219
222 tdev = tick_get_device(this_cpu); 220 tick_suspend_local();
223 if (tdev && !cpumask_equal(tdev->evtdev->cpumask, cpumask_of(this_cpu)))
224 tdev = NULL;
225 if (tdev) {
226 tdev_mode = tdev->evtdev->mode;
227 clockevents_set_mode(tdev->evtdev, CLOCK_EVT_MODE_SHUTDOWN);
228 }
229 221
230 ret = cpu_pm_enter(); 222 ret = cpu_pm_enter();
231 223
@@ -251,11 +243,7 @@ static int bL_switch_to(unsigned int new_cluster_id)
251 243
252 ret = cpu_pm_exit(); 244 ret = cpu_pm_exit();
253 245
254 if (tdev) { 246 tick_resume_local();
255 clockevents_set_mode(tdev->evtdev, tdev_mode);
256 clockevents_program_event(tdev->evtdev,
257 tdev->evtdev->next_event, 1);
258 }
259 247
260 trace_cpu_migrate_finish(ktime_get_real_ns(), ib_mpidr); 248 trace_cpu_migrate_finish(ktime_get_real_ns(), ib_mpidr);
261 local_fiq_enable(); 249 local_fiq_enable();
diff --git a/arch/arm/configs/badge4_defconfig b/arch/arm/configs/badge4_defconfig
index 0494c8f229a2..d59009878312 100644
--- a/arch/arm/configs/badge4_defconfig
+++ b/arch/arm/configs/badge4_defconfig
@@ -12,7 +12,6 @@ CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
12CONFIG_FPE_NWFPE=y 12CONFIG_FPE_NWFPE=y
13CONFIG_BINFMT_AOUT=m 13CONFIG_BINFMT_AOUT=m
14CONFIG_BINFMT_MISC=m 14CONFIG_BINFMT_MISC=m
15CONFIG_ARTHUR=m
16CONFIG_NET=y 15CONFIG_NET=y
17CONFIG_PACKET=y 16CONFIG_PACKET=y
18CONFIG_UNIX=y 17CONFIG_UNIX=y
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
new file mode 100644
index 000000000000..8da2207b0072
--- /dev/null
+++ b/arch/arm/crypto/Kconfig
@@ -0,0 +1,130 @@
1
2menuconfig ARM_CRYPTO
3 bool "ARM Accelerated Cryptographic Algorithms"
4 depends on ARM
5 help
6 Say Y here to choose from a selection of cryptographic algorithms
7 implemented using ARM specific CPU features or instructions.
8
9if ARM_CRYPTO
10
11config CRYPTO_SHA1_ARM
12 tristate "SHA1 digest algorithm (ARM-asm)"
13 select CRYPTO_SHA1
14 select CRYPTO_HASH
15 help
16 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
17 using optimized ARM assembler.
18
19config CRYPTO_SHA1_ARM_NEON
20 tristate "SHA1 digest algorithm (ARM NEON)"
21 depends on KERNEL_MODE_NEON
22 select CRYPTO_SHA1_ARM
23 select CRYPTO_SHA1
24 select CRYPTO_HASH
25 help
26 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
27 using optimized ARM NEON assembly, when NEON instructions are
28 available.
29
30config CRYPTO_SHA1_ARM_CE
31 tristate "SHA1 digest algorithm (ARM v8 Crypto Extensions)"
32 depends on KERNEL_MODE_NEON
33 select CRYPTO_SHA1_ARM
34 select CRYPTO_HASH
35 help
36 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
37 using special ARMv8 Crypto Extensions.
38
39config CRYPTO_SHA2_ARM_CE
40 tristate "SHA-224/256 digest algorithm (ARM v8 Crypto Extensions)"
41 depends on KERNEL_MODE_NEON
42 select CRYPTO_SHA256_ARM
43 select CRYPTO_HASH
44 help
45 SHA-256 secure hash standard (DFIPS 180-2) implemented
46 using special ARMv8 Crypto Extensions.
47
48config CRYPTO_SHA256_ARM
49 tristate "SHA-224/256 digest algorithm (ARM-asm and NEON)"
50 select CRYPTO_HASH
51 depends on !CPU_V7M
52 help
53 SHA-256 secure hash standard (DFIPS 180-2) implemented
54 using optimized ARM assembler and NEON, when available.
55
56config CRYPTO_SHA512_ARM_NEON
57 tristate "SHA384 and SHA512 digest algorithm (ARM NEON)"
58 depends on KERNEL_MODE_NEON
59 select CRYPTO_SHA512
60 select CRYPTO_HASH
61 help
62 SHA-512 secure hash standard (DFIPS 180-2) implemented
63 using ARM NEON instructions, when available.
64
65 This version of SHA implements a 512 bit hash with 256 bits of
66 security against collision attacks.
67
68 This code also includes SHA-384, a 384 bit hash with 192 bits
69 of security against collision attacks.
70
71config CRYPTO_AES_ARM
72 tristate "AES cipher algorithms (ARM-asm)"
73 depends on ARM
74 select CRYPTO_ALGAPI
75 select CRYPTO_AES
76 help
77 Use optimized AES assembler routines for ARM platforms.
78
79 AES cipher algorithms (FIPS-197). AES uses the Rijndael
80 algorithm.
81
82 Rijndael appears to be consistently a very good performer in
83 both hardware and software across a wide range of computing
84 environments regardless of its use in feedback or non-feedback
85 modes. Its key setup time is excellent, and its key agility is
86 good. Rijndael's very low memory requirements make it very well
87 suited for restricted-space environments, in which it also
88 demonstrates excellent performance. Rijndael's operations are
89 among the easiest to defend against power and timing attacks.
90
91 The AES specifies three key sizes: 128, 192 and 256 bits
92
93 See <http://csrc.nist.gov/encryption/aes/> for more information.
94
95config CRYPTO_AES_ARM_BS
96 tristate "Bit sliced AES using NEON instructions"
97 depends on KERNEL_MODE_NEON
98 select CRYPTO_ALGAPI
99 select CRYPTO_AES_ARM
100 select CRYPTO_ABLK_HELPER
101 help
102 Use a faster and more secure NEON based implementation of AES in CBC,
103 CTR and XTS modes
104
105 Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode
106 and for XTS mode encryption, CBC and XTS mode decryption speedup is
107 around 25%. (CBC encryption speed is not affected by this driver.)
108 This implementation does not rely on any lookup tables so it is
109 believed to be invulnerable to cache timing attacks.
110
111config CRYPTO_AES_ARM_CE
112 tristate "Accelerated AES using ARMv8 Crypto Extensions"
113 depends on KERNEL_MODE_NEON
114 select CRYPTO_ALGAPI
115 select CRYPTO_ABLK_HELPER
116 help
117 Use an implementation of AES in CBC, CTR and XTS modes that uses
118 ARMv8 Crypto Extensions
119
120config CRYPTO_GHASH_ARM_CE
121 tristate "PMULL-accelerated GHASH using ARMv8 Crypto Extensions"
122 depends on KERNEL_MODE_NEON
123 select CRYPTO_HASH
124 select CRYPTO_CRYPTD
125 help
126 Use an implementation of GHASH (used by the GCM AEAD chaining mode)
127 that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64)
128 that is part of the ARMv8 Crypto Extensions
129
130endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index b48fa341648d..6ea828241fcb 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -6,13 +6,35 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o
6obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o 6obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
7obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o 7obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
8obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o 8obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
9obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
9obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o 10obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o
10 11
12ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
13ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
14ce-obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o
15ce-obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
16
17ifneq ($(ce-obj-y)$(ce-obj-m),)
18ifeq ($(call as-instr,.fpu crypto-neon-fp-armv8,y,n),y)
19obj-y += $(ce-obj-y)
20obj-m += $(ce-obj-m)
21else
22$(warning These ARMv8 Crypto Extensions modules need binutils 2.23 or higher)
23$(warning $(ce-obj-y) $(ce-obj-m))
24endif
25endif
26
11aes-arm-y := aes-armv4.o aes_glue.o 27aes-arm-y := aes-armv4.o aes_glue.o
12aes-arm-bs-y := aesbs-core.o aesbs-glue.o 28aes-arm-bs-y := aesbs-core.o aesbs-glue.o
13sha1-arm-y := sha1-armv4-large.o sha1_glue.o 29sha1-arm-y := sha1-armv4-large.o sha1_glue.o
14sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o 30sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
31sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
32sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y)
15sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o 33sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o
34sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o
35sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o
36aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
37ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
16 38
17quiet_cmd_perl = PERL $@ 39quiet_cmd_perl = PERL $@
18 cmd_perl = $(PERL) $(<) > $(@) 40 cmd_perl = $(PERL) $(<) > $(@)
@@ -20,4 +42,7 @@ quiet_cmd_perl = PERL $@
20$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl 42$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl
21 $(call cmd,perl) 43 $(call cmd,perl)
22 44
23.PRECIOUS: $(obj)/aesbs-core.S 45$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl
46 $(call cmd,perl)
47
48.PRECIOUS: $(obj)/aesbs-core.S $(obj)/sha256-core.S
diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S
new file mode 100644
index 000000000000..8cfa468ee570
--- /dev/null
+++ b/arch/arm/crypto/aes-ce-core.S
@@ -0,0 +1,518 @@
1/*
2 * aes-ce-core.S - AES in CBC/CTR/XTS mode using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/linkage.h>
12#include <asm/assembler.h>
13
14 .text
15 .fpu crypto-neon-fp-armv8
16 .align 3
17
18 .macro enc_round, state, key
19 aese.8 \state, \key
20 aesmc.8 \state, \state
21 .endm
22
23 .macro dec_round, state, key
24 aesd.8 \state, \key
25 aesimc.8 \state, \state
26 .endm
27
28 .macro enc_dround, key1, key2
29 enc_round q0, \key1
30 enc_round q0, \key2
31 .endm
32
33 .macro dec_dround, key1, key2
34 dec_round q0, \key1
35 dec_round q0, \key2
36 .endm
37
38 .macro enc_fround, key1, key2, key3
39 enc_round q0, \key1
40 aese.8 q0, \key2
41 veor q0, q0, \key3
42 .endm
43
44 .macro dec_fround, key1, key2, key3
45 dec_round q0, \key1
46 aesd.8 q0, \key2
47 veor q0, q0, \key3
48 .endm
49
50 .macro enc_dround_3x, key1, key2
51 enc_round q0, \key1
52 enc_round q1, \key1
53 enc_round q2, \key1
54 enc_round q0, \key2
55 enc_round q1, \key2
56 enc_round q2, \key2
57 .endm
58
59 .macro dec_dround_3x, key1, key2
60 dec_round q0, \key1
61 dec_round q1, \key1
62 dec_round q2, \key1
63 dec_round q0, \key2
64 dec_round q1, \key2
65 dec_round q2, \key2
66 .endm
67
68 .macro enc_fround_3x, key1, key2, key3
69 enc_round q0, \key1
70 enc_round q1, \key1
71 enc_round q2, \key1
72 aese.8 q0, \key2
73 aese.8 q1, \key2
74 aese.8 q2, \key2
75 veor q0, q0, \key3
76 veor q1, q1, \key3
77 veor q2, q2, \key3
78 .endm
79
80 .macro dec_fround_3x, key1, key2, key3
81 dec_round q0, \key1
82 dec_round q1, \key1
83 dec_round q2, \key1
84 aesd.8 q0, \key2
85 aesd.8 q1, \key2
86 aesd.8 q2, \key2
87 veor q0, q0, \key3
88 veor q1, q1, \key3
89 veor q2, q2, \key3
90 .endm
91
92 .macro do_block, dround, fround
93 cmp r3, #12 @ which key size?
94 vld1.8 {q10-q11}, [ip]!
95 \dround q8, q9
96 vld1.8 {q12-q13}, [ip]!
97 \dround q10, q11
98 vld1.8 {q10-q11}, [ip]!
99 \dround q12, q13
100 vld1.8 {q12-q13}, [ip]!
101 \dround q10, q11
102 blo 0f @ AES-128: 10 rounds
103 vld1.8 {q10-q11}, [ip]!
104 beq 1f @ AES-192: 12 rounds
105 \dround q12, q13
106 vld1.8 {q12-q13}, [ip]
107 \dround q10, q11
1080: \fround q12, q13, q14
109 bx lr
110
1111: \dround q12, q13
112 \fround q10, q11, q14
113 bx lr
114 .endm
115
116 /*
117 * Internal, non-AAPCS compliant functions that implement the core AES
118 * transforms. These should preserve all registers except q0 - q2 and ip
119 * Arguments:
120 * q0 : first in/output block
121 * q1 : second in/output block (_3x version only)
122 * q2 : third in/output block (_3x version only)
123 * q8 : first round key
124 * q9 : secound round key
125 * ip : address of 3rd round key
126 * q14 : final round key
127 * r3 : number of rounds
128 */
129 .align 6
130aes_encrypt:
131 add ip, r2, #32 @ 3rd round key
132.Laes_encrypt_tweak:
133 do_block enc_dround, enc_fround
134ENDPROC(aes_encrypt)
135
136 .align 6
137aes_decrypt:
138 add ip, r2, #32 @ 3rd round key
139 do_block dec_dround, dec_fround
140ENDPROC(aes_decrypt)
141
142 .align 6
143aes_encrypt_3x:
144 add ip, r2, #32 @ 3rd round key
145 do_block enc_dround_3x, enc_fround_3x
146ENDPROC(aes_encrypt_3x)
147
148 .align 6
149aes_decrypt_3x:
150 add ip, r2, #32 @ 3rd round key
151 do_block dec_dround_3x, dec_fround_3x
152ENDPROC(aes_decrypt_3x)
153
154 .macro prepare_key, rk, rounds
155 add ip, \rk, \rounds, lsl #4
156 vld1.8 {q8-q9}, [\rk] @ load first 2 round keys
157 vld1.8 {q14}, [ip] @ load last round key
158 .endm
159
160 /*
161 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
162 * int blocks)
163 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
164 * int blocks)
165 */
166ENTRY(ce_aes_ecb_encrypt)
167 push {r4, lr}
168 ldr r4, [sp, #8]
169 prepare_key r2, r3
170.Lecbencloop3x:
171 subs r4, r4, #3
172 bmi .Lecbenc1x
173 vld1.8 {q0-q1}, [r1, :64]!
174 vld1.8 {q2}, [r1, :64]!
175 bl aes_encrypt_3x
176 vst1.8 {q0-q1}, [r0, :64]!
177 vst1.8 {q2}, [r0, :64]!
178 b .Lecbencloop3x
179.Lecbenc1x:
180 adds r4, r4, #3
181 beq .Lecbencout
182.Lecbencloop:
183 vld1.8 {q0}, [r1, :64]!
184 bl aes_encrypt
185 vst1.8 {q0}, [r0, :64]!
186 subs r4, r4, #1
187 bne .Lecbencloop
188.Lecbencout:
189 pop {r4, pc}
190ENDPROC(ce_aes_ecb_encrypt)
191
192ENTRY(ce_aes_ecb_decrypt)
193 push {r4, lr}
194 ldr r4, [sp, #8]
195 prepare_key r2, r3
196.Lecbdecloop3x:
197 subs r4, r4, #3
198 bmi .Lecbdec1x
199 vld1.8 {q0-q1}, [r1, :64]!
200 vld1.8 {q2}, [r1, :64]!
201 bl aes_decrypt_3x
202 vst1.8 {q0-q1}, [r0, :64]!
203 vst1.8 {q2}, [r0, :64]!
204 b .Lecbdecloop3x
205.Lecbdec1x:
206 adds r4, r4, #3
207 beq .Lecbdecout
208.Lecbdecloop:
209 vld1.8 {q0}, [r1, :64]!
210 bl aes_decrypt
211 vst1.8 {q0}, [r0, :64]!
212 subs r4, r4, #1
213 bne .Lecbdecloop
214.Lecbdecout:
215 pop {r4, pc}
216ENDPROC(ce_aes_ecb_decrypt)
217
218 /*
219 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
220 * int blocks, u8 iv[])
221 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
222 * int blocks, u8 iv[])
223 */
224ENTRY(ce_aes_cbc_encrypt)
225 push {r4-r6, lr}
226 ldrd r4, r5, [sp, #16]
227 vld1.8 {q0}, [r5]
228 prepare_key r2, r3
229.Lcbcencloop:
230 vld1.8 {q1}, [r1, :64]! @ get next pt block
231 veor q0, q0, q1 @ ..and xor with iv
232 bl aes_encrypt
233 vst1.8 {q0}, [r0, :64]!
234 subs r4, r4, #1
235 bne .Lcbcencloop
236 vst1.8 {q0}, [r5]
237 pop {r4-r6, pc}
238ENDPROC(ce_aes_cbc_encrypt)
239
240ENTRY(ce_aes_cbc_decrypt)
241 push {r4-r6, lr}
242 ldrd r4, r5, [sp, #16]
243 vld1.8 {q6}, [r5] @ keep iv in q6
244 prepare_key r2, r3
245.Lcbcdecloop3x:
246 subs r4, r4, #3
247 bmi .Lcbcdec1x
248 vld1.8 {q0-q1}, [r1, :64]!
249 vld1.8 {q2}, [r1, :64]!
250 vmov q3, q0
251 vmov q4, q1
252 vmov q5, q2
253 bl aes_decrypt_3x
254 veor q0, q0, q6
255 veor q1, q1, q3
256 veor q2, q2, q4
257 vmov q6, q5
258 vst1.8 {q0-q1}, [r0, :64]!
259 vst1.8 {q2}, [r0, :64]!
260 b .Lcbcdecloop3x
261.Lcbcdec1x:
262 adds r4, r4, #3
263 beq .Lcbcdecout
264 vmov q15, q14 @ preserve last round key
265.Lcbcdecloop:
266 vld1.8 {q0}, [r1, :64]! @ get next ct block
267 veor q14, q15, q6 @ combine prev ct with last key
268 vmov q6, q0
269 bl aes_decrypt
270 vst1.8 {q0}, [r0, :64]!
271 subs r4, r4, #1
272 bne .Lcbcdecloop
273.Lcbcdecout:
274 vst1.8 {q6}, [r5] @ keep iv in q6
275 pop {r4-r6, pc}
276ENDPROC(ce_aes_cbc_decrypt)
277
278 /*
279 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
280 * int blocks, u8 ctr[])
281 */
282ENTRY(ce_aes_ctr_encrypt)
283 push {r4-r6, lr}
284 ldrd r4, r5, [sp, #16]
285 vld1.8 {q6}, [r5] @ load ctr
286 prepare_key r2, r3
287 vmov r6, s27 @ keep swabbed ctr in r6
288 rev r6, r6
289 cmn r6, r4 @ 32 bit overflow?
290 bcs .Lctrloop
291.Lctrloop3x:
292 subs r4, r4, #3
293 bmi .Lctr1x
294 add r6, r6, #1
295 vmov q0, q6
296 vmov q1, q6
297 rev ip, r6
298 add r6, r6, #1
299 vmov q2, q6
300 vmov s7, ip
301 rev ip, r6
302 add r6, r6, #1
303 vmov s11, ip
304 vld1.8 {q3-q4}, [r1, :64]!
305 vld1.8 {q5}, [r1, :64]!
306 bl aes_encrypt_3x
307 veor q0, q0, q3
308 veor q1, q1, q4
309 veor q2, q2, q5
310 rev ip, r6
311 vst1.8 {q0-q1}, [r0, :64]!
312 vst1.8 {q2}, [r0, :64]!
313 vmov s27, ip
314 b .Lctrloop3x
315.Lctr1x:
316 adds r4, r4, #3
317 beq .Lctrout
318.Lctrloop:
319 vmov q0, q6
320 bl aes_encrypt
321 subs r4, r4, #1
322 bmi .Lctrhalfblock @ blocks < 0 means 1/2 block
323 vld1.8 {q3}, [r1, :64]!
324 veor q3, q0, q3
325 vst1.8 {q3}, [r0, :64]!
326
327 adds r6, r6, #1 @ increment BE ctr
328 rev ip, r6
329 vmov s27, ip
330 bcs .Lctrcarry
331 teq r4, #0
332 bne .Lctrloop
333.Lctrout:
334 vst1.8 {q6}, [r5]
335 pop {r4-r6, pc}
336
337.Lctrhalfblock:
338 vld1.8 {d1}, [r1, :64]
339 veor d0, d0, d1
340 vst1.8 {d0}, [r0, :64]
341 pop {r4-r6, pc}
342
343.Lctrcarry:
344 .irp sreg, s26, s25, s24
345 vmov ip, \sreg @ load next word of ctr
346 rev ip, ip @ ... to handle the carry
347 adds ip, ip, #1
348 rev ip, ip
349 vmov \sreg, ip
350 bcc 0f
351 .endr
3520: teq r4, #0
353 beq .Lctrout
354 b .Lctrloop
355ENDPROC(ce_aes_ctr_encrypt)
356
357 /*
358 * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
359 * int blocks, u8 iv[], u8 const rk2[], int first)
360 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
361 * int blocks, u8 iv[], u8 const rk2[], int first)
362 */
363
364 .macro next_tweak, out, in, const, tmp
365 vshr.s64 \tmp, \in, #63
366 vand \tmp, \tmp, \const
367 vadd.u64 \out, \in, \in
368 vext.8 \tmp, \tmp, \tmp, #8
369 veor \out, \out, \tmp
370 .endm
371
372 .align 3
373.Lxts_mul_x:
374 .quad 1, 0x87
375
376ce_aes_xts_init:
377 vldr d14, .Lxts_mul_x
378 vldr d15, .Lxts_mul_x + 8
379
380 ldrd r4, r5, [sp, #16] @ load args
381 ldr r6, [sp, #28]
382 vld1.8 {q0}, [r5] @ load iv
383 teq r6, #1 @ start of a block?
384 bxne lr
385
386 @ Encrypt the IV in q0 with the second AES key. This should only
387 @ be done at the start of a block.
388 ldr r6, [sp, #24] @ load AES key 2
389 prepare_key r6, r3
390 add ip, r6, #32 @ 3rd round key of key 2
391 b .Laes_encrypt_tweak @ tail call
392ENDPROC(ce_aes_xts_init)
393
394ENTRY(ce_aes_xts_encrypt)
395 push {r4-r6, lr}
396
397 bl ce_aes_xts_init @ run shared prologue
398 prepare_key r2, r3
399 vmov q3, q0
400
401 teq r6, #0 @ start of a block?
402 bne .Lxtsenc3x
403
404.Lxtsencloop3x:
405 next_tweak q3, q3, q7, q6
406.Lxtsenc3x:
407 subs r4, r4, #3
408 bmi .Lxtsenc1x
409 vld1.8 {q0-q1}, [r1, :64]! @ get 3 pt blocks
410 vld1.8 {q2}, [r1, :64]!
411 next_tweak q4, q3, q7, q6
412 veor q0, q0, q3
413 next_tweak q5, q4, q7, q6
414 veor q1, q1, q4
415 veor q2, q2, q5
416 bl aes_encrypt_3x
417 veor q0, q0, q3
418 veor q1, q1, q4
419 veor q2, q2, q5
420 vst1.8 {q0-q1}, [r0, :64]! @ write 3 ct blocks
421 vst1.8 {q2}, [r0, :64]!
422 vmov q3, q5
423 teq r4, #0
424 beq .Lxtsencout
425 b .Lxtsencloop3x
426.Lxtsenc1x:
427 adds r4, r4, #3
428 beq .Lxtsencout
429.Lxtsencloop:
430 vld1.8 {q0}, [r1, :64]!
431 veor q0, q0, q3
432 bl aes_encrypt
433 veor q0, q0, q3
434 vst1.8 {q0}, [r0, :64]!
435 subs r4, r4, #1
436 beq .Lxtsencout
437 next_tweak q3, q3, q7, q6
438 b .Lxtsencloop
439.Lxtsencout:
440 vst1.8 {q3}, [r5]
441 pop {r4-r6, pc}
442ENDPROC(ce_aes_xts_encrypt)
443
444
445ENTRY(ce_aes_xts_decrypt)
446 push {r4-r6, lr}
447
448 bl ce_aes_xts_init @ run shared prologue
449 prepare_key r2, r3
450 vmov q3, q0
451
452 teq r6, #0 @ start of a block?
453 bne .Lxtsdec3x
454
455.Lxtsdecloop3x:
456 next_tweak q3, q3, q7, q6
457.Lxtsdec3x:
458 subs r4, r4, #3
459 bmi .Lxtsdec1x
460 vld1.8 {q0-q1}, [r1, :64]! @ get 3 ct blocks
461 vld1.8 {q2}, [r1, :64]!
462 next_tweak q4, q3, q7, q6
463 veor q0, q0, q3
464 next_tweak q5, q4, q7, q6
465 veor q1, q1, q4
466 veor q2, q2, q5
467 bl aes_decrypt_3x
468 veor q0, q0, q3
469 veor q1, q1, q4
470 veor q2, q2, q5
471 vst1.8 {q0-q1}, [r0, :64]! @ write 3 pt blocks
472 vst1.8 {q2}, [r0, :64]!
473 vmov q3, q5
474 teq r4, #0
475 beq .Lxtsdecout
476 b .Lxtsdecloop3x
477.Lxtsdec1x:
478 adds r4, r4, #3
479 beq .Lxtsdecout
480.Lxtsdecloop:
481 vld1.8 {q0}, [r1, :64]!
482 veor q0, q0, q3
483 add ip, r2, #32 @ 3rd round key
484 bl aes_decrypt
485 veor q0, q0, q3
486 vst1.8 {q0}, [r0, :64]!
487 subs r4, r4, #1
488 beq .Lxtsdecout
489 next_tweak q3, q3, q7, q6
490 b .Lxtsdecloop
491.Lxtsdecout:
492 vst1.8 {q3}, [r5]
493 pop {r4-r6, pc}
494ENDPROC(ce_aes_xts_decrypt)
495
496 /*
497 * u32 ce_aes_sub(u32 input) - use the aese instruction to perform the
498 * AES sbox substitution on each byte in
499 * 'input'
500 */
501ENTRY(ce_aes_sub)
502 vdup.32 q1, r0
503 veor q0, q0, q0
504 aese.8 q0, q1
505 vmov r0, s0
506 bx lr
507ENDPROC(ce_aes_sub)
508
509 /*
510 * void ce_aes_invert(u8 *dst, u8 *src) - perform the Inverse MixColumns
511 * operation on round key *src
512 */
513ENTRY(ce_aes_invert)
514 vld1.8 {q0}, [r1]
515 aesimc.8 q0, q0
516 vst1.8 {q0}, [r0]
517 bx lr
518ENDPROC(ce_aes_invert)
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
new file mode 100644
index 000000000000..b445a5d56f43
--- /dev/null
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -0,0 +1,524 @@
1/*
2 * aes-ce-glue.c - wrapper code for ARMv8 AES
3 *
4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <asm/hwcap.h>
12#include <asm/neon.h>
13#include <asm/hwcap.h>
14#include <crypto/aes.h>
15#include <crypto/ablk_helper.h>
16#include <crypto/algapi.h>
17#include <linux/module.h>
18
19MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
20MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
21MODULE_LICENSE("GPL v2");
22
23/* defined in aes-ce-core.S */
24asmlinkage u32 ce_aes_sub(u32 input);
25asmlinkage void ce_aes_invert(void *dst, void *src);
26
27asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
28 int rounds, int blocks);
29asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
30 int rounds, int blocks);
31
32asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
33 int rounds, int blocks, u8 iv[]);
34asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
35 int rounds, int blocks, u8 iv[]);
36
37asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
38 int rounds, int blocks, u8 ctr[]);
39
40asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
41 int rounds, int blocks, u8 iv[],
42 u8 const rk2[], int first);
43asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
44 int rounds, int blocks, u8 iv[],
45 u8 const rk2[], int first);
46
47struct aes_block {
48 u8 b[AES_BLOCK_SIZE];
49};
50
51static int num_rounds(struct crypto_aes_ctx *ctx)
52{
53 /*
54 * # of rounds specified by AES:
55 * 128 bit key 10 rounds
56 * 192 bit key 12 rounds
57 * 256 bit key 14 rounds
58 * => n byte key => 6 + (n/4) rounds
59 */
60 return 6 + ctx->key_length / 4;
61}
62
63static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
64 unsigned int key_len)
65{
66 /*
67 * The AES key schedule round constants
68 */
69 static u8 const rcon[] = {
70 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
71 };
72
73 u32 kwords = key_len / sizeof(u32);
74 struct aes_block *key_enc, *key_dec;
75 int i, j;
76
77 if (key_len != AES_KEYSIZE_128 &&
78 key_len != AES_KEYSIZE_192 &&
79 key_len != AES_KEYSIZE_256)
80 return -EINVAL;
81
82 memcpy(ctx->key_enc, in_key, key_len);
83 ctx->key_length = key_len;
84
85 kernel_neon_begin();
86 for (i = 0; i < sizeof(rcon); i++) {
87 u32 *rki = ctx->key_enc + (i * kwords);
88 u32 *rko = rki + kwords;
89
90 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
91 rko[0] = rko[0] ^ rki[0] ^ rcon[i];
92 rko[1] = rko[0] ^ rki[1];
93 rko[2] = rko[1] ^ rki[2];
94 rko[3] = rko[2] ^ rki[3];
95
96 if (key_len == AES_KEYSIZE_192) {
97 if (i >= 7)
98 break;
99 rko[4] = rko[3] ^ rki[4];
100 rko[5] = rko[4] ^ rki[5];
101 } else if (key_len == AES_KEYSIZE_256) {
102 if (i >= 6)
103 break;
104 rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
105 rko[5] = rko[4] ^ rki[5];
106 rko[6] = rko[5] ^ rki[6];
107 rko[7] = rko[6] ^ rki[7];
108 }
109 }
110
111 /*
112 * Generate the decryption keys for the Equivalent Inverse Cipher.
113 * This involves reversing the order of the round keys, and applying
114 * the Inverse Mix Columns transformation on all but the first and
115 * the last one.
116 */
117 key_enc = (struct aes_block *)ctx->key_enc;
118 key_dec = (struct aes_block *)ctx->key_dec;
119 j = num_rounds(ctx);
120
121 key_dec[0] = key_enc[j];
122 for (i = 1, j--; j > 0; i++, j--)
123 ce_aes_invert(key_dec + i, key_enc + j);
124 key_dec[i] = key_enc[0];
125
126 kernel_neon_end();
127 return 0;
128}
129
130static int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
131 unsigned int key_len)
132{
133 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
134 int ret;
135
136 ret = ce_aes_expandkey(ctx, in_key, key_len);
137 if (!ret)
138 return 0;
139
140 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
141 return -EINVAL;
142}
143
144struct crypto_aes_xts_ctx {
145 struct crypto_aes_ctx key1;
146 struct crypto_aes_ctx __aligned(8) key2;
147};
148
149static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
150 unsigned int key_len)
151{
152 struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
153 int ret;
154
155 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
156 if (!ret)
157 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
158 key_len / 2);
159 if (!ret)
160 return 0;
161
162 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
163 return -EINVAL;
164}
165
166static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
167 struct scatterlist *src, unsigned int nbytes)
168{
169 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
170 struct blkcipher_walk walk;
171 unsigned int blocks;
172 int err;
173
174 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
175 blkcipher_walk_init(&walk, dst, src, nbytes);
176 err = blkcipher_walk_virt(desc, &walk);
177
178 kernel_neon_begin();
179 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
180 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
181 (u8 *)ctx->key_enc, num_rounds(ctx), blocks);
182 err = blkcipher_walk_done(desc, &walk,
183 walk.nbytes % AES_BLOCK_SIZE);
184 }
185 kernel_neon_end();
186 return err;
187}
188
189static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
190 struct scatterlist *src, unsigned int nbytes)
191{
192 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
193 struct blkcipher_walk walk;
194 unsigned int blocks;
195 int err;
196
197 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
198 blkcipher_walk_init(&walk, dst, src, nbytes);
199 err = blkcipher_walk_virt(desc, &walk);
200
201 kernel_neon_begin();
202 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
203 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
204 (u8 *)ctx->key_dec, num_rounds(ctx), blocks);
205 err = blkcipher_walk_done(desc, &walk,
206 walk.nbytes % AES_BLOCK_SIZE);
207 }
208 kernel_neon_end();
209 return err;
210}
211
212static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
213 struct scatterlist *src, unsigned int nbytes)
214{
215 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
216 struct blkcipher_walk walk;
217 unsigned int blocks;
218 int err;
219
220 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
221 blkcipher_walk_init(&walk, dst, src, nbytes);
222 err = blkcipher_walk_virt(desc, &walk);
223
224 kernel_neon_begin();
225 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
226 ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
227 (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
228 walk.iv);
229 err = blkcipher_walk_done(desc, &walk,
230 walk.nbytes % AES_BLOCK_SIZE);
231 }
232 kernel_neon_end();
233 return err;
234}
235
236static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
237 struct scatterlist *src, unsigned int nbytes)
238{
239 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
240 struct blkcipher_walk walk;
241 unsigned int blocks;
242 int err;
243
244 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
245 blkcipher_walk_init(&walk, dst, src, nbytes);
246 err = blkcipher_walk_virt(desc, &walk);
247
248 kernel_neon_begin();
249 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
250 ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
251 (u8 *)ctx->key_dec, num_rounds(ctx), blocks,
252 walk.iv);
253 err = blkcipher_walk_done(desc, &walk,
254 walk.nbytes % AES_BLOCK_SIZE);
255 }
256 kernel_neon_end();
257 return err;
258}
259
260static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
261 struct scatterlist *src, unsigned int nbytes)
262{
263 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
264 struct blkcipher_walk walk;
265 int err, blocks;
266
267 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
268 blkcipher_walk_init(&walk, dst, src, nbytes);
269 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
270
271 kernel_neon_begin();
272 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
273 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
274 (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
275 walk.iv);
276 nbytes -= blocks * AES_BLOCK_SIZE;
277 if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
278 break;
279 err = blkcipher_walk_done(desc, &walk,
280 walk.nbytes % AES_BLOCK_SIZE);
281 }
282 if (nbytes) {
283 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
284 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
285 u8 __aligned(8) tail[AES_BLOCK_SIZE];
286
287 /*
288 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
289 * to tell aes_ctr_encrypt() to only read half a block.
290 */
291 blocks = (nbytes <= 8) ? -1 : 1;
292
293 ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc,
294 num_rounds(ctx), blocks, walk.iv);
295 memcpy(tdst, tail, nbytes);
296 err = blkcipher_walk_done(desc, &walk, 0);
297 }
298 kernel_neon_end();
299
300 return err;
301}
302
303static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
304 struct scatterlist *src, unsigned int nbytes)
305{
306 struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
307 int err, first, rounds = num_rounds(&ctx->key1);
308 struct blkcipher_walk walk;
309 unsigned int blocks;
310
311 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
312 blkcipher_walk_init(&walk, dst, src, nbytes);
313 err = blkcipher_walk_virt(desc, &walk);
314
315 kernel_neon_begin();
316 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
317 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
318 (u8 *)ctx->key1.key_enc, rounds, blocks,
319 walk.iv, (u8 *)ctx->key2.key_enc, first);
320 err = blkcipher_walk_done(desc, &walk,
321 walk.nbytes % AES_BLOCK_SIZE);
322 }
323 kernel_neon_end();
324
325 return err;
326}
327
328static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
329 struct scatterlist *src, unsigned int nbytes)
330{
331 struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
332 int err, first, rounds = num_rounds(&ctx->key1);
333 struct blkcipher_walk walk;
334 unsigned int blocks;
335
336 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
337 blkcipher_walk_init(&walk, dst, src, nbytes);
338 err = blkcipher_walk_virt(desc, &walk);
339
340 kernel_neon_begin();
341 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
342 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
343 (u8 *)ctx->key1.key_dec, rounds, blocks,
344 walk.iv, (u8 *)ctx->key2.key_enc, first);
345 err = blkcipher_walk_done(desc, &walk,
346 walk.nbytes % AES_BLOCK_SIZE);
347 }
348 kernel_neon_end();
349
350 return err;
351}
352
353static struct crypto_alg aes_algs[] = { {
354 .cra_name = "__ecb-aes-ce",
355 .cra_driver_name = "__driver-ecb-aes-ce",
356 .cra_priority = 0,
357 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
358 CRYPTO_ALG_INTERNAL,
359 .cra_blocksize = AES_BLOCK_SIZE,
360 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
361 .cra_alignmask = 7,
362 .cra_type = &crypto_blkcipher_type,
363 .cra_module = THIS_MODULE,
364 .cra_blkcipher = {
365 .min_keysize = AES_MIN_KEY_SIZE,
366 .max_keysize = AES_MAX_KEY_SIZE,
367 .ivsize = AES_BLOCK_SIZE,
368 .setkey = ce_aes_setkey,
369 .encrypt = ecb_encrypt,
370 .decrypt = ecb_decrypt,
371 },
372}, {
373 .cra_name = "__cbc-aes-ce",
374 .cra_driver_name = "__driver-cbc-aes-ce",
375 .cra_priority = 0,
376 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
377 CRYPTO_ALG_INTERNAL,
378 .cra_blocksize = AES_BLOCK_SIZE,
379 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
380 .cra_alignmask = 7,
381 .cra_type = &crypto_blkcipher_type,
382 .cra_module = THIS_MODULE,
383 .cra_blkcipher = {
384 .min_keysize = AES_MIN_KEY_SIZE,
385 .max_keysize = AES_MAX_KEY_SIZE,
386 .ivsize = AES_BLOCK_SIZE,
387 .setkey = ce_aes_setkey,
388 .encrypt = cbc_encrypt,
389 .decrypt = cbc_decrypt,
390 },
391}, {
392 .cra_name = "__ctr-aes-ce",
393 .cra_driver_name = "__driver-ctr-aes-ce",
394 .cra_priority = 0,
395 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
396 CRYPTO_ALG_INTERNAL,
397 .cra_blocksize = 1,
398 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
399 .cra_alignmask = 7,
400 .cra_type = &crypto_blkcipher_type,
401 .cra_module = THIS_MODULE,
402 .cra_blkcipher = {
403 .min_keysize = AES_MIN_KEY_SIZE,
404 .max_keysize = AES_MAX_KEY_SIZE,
405 .ivsize = AES_BLOCK_SIZE,
406 .setkey = ce_aes_setkey,
407 .encrypt = ctr_encrypt,
408 .decrypt = ctr_encrypt,
409 },
410}, {
411 .cra_name = "__xts-aes-ce",
412 .cra_driver_name = "__driver-xts-aes-ce",
413 .cra_priority = 0,
414 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
415 CRYPTO_ALG_INTERNAL,
416 .cra_blocksize = AES_BLOCK_SIZE,
417 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
418 .cra_alignmask = 7,
419 .cra_type = &crypto_blkcipher_type,
420 .cra_module = THIS_MODULE,
421 .cra_blkcipher = {
422 .min_keysize = 2 * AES_MIN_KEY_SIZE,
423 .max_keysize = 2 * AES_MAX_KEY_SIZE,
424 .ivsize = AES_BLOCK_SIZE,
425 .setkey = xts_set_key,
426 .encrypt = xts_encrypt,
427 .decrypt = xts_decrypt,
428 },
429}, {
430 .cra_name = "ecb(aes)",
431 .cra_driver_name = "ecb-aes-ce",
432 .cra_priority = 300,
433 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
434 .cra_blocksize = AES_BLOCK_SIZE,
435 .cra_ctxsize = sizeof(struct async_helper_ctx),
436 .cra_alignmask = 7,
437 .cra_type = &crypto_ablkcipher_type,
438 .cra_module = THIS_MODULE,
439 .cra_init = ablk_init,
440 .cra_exit = ablk_exit,
441 .cra_ablkcipher = {
442 .min_keysize = AES_MIN_KEY_SIZE,
443 .max_keysize = AES_MAX_KEY_SIZE,
444 .ivsize = AES_BLOCK_SIZE,
445 .setkey = ablk_set_key,
446 .encrypt = ablk_encrypt,
447 .decrypt = ablk_decrypt,
448 }
449}, {
450 .cra_name = "cbc(aes)",
451 .cra_driver_name = "cbc-aes-ce",
452 .cra_priority = 300,
453 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
454 .cra_blocksize = AES_BLOCK_SIZE,
455 .cra_ctxsize = sizeof(struct async_helper_ctx),
456 .cra_alignmask = 7,
457 .cra_type = &crypto_ablkcipher_type,
458 .cra_module = THIS_MODULE,
459 .cra_init = ablk_init,
460 .cra_exit = ablk_exit,
461 .cra_ablkcipher = {
462 .min_keysize = AES_MIN_KEY_SIZE,
463 .max_keysize = AES_MAX_KEY_SIZE,
464 .ivsize = AES_BLOCK_SIZE,
465 .setkey = ablk_set_key,
466 .encrypt = ablk_encrypt,
467 .decrypt = ablk_decrypt,
468 }
469}, {
470 .cra_name = "ctr(aes)",
471 .cra_driver_name = "ctr-aes-ce",
472 .cra_priority = 300,
473 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
474 .cra_blocksize = 1,
475 .cra_ctxsize = sizeof(struct async_helper_ctx),
476 .cra_alignmask = 7,
477 .cra_type = &crypto_ablkcipher_type,
478 .cra_module = THIS_MODULE,
479 .cra_init = ablk_init,
480 .cra_exit = ablk_exit,
481 .cra_ablkcipher = {
482 .min_keysize = AES_MIN_KEY_SIZE,
483 .max_keysize = AES_MAX_KEY_SIZE,
484 .ivsize = AES_BLOCK_SIZE,
485 .setkey = ablk_set_key,
486 .encrypt = ablk_encrypt,
487 .decrypt = ablk_decrypt,
488 }
489}, {
490 .cra_name = "xts(aes)",
491 .cra_driver_name = "xts-aes-ce",
492 .cra_priority = 300,
493 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
494 .cra_blocksize = AES_BLOCK_SIZE,
495 .cra_ctxsize = sizeof(struct async_helper_ctx),
496 .cra_alignmask = 7,
497 .cra_type = &crypto_ablkcipher_type,
498 .cra_module = THIS_MODULE,
499 .cra_init = ablk_init,
500 .cra_exit = ablk_exit,
501 .cra_ablkcipher = {
502 .min_keysize = 2 * AES_MIN_KEY_SIZE,
503 .max_keysize = 2 * AES_MAX_KEY_SIZE,
504 .ivsize = AES_BLOCK_SIZE,
505 .setkey = ablk_set_key,
506 .encrypt = ablk_encrypt,
507 .decrypt = ablk_decrypt,
508 }
509} };
510
511static int __init aes_init(void)
512{
513 if (!(elf_hwcap2 & HWCAP2_AES))
514 return -ENODEV;
515 return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
516}
517
518static void __exit aes_exit(void)
519{
520 crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
521}
522
523module_init(aes_init);
524module_exit(aes_exit);
diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c
index 15468fbbdea3..6d685298690e 100644
--- a/arch/arm/crypto/aesbs-glue.c
+++ b/arch/arm/crypto/aesbs-glue.c
@@ -301,7 +301,8 @@ static struct crypto_alg aesbs_algs[] = { {
301 .cra_name = "__cbc-aes-neonbs", 301 .cra_name = "__cbc-aes-neonbs",
302 .cra_driver_name = "__driver-cbc-aes-neonbs", 302 .cra_driver_name = "__driver-cbc-aes-neonbs",
303 .cra_priority = 0, 303 .cra_priority = 0,
304 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 304 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
305 CRYPTO_ALG_INTERNAL,
305 .cra_blocksize = AES_BLOCK_SIZE, 306 .cra_blocksize = AES_BLOCK_SIZE,
306 .cra_ctxsize = sizeof(struct aesbs_cbc_ctx), 307 .cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
307 .cra_alignmask = 7, 308 .cra_alignmask = 7,
@@ -319,7 +320,8 @@ static struct crypto_alg aesbs_algs[] = { {
319 .cra_name = "__ctr-aes-neonbs", 320 .cra_name = "__ctr-aes-neonbs",
320 .cra_driver_name = "__driver-ctr-aes-neonbs", 321 .cra_driver_name = "__driver-ctr-aes-neonbs",
321 .cra_priority = 0, 322 .cra_priority = 0,
322 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 323 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
324 CRYPTO_ALG_INTERNAL,
323 .cra_blocksize = 1, 325 .cra_blocksize = 1,
324 .cra_ctxsize = sizeof(struct aesbs_ctr_ctx), 326 .cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
325 .cra_alignmask = 7, 327 .cra_alignmask = 7,
@@ -337,7 +339,8 @@ static struct crypto_alg aesbs_algs[] = { {
337 .cra_name = "__xts-aes-neonbs", 339 .cra_name = "__xts-aes-neonbs",
338 .cra_driver_name = "__driver-xts-aes-neonbs", 340 .cra_driver_name = "__driver-xts-aes-neonbs",
339 .cra_priority = 0, 341 .cra_priority = 0,
340 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 342 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
343 CRYPTO_ALG_INTERNAL,
341 .cra_blocksize = AES_BLOCK_SIZE, 344 .cra_blocksize = AES_BLOCK_SIZE,
342 .cra_ctxsize = sizeof(struct aesbs_xts_ctx), 345 .cra_ctxsize = sizeof(struct aesbs_xts_ctx),
343 .cra_alignmask = 7, 346 .cra_alignmask = 7,
diff --git a/arch/arm/crypto/ghash-ce-core.S b/arch/arm/crypto/ghash-ce-core.S
new file mode 100644
index 000000000000..f6ab8bcc9efe
--- /dev/null
+++ b/arch/arm/crypto/ghash-ce-core.S
@@ -0,0 +1,94 @@
1/*
2 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
3 *
4 * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <linux/linkage.h>
12#include <asm/assembler.h>
13
14 SHASH .req q0
15 SHASH2 .req q1
16 T1 .req q2
17 T2 .req q3
18 MASK .req q4
19 XL .req q5
20 XM .req q6
21 XH .req q7
22 IN1 .req q7
23
24 SHASH_L .req d0
25 SHASH_H .req d1
26 SHASH2_L .req d2
27 T1_L .req d4
28 MASK_L .req d8
29 XL_L .req d10
30 XL_H .req d11
31 XM_L .req d12
32 XM_H .req d13
33 XH_L .req d14
34
35 .text
36 .fpu crypto-neon-fp-armv8
37
38 /*
39 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
40 * struct ghash_key const *k, const char *head)
41 */
42ENTRY(pmull_ghash_update)
43 vld1.64 {SHASH}, [r3]
44 vld1.64 {XL}, [r1]
45 vmov.i8 MASK, #0xe1
46 vext.8 SHASH2, SHASH, SHASH, #8
47 vshl.u64 MASK, MASK, #57
48 veor SHASH2, SHASH2, SHASH
49
50 /* do the head block first, if supplied */
51 ldr ip, [sp]
52 teq ip, #0
53 beq 0f
54 vld1.64 {T1}, [ip]
55 teq r0, #0
56 b 1f
57
580: vld1.64 {T1}, [r2]!
59 subs r0, r0, #1
60
611: /* multiply XL by SHASH in GF(2^128) */
62#ifndef CONFIG_CPU_BIG_ENDIAN
63 vrev64.8 T1, T1
64#endif
65 vext.8 T2, XL, XL, #8
66 vext.8 IN1, T1, T1, #8
67 veor T1, T1, T2
68 veor XL, XL, IN1
69
70 vmull.p64 XH, SHASH_H, XL_H @ a1 * b1
71 veor T1, T1, XL
72 vmull.p64 XL, SHASH_L, XL_L @ a0 * b0
73 vmull.p64 XM, SHASH2_L, T1_L @ (a1 + a0)(b1 + b0)
74
75 vext.8 T1, XL, XH, #8
76 veor T2, XL, XH
77 veor XM, XM, T1
78 veor XM, XM, T2
79 vmull.p64 T2, XL_L, MASK_L
80
81 vmov XH_L, XM_H
82 vmov XM_H, XL_L
83
84 veor XL, XM, T2
85 vext.8 T2, XL, XL, #8
86 vmull.p64 XL, XL_L, MASK_L
87 veor T2, T2, XH
88 veor XL, XL, T2
89
90 bne 0b
91
92 vst1.64 {XL}, [r1]
93 bx lr
94ENDPROC(pmull_ghash_update)
diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
new file mode 100644
index 000000000000..03a39fe29246
--- /dev/null
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -0,0 +1,320 @@
1/*
2 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
3 *
4 * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <asm/hwcap.h>
12#include <asm/neon.h>
13#include <asm/simd.h>
14#include <asm/unaligned.h>
15#include <crypto/cryptd.h>
16#include <crypto/internal/hash.h>
17#include <crypto/gf128mul.h>
18#include <linux/crypto.h>
19#include <linux/module.h>
20
21MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
22MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23MODULE_LICENSE("GPL v2");
24
25#define GHASH_BLOCK_SIZE 16
26#define GHASH_DIGEST_SIZE 16
27
28struct ghash_key {
29 u64 a;
30 u64 b;
31};
32
33struct ghash_desc_ctx {
34 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
35 u8 buf[GHASH_BLOCK_SIZE];
36 u32 count;
37};
38
39struct ghash_async_ctx {
40 struct cryptd_ahash *cryptd_tfm;
41};
42
43asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
44 struct ghash_key const *k, const char *head);
45
46static int ghash_init(struct shash_desc *desc)
47{
48 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
49
50 *ctx = (struct ghash_desc_ctx){};
51 return 0;
52}
53
54static int ghash_update(struct shash_desc *desc, const u8 *src,
55 unsigned int len)
56{
57 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
58 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
59
60 ctx->count += len;
61
62 if ((partial + len) >= GHASH_BLOCK_SIZE) {
63 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
64 int blocks;
65
66 if (partial) {
67 int p = GHASH_BLOCK_SIZE - partial;
68
69 memcpy(ctx->buf + partial, src, p);
70 src += p;
71 len -= p;
72 }
73
74 blocks = len / GHASH_BLOCK_SIZE;
75 len %= GHASH_BLOCK_SIZE;
76
77 kernel_neon_begin();
78 pmull_ghash_update(blocks, ctx->digest, src, key,
79 partial ? ctx->buf : NULL);
80 kernel_neon_end();
81 src += blocks * GHASH_BLOCK_SIZE;
82 partial = 0;
83 }
84 if (len)
85 memcpy(ctx->buf + partial, src, len);
86 return 0;
87}
88
89static int ghash_final(struct shash_desc *desc, u8 *dst)
90{
91 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
92 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
93
94 if (partial) {
95 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
96
97 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
98 kernel_neon_begin();
99 pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
100 kernel_neon_end();
101 }
102 put_unaligned_be64(ctx->digest[1], dst);
103 put_unaligned_be64(ctx->digest[0], dst + 8);
104
105 *ctx = (struct ghash_desc_ctx){};
106 return 0;
107}
108
109static int ghash_setkey(struct crypto_shash *tfm,
110 const u8 *inkey, unsigned int keylen)
111{
112 struct ghash_key *key = crypto_shash_ctx(tfm);
113 u64 a, b;
114
115 if (keylen != GHASH_BLOCK_SIZE) {
116 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
117 return -EINVAL;
118 }
119
120 /* perform multiplication by 'x' in GF(2^128) */
121 b = get_unaligned_be64(inkey);
122 a = get_unaligned_be64(inkey + 8);
123
124 key->a = (a << 1) | (b >> 63);
125 key->b = (b << 1) | (a >> 63);
126
127 if (b >> 63)
128 key->b ^= 0xc200000000000000UL;
129
130 return 0;
131}
132
133static struct shash_alg ghash_alg = {
134 .digestsize = GHASH_DIGEST_SIZE,
135 .init = ghash_init,
136 .update = ghash_update,
137 .final = ghash_final,
138 .setkey = ghash_setkey,
139 .descsize = sizeof(struct ghash_desc_ctx),
140 .base = {
141 .cra_name = "ghash",
142 .cra_driver_name = "__driver-ghash-ce",
143 .cra_priority = 0,
144 .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL,
145 .cra_blocksize = GHASH_BLOCK_SIZE,
146 .cra_ctxsize = sizeof(struct ghash_key),
147 .cra_module = THIS_MODULE,
148 },
149};
150
151static int ghash_async_init(struct ahash_request *req)
152{
153 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
154 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
155 struct ahash_request *cryptd_req = ahash_request_ctx(req);
156 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
157
158 if (!may_use_simd()) {
159 memcpy(cryptd_req, req, sizeof(*req));
160 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
161 return crypto_ahash_init(cryptd_req);
162 } else {
163 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
164 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
165
166 desc->tfm = child;
167 desc->flags = req->base.flags;
168 return crypto_shash_init(desc);
169 }
170}
171
172static int ghash_async_update(struct ahash_request *req)
173{
174 struct ahash_request *cryptd_req = ahash_request_ctx(req);
175
176 if (!may_use_simd()) {
177 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
178 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
179 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
180
181 memcpy(cryptd_req, req, sizeof(*req));
182 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
183 return crypto_ahash_update(cryptd_req);
184 } else {
185 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
186 return shash_ahash_update(req, desc);
187 }
188}
189
190static int ghash_async_final(struct ahash_request *req)
191{
192 struct ahash_request *cryptd_req = ahash_request_ctx(req);
193
194 if (!may_use_simd()) {
195 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
196 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
197 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
198
199 memcpy(cryptd_req, req, sizeof(*req));
200 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
201 return crypto_ahash_final(cryptd_req);
202 } else {
203 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
204 return crypto_shash_final(desc, req->result);
205 }
206}
207
208static int ghash_async_digest(struct ahash_request *req)
209{
210 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
211 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
212 struct ahash_request *cryptd_req = ahash_request_ctx(req);
213 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
214
215 if (!may_use_simd()) {
216 memcpy(cryptd_req, req, sizeof(*req));
217 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
218 return crypto_ahash_digest(cryptd_req);
219 } else {
220 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
221 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
222
223 desc->tfm = child;
224 desc->flags = req->base.flags;
225 return shash_ahash_digest(req, desc);
226 }
227}
228
229static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
230 unsigned int keylen)
231{
232 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
233 struct crypto_ahash *child = &ctx->cryptd_tfm->base;
234 int err;
235
236 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
237 crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
238 & CRYPTO_TFM_REQ_MASK);
239 err = crypto_ahash_setkey(child, key, keylen);
240 crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
241 & CRYPTO_TFM_RES_MASK);
242
243 return err;
244}
245
246static int ghash_async_init_tfm(struct crypto_tfm *tfm)
247{
248 struct cryptd_ahash *cryptd_tfm;
249 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
250
251 cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce",
252 CRYPTO_ALG_INTERNAL,
253 CRYPTO_ALG_INTERNAL);
254 if (IS_ERR(cryptd_tfm))
255 return PTR_ERR(cryptd_tfm);
256 ctx->cryptd_tfm = cryptd_tfm;
257 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
258 sizeof(struct ahash_request) +
259 crypto_ahash_reqsize(&cryptd_tfm->base));
260
261 return 0;
262}
263
264static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
265{
266 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
267
268 cryptd_free_ahash(ctx->cryptd_tfm);
269}
270
271static struct ahash_alg ghash_async_alg = {
272 .init = ghash_async_init,
273 .update = ghash_async_update,
274 .final = ghash_async_final,
275 .setkey = ghash_async_setkey,
276 .digest = ghash_async_digest,
277 .halg.digestsize = GHASH_DIGEST_SIZE,
278 .halg.base = {
279 .cra_name = "ghash",
280 .cra_driver_name = "ghash-ce",
281 .cra_priority = 300,
282 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
283 .cra_blocksize = GHASH_BLOCK_SIZE,
284 .cra_type = &crypto_ahash_type,
285 .cra_ctxsize = sizeof(struct ghash_async_ctx),
286 .cra_module = THIS_MODULE,
287 .cra_init = ghash_async_init_tfm,
288 .cra_exit = ghash_async_exit_tfm,
289 },
290};
291
292static int __init ghash_ce_mod_init(void)
293{
294 int err;
295
296 if (!(elf_hwcap2 & HWCAP2_PMULL))
297 return -ENODEV;
298
299 err = crypto_register_shash(&ghash_alg);
300 if (err)
301 return err;
302 err = crypto_register_ahash(&ghash_async_alg);
303 if (err)
304 goto err_shash;
305
306 return 0;
307
308err_shash:
309 crypto_unregister_shash(&ghash_alg);
310 return err;
311}
312
313static void __exit ghash_ce_mod_exit(void)
314{
315 crypto_unregister_ahash(&ghash_async_alg);
316 crypto_unregister_shash(&ghash_alg);
317}
318
319module_init(ghash_ce_mod_init);
320module_exit(ghash_ce_mod_exit);
diff --git a/arch/arm/crypto/sha1-ce-core.S b/arch/arm/crypto/sha1-ce-core.S
new file mode 100644
index 000000000000..b623f51ccbcf
--- /dev/null
+++ b/arch/arm/crypto/sha1-ce-core.S
@@ -0,0 +1,125 @@
1/*
2 * sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd.
5 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/linkage.h>
13#include <asm/assembler.h>
14
15 .text
16 .fpu crypto-neon-fp-armv8
17
18 k0 .req q0
19 k1 .req q1
20 k2 .req q2
21 k3 .req q3
22
23 ta0 .req q4
24 ta1 .req q5
25 tb0 .req q5
26 tb1 .req q4
27
28 dga .req q6
29 dgb .req q7
30 dgbs .req s28
31
32 dg0 .req q12
33 dg1a0 .req q13
34 dg1a1 .req q14
35 dg1b0 .req q14
36 dg1b1 .req q13
37
38 .macro add_only, op, ev, rc, s0, dg1
39 .ifnb \s0
40 vadd.u32 tb\ev, q\s0, \rc
41 .endif
42 sha1h.32 dg1b\ev, dg0
43 .ifb \dg1
44 sha1\op\().32 dg0, dg1a\ev, ta\ev
45 .else
46 sha1\op\().32 dg0, \dg1, ta\ev
47 .endif
48 .endm
49
50 .macro add_update, op, ev, rc, s0, s1, s2, s3, dg1
51 sha1su0.32 q\s0, q\s1, q\s2
52 add_only \op, \ev, \rc, \s1, \dg1
53 sha1su1.32 q\s0, q\s3
54 .endm
55
56 .align 6
57.Lsha1_rcon:
58 .word 0x5a827999, 0x5a827999, 0x5a827999, 0x5a827999
59 .word 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1
60 .word 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc
61 .word 0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6
62
63 /*
64 * void sha1_ce_transform(struct sha1_state *sst, u8 const *src,
65 * int blocks);
66 */
67ENTRY(sha1_ce_transform)
68 /* load round constants */
69 adr ip, .Lsha1_rcon
70 vld1.32 {k0-k1}, [ip, :128]!
71 vld1.32 {k2-k3}, [ip, :128]
72
73 /* load state */
74 vld1.32 {dga}, [r0]
75 vldr dgbs, [r0, #16]
76
77 /* load input */
780: vld1.32 {q8-q9}, [r1]!
79 vld1.32 {q10-q11}, [r1]!
80 subs r2, r2, #1
81
82#ifndef CONFIG_CPU_BIG_ENDIAN
83 vrev32.8 q8, q8
84 vrev32.8 q9, q9
85 vrev32.8 q10, q10
86 vrev32.8 q11, q11
87#endif
88
89 vadd.u32 ta0, q8, k0
90 vmov dg0, dga
91
92 add_update c, 0, k0, 8, 9, 10, 11, dgb
93 add_update c, 1, k0, 9, 10, 11, 8
94 add_update c, 0, k0, 10, 11, 8, 9
95 add_update c, 1, k0, 11, 8, 9, 10
96 add_update c, 0, k1, 8, 9, 10, 11
97
98 add_update p, 1, k1, 9, 10, 11, 8
99 add_update p, 0, k1, 10, 11, 8, 9
100 add_update p, 1, k1, 11, 8, 9, 10
101 add_update p, 0, k1, 8, 9, 10, 11
102 add_update p, 1, k2, 9, 10, 11, 8
103
104 add_update m, 0, k2, 10, 11, 8, 9
105 add_update m, 1, k2, 11, 8, 9, 10
106 add_update m, 0, k2, 8, 9, 10, 11
107 add_update m, 1, k2, 9, 10, 11, 8
108 add_update m, 0, k3, 10, 11, 8, 9
109
110 add_update p, 1, k3, 11, 8, 9, 10
111 add_only p, 0, k3, 9
112 add_only p, 1, k3, 10
113 add_only p, 0, k3, 11
114 add_only p, 1
115
116 /* update state */
117 vadd.u32 dga, dga, dg0
118 vadd.u32 dgb, dgb, dg1a0
119 bne 0b
120
121 /* store new state */
122 vst1.32 {dga}, [r0]
123 vstr dgbs, [r0, #16]
124 bx lr
125ENDPROC(sha1_ce_transform)
diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c
new file mode 100644
index 000000000000..80bc2fcd241a
--- /dev/null
+++ b/arch/arm/crypto/sha1-ce-glue.c
@@ -0,0 +1,96 @@
1/*
2 * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <crypto/internal/hash.h>
12#include <crypto/sha.h>
13#include <crypto/sha1_base.h>
14#include <linux/crypto.h>
15#include <linux/module.h>
16
17#include <asm/hwcap.h>
18#include <asm/neon.h>
19#include <asm/simd.h>
20
21#include "sha1.h"
22
23MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
24MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
25MODULE_LICENSE("GPL v2");
26
27asmlinkage void sha1_ce_transform(struct sha1_state *sst, u8 const *src,
28 int blocks);
29
30static int sha1_ce_update(struct shash_desc *desc, const u8 *data,
31 unsigned int len)
32{
33 struct sha1_state *sctx = shash_desc_ctx(desc);
34
35 if (!may_use_simd() ||
36 (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
37 return sha1_update_arm(desc, data, len);
38
39 kernel_neon_begin();
40 sha1_base_do_update(desc, data, len, sha1_ce_transform);
41 kernel_neon_end();
42
43 return 0;
44}
45
46static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
47 unsigned int len, u8 *out)
48{
49 if (!may_use_simd())
50 return sha1_finup_arm(desc, data, len, out);
51
52 kernel_neon_begin();
53 if (len)
54 sha1_base_do_update(desc, data, len, sha1_ce_transform);
55 sha1_base_do_finalize(desc, sha1_ce_transform);
56 kernel_neon_end();
57
58 return sha1_base_finish(desc, out);
59}
60
61static int sha1_ce_final(struct shash_desc *desc, u8 *out)
62{
63 return sha1_ce_finup(desc, NULL, 0, out);
64}
65
66static struct shash_alg alg = {
67 .init = sha1_base_init,
68 .update = sha1_ce_update,
69 .final = sha1_ce_final,
70 .finup = sha1_ce_finup,
71 .descsize = sizeof(struct sha1_state),
72 .digestsize = SHA1_DIGEST_SIZE,
73 .base = {
74 .cra_name = "sha1",
75 .cra_driver_name = "sha1-ce",
76 .cra_priority = 200,
77 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
78 .cra_blocksize = SHA1_BLOCK_SIZE,
79 .cra_module = THIS_MODULE,
80 }
81};
82
83static int __init sha1_ce_mod_init(void)
84{
85 if (!(elf_hwcap2 & HWCAP2_SHA1))
86 return -ENODEV;
87 return crypto_register_shash(&alg);
88}
89
90static void __exit sha1_ce_mod_fini(void)
91{
92 crypto_unregister_shash(&alg);
93}
94
95module_init(sha1_ce_mod_init);
96module_exit(sha1_ce_mod_fini);
diff --git a/arch/arm/include/asm/crypto/sha1.h b/arch/arm/crypto/sha1.h
index 75e6a417416b..ffd8bd08b1a7 100644
--- a/arch/arm/include/asm/crypto/sha1.h
+++ b/arch/arm/crypto/sha1.h
@@ -7,4 +7,7 @@
7extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, 7extern int sha1_update_arm(struct shash_desc *desc, const u8 *data,
8 unsigned int len); 8 unsigned int len);
9 9
10extern int sha1_finup_arm(struct shash_desc *desc, const u8 *data,
11 unsigned int len, u8 *out);
12
10#endif 13#endif
diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c
index e31b0440c613..6fc73bf8766d 100644
--- a/arch/arm/crypto/sha1_glue.c
+++ b/arch/arm/crypto/sha1_glue.c
@@ -22,127 +22,47 @@
22#include <linux/cryptohash.h> 22#include <linux/cryptohash.h>
23#include <linux/types.h> 23#include <linux/types.h>
24#include <crypto/sha.h> 24#include <crypto/sha.h>
25#include <crypto/sha1_base.h>
25#include <asm/byteorder.h> 26#include <asm/byteorder.h>
26#include <asm/crypto/sha1.h>
27 27
28#include "sha1.h"
28 29
29asmlinkage void sha1_block_data_order(u32 *digest, 30asmlinkage void sha1_block_data_order(u32 *digest,
30 const unsigned char *data, unsigned int rounds); 31 const unsigned char *data, unsigned int rounds);
31 32
32
33static int sha1_init(struct shash_desc *desc)
34{
35 struct sha1_state *sctx = shash_desc_ctx(desc);
36
37 *sctx = (struct sha1_state){
38 .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
39 };
40
41 return 0;
42}
43
44
45static int __sha1_update(struct sha1_state *sctx, const u8 *data,
46 unsigned int len, unsigned int partial)
47{
48 unsigned int done = 0;
49
50 sctx->count += len;
51
52 if (partial) {
53 done = SHA1_BLOCK_SIZE - partial;
54 memcpy(sctx->buffer + partial, data, done);
55 sha1_block_data_order(sctx->state, sctx->buffer, 1);
56 }
57
58 if (len - done >= SHA1_BLOCK_SIZE) {
59 const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE;
60 sha1_block_data_order(sctx->state, data + done, rounds);
61 done += rounds * SHA1_BLOCK_SIZE;
62 }
63
64 memcpy(sctx->buffer, data + done, len - done);
65 return 0;
66}
67
68
69int sha1_update_arm(struct shash_desc *desc, const u8 *data, 33int sha1_update_arm(struct shash_desc *desc, const u8 *data,
70 unsigned int len) 34 unsigned int len)
71{ 35{
72 struct sha1_state *sctx = shash_desc_ctx(desc); 36 /* make sure casting to sha1_block_fn() is safe */
73 unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; 37 BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
74 int res;
75 38
76 /* Handle the fast case right here */ 39 return sha1_base_do_update(desc, data, len,
77 if (partial + len < SHA1_BLOCK_SIZE) { 40 (sha1_block_fn *)sha1_block_data_order);
78 sctx->count += len;
79 memcpy(sctx->buffer + partial, data, len);
80 return 0;
81 }
82 res = __sha1_update(sctx, data, len, partial);
83 return res;
84} 41}
85EXPORT_SYMBOL_GPL(sha1_update_arm); 42EXPORT_SYMBOL_GPL(sha1_update_arm);
86 43
87
88/* Add padding and return the message digest. */
89static int sha1_final(struct shash_desc *desc, u8 *out) 44static int sha1_final(struct shash_desc *desc, u8 *out)
90{ 45{
91 struct sha1_state *sctx = shash_desc_ctx(desc); 46 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_block_data_order);
92 unsigned int i, index, padlen; 47 return sha1_base_finish(desc, out);
93 __be32 *dst = (__be32 *)out;
94 __be64 bits;
95 static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };
96
97 bits = cpu_to_be64(sctx->count << 3);
98
99 /* Pad out to 56 mod 64 and append length */
100 index = sctx->count % SHA1_BLOCK_SIZE;
101 padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index);
102 /* We need to fill a whole block for __sha1_update() */
103 if (padlen <= 56) {
104 sctx->count += padlen;
105 memcpy(sctx->buffer + index, padding, padlen);
106 } else {
107 __sha1_update(sctx, padding, padlen, index);
108 }
109 __sha1_update(sctx, (const u8 *)&bits, sizeof(bits), 56);
110
111 /* Store state in digest */
112 for (i = 0; i < 5; i++)
113 dst[i] = cpu_to_be32(sctx->state[i]);
114
115 /* Wipe context */
116 memset(sctx, 0, sizeof(*sctx));
117 return 0;
118} 48}
119 49
120 50int sha1_finup_arm(struct shash_desc *desc, const u8 *data,
121static int sha1_export(struct shash_desc *desc, void *out) 51 unsigned int len, u8 *out)
122{ 52{
123 struct sha1_state *sctx = shash_desc_ctx(desc); 53 sha1_base_do_update(desc, data, len,
124 memcpy(out, sctx, sizeof(*sctx)); 54 (sha1_block_fn *)sha1_block_data_order);
125 return 0; 55 return sha1_final(desc, out);
126} 56}
127 57EXPORT_SYMBOL_GPL(sha1_finup_arm);
128
129static int sha1_import(struct shash_desc *desc, const void *in)
130{
131 struct sha1_state *sctx = shash_desc_ctx(desc);
132 memcpy(sctx, in, sizeof(*sctx));
133 return 0;
134}
135
136 58
137static struct shash_alg alg = { 59static struct shash_alg alg = {
138 .digestsize = SHA1_DIGEST_SIZE, 60 .digestsize = SHA1_DIGEST_SIZE,
139 .init = sha1_init, 61 .init = sha1_base_init,
140 .update = sha1_update_arm, 62 .update = sha1_update_arm,
141 .final = sha1_final, 63 .final = sha1_final,
142 .export = sha1_export, 64 .finup = sha1_finup_arm,
143 .import = sha1_import,
144 .descsize = sizeof(struct sha1_state), 65 .descsize = sizeof(struct sha1_state),
145 .statesize = sizeof(struct sha1_state),
146 .base = { 66 .base = {
147 .cra_name = "sha1", 67 .cra_name = "sha1",
148 .cra_driver_name= "sha1-asm", 68 .cra_driver_name= "sha1-asm",
diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c
index 0b0083757d47..4e22f122f966 100644
--- a/arch/arm/crypto/sha1_neon_glue.c
+++ b/arch/arm/crypto/sha1_neon_glue.c
@@ -25,147 +25,60 @@
25#include <linux/cryptohash.h> 25#include <linux/cryptohash.h>
26#include <linux/types.h> 26#include <linux/types.h>
27#include <crypto/sha.h> 27#include <crypto/sha.h>
28#include <asm/byteorder.h> 28#include <crypto/sha1_base.h>
29#include <asm/neon.h> 29#include <asm/neon.h>
30#include <asm/simd.h> 30#include <asm/simd.h>
31#include <asm/crypto/sha1.h>
32 31
32#include "sha1.h"
33 33
34asmlinkage void sha1_transform_neon(void *state_h, const char *data, 34asmlinkage void sha1_transform_neon(void *state_h, const char *data,
35 unsigned int rounds); 35 unsigned int rounds);
36 36
37
38static int sha1_neon_init(struct shash_desc *desc)
39{
40 struct sha1_state *sctx = shash_desc_ctx(desc);
41
42 *sctx = (struct sha1_state){
43 .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
44 };
45
46 return 0;
47}
48
49static int __sha1_neon_update(struct shash_desc *desc, const u8 *data,
50 unsigned int len, unsigned int partial)
51{
52 struct sha1_state *sctx = shash_desc_ctx(desc);
53 unsigned int done = 0;
54
55 sctx->count += len;
56
57 if (partial) {
58 done = SHA1_BLOCK_SIZE - partial;
59 memcpy(sctx->buffer + partial, data, done);
60 sha1_transform_neon(sctx->state, sctx->buffer, 1);
61 }
62
63 if (len - done >= SHA1_BLOCK_SIZE) {
64 const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE;
65
66 sha1_transform_neon(sctx->state, data + done, rounds);
67 done += rounds * SHA1_BLOCK_SIZE;
68 }
69
70 memcpy(sctx->buffer, data + done, len - done);
71
72 return 0;
73}
74
75static int sha1_neon_update(struct shash_desc *desc, const u8 *data, 37static int sha1_neon_update(struct shash_desc *desc, const u8 *data,
76 unsigned int len) 38 unsigned int len)
77{ 39{
78 struct sha1_state *sctx = shash_desc_ctx(desc); 40 struct sha1_state *sctx = shash_desc_ctx(desc);
79 unsigned int partial = sctx->count % SHA1_BLOCK_SIZE;
80 int res;
81 41
82 /* Handle the fast case right here */ 42 if (!may_use_simd() ||
83 if (partial + len < SHA1_BLOCK_SIZE) { 43 (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
84 sctx->count += len; 44 return sha1_update_arm(desc, data, len);
85 memcpy(sctx->buffer + partial, data, len);
86 45
87 return 0; 46 kernel_neon_begin();
88 } 47 sha1_base_do_update(desc, data, len,
89 48 (sha1_block_fn *)sha1_transform_neon);
90 if (!may_use_simd()) { 49 kernel_neon_end();
91 res = sha1_update_arm(desc, data, len);
92 } else {
93 kernel_neon_begin();
94 res = __sha1_neon_update(desc, data, len, partial);
95 kernel_neon_end();
96 }
97
98 return res;
99}
100
101
102/* Add padding and return the message digest. */
103static int sha1_neon_final(struct shash_desc *desc, u8 *out)
104{
105 struct sha1_state *sctx = shash_desc_ctx(desc);
106 unsigned int i, index, padlen;
107 __be32 *dst = (__be32 *)out;
108 __be64 bits;
109 static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };
110
111 bits = cpu_to_be64(sctx->count << 3);
112
113 /* Pad out to 56 mod 64 and append length */
114 index = sctx->count % SHA1_BLOCK_SIZE;
115 padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index);
116 if (!may_use_simd()) {
117 sha1_update_arm(desc, padding, padlen);
118 sha1_update_arm(desc, (const u8 *)&bits, sizeof(bits));
119 } else {
120 kernel_neon_begin();
121 /* We need to fill a whole block for __sha1_neon_update() */
122 if (padlen <= 56) {
123 sctx->count += padlen;
124 memcpy(sctx->buffer + index, padding, padlen);
125 } else {
126 __sha1_neon_update(desc, padding, padlen, index);
127 }
128 __sha1_neon_update(desc, (const u8 *)&bits, sizeof(bits), 56);
129 kernel_neon_end();
130 }
131
132 /* Store state in digest */
133 for (i = 0; i < 5; i++)
134 dst[i] = cpu_to_be32(sctx->state[i]);
135
136 /* Wipe context */
137 memset(sctx, 0, sizeof(*sctx));
138 50
139 return 0; 51 return 0;
140} 52}
141 53
142static int sha1_neon_export(struct shash_desc *desc, void *out) 54static int sha1_neon_finup(struct shash_desc *desc, const u8 *data,
55 unsigned int len, u8 *out)
143{ 56{
144 struct sha1_state *sctx = shash_desc_ctx(desc); 57 if (!may_use_simd())
58 return sha1_finup_arm(desc, data, len, out);
145 59
146 memcpy(out, sctx, sizeof(*sctx)); 60 kernel_neon_begin();
61 if (len)
62 sha1_base_do_update(desc, data, len,
63 (sha1_block_fn *)sha1_transform_neon);
64 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_neon);
65 kernel_neon_end();
147 66
148 return 0; 67 return sha1_base_finish(desc, out);
149} 68}
150 69
151static int sha1_neon_import(struct shash_desc *desc, const void *in) 70static int sha1_neon_final(struct shash_desc *desc, u8 *out)
152{ 71{
153 struct sha1_state *sctx = shash_desc_ctx(desc); 72 return sha1_neon_finup(desc, NULL, 0, out);
154
155 memcpy(sctx, in, sizeof(*sctx));
156
157 return 0;
158} 73}
159 74
160static struct shash_alg alg = { 75static struct shash_alg alg = {
161 .digestsize = SHA1_DIGEST_SIZE, 76 .digestsize = SHA1_DIGEST_SIZE,
162 .init = sha1_neon_init, 77 .init = sha1_base_init,
163 .update = sha1_neon_update, 78 .update = sha1_neon_update,
164 .final = sha1_neon_final, 79 .final = sha1_neon_final,
165 .export = sha1_neon_export, 80 .finup = sha1_neon_finup,
166 .import = sha1_neon_import,
167 .descsize = sizeof(struct sha1_state), 81 .descsize = sizeof(struct sha1_state),
168 .statesize = sizeof(struct sha1_state),
169 .base = { 82 .base = {
170 .cra_name = "sha1", 83 .cra_name = "sha1",
171 .cra_driver_name = "sha1-neon", 84 .cra_driver_name = "sha1-neon",
diff --git a/arch/arm/crypto/sha2-ce-core.S b/arch/arm/crypto/sha2-ce-core.S
new file mode 100644
index 000000000000..87ec11a5f405
--- /dev/null
+++ b/arch/arm/crypto/sha2-ce-core.S
@@ -0,0 +1,125 @@
1/*
2 * sha2-ce-core.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd.
5 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/linkage.h>
13#include <asm/assembler.h>
14
15 .text
16 .fpu crypto-neon-fp-armv8
17
18 k0 .req q7
19 k1 .req q8
20 rk .req r3
21
22 ta0 .req q9
23 ta1 .req q10
24 tb0 .req q10
25 tb1 .req q9
26
27 dga .req q11
28 dgb .req q12
29
30 dg0 .req q13
31 dg1 .req q14
32 dg2 .req q15
33
34 .macro add_only, ev, s0
35 vmov dg2, dg0
36 .ifnb \s0
37 vld1.32 {k\ev}, [rk, :128]!
38 .endif
39 sha256h.32 dg0, dg1, tb\ev
40 sha256h2.32 dg1, dg2, tb\ev
41 .ifnb \s0
42 vadd.u32 ta\ev, q\s0, k\ev
43 .endif
44 .endm
45
46 .macro add_update, ev, s0, s1, s2, s3
47 sha256su0.32 q\s0, q\s1
48 add_only \ev, \s1
49 sha256su1.32 q\s0, q\s2, q\s3
50 .endm
51
52 .align 6
53.Lsha256_rcon:
54 .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
55 .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
56 .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
57 .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
58 .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
59 .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
60 .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
61 .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
62 .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
63 .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
64 .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
65 .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
66 .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
67 .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
68 .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
69 .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
70
71 /*
72 * void sha2_ce_transform(struct sha256_state *sst, u8 const *src,
73 int blocks);
74 */
75ENTRY(sha2_ce_transform)
76 /* load state */
77 vld1.32 {dga-dgb}, [r0]
78
79 /* load input */
800: vld1.32 {q0-q1}, [r1]!
81 vld1.32 {q2-q3}, [r1]!
82 subs r2, r2, #1
83
84#ifndef CONFIG_CPU_BIG_ENDIAN
85 vrev32.8 q0, q0
86 vrev32.8 q1, q1
87 vrev32.8 q2, q2
88 vrev32.8 q3, q3
89#endif
90
91 /* load first round constant */
92 adr rk, .Lsha256_rcon
93 vld1.32 {k0}, [rk, :128]!
94
95 vadd.u32 ta0, q0, k0
96 vmov dg0, dga
97 vmov dg1, dgb
98
99 add_update 1, 0, 1, 2, 3
100 add_update 0, 1, 2, 3, 0
101 add_update 1, 2, 3, 0, 1
102 add_update 0, 3, 0, 1, 2
103 add_update 1, 0, 1, 2, 3
104 add_update 0, 1, 2, 3, 0
105 add_update 1, 2, 3, 0, 1
106 add_update 0, 3, 0, 1, 2
107 add_update 1, 0, 1, 2, 3
108 add_update 0, 1, 2, 3, 0
109 add_update 1, 2, 3, 0, 1
110 add_update 0, 3, 0, 1, 2
111
112 add_only 1, 1
113 add_only 0, 2
114 add_only 1, 3
115 add_only 0
116
117 /* update state */
118 vadd.u32 dga, dga, dg0
119 vadd.u32 dgb, dgb, dg1
120 bne 0b
121
122 /* store new state */
123 vst1.32 {dga-dgb}, [r0]
124 bx lr
125ENDPROC(sha2_ce_transform)
diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c
new file mode 100644
index 000000000000..0755b2d657f3
--- /dev/null
+++ b/arch/arm/crypto/sha2-ce-glue.c
@@ -0,0 +1,114 @@
1/*
2 * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <crypto/internal/hash.h>
12#include <crypto/sha.h>
13#include <crypto/sha256_base.h>
14#include <linux/crypto.h>
15#include <linux/module.h>
16
17#include <asm/hwcap.h>
18#include <asm/simd.h>
19#include <asm/neon.h>
20#include <asm/unaligned.h>
21
22#include "sha256_glue.h"
23
24MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
25MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
26MODULE_LICENSE("GPL v2");
27
28asmlinkage void sha2_ce_transform(struct sha256_state *sst, u8 const *src,
29 int blocks);
30
31static int sha2_ce_update(struct shash_desc *desc, const u8 *data,
32 unsigned int len)
33{
34 struct sha256_state *sctx = shash_desc_ctx(desc);
35
36 if (!may_use_simd() ||
37 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
38 return crypto_sha256_arm_update(desc, data, len);
39
40 kernel_neon_begin();
41 sha256_base_do_update(desc, data, len,
42 (sha256_block_fn *)sha2_ce_transform);
43 kernel_neon_end();
44
45 return 0;
46}
47
48static int sha2_ce_finup(struct shash_desc *desc, const u8 *data,
49 unsigned int len, u8 *out)
50{
51 if (!may_use_simd())
52 return crypto_sha256_arm_finup(desc, data, len, out);
53
54 kernel_neon_begin();
55 if (len)
56 sha256_base_do_update(desc, data, len,
57 (sha256_block_fn *)sha2_ce_transform);
58 sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform);
59 kernel_neon_end();
60
61 return sha256_base_finish(desc, out);
62}
63
64static int sha2_ce_final(struct shash_desc *desc, u8 *out)
65{
66 return sha2_ce_finup(desc, NULL, 0, out);
67}
68
69static struct shash_alg algs[] = { {
70 .init = sha224_base_init,
71 .update = sha2_ce_update,
72 .final = sha2_ce_final,
73 .finup = sha2_ce_finup,
74 .descsize = sizeof(struct sha256_state),
75 .digestsize = SHA224_DIGEST_SIZE,
76 .base = {
77 .cra_name = "sha224",
78 .cra_driver_name = "sha224-ce",
79 .cra_priority = 300,
80 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
81 .cra_blocksize = SHA256_BLOCK_SIZE,
82 .cra_module = THIS_MODULE,
83 }
84}, {
85 .init = sha256_base_init,
86 .update = sha2_ce_update,
87 .final = sha2_ce_final,
88 .finup = sha2_ce_finup,
89 .descsize = sizeof(struct sha256_state),
90 .digestsize = SHA256_DIGEST_SIZE,
91 .base = {
92 .cra_name = "sha256",
93 .cra_driver_name = "sha256-ce",
94 .cra_priority = 300,
95 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
96 .cra_blocksize = SHA256_BLOCK_SIZE,
97 .cra_module = THIS_MODULE,
98 }
99} };
100
101static int __init sha2_ce_mod_init(void)
102{
103 if (!(elf_hwcap2 & HWCAP2_SHA2))
104 return -ENODEV;
105 return crypto_register_shashes(algs, ARRAY_SIZE(algs));
106}
107
108static void __exit sha2_ce_mod_fini(void)
109{
110 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
111}
112
113module_init(sha2_ce_mod_init);
114module_exit(sha2_ce_mod_fini);
diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/crypto/sha256-armv4.pl
new file mode 100644
index 000000000000..fac0533ea633
--- /dev/null
+++ b/arch/arm/crypto/sha256-armv4.pl
@@ -0,0 +1,716 @@
1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8#
9# Permission to use under GPL terms is granted.
10# ====================================================================
11
12# SHA256 block procedure for ARMv4. May 2007.
13
14# Performance is ~2x better than gcc 3.4 generated code and in "abso-
15# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
16# byte [on single-issue Xscale PXA250 core].
17
18# July 2010.
19#
20# Rescheduling for dual-issue pipeline resulted in 22% improvement on
21# Cortex A8 core and ~20 cycles per processed byte.
22
23# February 2011.
24#
25# Profiler-assisted and platform-specific optimization resulted in 16%
26# improvement on Cortex A8 core and ~15.4 cycles per processed byte.
27
28# September 2013.
29#
30# Add NEON implementation. On Cortex A8 it was measured to process one
31# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
32# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
33# code (meaning that latter performs sub-optimally, nothing was done
34# about it).
35
36# May 2014.
37#
38# Add ARMv8 code path performing at 2.0 cpb on Apple A7.
39
40while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
41open STDOUT,">$output";
42
43$ctx="r0"; $t0="r0";
44$inp="r1"; $t4="r1";
45$len="r2"; $t1="r2";
46$T1="r3"; $t3="r3";
47$A="r4";
48$B="r5";
49$C="r6";
50$D="r7";
51$E="r8";
52$F="r9";
53$G="r10";
54$H="r11";
55@V=($A,$B,$C,$D,$E,$F,$G,$H);
56$t2="r12";
57$Ktbl="r14";
58
59@Sigma0=( 2,13,22);
60@Sigma1=( 6,11,25);
61@sigma0=( 7,18, 3);
62@sigma1=(17,19,10);
63
64sub BODY_00_15 {
65my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
66
67$code.=<<___ if ($i<16);
68#if __ARM_ARCH__>=7
69 @ ldr $t1,[$inp],#4 @ $i
70# if $i==15
71 str $inp,[sp,#17*4] @ make room for $t4
72# endif
73 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
74 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
75 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
76# ifndef __ARMEB__
77 rev $t1,$t1
78# endif
79#else
80 @ ldrb $t1,[$inp,#3] @ $i
81 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
82 ldrb $t2,[$inp,#2]
83 ldrb $t0,[$inp,#1]
84 orr $t1,$t1,$t2,lsl#8
85 ldrb $t2,[$inp],#4
86 orr $t1,$t1,$t0,lsl#16
87# if $i==15
88 str $inp,[sp,#17*4] @ make room for $t4
89# endif
90 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
91 orr $t1,$t1,$t2,lsl#24
92 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
93#endif
94___
95$code.=<<___;
96 ldr $t2,[$Ktbl],#4 @ *K256++
97 add $h,$h,$t1 @ h+=X[i]
98 str $t1,[sp,#`$i%16`*4]
99 eor $t1,$f,$g
100 add $h,$h,$t0,ror#$Sigma1[0] @ h+=Sigma1(e)
101 and $t1,$t1,$e
102 add $h,$h,$t2 @ h+=K256[i]
103 eor $t1,$t1,$g @ Ch(e,f,g)
104 eor $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
105 add $h,$h,$t1 @ h+=Ch(e,f,g)
106#if $i==31
107 and $t2,$t2,#0xff
108 cmp $t2,#0xf2 @ done?
109#endif
110#if $i<15
111# if __ARM_ARCH__>=7
112 ldr $t1,[$inp],#4 @ prefetch
113# else
114 ldrb $t1,[$inp,#3]
115# endif
116 eor $t2,$a,$b @ a^b, b^c in next round
117#else
118 ldr $t1,[sp,#`($i+2)%16`*4] @ from future BODY_16_xx
119 eor $t2,$a,$b @ a^b, b^c in next round
120 ldr $t4,[sp,#`($i+15)%16`*4] @ from future BODY_16_xx
121#endif
122 eor $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]` @ Sigma0(a)
123 and $t3,$t3,$t2 @ (b^c)&=(a^b)
124 add $d,$d,$h @ d+=h
125 eor $t3,$t3,$b @ Maj(a,b,c)
126 add $h,$h,$t0,ror#$Sigma0[0] @ h+=Sigma0(a)
127 @ add $h,$h,$t3 @ h+=Maj(a,b,c)
128___
129 ($t2,$t3)=($t3,$t2);
130}
131
132sub BODY_16_XX {
133my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
134
135$code.=<<___;
136 @ ldr $t1,[sp,#`($i+1)%16`*4] @ $i
137 @ ldr $t4,[sp,#`($i+14)%16`*4]
138 mov $t0,$t1,ror#$sigma0[0]
139 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
140 mov $t2,$t4,ror#$sigma1[0]
141 eor $t0,$t0,$t1,ror#$sigma0[1]
142 eor $t2,$t2,$t4,ror#$sigma1[1]
143 eor $t0,$t0,$t1,lsr#$sigma0[2] @ sigma0(X[i+1])
144 ldr $t1,[sp,#`($i+0)%16`*4]
145 eor $t2,$t2,$t4,lsr#$sigma1[2] @ sigma1(X[i+14])
146 ldr $t4,[sp,#`($i+9)%16`*4]
147
148 add $t2,$t2,$t0
149 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` @ from BODY_00_15
150 add $t1,$t1,$t2
151 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
152 add $t1,$t1,$t4 @ X[i]
153___
154 &BODY_00_15(@_);
155}
156
157$code=<<___;
158#ifndef __KERNEL__
159# include "arm_arch.h"
160#else
161# define __ARM_ARCH__ __LINUX_ARM_ARCH__
162# define __ARM_MAX_ARCH__ 7
163#endif
164
165.text
166#if __ARM_ARCH__<7
167.code 32
168#else
169.syntax unified
170# ifdef __thumb2__
171# define adrl adr
172.thumb
173# else
174.code 32
175# endif
176#endif
177
178.type K256,%object
179.align 5
180K256:
181.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
182.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
183.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
184.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
185.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
186.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
187.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
188.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
189.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
190.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
191.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
192.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
193.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
194.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
195.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
196.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
197.size K256,.-K256
198.word 0 @ terminator
199#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
200.LOPENSSL_armcap:
201.word OPENSSL_armcap_P-sha256_block_data_order
202#endif
203.align 5
204
205.global sha256_block_data_order
206.type sha256_block_data_order,%function
207sha256_block_data_order:
208#if __ARM_ARCH__<7
209 sub r3,pc,#8 @ sha256_block_data_order
210#else
211 adr r3,sha256_block_data_order
212#endif
213#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
214 ldr r12,.LOPENSSL_armcap
215 ldr r12,[r3,r12] @ OPENSSL_armcap_P
216 tst r12,#ARMV8_SHA256
217 bne .LARMv8
218 tst r12,#ARMV7_NEON
219 bne .LNEON
220#endif
221 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
222 stmdb sp!,{$ctx,$inp,$len,r4-r11,lr}
223 ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
224 sub $Ktbl,r3,#256+32 @ K256
225 sub sp,sp,#16*4 @ alloca(X[16])
226.Loop:
227# if __ARM_ARCH__>=7
228 ldr $t1,[$inp],#4
229# else
230 ldrb $t1,[$inp,#3]
231# endif
232 eor $t3,$B,$C @ magic
233 eor $t2,$t2,$t2
234___
235for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
236$code.=".Lrounds_16_xx:\n";
237for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
238$code.=<<___;
239#if __ARM_ARCH__>=7
240 ite eq @ Thumb2 thing, sanity check in ARM
241#endif
242 ldreq $t3,[sp,#16*4] @ pull ctx
243 bne .Lrounds_16_xx
244
245 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
246 ldr $t0,[$t3,#0]
247 ldr $t1,[$t3,#4]
248 ldr $t2,[$t3,#8]
249 add $A,$A,$t0
250 ldr $t0,[$t3,#12]
251 add $B,$B,$t1
252 ldr $t1,[$t3,#16]
253 add $C,$C,$t2
254 ldr $t2,[$t3,#20]
255 add $D,$D,$t0
256 ldr $t0,[$t3,#24]
257 add $E,$E,$t1
258 ldr $t1,[$t3,#28]
259 add $F,$F,$t2
260 ldr $inp,[sp,#17*4] @ pull inp
261 ldr $t2,[sp,#18*4] @ pull inp+len
262 add $G,$G,$t0
263 add $H,$H,$t1
264 stmia $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
265 cmp $inp,$t2
266 sub $Ktbl,$Ktbl,#256 @ rewind Ktbl
267 bne .Loop
268
269 add sp,sp,#`16+3`*4 @ destroy frame
270#if __ARM_ARCH__>=5
271 ldmia sp!,{r4-r11,pc}
272#else
273 ldmia sp!,{r4-r11,lr}
274 tst lr,#1
275 moveq pc,lr @ be binary compatible with V4, yet
276 bx lr @ interoperable with Thumb ISA:-)
277#endif
278.size sha256_block_data_order,.-sha256_block_data_order
279___
280######################################################################
281# NEON stuff
282#
283{{{
284my @X=map("q$_",(0..3));
285my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
286my $Xfer=$t4;
287my $j=0;
288
289sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
290sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
291
292sub AUTOLOAD() # thunk [simplified] x86-style perlasm
293{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
294 my $arg = pop;
295 $arg = "#$arg" if ($arg*1 eq $arg);
296 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
297}
298
299sub Xupdate()
300{ use integer;
301 my $body = shift;
302 my @insns = (&$body,&$body,&$body,&$body);
303 my ($a,$b,$c,$d,$e,$f,$g,$h);
304
305 &vext_8 ($T0,@X[0],@X[1],4); # X[1..4]
306 eval(shift(@insns));
307 eval(shift(@insns));
308 eval(shift(@insns));
309 &vext_8 ($T1,@X[2],@X[3],4); # X[9..12]
310 eval(shift(@insns));
311 eval(shift(@insns));
312 eval(shift(@insns));
313 &vshr_u32 ($T2,$T0,$sigma0[0]);
314 eval(shift(@insns));
315 eval(shift(@insns));
316 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += X[9..12]
317 eval(shift(@insns));
318 eval(shift(@insns));
319 &vshr_u32 ($T1,$T0,$sigma0[2]);
320 eval(shift(@insns));
321 eval(shift(@insns));
322 &vsli_32 ($T2,$T0,32-$sigma0[0]);
323 eval(shift(@insns));
324 eval(shift(@insns));
325 &vshr_u32 ($T3,$T0,$sigma0[1]);
326 eval(shift(@insns));
327 eval(shift(@insns));
328 &veor ($T1,$T1,$T2);
329 eval(shift(@insns));
330 eval(shift(@insns));
331 &vsli_32 ($T3,$T0,32-$sigma0[1]);
332 eval(shift(@insns));
333 eval(shift(@insns));
334 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[0]);
335 eval(shift(@insns));
336 eval(shift(@insns));
337 &veor ($T1,$T1,$T3); # sigma0(X[1..4])
338 eval(shift(@insns));
339 eval(shift(@insns));
340 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[0]);
341 eval(shift(@insns));
342 eval(shift(@insns));
343 &vshr_u32 ($T5,&Dhi(@X[3]),$sigma1[2]);
344 eval(shift(@insns));
345 eval(shift(@insns));
346 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4])
347 eval(shift(@insns));
348 eval(shift(@insns));
349 &veor ($T5,$T5,$T4);
350 eval(shift(@insns));
351 eval(shift(@insns));
352 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[1]);
353 eval(shift(@insns));
354 eval(shift(@insns));
355 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[1]);
356 eval(shift(@insns));
357 eval(shift(@insns));
358 &veor ($T5,$T5,$T4); # sigma1(X[14..15])
359 eval(shift(@insns));
360 eval(shift(@insns));
361 &vadd_i32 (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
362 eval(shift(@insns));
363 eval(shift(@insns));
364 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[0]);
365 eval(shift(@insns));
366 eval(shift(@insns));
367 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[0]);
368 eval(shift(@insns));
369 eval(shift(@insns));
370 &vshr_u32 ($T5,&Dlo(@X[0]),$sigma1[2]);
371 eval(shift(@insns));
372 eval(shift(@insns));
373 &veor ($T5,$T5,$T4);
374 eval(shift(@insns));
375 eval(shift(@insns));
376 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[1]);
377 eval(shift(@insns));
378 eval(shift(@insns));
379 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
380 eval(shift(@insns));
381 eval(shift(@insns));
382 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[1]);
383 eval(shift(@insns));
384 eval(shift(@insns));
385 &veor ($T5,$T5,$T4); # sigma1(X[16..17])
386 eval(shift(@insns));
387 eval(shift(@insns));
388 &vadd_i32 (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
389 eval(shift(@insns));
390 eval(shift(@insns));
391 &vadd_i32 ($T0,$T0,@X[0]);
392 while($#insns>=2) { eval(shift(@insns)); }
393 &vst1_32 ("{$T0}","[$Xfer,:128]!");
394 eval(shift(@insns));
395 eval(shift(@insns));
396
397 push(@X,shift(@X)); # "rotate" X[]
398}
399
400sub Xpreload()
401{ use integer;
402 my $body = shift;
403 my @insns = (&$body,&$body,&$body,&$body);
404 my ($a,$b,$c,$d,$e,$f,$g,$h);
405
406 eval(shift(@insns));
407 eval(shift(@insns));
408 eval(shift(@insns));
409 eval(shift(@insns));
410 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
411 eval(shift(@insns));
412 eval(shift(@insns));
413 eval(shift(@insns));
414 eval(shift(@insns));
415 &vrev32_8 (@X[0],@X[0]);
416 eval(shift(@insns));
417 eval(shift(@insns));
418 eval(shift(@insns));
419 eval(shift(@insns));
420 &vadd_i32 ($T0,$T0,@X[0]);
421 foreach (@insns) { eval; } # remaining instructions
422 &vst1_32 ("{$T0}","[$Xfer,:128]!");
423
424 push(@X,shift(@X)); # "rotate" X[]
425}
426
427sub body_00_15 () {
428 (
429 '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
430 '&add ($h,$h,$t1)', # h+=X[i]+K[i]
431 '&eor ($t1,$f,$g)',
432 '&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
433 '&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past
434 '&and ($t1,$t1,$e)',
435 '&eor ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e)
436 '&eor ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
437 '&eor ($t1,$t1,$g)', # Ch(e,f,g)
438 '&add ($h,$h,$t2,"ror#$Sigma1[0]")', # h+=Sigma1(e)
439 '&eor ($t2,$a,$b)', # a^b, b^c in next round
440 '&eor ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a)
441 '&add ($h,$h,$t1)', # h+=Ch(e,f,g)
442 '&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'.
443 '&ldr ($t1,"[$Ktbl]") if ($j==15);'.
444 '&ldr ($t1,"[sp,#64]") if ($j==31)',
445 '&and ($t3,$t3,$t2)', # (b^c)&=(a^b)
446 '&add ($d,$d,$h)', # d+=h
447 '&add ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
448 '&eor ($t3,$t3,$b)', # Maj(a,b,c)
449 '$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
450 )
451}
452
453$code.=<<___;
454#if __ARM_MAX_ARCH__>=7
455.arch armv7-a
456.fpu neon
457
458.global sha256_block_data_order_neon
459.type sha256_block_data_order_neon,%function
460.align 4
461sha256_block_data_order_neon:
462.LNEON:
463 stmdb sp!,{r4-r12,lr}
464
465 sub $H,sp,#16*4+16
466 adrl $Ktbl,K256
467 bic $H,$H,#15 @ align for 128-bit stores
468 mov $t2,sp
469 mov sp,$H @ alloca
470 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
471
472 vld1.8 {@X[0]},[$inp]!
473 vld1.8 {@X[1]},[$inp]!
474 vld1.8 {@X[2]},[$inp]!
475 vld1.8 {@X[3]},[$inp]!
476 vld1.32 {$T0},[$Ktbl,:128]!
477 vld1.32 {$T1},[$Ktbl,:128]!
478 vld1.32 {$T2},[$Ktbl,:128]!
479 vld1.32 {$T3},[$Ktbl,:128]!
480 vrev32.8 @X[0],@X[0] @ yes, even on
481 str $ctx,[sp,#64]
482 vrev32.8 @X[1],@X[1] @ big-endian
483 str $inp,[sp,#68]
484 mov $Xfer,sp
485 vrev32.8 @X[2],@X[2]
486 str $len,[sp,#72]
487 vrev32.8 @X[3],@X[3]
488 str $t2,[sp,#76] @ save original sp
489 vadd.i32 $T0,$T0,@X[0]
490 vadd.i32 $T1,$T1,@X[1]
491 vst1.32 {$T0},[$Xfer,:128]!
492 vadd.i32 $T2,$T2,@X[2]
493 vst1.32 {$T1},[$Xfer,:128]!
494 vadd.i32 $T3,$T3,@X[3]
495 vst1.32 {$T2},[$Xfer,:128]!
496 vst1.32 {$T3},[$Xfer,:128]!
497
498 ldmia $ctx,{$A-$H}
499 sub $Xfer,$Xfer,#64
500 ldr $t1,[sp,#0]
501 eor $t2,$t2,$t2
502 eor $t3,$B,$C
503 b .L_00_48
504
505.align 4
506.L_00_48:
507___
508 &Xupdate(\&body_00_15);
509 &Xupdate(\&body_00_15);
510 &Xupdate(\&body_00_15);
511 &Xupdate(\&body_00_15);
512$code.=<<___;
513 teq $t1,#0 @ check for K256 terminator
514 ldr $t1,[sp,#0]
515 sub $Xfer,$Xfer,#64
516 bne .L_00_48
517
518 ldr $inp,[sp,#68]
519 ldr $t0,[sp,#72]
520 sub $Ktbl,$Ktbl,#256 @ rewind $Ktbl
521 teq $inp,$t0
522 it eq
523 subeq $inp,$inp,#64 @ avoid SEGV
524 vld1.8 {@X[0]},[$inp]! @ load next input block
525 vld1.8 {@X[1]},[$inp]!
526 vld1.8 {@X[2]},[$inp]!
527 vld1.8 {@X[3]},[$inp]!
528 it ne
529 strne $inp,[sp,#68]
530 mov $Xfer,sp
531___
532 &Xpreload(\&body_00_15);
533 &Xpreload(\&body_00_15);
534 &Xpreload(\&body_00_15);
535 &Xpreload(\&body_00_15);
536$code.=<<___;
537 ldr $t0,[$t1,#0]
538 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
539 ldr $t2,[$t1,#4]
540 ldr $t3,[$t1,#8]
541 ldr $t4,[$t1,#12]
542 add $A,$A,$t0 @ accumulate
543 ldr $t0,[$t1,#16]
544 add $B,$B,$t2
545 ldr $t2,[$t1,#20]
546 add $C,$C,$t3
547 ldr $t3,[$t1,#24]
548 add $D,$D,$t4
549 ldr $t4,[$t1,#28]
550 add $E,$E,$t0
551 str $A,[$t1],#4
552 add $F,$F,$t2
553 str $B,[$t1],#4
554 add $G,$G,$t3
555 str $C,[$t1],#4
556 add $H,$H,$t4
557 str $D,[$t1],#4
558 stmia $t1,{$E-$H}
559
560 ittte ne
561 movne $Xfer,sp
562 ldrne $t1,[sp,#0]
563 eorne $t2,$t2,$t2
564 ldreq sp,[sp,#76] @ restore original sp
565 itt ne
566 eorne $t3,$B,$C
567 bne .L_00_48
568
569 ldmia sp!,{r4-r12,pc}
570.size sha256_block_data_order_neon,.-sha256_block_data_order_neon
571#endif
572___
573}}}
574######################################################################
575# ARMv8 stuff
576#
577{{{
578my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
579my @MSG=map("q$_",(8..11));
580my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
581my $Ktbl="r3";
582
583$code.=<<___;
584#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
585
586# ifdef __thumb2__
587# define INST(a,b,c,d) .byte c,d|0xc,a,b
588# else
589# define INST(a,b,c,d) .byte a,b,c,d
590# endif
591
592.type sha256_block_data_order_armv8,%function
593.align 5
594sha256_block_data_order_armv8:
595.LARMv8:
596 vld1.32 {$ABCD,$EFGH},[$ctx]
597# ifdef __thumb2__
598 adr $Ktbl,.LARMv8
599 sub $Ktbl,$Ktbl,#.LARMv8-K256
600# else
601 adrl $Ktbl,K256
602# endif
603 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
604
605.Loop_v8:
606 vld1.8 {@MSG[0]-@MSG[1]},[$inp]!
607 vld1.8 {@MSG[2]-@MSG[3]},[$inp]!
608 vld1.32 {$W0},[$Ktbl]!
609 vrev32.8 @MSG[0],@MSG[0]
610 vrev32.8 @MSG[1],@MSG[1]
611 vrev32.8 @MSG[2],@MSG[2]
612 vrev32.8 @MSG[3],@MSG[3]
613 vmov $ABCD_SAVE,$ABCD @ offload
614 vmov $EFGH_SAVE,$EFGH
615 teq $inp,$len
616___
617for($i=0;$i<12;$i++) {
618$code.=<<___;
619 vld1.32 {$W1},[$Ktbl]!
620 vadd.i32 $W0,$W0,@MSG[0]
621 sha256su0 @MSG[0],@MSG[1]
622 vmov $abcd,$ABCD
623 sha256h $ABCD,$EFGH,$W0
624 sha256h2 $EFGH,$abcd,$W0
625 sha256su1 @MSG[0],@MSG[2],@MSG[3]
626___
627 ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
628}
629$code.=<<___;
630 vld1.32 {$W1},[$Ktbl]!
631 vadd.i32 $W0,$W0,@MSG[0]
632 vmov $abcd,$ABCD
633 sha256h $ABCD,$EFGH,$W0
634 sha256h2 $EFGH,$abcd,$W0
635
636 vld1.32 {$W0},[$Ktbl]!
637 vadd.i32 $W1,$W1,@MSG[1]
638 vmov $abcd,$ABCD
639 sha256h $ABCD,$EFGH,$W1
640 sha256h2 $EFGH,$abcd,$W1
641
642 vld1.32 {$W1},[$Ktbl]
643 vadd.i32 $W0,$W0,@MSG[2]
644 sub $Ktbl,$Ktbl,#256-16 @ rewind
645 vmov $abcd,$ABCD
646 sha256h $ABCD,$EFGH,$W0
647 sha256h2 $EFGH,$abcd,$W0
648
649 vadd.i32 $W1,$W1,@MSG[3]
650 vmov $abcd,$ABCD
651 sha256h $ABCD,$EFGH,$W1
652 sha256h2 $EFGH,$abcd,$W1
653
654 vadd.i32 $ABCD,$ABCD,$ABCD_SAVE
655 vadd.i32 $EFGH,$EFGH,$EFGH_SAVE
656 it ne
657 bne .Loop_v8
658
659 vst1.32 {$ABCD,$EFGH},[$ctx]
660
661 ret @ bx lr
662.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
663#endif
664___
665}}}
666$code.=<<___;
667.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
668.align 2
669#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
670.comm OPENSSL_armcap_P,4,4
671#endif
672___
673
674open SELF,$0;
675while(<SELF>) {
676 next if (/^#!/);
677 last if (!s/^#/@/ and !/^$/);
678 print;
679}
680close SELF;
681
682{ my %opcode = (
683 "sha256h" => 0xf3000c40, "sha256h2" => 0xf3100c40,
684 "sha256su0" => 0xf3ba03c0, "sha256su1" => 0xf3200c40 );
685
686 sub unsha256 {
687 my ($mnemonic,$arg)=@_;
688
689 if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
690 my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
691 |(($2&7)<<17)|(($2&8)<<4)
692 |(($3&7)<<1) |(($3&8)<<2);
693 # since ARMv7 instructions are always encoded little-endian.
694 # correct solution is to use .inst directive, but older
695 # assemblers don't implement it:-(
696 sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
697 $word&0xff,($word>>8)&0xff,
698 ($word>>16)&0xff,($word>>24)&0xff,
699 $mnemonic,$arg;
700 }
701 }
702}
703
704foreach (split($/,$code)) {
705
706 s/\`([^\`]*)\`/eval $1/geo;
707
708 s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
709
710 s/\bret\b/bx lr/go or
711 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
712
713 print $_,"\n";
714}
715
716close STDOUT; # enforce flush
diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped
new file mode 100644
index 000000000000..555a1a8eec90
--- /dev/null
+++ b/arch/arm/crypto/sha256-core.S_shipped
@@ -0,0 +1,2808 @@
1
2@ ====================================================================
3@ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
4@ project. The module is, however, dual licensed under OpenSSL and
5@ CRYPTOGAMS licenses depending on where you obtain it. For further
6@ details see http://www.openssl.org/~appro/cryptogams/.
7@
8@ Permission to use under GPL terms is granted.
9@ ====================================================================
10
11@ SHA256 block procedure for ARMv4. May 2007.
12
13@ Performance is ~2x better than gcc 3.4 generated code and in "abso-
14@ lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
15@ byte [on single-issue Xscale PXA250 core].
16
17@ July 2010.
18@
19@ Rescheduling for dual-issue pipeline resulted in 22% improvement on
20@ Cortex A8 core and ~20 cycles per processed byte.
21
22@ February 2011.
23@
24@ Profiler-assisted and platform-specific optimization resulted in 16%
25@ improvement on Cortex A8 core and ~15.4 cycles per processed byte.
26
27@ September 2013.
28@
29@ Add NEON implementation. On Cortex A8 it was measured to process one
30@ byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
31@ S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
32@ code (meaning that latter performs sub-optimally, nothing was done
33@ about it).
34
35@ May 2014.
36@
37@ Add ARMv8 code path performing at 2.0 cpb on Apple A7.
38
39#ifndef __KERNEL__
40# include "arm_arch.h"
41#else
42# define __ARM_ARCH__ __LINUX_ARM_ARCH__
43# define __ARM_MAX_ARCH__ 7
44#endif
45
46.text
47#if __ARM_ARCH__<7
48.code 32
49#else
50.syntax unified
51# ifdef __thumb2__
52# define adrl adr
53.thumb
54# else
55.code 32
56# endif
57#endif
58
59.type K256,%object
60.align 5
61K256:
62.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
63.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
64.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
65.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
66.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
67.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
68.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
69.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
70.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
71.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
72.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
73.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
74.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
75.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
76.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
77.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
78.size K256,.-K256
79.word 0 @ terminator
80#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
81.LOPENSSL_armcap:
82.word OPENSSL_armcap_P-sha256_block_data_order
83#endif
84.align 5
85
86.global sha256_block_data_order
87.type sha256_block_data_order,%function
88sha256_block_data_order:
89#if __ARM_ARCH__<7
90 sub r3,pc,#8 @ sha256_block_data_order
91#else
92 adr r3,sha256_block_data_order
93#endif
94#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
95 ldr r12,.LOPENSSL_armcap
96 ldr r12,[r3,r12] @ OPENSSL_armcap_P
97 tst r12,#ARMV8_SHA256
98 bne .LARMv8
99 tst r12,#ARMV7_NEON
100 bne .LNEON
101#endif
102 add r2,r1,r2,lsl#6 @ len to point at the end of inp
103 stmdb sp!,{r0,r1,r2,r4-r11,lr}
104 ldmia r0,{r4,r5,r6,r7,r8,r9,r10,r11}
105 sub r14,r3,#256+32 @ K256
106 sub sp,sp,#16*4 @ alloca(X[16])
107.Loop:
108# if __ARM_ARCH__>=7
109 ldr r2,[r1],#4
110# else
111 ldrb r2,[r1,#3]
112# endif
113 eor r3,r5,r6 @ magic
114 eor r12,r12,r12
115#if __ARM_ARCH__>=7
116 @ ldr r2,[r1],#4 @ 0
117# if 0==15
118 str r1,[sp,#17*4] @ make room for r1
119# endif
120 eor r0,r8,r8,ror#5
121 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
122 eor r0,r0,r8,ror#19 @ Sigma1(e)
123# ifndef __ARMEB__
124 rev r2,r2
125# endif
126#else
127 @ ldrb r2,[r1,#3] @ 0
128 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
129 ldrb r12,[r1,#2]
130 ldrb r0,[r1,#1]
131 orr r2,r2,r12,lsl#8
132 ldrb r12,[r1],#4
133 orr r2,r2,r0,lsl#16
134# if 0==15
135 str r1,[sp,#17*4] @ make room for r1
136# endif
137 eor r0,r8,r8,ror#5
138 orr r2,r2,r12,lsl#24
139 eor r0,r0,r8,ror#19 @ Sigma1(e)
140#endif
141 ldr r12,[r14],#4 @ *K256++
142 add r11,r11,r2 @ h+=X[i]
143 str r2,[sp,#0*4]
144 eor r2,r9,r10
145 add r11,r11,r0,ror#6 @ h+=Sigma1(e)
146 and r2,r2,r8
147 add r11,r11,r12 @ h+=K256[i]
148 eor r2,r2,r10 @ Ch(e,f,g)
149 eor r0,r4,r4,ror#11
150 add r11,r11,r2 @ h+=Ch(e,f,g)
151#if 0==31
152 and r12,r12,#0xff
153 cmp r12,#0xf2 @ done?
154#endif
155#if 0<15
156# if __ARM_ARCH__>=7
157 ldr r2,[r1],#4 @ prefetch
158# else
159 ldrb r2,[r1,#3]
160# endif
161 eor r12,r4,r5 @ a^b, b^c in next round
162#else
163 ldr r2,[sp,#2*4] @ from future BODY_16_xx
164 eor r12,r4,r5 @ a^b, b^c in next round
165 ldr r1,[sp,#15*4] @ from future BODY_16_xx
166#endif
167 eor r0,r0,r4,ror#20 @ Sigma0(a)
168 and r3,r3,r12 @ (b^c)&=(a^b)
169 add r7,r7,r11 @ d+=h
170 eor r3,r3,r5 @ Maj(a,b,c)
171 add r11,r11,r0,ror#2 @ h+=Sigma0(a)
172 @ add r11,r11,r3 @ h+=Maj(a,b,c)
173#if __ARM_ARCH__>=7
174 @ ldr r2,[r1],#4 @ 1
175# if 1==15
176 str r1,[sp,#17*4] @ make room for r1
177# endif
178 eor r0,r7,r7,ror#5
179 add r11,r11,r3 @ h+=Maj(a,b,c) from the past
180 eor r0,r0,r7,ror#19 @ Sigma1(e)
181# ifndef __ARMEB__
182 rev r2,r2
183# endif
184#else
185 @ ldrb r2,[r1,#3] @ 1
186 add r11,r11,r3 @ h+=Maj(a,b,c) from the past
187 ldrb r3,[r1,#2]
188 ldrb r0,[r1,#1]
189 orr r2,r2,r3,lsl#8
190 ldrb r3,[r1],#4
191 orr r2,r2,r0,lsl#16
192# if 1==15
193 str r1,[sp,#17*4] @ make room for r1
194# endif
195 eor r0,r7,r7,ror#5
196 orr r2,r2,r3,lsl#24
197 eor r0,r0,r7,ror#19 @ Sigma1(e)
198#endif
199 ldr r3,[r14],#4 @ *K256++
200 add r10,r10,r2 @ h+=X[i]
201 str r2,[sp,#1*4]
202 eor r2,r8,r9
203 add r10,r10,r0,ror#6 @ h+=Sigma1(e)
204 and r2,r2,r7
205 add r10,r10,r3 @ h+=K256[i]
206 eor r2,r2,r9 @ Ch(e,f,g)
207 eor r0,r11,r11,ror#11
208 add r10,r10,r2 @ h+=Ch(e,f,g)
209#if 1==31
210 and r3,r3,#0xff
211 cmp r3,#0xf2 @ done?
212#endif
213#if 1<15
214# if __ARM_ARCH__>=7
215 ldr r2,[r1],#4 @ prefetch
216# else
217 ldrb r2,[r1,#3]
218# endif
219 eor r3,r11,r4 @ a^b, b^c in next round
220#else
221 ldr r2,[sp,#3*4] @ from future BODY_16_xx
222 eor r3,r11,r4 @ a^b, b^c in next round
223 ldr r1,[sp,#0*4] @ from future BODY_16_xx
224#endif
225 eor r0,r0,r11,ror#20 @ Sigma0(a)
226 and r12,r12,r3 @ (b^c)&=(a^b)
227 add r6,r6,r10 @ d+=h
228 eor r12,r12,r4 @ Maj(a,b,c)
229 add r10,r10,r0,ror#2 @ h+=Sigma0(a)
230 @ add r10,r10,r12 @ h+=Maj(a,b,c)
231#if __ARM_ARCH__>=7
232 @ ldr r2,[r1],#4 @ 2
233# if 2==15
234 str r1,[sp,#17*4] @ make room for r1
235# endif
236 eor r0,r6,r6,ror#5
237 add r10,r10,r12 @ h+=Maj(a,b,c) from the past
238 eor r0,r0,r6,ror#19 @ Sigma1(e)
239# ifndef __ARMEB__
240 rev r2,r2
241# endif
242#else
243 @ ldrb r2,[r1,#3] @ 2
244 add r10,r10,r12 @ h+=Maj(a,b,c) from the past
245 ldrb r12,[r1,#2]
246 ldrb r0,[r1,#1]
247 orr r2,r2,r12,lsl#8
248 ldrb r12,[r1],#4
249 orr r2,r2,r0,lsl#16
250# if 2==15
251 str r1,[sp,#17*4] @ make room for r1
252# endif
253 eor r0,r6,r6,ror#5
254 orr r2,r2,r12,lsl#24
255 eor r0,r0,r6,ror#19 @ Sigma1(e)
256#endif
257 ldr r12,[r14],#4 @ *K256++
258 add r9,r9,r2 @ h+=X[i]
259 str r2,[sp,#2*4]
260 eor r2,r7,r8
261 add r9,r9,r0,ror#6 @ h+=Sigma1(e)
262 and r2,r2,r6
263 add r9,r9,r12 @ h+=K256[i]
264 eor r2,r2,r8 @ Ch(e,f,g)
265 eor r0,r10,r10,ror#11
266 add r9,r9,r2 @ h+=Ch(e,f,g)
267#if 2==31
268 and r12,r12,#0xff
269 cmp r12,#0xf2 @ done?
270#endif
271#if 2<15
272# if __ARM_ARCH__>=7
273 ldr r2,[r1],#4 @ prefetch
274# else
275 ldrb r2,[r1,#3]
276# endif
277 eor r12,r10,r11 @ a^b, b^c in next round
278#else
279 ldr r2,[sp,#4*4] @ from future BODY_16_xx
280 eor r12,r10,r11 @ a^b, b^c in next round
281 ldr r1,[sp,#1*4] @ from future BODY_16_xx
282#endif
283 eor r0,r0,r10,ror#20 @ Sigma0(a)
284 and r3,r3,r12 @ (b^c)&=(a^b)
285 add r5,r5,r9 @ d+=h
286 eor r3,r3,r11 @ Maj(a,b,c)
287 add r9,r9,r0,ror#2 @ h+=Sigma0(a)
288 @ add r9,r9,r3 @ h+=Maj(a,b,c)
289#if __ARM_ARCH__>=7
290 @ ldr r2,[r1],#4 @ 3
291# if 3==15
292 str r1,[sp,#17*4] @ make room for r1
293# endif
294 eor r0,r5,r5,ror#5
295 add r9,r9,r3 @ h+=Maj(a,b,c) from the past
296 eor r0,r0,r5,ror#19 @ Sigma1(e)
297# ifndef __ARMEB__
298 rev r2,r2
299# endif
300#else
301 @ ldrb r2,[r1,#3] @ 3
302 add r9,r9,r3 @ h+=Maj(a,b,c) from the past
303 ldrb r3,[r1,#2]
304 ldrb r0,[r1,#1]
305 orr r2,r2,r3,lsl#8
306 ldrb r3,[r1],#4
307 orr r2,r2,r0,lsl#16
308# if 3==15
309 str r1,[sp,#17*4] @ make room for r1
310# endif
311 eor r0,r5,r5,ror#5
312 orr r2,r2,r3,lsl#24
313 eor r0,r0,r5,ror#19 @ Sigma1(e)
314#endif
315 ldr r3,[r14],#4 @ *K256++
316 add r8,r8,r2 @ h+=X[i]
317 str r2,[sp,#3*4]
318 eor r2,r6,r7
319 add r8,r8,r0,ror#6 @ h+=Sigma1(e)
320 and r2,r2,r5
321 add r8,r8,r3 @ h+=K256[i]
322 eor r2,r2,r7 @ Ch(e,f,g)
323 eor r0,r9,r9,ror#11
324 add r8,r8,r2 @ h+=Ch(e,f,g)
325#if 3==31
326 and r3,r3,#0xff
327 cmp r3,#0xf2 @ done?
328#endif
329#if 3<15
330# if __ARM_ARCH__>=7
331 ldr r2,[r1],#4 @ prefetch
332# else
333 ldrb r2,[r1,#3]
334# endif
335 eor r3,r9,r10 @ a^b, b^c in next round
336#else
337 ldr r2,[sp,#5*4] @ from future BODY_16_xx
338 eor r3,r9,r10 @ a^b, b^c in next round
339 ldr r1,[sp,#2*4] @ from future BODY_16_xx
340#endif
341 eor r0,r0,r9,ror#20 @ Sigma0(a)
342 and r12,r12,r3 @ (b^c)&=(a^b)
343 add r4,r4,r8 @ d+=h
344 eor r12,r12,r10 @ Maj(a,b,c)
345 add r8,r8,r0,ror#2 @ h+=Sigma0(a)
346 @ add r8,r8,r12 @ h+=Maj(a,b,c)
347#if __ARM_ARCH__>=7
348 @ ldr r2,[r1],#4 @ 4
349# if 4==15
350 str r1,[sp,#17*4] @ make room for r1
351# endif
352 eor r0,r4,r4,ror#5
353 add r8,r8,r12 @ h+=Maj(a,b,c) from the past
354 eor r0,r0,r4,ror#19 @ Sigma1(e)
355# ifndef __ARMEB__
356 rev r2,r2
357# endif
358#else
359 @ ldrb r2,[r1,#3] @ 4
360 add r8,r8,r12 @ h+=Maj(a,b,c) from the past
361 ldrb r12,[r1,#2]
362 ldrb r0,[r1,#1]
363 orr r2,r2,r12,lsl#8
364 ldrb r12,[r1],#4
365 orr r2,r2,r0,lsl#16
366# if 4==15
367 str r1,[sp,#17*4] @ make room for r1
368# endif
369 eor r0,r4,r4,ror#5
370 orr r2,r2,r12,lsl#24
371 eor r0,r0,r4,ror#19 @ Sigma1(e)
372#endif
373 ldr r12,[r14],#4 @ *K256++
374 add r7,r7,r2 @ h+=X[i]
375 str r2,[sp,#4*4]
376 eor r2,r5,r6
377 add r7,r7,r0,ror#6 @ h+=Sigma1(e)
378 and r2,r2,r4
379 add r7,r7,r12 @ h+=K256[i]
380 eor r2,r2,r6 @ Ch(e,f,g)
381 eor r0,r8,r8,ror#11
382 add r7,r7,r2 @ h+=Ch(e,f,g)
383#if 4==31
384 and r12,r12,#0xff
385 cmp r12,#0xf2 @ done?
386#endif
387#if 4<15
388# if __ARM_ARCH__>=7
389 ldr r2,[r1],#4 @ prefetch
390# else
391 ldrb r2,[r1,#3]
392# endif
393 eor r12,r8,r9 @ a^b, b^c in next round
394#else
395 ldr r2,[sp,#6*4] @ from future BODY_16_xx
396 eor r12,r8,r9 @ a^b, b^c in next round
397 ldr r1,[sp,#3*4] @ from future BODY_16_xx
398#endif
399 eor r0,r0,r8,ror#20 @ Sigma0(a)
400 and r3,r3,r12 @ (b^c)&=(a^b)
401 add r11,r11,r7 @ d+=h
402 eor r3,r3,r9 @ Maj(a,b,c)
403 add r7,r7,r0,ror#2 @ h+=Sigma0(a)
404 @ add r7,r7,r3 @ h+=Maj(a,b,c)
405#if __ARM_ARCH__>=7
406 @ ldr r2,[r1],#4 @ 5
407# if 5==15
408 str r1,[sp,#17*4] @ make room for r1
409# endif
410 eor r0,r11,r11,ror#5
411 add r7,r7,r3 @ h+=Maj(a,b,c) from the past
412 eor r0,r0,r11,ror#19 @ Sigma1(e)
413# ifndef __ARMEB__
414 rev r2,r2
415# endif
416#else
417 @ ldrb r2,[r1,#3] @ 5
418 add r7,r7,r3 @ h+=Maj(a,b,c) from the past
419 ldrb r3,[r1,#2]
420 ldrb r0,[r1,#1]
421 orr r2,r2,r3,lsl#8
422 ldrb r3,[r1],#4
423 orr r2,r2,r0,lsl#16
424# if 5==15
425 str r1,[sp,#17*4] @ make room for r1
426# endif
427 eor r0,r11,r11,ror#5
428 orr r2,r2,r3,lsl#24
429 eor r0,r0,r11,ror#19 @ Sigma1(e)
430#endif
431 ldr r3,[r14],#4 @ *K256++
432 add r6,r6,r2 @ h+=X[i]
433 str r2,[sp,#5*4]
434 eor r2,r4,r5
435 add r6,r6,r0,ror#6 @ h+=Sigma1(e)
436 and r2,r2,r11
437 add r6,r6,r3 @ h+=K256[i]
438 eor r2,r2,r5 @ Ch(e,f,g)
439 eor r0,r7,r7,ror#11
440 add r6,r6,r2 @ h+=Ch(e,f,g)
441#if 5==31
442 and r3,r3,#0xff
443 cmp r3,#0xf2 @ done?
444#endif
445#if 5<15
446# if __ARM_ARCH__>=7
447 ldr r2,[r1],#4 @ prefetch
448# else
449 ldrb r2,[r1,#3]
450# endif
451 eor r3,r7,r8 @ a^b, b^c in next round
452#else
453 ldr r2,[sp,#7*4] @ from future BODY_16_xx
454 eor r3,r7,r8 @ a^b, b^c in next round
455 ldr r1,[sp,#4*4] @ from future BODY_16_xx
456#endif
457 eor r0,r0,r7,ror#20 @ Sigma0(a)
458 and r12,r12,r3 @ (b^c)&=(a^b)
459 add r10,r10,r6 @ d+=h
460 eor r12,r12,r8 @ Maj(a,b,c)
461 add r6,r6,r0,ror#2 @ h+=Sigma0(a)
462 @ add r6,r6,r12 @ h+=Maj(a,b,c)
463#if __ARM_ARCH__>=7
464 @ ldr r2,[r1],#4 @ 6
465# if 6==15
466 str r1,[sp,#17*4] @ make room for r1
467# endif
468 eor r0,r10,r10,ror#5
469 add r6,r6,r12 @ h+=Maj(a,b,c) from the past
470 eor r0,r0,r10,ror#19 @ Sigma1(e)
471# ifndef __ARMEB__
472 rev r2,r2
473# endif
474#else
475 @ ldrb r2,[r1,#3] @ 6
476 add r6,r6,r12 @ h+=Maj(a,b,c) from the past
477 ldrb r12,[r1,#2]
478 ldrb r0,[r1,#1]
479 orr r2,r2,r12,lsl#8
480 ldrb r12,[r1],#4
481 orr r2,r2,r0,lsl#16
482# if 6==15
483 str r1,[sp,#17*4] @ make room for r1
484# endif
485 eor r0,r10,r10,ror#5
486 orr r2,r2,r12,lsl#24
487 eor r0,r0,r10,ror#19 @ Sigma1(e)
488#endif
489 ldr r12,[r14],#4 @ *K256++
490 add r5,r5,r2 @ h+=X[i]
491 str r2,[sp,#6*4]
492 eor r2,r11,r4
493 add r5,r5,r0,ror#6 @ h+=Sigma1(e)
494 and r2,r2,r10
495 add r5,r5,r12 @ h+=K256[i]
496 eor r2,r2,r4 @ Ch(e,f,g)
497 eor r0,r6,r6,ror#11
498 add r5,r5,r2 @ h+=Ch(e,f,g)
499#if 6==31
500 and r12,r12,#0xff
501 cmp r12,#0xf2 @ done?
502#endif
503#if 6<15
504# if __ARM_ARCH__>=7
505 ldr r2,[r1],#4 @ prefetch
506# else
507 ldrb r2,[r1,#3]
508# endif
509 eor r12,r6,r7 @ a^b, b^c in next round
510#else
511 ldr r2,[sp,#8*4] @ from future BODY_16_xx
512 eor r12,r6,r7 @ a^b, b^c in next round
513 ldr r1,[sp,#5*4] @ from future BODY_16_xx
514#endif
515 eor r0,r0,r6,ror#20 @ Sigma0(a)
516 and r3,r3,r12 @ (b^c)&=(a^b)
517 add r9,r9,r5 @ d+=h
518 eor r3,r3,r7 @ Maj(a,b,c)
519 add r5,r5,r0,ror#2 @ h+=Sigma0(a)
520 @ add r5,r5,r3 @ h+=Maj(a,b,c)
521#if __ARM_ARCH__>=7
522 @ ldr r2,[r1],#4 @ 7
523# if 7==15
524 str r1,[sp,#17*4] @ make room for r1
525# endif
526 eor r0,r9,r9,ror#5
527 add r5,r5,r3 @ h+=Maj(a,b,c) from the past
528 eor r0,r0,r9,ror#19 @ Sigma1(e)
529# ifndef __ARMEB__
530 rev r2,r2
531# endif
532#else
533 @ ldrb r2,[r1,#3] @ 7
534 add r5,r5,r3 @ h+=Maj(a,b,c) from the past
535 ldrb r3,[r1,#2]
536 ldrb r0,[r1,#1]
537 orr r2,r2,r3,lsl#8
538 ldrb r3,[r1],#4
539 orr r2,r2,r0,lsl#16
540# if 7==15
541 str r1,[sp,#17*4] @ make room for r1
542# endif
543 eor r0,r9,r9,ror#5
544 orr r2,r2,r3,lsl#24
545 eor r0,r0,r9,ror#19 @ Sigma1(e)
546#endif
547 ldr r3,[r14],#4 @ *K256++
548 add r4,r4,r2 @ h+=X[i]
549 str r2,[sp,#7*4]
550 eor r2,r10,r11
551 add r4,r4,r0,ror#6 @ h+=Sigma1(e)
552 and r2,r2,r9
553 add r4,r4,r3 @ h+=K256[i]
554 eor r2,r2,r11 @ Ch(e,f,g)
555 eor r0,r5,r5,ror#11
556 add r4,r4,r2 @ h+=Ch(e,f,g)
557#if 7==31
558 and r3,r3,#0xff
559 cmp r3,#0xf2 @ done?
560#endif
561#if 7<15
562# if __ARM_ARCH__>=7
563 ldr r2,[r1],#4 @ prefetch
564# else
565 ldrb r2,[r1,#3]
566# endif
567 eor r3,r5,r6 @ a^b, b^c in next round
568#else
569 ldr r2,[sp,#9*4] @ from future BODY_16_xx
570 eor r3,r5,r6 @ a^b, b^c in next round
571 ldr r1,[sp,#6*4] @ from future BODY_16_xx
572#endif
573 eor r0,r0,r5,ror#20 @ Sigma0(a)
574 and r12,r12,r3 @ (b^c)&=(a^b)
575 add r8,r8,r4 @ d+=h
576 eor r12,r12,r6 @ Maj(a,b,c)
577 add r4,r4,r0,ror#2 @ h+=Sigma0(a)
578 @ add r4,r4,r12 @ h+=Maj(a,b,c)
579#if __ARM_ARCH__>=7
580 @ ldr r2,[r1],#4 @ 8
581# if 8==15
582 str r1,[sp,#17*4] @ make room for r1
583# endif
584 eor r0,r8,r8,ror#5
585 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
586 eor r0,r0,r8,ror#19 @ Sigma1(e)
587# ifndef __ARMEB__
588 rev r2,r2
589# endif
590#else
591 @ ldrb r2,[r1,#3] @ 8
592 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
593 ldrb r12,[r1,#2]
594 ldrb r0,[r1,#1]
595 orr r2,r2,r12,lsl#8
596 ldrb r12,[r1],#4
597 orr r2,r2,r0,lsl#16
598# if 8==15
599 str r1,[sp,#17*4] @ make room for r1
600# endif
601 eor r0,r8,r8,ror#5
602 orr r2,r2,r12,lsl#24
603 eor r0,r0,r8,ror#19 @ Sigma1(e)
604#endif
605 ldr r12,[r14],#4 @ *K256++
606 add r11,r11,r2 @ h+=X[i]
607 str r2,[sp,#8*4]
608 eor r2,r9,r10
609 add r11,r11,r0,ror#6 @ h+=Sigma1(e)
610 and r2,r2,r8
611 add r11,r11,r12 @ h+=K256[i]
612 eor r2,r2,r10 @ Ch(e,f,g)
613 eor r0,r4,r4,ror#11
614 add r11,r11,r2 @ h+=Ch(e,f,g)
615#if 8==31
616 and r12,r12,#0xff
617 cmp r12,#0xf2 @ done?
618#endif
619#if 8<15
620# if __ARM_ARCH__>=7
621 ldr r2,[r1],#4 @ prefetch
622# else
623 ldrb r2,[r1,#3]
624# endif
625 eor r12,r4,r5 @ a^b, b^c in next round
626#else
627 ldr r2,[sp,#10*4] @ from future BODY_16_xx
628 eor r12,r4,r5 @ a^b, b^c in next round
629 ldr r1,[sp,#7*4] @ from future BODY_16_xx
630#endif
631 eor r0,r0,r4,ror#20 @ Sigma0(a)
632 and r3,r3,r12 @ (b^c)&=(a^b)
633 add r7,r7,r11 @ d+=h
634 eor r3,r3,r5 @ Maj(a,b,c)
635 add r11,r11,r0,ror#2 @ h+=Sigma0(a)
636 @ add r11,r11,r3 @ h+=Maj(a,b,c)
637#if __ARM_ARCH__>=7
638 @ ldr r2,[r1],#4 @ 9
639# if 9==15
640 str r1,[sp,#17*4] @ make room for r1
641# endif
642 eor r0,r7,r7,ror#5
643 add r11,r11,r3 @ h+=Maj(a,b,c) from the past
644 eor r0,r0,r7,ror#19 @ Sigma1(e)
645# ifndef __ARMEB__
646 rev r2,r2
647# endif
648#else
649 @ ldrb r2,[r1,#3] @ 9
650 add r11,r11,r3 @ h+=Maj(a,b,c) from the past
651 ldrb r3,[r1,#2]
652 ldrb r0,[r1,#1]
653 orr r2,r2,r3,lsl#8
654 ldrb r3,[r1],#4
655 orr r2,r2,r0,lsl#16
656# if 9==15
657 str r1,[sp,#17*4] @ make room for r1
658# endif
659 eor r0,r7,r7,ror#5
660 orr r2,r2,r3,lsl#24
661 eor r0,r0,r7,ror#19 @ Sigma1(e)
662#endif
663 ldr r3,[r14],#4 @ *K256++
664 add r10,r10,r2 @ h+=X[i]
665 str r2,[sp,#9*4]
666 eor r2,r8,r9
667 add r10,r10,r0,ror#6 @ h+=Sigma1(e)
668 and r2,r2,r7
669 add r10,r10,r3 @ h+=K256[i]
670 eor r2,r2,r9 @ Ch(e,f,g)
671 eor r0,r11,r11,ror#11
672 add r10,r10,r2 @ h+=Ch(e,f,g)
673#if 9==31
674 and r3,r3,#0xff
675 cmp r3,#0xf2 @ done?
676#endif
677#if 9<15
678# if __ARM_ARCH__>=7
679 ldr r2,[r1],#4 @ prefetch
680# else
681 ldrb r2,[r1,#3]
682# endif
683 eor r3,r11,r4 @ a^b, b^c in next round
684#else
685 ldr r2,[sp,#11*4] @ from future BODY_16_xx
686 eor r3,r11,r4 @ a^b, b^c in next round
687 ldr r1,[sp,#8*4] @ from future BODY_16_xx
688#endif
689 eor r0,r0,r11,ror#20 @ Sigma0(a)
690 and r12,r12,r3 @ (b^c)&=(a^b)
691 add r6,r6,r10 @ d+=h
692 eor r12,r12,r4 @ Maj(a,b,c)
693 add r10,r10,r0,ror#2 @ h+=Sigma0(a)
694 @ add r10,r10,r12 @ h+=Maj(a,b,c)
695#if __ARM_ARCH__>=7
696 @ ldr r2,[r1],#4 @ 10
697# if 10==15
698 str r1,[sp,#17*4] @ make room for r1
699# endif
700 eor r0,r6,r6,ror#5
701 add r10,r10,r12 @ h+=Maj(a,b,c) from the past
702 eor r0,r0,r6,ror#19 @ Sigma1(e)
703# ifndef __ARMEB__
704 rev r2,r2
705# endif
706#else
707 @ ldrb r2,[r1,#3] @ 10
708 add r10,r10,r12 @ h+=Maj(a,b,c) from the past
709 ldrb r12,[r1,#2]
710 ldrb r0,[r1,#1]
711 orr r2,r2,r12,lsl#8
712 ldrb r12,[r1],#4
713 orr r2,r2,r0,lsl#16
714# if 10==15
715 str r1,[sp,#17*4] @ make room for r1
716# endif
717 eor r0,r6,r6,ror#5
718 orr r2,r2,r12,lsl#24
719 eor r0,r0,r6,ror#19 @ Sigma1(e)
720#endif
721 ldr r12,[r14],#4 @ *K256++
722 add r9,r9,r2 @ h+=X[i]
723 str r2,[sp,#10*4]
724 eor r2,r7,r8
725 add r9,r9,r0,ror#6 @ h+=Sigma1(e)
726 and r2,r2,r6
727 add r9,r9,r12 @ h+=K256[i]
728 eor r2,r2,r8 @ Ch(e,f,g)
729 eor r0,r10,r10,ror#11
730 add r9,r9,r2 @ h+=Ch(e,f,g)
731#if 10==31
732 and r12,r12,#0xff
733 cmp r12,#0xf2 @ done?
734#endif
735#if 10<15
736# if __ARM_ARCH__>=7
737 ldr r2,[r1],#4 @ prefetch
738# else
739 ldrb r2,[r1,#3]
740# endif
741 eor r12,r10,r11 @ a^b, b^c in next round
742#else
743 ldr r2,[sp,#12*4] @ from future BODY_16_xx
744 eor r12,r10,r11 @ a^b, b^c in next round
745 ldr r1,[sp,#9*4] @ from future BODY_16_xx
746#endif
747 eor r0,r0,r10,ror#20 @ Sigma0(a)
748 and r3,r3,r12 @ (b^c)&=(a^b)
749 add r5,r5,r9 @ d+=h
750 eor r3,r3,r11 @ Maj(a,b,c)
751 add r9,r9,r0,ror#2 @ h+=Sigma0(a)
752 @ add r9,r9,r3 @ h+=Maj(a,b,c)
753#if __ARM_ARCH__>=7
754 @ ldr r2,[r1],#4 @ 11
755# if 11==15
756 str r1,[sp,#17*4] @ make room for r1
757# endif
758 eor r0,r5,r5,ror#5
759 add r9,r9,r3 @ h+=Maj(a,b,c) from the past
760 eor r0,r0,r5,ror#19 @ Sigma1(e)
761# ifndef __ARMEB__
762 rev r2,r2
763# endif
764#else
765 @ ldrb r2,[r1,#3] @ 11
766 add r9,r9,r3 @ h+=Maj(a,b,c) from the past
767 ldrb r3,[r1,#2]
768 ldrb r0,[r1,#1]
769 orr r2,r2,r3,lsl#8
770 ldrb r3,[r1],#4
771 orr r2,r2,r0,lsl#16
772# if 11==15
773 str r1,[sp,#17*4] @ make room for r1
774# endif
775 eor r0,r5,r5,ror#5
776 orr r2,r2,r3,lsl#24
777 eor r0,r0,r5,ror#19 @ Sigma1(e)
778#endif
779 ldr r3,[r14],#4 @ *K256++
780 add r8,r8,r2 @ h+=X[i]
781 str r2,[sp,#11*4]
782 eor r2,r6,r7
783 add r8,r8,r0,ror#6 @ h+=Sigma1(e)
784 and r2,r2,r5
785 add r8,r8,r3 @ h+=K256[i]
786 eor r2,r2,r7 @ Ch(e,f,g)
787 eor r0,r9,r9,ror#11
788 add r8,r8,r2 @ h+=Ch(e,f,g)
789#if 11==31
790 and r3,r3,#0xff
791 cmp r3,#0xf2 @ done?
792#endif
793#if 11<15
794# if __ARM_ARCH__>=7
795 ldr r2,[r1],#4 @ prefetch
796# else
797 ldrb r2,[r1,#3]
798# endif
799 eor r3,r9,r10 @ a^b, b^c in next round
800#else
801 ldr r2,[sp,#13*4] @ from future BODY_16_xx
802 eor r3,r9,r10 @ a^b, b^c in next round
803 ldr r1,[sp,#10*4] @ from future BODY_16_xx
804#endif
805 eor r0,r0,r9,ror#20 @ Sigma0(a)
806 and r12,r12,r3 @ (b^c)&=(a^b)
807 add r4,r4,r8 @ d+=h
808 eor r12,r12,r10 @ Maj(a,b,c)
809 add r8,r8,r0,ror#2 @ h+=Sigma0(a)
810 @ add r8,r8,r12 @ h+=Maj(a,b,c)
811#if __ARM_ARCH__>=7
812 @ ldr r2,[r1],#4 @ 12
813# if 12==15
814 str r1,[sp,#17*4] @ make room for r1
815# endif
816 eor r0,r4,r4,ror#5
817 add r8,r8,r12 @ h+=Maj(a,b,c) from the past
818 eor r0,r0,r4,ror#19 @ Sigma1(e)
819# ifndef __ARMEB__
820 rev r2,r2
821# endif
822#else
823 @ ldrb r2,[r1,#3] @ 12
824 add r8,r8,r12 @ h+=Maj(a,b,c) from the past
825 ldrb r12,[r1,#2]
826 ldrb r0,[r1,#1]
827 orr r2,r2,r12,lsl#8
828 ldrb r12,[r1],#4
829 orr r2,r2,r0,lsl#16
830# if 12==15
831 str r1,[sp,#17*4] @ make room for r1
832# endif
833 eor r0,r4,r4,ror#5
834 orr r2,r2,r12,lsl#24
835 eor r0,r0,r4,ror#19 @ Sigma1(e)
836#endif
837 ldr r12,[r14],#4 @ *K256++
838 add r7,r7,r2 @ h+=X[i]
839 str r2,[sp,#12*4]
840 eor r2,r5,r6
841 add r7,r7,r0,ror#6 @ h+=Sigma1(e)
842 and r2,r2,r4
843 add r7,r7,r12 @ h+=K256[i]
844 eor r2,r2,r6 @ Ch(e,f,g)
845 eor r0,r8,r8,ror#11
846 add r7,r7,r2 @ h+=Ch(e,f,g)
847#if 12==31
848 and r12,r12,#0xff
849 cmp r12,#0xf2 @ done?
850#endif
851#if 12<15
852# if __ARM_ARCH__>=7
853 ldr r2,[r1],#4 @ prefetch
854# else
855 ldrb r2,[r1,#3]
856# endif
857 eor r12,r8,r9 @ a^b, b^c in next round
858#else
859 ldr r2,[sp,#14*4] @ from future BODY_16_xx
860 eor r12,r8,r9 @ a^b, b^c in next round
861 ldr r1,[sp,#11*4] @ from future BODY_16_xx
862#endif
863 eor r0,r0,r8,ror#20 @ Sigma0(a)
864 and r3,r3,r12 @ (b^c)&=(a^b)
865 add r11,r11,r7 @ d+=h
866 eor r3,r3,r9 @ Maj(a,b,c)
867 add r7,r7,r0,ror#2 @ h+=Sigma0(a)
868 @ add r7,r7,r3 @ h+=Maj(a,b,c)
869#if __ARM_ARCH__>=7
870 @ ldr r2,[r1],#4 @ 13
871# if 13==15
872 str r1,[sp,#17*4] @ make room for r1
873# endif
874 eor r0,r11,r11,ror#5
875 add r7,r7,r3 @ h+=Maj(a,b,c) from the past
876 eor r0,r0,r11,ror#19 @ Sigma1(e)
877# ifndef __ARMEB__
878 rev r2,r2
879# endif
880#else
881 @ ldrb r2,[r1,#3] @ 13
882 add r7,r7,r3 @ h+=Maj(a,b,c) from the past
883 ldrb r3,[r1,#2]
884 ldrb r0,[r1,#1]
885 orr r2,r2,r3,lsl#8
886 ldrb r3,[r1],#4
887 orr r2,r2,r0,lsl#16
888# if 13==15
889 str r1,[sp,#17*4] @ make room for r1
890# endif
891 eor r0,r11,r11,ror#5
892 orr r2,r2,r3,lsl#24
893 eor r0,r0,r11,ror#19 @ Sigma1(e)
894#endif
895 ldr r3,[r14],#4 @ *K256++
896 add r6,r6,r2 @ h+=X[i]
897 str r2,[sp,#13*4]
898 eor r2,r4,r5
899 add r6,r6,r0,ror#6 @ h+=Sigma1(e)
900 and r2,r2,r11
901 add r6,r6,r3 @ h+=K256[i]
902 eor r2,r2,r5 @ Ch(e,f,g)
903 eor r0,r7,r7,ror#11
904 add r6,r6,r2 @ h+=Ch(e,f,g)
905#if 13==31
906 and r3,r3,#0xff
907 cmp r3,#0xf2 @ done?
908#endif
909#if 13<15
910# if __ARM_ARCH__>=7
911 ldr r2,[r1],#4 @ prefetch
912# else
913 ldrb r2,[r1,#3]
914# endif
915 eor r3,r7,r8 @ a^b, b^c in next round
916#else
917 ldr r2,[sp,#15*4] @ from future BODY_16_xx
918 eor r3,r7,r8 @ a^b, b^c in next round
919 ldr r1,[sp,#12*4] @ from future BODY_16_xx
920#endif
921 eor r0,r0,r7,ror#20 @ Sigma0(a)
922 and r12,r12,r3 @ (b^c)&=(a^b)
923 add r10,r10,r6 @ d+=h
924 eor r12,r12,r8 @ Maj(a,b,c)
925 add r6,r6,r0,ror#2 @ h+=Sigma0(a)
926 @ add r6,r6,r12 @ h+=Maj(a,b,c)
927#if __ARM_ARCH__>=7
928 @ ldr r2,[r1],#4 @ 14
929# if 14==15
930 str r1,[sp,#17*4] @ make room for r1
931# endif
932 eor r0,r10,r10,ror#5
933 add r6,r6,r12 @ h+=Maj(a,b,c) from the past
934 eor r0,r0,r10,ror#19 @ Sigma1(e)
935# ifndef __ARMEB__
936 rev r2,r2
937# endif
938#else
939 @ ldrb r2,[r1,#3] @ 14
940 add r6,r6,r12 @ h+=Maj(a,b,c) from the past
941 ldrb r12,[r1,#2]
942 ldrb r0,[r1,#1]
943 orr r2,r2,r12,lsl#8
944 ldrb r12,[r1],#4
945 orr r2,r2,r0,lsl#16
946# if 14==15
947 str r1,[sp,#17*4] @ make room for r1
948# endif
949 eor r0,r10,r10,ror#5
950 orr r2,r2,r12,lsl#24
951 eor r0,r0,r10,ror#19 @ Sigma1(e)
952#endif
953 ldr r12,[r14],#4 @ *K256++
954 add r5,r5,r2 @ h+=X[i]
955 str r2,[sp,#14*4]
956 eor r2,r11,r4
957 add r5,r5,r0,ror#6 @ h+=Sigma1(e)
958 and r2,r2,r10
959 add r5,r5,r12 @ h+=K256[i]
960 eor r2,r2,r4 @ Ch(e,f,g)
961 eor r0,r6,r6,ror#11
962 add r5,r5,r2 @ h+=Ch(e,f,g)
963#if 14==31
964 and r12,r12,#0xff
965 cmp r12,#0xf2 @ done?
966#endif
967#if 14<15
968# if __ARM_ARCH__>=7
969 ldr r2,[r1],#4 @ prefetch
970# else
971 ldrb r2,[r1,#3]
972# endif
973 eor r12,r6,r7 @ a^b, b^c in next round
974#else
975 ldr r2,[sp,#0*4] @ from future BODY_16_xx
976 eor r12,r6,r7 @ a^b, b^c in next round
977 ldr r1,[sp,#13*4] @ from future BODY_16_xx
978#endif
979 eor r0,r0,r6,ror#20 @ Sigma0(a)
980 and r3,r3,r12 @ (b^c)&=(a^b)
981 add r9,r9,r5 @ d+=h
982 eor r3,r3,r7 @ Maj(a,b,c)
983 add r5,r5,r0,ror#2 @ h+=Sigma0(a)
984 @ add r5,r5,r3 @ h+=Maj(a,b,c)
985#if __ARM_ARCH__>=7
986 @ ldr r2,[r1],#4 @ 15
987# if 15==15
988 str r1,[sp,#17*4] @ make room for r1
989# endif
990 eor r0,r9,r9,ror#5
991 add r5,r5,r3 @ h+=Maj(a,b,c) from the past
992 eor r0,r0,r9,ror#19 @ Sigma1(e)
993# ifndef __ARMEB__
994 rev r2,r2
995# endif
996#else
997 @ ldrb r2,[r1,#3] @ 15
998 add r5,r5,r3 @ h+=Maj(a,b,c) from the past
999 ldrb r3,[r1,#2]
1000 ldrb r0,[r1,#1]
1001 orr r2,r2,r3,lsl#8
1002 ldrb r3,[r1],#4
1003 orr r2,r2,r0,lsl#16
1004# if 15==15
1005 str r1,[sp,#17*4] @ make room for r1
1006# endif
1007 eor r0,r9,r9,ror#5
1008 orr r2,r2,r3,lsl#24
1009 eor r0,r0,r9,ror#19 @ Sigma1(e)
1010#endif
1011 ldr r3,[r14],#4 @ *K256++
1012 add r4,r4,r2 @ h+=X[i]
1013 str r2,[sp,#15*4]
1014 eor r2,r10,r11
1015 add r4,r4,r0,ror#6 @ h+=Sigma1(e)
1016 and r2,r2,r9
1017 add r4,r4,r3 @ h+=K256[i]
1018 eor r2,r2,r11 @ Ch(e,f,g)
1019 eor r0,r5,r5,ror#11
1020 add r4,r4,r2 @ h+=Ch(e,f,g)
1021#if 15==31
1022 and r3,r3,#0xff
1023 cmp r3,#0xf2 @ done?
1024#endif
1025#if 15<15
1026# if __ARM_ARCH__>=7
1027 ldr r2,[r1],#4 @ prefetch
1028# else
1029 ldrb r2,[r1,#3]
1030# endif
1031 eor r3,r5,r6 @ a^b, b^c in next round
1032#else
1033 ldr r2,[sp,#1*4] @ from future BODY_16_xx
1034 eor r3,r5,r6 @ a^b, b^c in next round
1035 ldr r1,[sp,#14*4] @ from future BODY_16_xx
1036#endif
1037 eor r0,r0,r5,ror#20 @ Sigma0(a)
1038 and r12,r12,r3 @ (b^c)&=(a^b)
1039 add r8,r8,r4 @ d+=h
1040 eor r12,r12,r6 @ Maj(a,b,c)
1041 add r4,r4,r0,ror#2 @ h+=Sigma0(a)
1042 @ add r4,r4,r12 @ h+=Maj(a,b,c)
1043.Lrounds_16_xx:
1044 @ ldr r2,[sp,#1*4] @ 16
1045 @ ldr r1,[sp,#14*4]
1046 mov r0,r2,ror#7
1047 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
1048 mov r12,r1,ror#17
1049 eor r0,r0,r2,ror#18
1050 eor r12,r12,r1,ror#19
1051 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1052 ldr r2,[sp,#0*4]
1053 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1054 ldr r1,[sp,#9*4]
1055
1056 add r12,r12,r0
1057 eor r0,r8,r8,ror#5 @ from BODY_00_15
1058 add r2,r2,r12
1059 eor r0,r0,r8,ror#19 @ Sigma1(e)
1060 add r2,r2,r1 @ X[i]
1061 ldr r12,[r14],#4 @ *K256++
1062 add r11,r11,r2 @ h+=X[i]
1063 str r2,[sp,#0*4]
1064 eor r2,r9,r10
1065 add r11,r11,r0,ror#6 @ h+=Sigma1(e)
1066 and r2,r2,r8
1067 add r11,r11,r12 @ h+=K256[i]
1068 eor r2,r2,r10 @ Ch(e,f,g)
1069 eor r0,r4,r4,ror#11
1070 add r11,r11,r2 @ h+=Ch(e,f,g)
1071#if 16==31
1072 and r12,r12,#0xff
1073 cmp r12,#0xf2 @ done?
1074#endif
1075#if 16<15
1076# if __ARM_ARCH__>=7
1077 ldr r2,[r1],#4 @ prefetch
1078# else
1079 ldrb r2,[r1,#3]
1080# endif
1081 eor r12,r4,r5 @ a^b, b^c in next round
1082#else
1083 ldr r2,[sp,#2*4] @ from future BODY_16_xx
1084 eor r12,r4,r5 @ a^b, b^c in next round
1085 ldr r1,[sp,#15*4] @ from future BODY_16_xx
1086#endif
1087 eor r0,r0,r4,ror#20 @ Sigma0(a)
1088 and r3,r3,r12 @ (b^c)&=(a^b)
1089 add r7,r7,r11 @ d+=h
1090 eor r3,r3,r5 @ Maj(a,b,c)
1091 add r11,r11,r0,ror#2 @ h+=Sigma0(a)
1092 @ add r11,r11,r3 @ h+=Maj(a,b,c)
1093 @ ldr r2,[sp,#2*4] @ 17
1094 @ ldr r1,[sp,#15*4]
1095 mov r0,r2,ror#7
1096 add r11,r11,r3 @ h+=Maj(a,b,c) from the past
1097 mov r3,r1,ror#17
1098 eor r0,r0,r2,ror#18
1099 eor r3,r3,r1,ror#19
1100 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1101 ldr r2,[sp,#1*4]
1102 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1103 ldr r1,[sp,#10*4]
1104
1105 add r3,r3,r0
1106 eor r0,r7,r7,ror#5 @ from BODY_00_15
1107 add r2,r2,r3
1108 eor r0,r0,r7,ror#19 @ Sigma1(e)
1109 add r2,r2,r1 @ X[i]
1110 ldr r3,[r14],#4 @ *K256++
1111 add r10,r10,r2 @ h+=X[i]
1112 str r2,[sp,#1*4]
1113 eor r2,r8,r9
1114 add r10,r10,r0,ror#6 @ h+=Sigma1(e)
1115 and r2,r2,r7
1116 add r10,r10,r3 @ h+=K256[i]
1117 eor r2,r2,r9 @ Ch(e,f,g)
1118 eor r0,r11,r11,ror#11
1119 add r10,r10,r2 @ h+=Ch(e,f,g)
1120#if 17==31
1121 and r3,r3,#0xff
1122 cmp r3,#0xf2 @ done?
1123#endif
1124#if 17<15
1125# if __ARM_ARCH__>=7
1126 ldr r2,[r1],#4 @ prefetch
1127# else
1128 ldrb r2,[r1,#3]
1129# endif
1130 eor r3,r11,r4 @ a^b, b^c in next round
1131#else
1132 ldr r2,[sp,#3*4] @ from future BODY_16_xx
1133 eor r3,r11,r4 @ a^b, b^c in next round
1134 ldr r1,[sp,#0*4] @ from future BODY_16_xx
1135#endif
1136 eor r0,r0,r11,ror#20 @ Sigma0(a)
1137 and r12,r12,r3 @ (b^c)&=(a^b)
1138 add r6,r6,r10 @ d+=h
1139 eor r12,r12,r4 @ Maj(a,b,c)
1140 add r10,r10,r0,ror#2 @ h+=Sigma0(a)
1141 @ add r10,r10,r12 @ h+=Maj(a,b,c)
1142 @ ldr r2,[sp,#3*4] @ 18
1143 @ ldr r1,[sp,#0*4]
1144 mov r0,r2,ror#7
1145 add r10,r10,r12 @ h+=Maj(a,b,c) from the past
1146 mov r12,r1,ror#17
1147 eor r0,r0,r2,ror#18
1148 eor r12,r12,r1,ror#19
1149 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1150 ldr r2,[sp,#2*4]
1151 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1152 ldr r1,[sp,#11*4]
1153
1154 add r12,r12,r0
1155 eor r0,r6,r6,ror#5 @ from BODY_00_15
1156 add r2,r2,r12
1157 eor r0,r0,r6,ror#19 @ Sigma1(e)
1158 add r2,r2,r1 @ X[i]
1159 ldr r12,[r14],#4 @ *K256++
1160 add r9,r9,r2 @ h+=X[i]
1161 str r2,[sp,#2*4]
1162 eor r2,r7,r8
1163 add r9,r9,r0,ror#6 @ h+=Sigma1(e)
1164 and r2,r2,r6
1165 add r9,r9,r12 @ h+=K256[i]
1166 eor r2,r2,r8 @ Ch(e,f,g)
1167 eor r0,r10,r10,ror#11
1168 add r9,r9,r2 @ h+=Ch(e,f,g)
1169#if 18==31
1170 and r12,r12,#0xff
1171 cmp r12,#0xf2 @ done?
1172#endif
1173#if 18<15
1174# if __ARM_ARCH__>=7
1175 ldr r2,[r1],#4 @ prefetch
1176# else
1177 ldrb r2,[r1,#3]
1178# endif
1179 eor r12,r10,r11 @ a^b, b^c in next round
1180#else
1181 ldr r2,[sp,#4*4] @ from future BODY_16_xx
1182 eor r12,r10,r11 @ a^b, b^c in next round
1183 ldr r1,[sp,#1*4] @ from future BODY_16_xx
1184#endif
1185 eor r0,r0,r10,ror#20 @ Sigma0(a)
1186 and r3,r3,r12 @ (b^c)&=(a^b)
1187 add r5,r5,r9 @ d+=h
1188 eor r3,r3,r11 @ Maj(a,b,c)
1189 add r9,r9,r0,ror#2 @ h+=Sigma0(a)
1190 @ add r9,r9,r3 @ h+=Maj(a,b,c)
1191 @ ldr r2,[sp,#4*4] @ 19
1192 @ ldr r1,[sp,#1*4]
1193 mov r0,r2,ror#7
1194 add r9,r9,r3 @ h+=Maj(a,b,c) from the past
1195 mov r3,r1,ror#17
1196 eor r0,r0,r2,ror#18
1197 eor r3,r3,r1,ror#19
1198 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1199 ldr r2,[sp,#3*4]
1200 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1201 ldr r1,[sp,#12*4]
1202
1203 add r3,r3,r0
1204 eor r0,r5,r5,ror#5 @ from BODY_00_15
1205 add r2,r2,r3
1206 eor r0,r0,r5,ror#19 @ Sigma1(e)
1207 add r2,r2,r1 @ X[i]
1208 ldr r3,[r14],#4 @ *K256++
1209 add r8,r8,r2 @ h+=X[i]
1210 str r2,[sp,#3*4]
1211 eor r2,r6,r7
1212 add r8,r8,r0,ror#6 @ h+=Sigma1(e)
1213 and r2,r2,r5
1214 add r8,r8,r3 @ h+=K256[i]
1215 eor r2,r2,r7 @ Ch(e,f,g)
1216 eor r0,r9,r9,ror#11
1217 add r8,r8,r2 @ h+=Ch(e,f,g)
1218#if 19==31
1219 and r3,r3,#0xff
1220 cmp r3,#0xf2 @ done?
1221#endif
1222#if 19<15
1223# if __ARM_ARCH__>=7
1224 ldr r2,[r1],#4 @ prefetch
1225# else
1226 ldrb r2,[r1,#3]
1227# endif
1228 eor r3,r9,r10 @ a^b, b^c in next round
1229#else
1230 ldr r2,[sp,#5*4] @ from future BODY_16_xx
1231 eor r3,r9,r10 @ a^b, b^c in next round
1232 ldr r1,[sp,#2*4] @ from future BODY_16_xx
1233#endif
1234 eor r0,r0,r9,ror#20 @ Sigma0(a)
1235 and r12,r12,r3 @ (b^c)&=(a^b)
1236 add r4,r4,r8 @ d+=h
1237 eor r12,r12,r10 @ Maj(a,b,c)
1238 add r8,r8,r0,ror#2 @ h+=Sigma0(a)
1239 @ add r8,r8,r12 @ h+=Maj(a,b,c)
1240 @ ldr r2,[sp,#5*4] @ 20
1241 @ ldr r1,[sp,#2*4]
1242 mov r0,r2,ror#7
1243 add r8,r8,r12 @ h+=Maj(a,b,c) from the past
1244 mov r12,r1,ror#17
1245 eor r0,r0,r2,ror#18
1246 eor r12,r12,r1,ror#19
1247 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1248 ldr r2,[sp,#4*4]
1249 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1250 ldr r1,[sp,#13*4]
1251
1252 add r12,r12,r0
1253 eor r0,r4,r4,ror#5 @ from BODY_00_15
1254 add r2,r2,r12
1255 eor r0,r0,r4,ror#19 @ Sigma1(e)
1256 add r2,r2,r1 @ X[i]
1257 ldr r12,[r14],#4 @ *K256++
1258 add r7,r7,r2 @ h+=X[i]
1259 str r2,[sp,#4*4]
1260 eor r2,r5,r6
1261 add r7,r7,r0,ror#6 @ h+=Sigma1(e)
1262 and r2,r2,r4
1263 add r7,r7,r12 @ h+=K256[i]
1264 eor r2,r2,r6 @ Ch(e,f,g)
1265 eor r0,r8,r8,ror#11
1266 add r7,r7,r2 @ h+=Ch(e,f,g)
1267#if 20==31
1268 and r12,r12,#0xff
1269 cmp r12,#0xf2 @ done?
1270#endif
1271#if 20<15
1272# if __ARM_ARCH__>=7
1273 ldr r2,[r1],#4 @ prefetch
1274# else
1275 ldrb r2,[r1,#3]
1276# endif
1277 eor r12,r8,r9 @ a^b, b^c in next round
1278#else
1279 ldr r2,[sp,#6*4] @ from future BODY_16_xx
1280 eor r12,r8,r9 @ a^b, b^c in next round
1281 ldr r1,[sp,#3*4] @ from future BODY_16_xx
1282#endif
1283 eor r0,r0,r8,ror#20 @ Sigma0(a)
1284 and r3,r3,r12 @ (b^c)&=(a^b)
1285 add r11,r11,r7 @ d+=h
1286 eor r3,r3,r9 @ Maj(a,b,c)
1287 add r7,r7,r0,ror#2 @ h+=Sigma0(a)
1288 @ add r7,r7,r3 @ h+=Maj(a,b,c)
1289 @ ldr r2,[sp,#6*4] @ 21
1290 @ ldr r1,[sp,#3*4]
1291 mov r0,r2,ror#7
1292 add r7,r7,r3 @ h+=Maj(a,b,c) from the past
1293 mov r3,r1,ror#17
1294 eor r0,r0,r2,ror#18
1295 eor r3,r3,r1,ror#19
1296 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1297 ldr r2,[sp,#5*4]
1298 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1299 ldr r1,[sp,#14*4]
1300
1301 add r3,r3,r0
1302 eor r0,r11,r11,ror#5 @ from BODY_00_15
1303 add r2,r2,r3
1304 eor r0,r0,r11,ror#19 @ Sigma1(e)
1305 add r2,r2,r1 @ X[i]
1306 ldr r3,[r14],#4 @ *K256++
1307 add r6,r6,r2 @ h+=X[i]
1308 str r2,[sp,#5*4]
1309 eor r2,r4,r5
1310 add r6,r6,r0,ror#6 @ h+=Sigma1(e)
1311 and r2,r2,r11
1312 add r6,r6,r3 @ h+=K256[i]
1313 eor r2,r2,r5 @ Ch(e,f,g)
1314 eor r0,r7,r7,ror#11
1315 add r6,r6,r2 @ h+=Ch(e,f,g)
1316#if 21==31
1317 and r3,r3,#0xff
1318 cmp r3,#0xf2 @ done?
1319#endif
1320#if 21<15
1321# if __ARM_ARCH__>=7
1322 ldr r2,[r1],#4 @ prefetch
1323# else
1324 ldrb r2,[r1,#3]
1325# endif
1326 eor r3,r7,r8 @ a^b, b^c in next round
1327#else
1328 ldr r2,[sp,#7*4] @ from future BODY_16_xx
1329 eor r3,r7,r8 @ a^b, b^c in next round
1330 ldr r1,[sp,#4*4] @ from future BODY_16_xx
1331#endif
1332 eor r0,r0,r7,ror#20 @ Sigma0(a)
1333 and r12,r12,r3 @ (b^c)&=(a^b)
1334 add r10,r10,r6 @ d+=h
1335 eor r12,r12,r8 @ Maj(a,b,c)
1336 add r6,r6,r0,ror#2 @ h+=Sigma0(a)
1337 @ add r6,r6,r12 @ h+=Maj(a,b,c)
1338 @ ldr r2,[sp,#7*4] @ 22
1339 @ ldr r1,[sp,#4*4]
1340 mov r0,r2,ror#7
1341 add r6,r6,r12 @ h+=Maj(a,b,c) from the past
1342 mov r12,r1,ror#17
1343 eor r0,r0,r2,ror#18
1344 eor r12,r12,r1,ror#19
1345 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1346 ldr r2,[sp,#6*4]
1347 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1348 ldr r1,[sp,#15*4]
1349
1350 add r12,r12,r0
1351 eor r0,r10,r10,ror#5 @ from BODY_00_15
1352 add r2,r2,r12
1353 eor r0,r0,r10,ror#19 @ Sigma1(e)
1354 add r2,r2,r1 @ X[i]
1355 ldr r12,[r14],#4 @ *K256++
1356 add r5,r5,r2 @ h+=X[i]
1357 str r2,[sp,#6*4]
1358 eor r2,r11,r4
1359 add r5,r5,r0,ror#6 @ h+=Sigma1(e)
1360 and r2,r2,r10
1361 add r5,r5,r12 @ h+=K256[i]
1362 eor r2,r2,r4 @ Ch(e,f,g)
1363 eor r0,r6,r6,ror#11
1364 add r5,r5,r2 @ h+=Ch(e,f,g)
1365#if 22==31
1366 and r12,r12,#0xff
1367 cmp r12,#0xf2 @ done?
1368#endif
1369#if 22<15
1370# if __ARM_ARCH__>=7
1371 ldr r2,[r1],#4 @ prefetch
1372# else
1373 ldrb r2,[r1,#3]
1374# endif
1375 eor r12,r6,r7 @ a^b, b^c in next round
1376#else
1377 ldr r2,[sp,#8*4] @ from future BODY_16_xx
1378 eor r12,r6,r7 @ a^b, b^c in next round
1379 ldr r1,[sp,#5*4] @ from future BODY_16_xx
1380#endif
1381 eor r0,r0,r6,ror#20 @ Sigma0(a)
1382 and r3,r3,r12 @ (b^c)&=(a^b)
1383 add r9,r9,r5 @ d+=h
1384 eor r3,r3,r7 @ Maj(a,b,c)
1385 add r5,r5,r0,ror#2 @ h+=Sigma0(a)
1386 @ add r5,r5,r3 @ h+=Maj(a,b,c)
1387 @ ldr r2,[sp,#8*4] @ 23
1388 @ ldr r1,[sp,#5*4]
1389 mov r0,r2,ror#7
1390 add r5,r5,r3 @ h+=Maj(a,b,c) from the past
1391 mov r3,r1,ror#17
1392 eor r0,r0,r2,ror#18
1393 eor r3,r3,r1,ror#19
1394 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1395 ldr r2,[sp,#7*4]
1396 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1397 ldr r1,[sp,#0*4]
1398
1399 add r3,r3,r0
1400 eor r0,r9,r9,ror#5 @ from BODY_00_15
1401 add r2,r2,r3
1402 eor r0,r0,r9,ror#19 @ Sigma1(e)
1403 add r2,r2,r1 @ X[i]
1404 ldr r3,[r14],#4 @ *K256++
1405 add r4,r4,r2 @ h+=X[i]
1406 str r2,[sp,#7*4]
1407 eor r2,r10,r11
1408 add r4,r4,r0,ror#6 @ h+=Sigma1(e)
1409 and r2,r2,r9
1410 add r4,r4,r3 @ h+=K256[i]
1411 eor r2,r2,r11 @ Ch(e,f,g)
1412 eor r0,r5,r5,ror#11
1413 add r4,r4,r2 @ h+=Ch(e,f,g)
1414#if 23==31
1415 and r3,r3,#0xff
1416 cmp r3,#0xf2 @ done?
1417#endif
1418#if 23<15
1419# if __ARM_ARCH__>=7
1420 ldr r2,[r1],#4 @ prefetch
1421# else
1422 ldrb r2,[r1,#3]
1423# endif
1424 eor r3,r5,r6 @ a^b, b^c in next round
1425#else
1426 ldr r2,[sp,#9*4] @ from future BODY_16_xx
1427 eor r3,r5,r6 @ a^b, b^c in next round
1428 ldr r1,[sp,#6*4] @ from future BODY_16_xx
1429#endif
1430 eor r0,r0,r5,ror#20 @ Sigma0(a)
1431 and r12,r12,r3 @ (b^c)&=(a^b)
1432 add r8,r8,r4 @ d+=h
1433 eor r12,r12,r6 @ Maj(a,b,c)
1434 add r4,r4,r0,ror#2 @ h+=Sigma0(a)
1435 @ add r4,r4,r12 @ h+=Maj(a,b,c)
1436 @ ldr r2,[sp,#9*4] @ 24
1437 @ ldr r1,[sp,#6*4]
1438 mov r0,r2,ror#7
1439 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
1440 mov r12,r1,ror#17
1441 eor r0,r0,r2,ror#18
1442 eor r12,r12,r1,ror#19
1443 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1444 ldr r2,[sp,#8*4]
1445 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1446 ldr r1,[sp,#1*4]
1447
1448 add r12,r12,r0
1449 eor r0,r8,r8,ror#5 @ from BODY_00_15
1450 add r2,r2,r12
1451 eor r0,r0,r8,ror#19 @ Sigma1(e)
1452 add r2,r2,r1 @ X[i]
1453 ldr r12,[r14],#4 @ *K256++
1454 add r11,r11,r2 @ h+=X[i]
1455 str r2,[sp,#8*4]
1456 eor r2,r9,r10
1457 add r11,r11,r0,ror#6 @ h+=Sigma1(e)
1458 and r2,r2,r8
1459 add r11,r11,r12 @ h+=K256[i]
1460 eor r2,r2,r10 @ Ch(e,f,g)
1461 eor r0,r4,r4,ror#11
1462 add r11,r11,r2 @ h+=Ch(e,f,g)
1463#if 24==31
1464 and r12,r12,#0xff
1465 cmp r12,#0xf2 @ done?
1466#endif
1467#if 24<15
1468# if __ARM_ARCH__>=7
1469 ldr r2,[r1],#4 @ prefetch
1470# else
1471 ldrb r2,[r1,#3]
1472# endif
1473 eor r12,r4,r5 @ a^b, b^c in next round
1474#else
1475 ldr r2,[sp,#10*4] @ from future BODY_16_xx
1476 eor r12,r4,r5 @ a^b, b^c in next round
1477 ldr r1,[sp,#7*4] @ from future BODY_16_xx
1478#endif
1479 eor r0,r0,r4,ror#20 @ Sigma0(a)
1480 and r3,r3,r12 @ (b^c)&=(a^b)
1481 add r7,r7,r11 @ d+=h
1482 eor r3,r3,r5 @ Maj(a,b,c)
1483 add r11,r11,r0,ror#2 @ h+=Sigma0(a)
1484 @ add r11,r11,r3 @ h+=Maj(a,b,c)
1485 @ ldr r2,[sp,#10*4] @ 25
1486 @ ldr r1,[sp,#7*4]
1487 mov r0,r2,ror#7
1488 add r11,r11,r3 @ h+=Maj(a,b,c) from the past
1489 mov r3,r1,ror#17
1490 eor r0,r0,r2,ror#18
1491 eor r3,r3,r1,ror#19
1492 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1493 ldr r2,[sp,#9*4]
1494 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1495 ldr r1,[sp,#2*4]
1496
1497 add r3,r3,r0
1498 eor r0,r7,r7,ror#5 @ from BODY_00_15
1499 add r2,r2,r3
1500 eor r0,r0,r7,ror#19 @ Sigma1(e)
1501 add r2,r2,r1 @ X[i]
1502 ldr r3,[r14],#4 @ *K256++
1503 add r10,r10,r2 @ h+=X[i]
1504 str r2,[sp,#9*4]
1505 eor r2,r8,r9
1506 add r10,r10,r0,ror#6 @ h+=Sigma1(e)
1507 and r2,r2,r7
1508 add r10,r10,r3 @ h+=K256[i]
1509 eor r2,r2,r9 @ Ch(e,f,g)
1510 eor r0,r11,r11,ror#11
1511 add r10,r10,r2 @ h+=Ch(e,f,g)
1512#if 25==31
1513 and r3,r3,#0xff
1514 cmp r3,#0xf2 @ done?
1515#endif
1516#if 25<15
1517# if __ARM_ARCH__>=7
1518 ldr r2,[r1],#4 @ prefetch
1519# else
1520 ldrb r2,[r1,#3]
1521# endif
1522 eor r3,r11,r4 @ a^b, b^c in next round
1523#else
1524 ldr r2,[sp,#11*4] @ from future BODY_16_xx
1525 eor r3,r11,r4 @ a^b, b^c in next round
1526 ldr r1,[sp,#8*4] @ from future BODY_16_xx
1527#endif
1528 eor r0,r0,r11,ror#20 @ Sigma0(a)
1529 and r12,r12,r3 @ (b^c)&=(a^b)
1530 add r6,r6,r10 @ d+=h
1531 eor r12,r12,r4 @ Maj(a,b,c)
1532 add r10,r10,r0,ror#2 @ h+=Sigma0(a)
1533 @ add r10,r10,r12 @ h+=Maj(a,b,c)
1534 @ ldr r2,[sp,#11*4] @ 26
1535 @ ldr r1,[sp,#8*4]
1536 mov r0,r2,ror#7
1537 add r10,r10,r12 @ h+=Maj(a,b,c) from the past
1538 mov r12,r1,ror#17
1539 eor r0,r0,r2,ror#18
1540 eor r12,r12,r1,ror#19
1541 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1542 ldr r2,[sp,#10*4]
1543 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1544 ldr r1,[sp,#3*4]
1545
1546 add r12,r12,r0
1547 eor r0,r6,r6,ror#5 @ from BODY_00_15
1548 add r2,r2,r12
1549 eor r0,r0,r6,ror#19 @ Sigma1(e)
1550 add r2,r2,r1 @ X[i]
1551 ldr r12,[r14],#4 @ *K256++
1552 add r9,r9,r2 @ h+=X[i]
1553 str r2,[sp,#10*4]
1554 eor r2,r7,r8
1555 add r9,r9,r0,ror#6 @ h+=Sigma1(e)
1556 and r2,r2,r6
1557 add r9,r9,r12 @ h+=K256[i]
1558 eor r2,r2,r8 @ Ch(e,f,g)
1559 eor r0,r10,r10,ror#11
1560 add r9,r9,r2 @ h+=Ch(e,f,g)
1561#if 26==31
1562 and r12,r12,#0xff
1563 cmp r12,#0xf2 @ done?
1564#endif
1565#if 26<15
1566# if __ARM_ARCH__>=7
1567 ldr r2,[r1],#4 @ prefetch
1568# else
1569 ldrb r2,[r1,#3]
1570# endif
1571 eor r12,r10,r11 @ a^b, b^c in next round
1572#else
1573 ldr r2,[sp,#12*4] @ from future BODY_16_xx
1574 eor r12,r10,r11 @ a^b, b^c in next round
1575 ldr r1,[sp,#9*4] @ from future BODY_16_xx
1576#endif
1577 eor r0,r0,r10,ror#20 @ Sigma0(a)
1578 and r3,r3,r12 @ (b^c)&=(a^b)
1579 add r5,r5,r9 @ d+=h
1580 eor r3,r3,r11 @ Maj(a,b,c)
1581 add r9,r9,r0,ror#2 @ h+=Sigma0(a)
1582 @ add r9,r9,r3 @ h+=Maj(a,b,c)
1583 @ ldr r2,[sp,#12*4] @ 27
1584 @ ldr r1,[sp,#9*4]
1585 mov r0,r2,ror#7
1586 add r9,r9,r3 @ h+=Maj(a,b,c) from the past
1587 mov r3,r1,ror#17
1588 eor r0,r0,r2,ror#18
1589 eor r3,r3,r1,ror#19
1590 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1591 ldr r2,[sp,#11*4]
1592 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1593 ldr r1,[sp,#4*4]
1594
1595 add r3,r3,r0
1596 eor r0,r5,r5,ror#5 @ from BODY_00_15
1597 add r2,r2,r3
1598 eor r0,r0,r5,ror#19 @ Sigma1(e)
1599 add r2,r2,r1 @ X[i]
1600 ldr r3,[r14],#4 @ *K256++
1601 add r8,r8,r2 @ h+=X[i]
1602 str r2,[sp,#11*4]
1603 eor r2,r6,r7
1604 add r8,r8,r0,ror#6 @ h+=Sigma1(e)
1605 and r2,r2,r5
1606 add r8,r8,r3 @ h+=K256[i]
1607 eor r2,r2,r7 @ Ch(e,f,g)
1608 eor r0,r9,r9,ror#11
1609 add r8,r8,r2 @ h+=Ch(e,f,g)
1610#if 27==31
1611 and r3,r3,#0xff
1612 cmp r3,#0xf2 @ done?
1613#endif
1614#if 27<15
1615# if __ARM_ARCH__>=7
1616 ldr r2,[r1],#4 @ prefetch
1617# else
1618 ldrb r2,[r1,#3]
1619# endif
1620 eor r3,r9,r10 @ a^b, b^c in next round
1621#else
1622 ldr r2,[sp,#13*4] @ from future BODY_16_xx
1623 eor r3,r9,r10 @ a^b, b^c in next round
1624 ldr r1,[sp,#10*4] @ from future BODY_16_xx
1625#endif
1626 eor r0,r0,r9,ror#20 @ Sigma0(a)
1627 and r12,r12,r3 @ (b^c)&=(a^b)
1628 add r4,r4,r8 @ d+=h
1629 eor r12,r12,r10 @ Maj(a,b,c)
1630 add r8,r8,r0,ror#2 @ h+=Sigma0(a)
1631 @ add r8,r8,r12 @ h+=Maj(a,b,c)
1632 @ ldr r2,[sp,#13*4] @ 28
1633 @ ldr r1,[sp,#10*4]
1634 mov r0,r2,ror#7
1635 add r8,r8,r12 @ h+=Maj(a,b,c) from the past
1636 mov r12,r1,ror#17
1637 eor r0,r0,r2,ror#18
1638 eor r12,r12,r1,ror#19
1639 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1640 ldr r2,[sp,#12*4]
1641 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1642 ldr r1,[sp,#5*4]
1643
1644 add r12,r12,r0
1645 eor r0,r4,r4,ror#5 @ from BODY_00_15
1646 add r2,r2,r12
1647 eor r0,r0,r4,ror#19 @ Sigma1(e)
1648 add r2,r2,r1 @ X[i]
1649 ldr r12,[r14],#4 @ *K256++
1650 add r7,r7,r2 @ h+=X[i]
1651 str r2,[sp,#12*4]
1652 eor r2,r5,r6
1653 add r7,r7,r0,ror#6 @ h+=Sigma1(e)
1654 and r2,r2,r4
1655 add r7,r7,r12 @ h+=K256[i]
1656 eor r2,r2,r6 @ Ch(e,f,g)
1657 eor r0,r8,r8,ror#11
1658 add r7,r7,r2 @ h+=Ch(e,f,g)
1659#if 28==31
1660 and r12,r12,#0xff
1661 cmp r12,#0xf2 @ done?
1662#endif
1663#if 28<15
1664# if __ARM_ARCH__>=7
1665 ldr r2,[r1],#4 @ prefetch
1666# else
1667 ldrb r2,[r1,#3]
1668# endif
1669 eor r12,r8,r9 @ a^b, b^c in next round
1670#else
1671 ldr r2,[sp,#14*4] @ from future BODY_16_xx
1672 eor r12,r8,r9 @ a^b, b^c in next round
1673 ldr r1,[sp,#11*4] @ from future BODY_16_xx
1674#endif
1675 eor r0,r0,r8,ror#20 @ Sigma0(a)
1676 and r3,r3,r12 @ (b^c)&=(a^b)
1677 add r11,r11,r7 @ d+=h
1678 eor r3,r3,r9 @ Maj(a,b,c)
1679 add r7,r7,r0,ror#2 @ h+=Sigma0(a)
1680 @ add r7,r7,r3 @ h+=Maj(a,b,c)
1681 @ ldr r2,[sp,#14*4] @ 29
1682 @ ldr r1,[sp,#11*4]
1683 mov r0,r2,ror#7
1684 add r7,r7,r3 @ h+=Maj(a,b,c) from the past
1685 mov r3,r1,ror#17
1686 eor r0,r0,r2,ror#18
1687 eor r3,r3,r1,ror#19
1688 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1689 ldr r2,[sp,#13*4]
1690 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1691 ldr r1,[sp,#6*4]
1692
1693 add r3,r3,r0
1694 eor r0,r11,r11,ror#5 @ from BODY_00_15
1695 add r2,r2,r3
1696 eor r0,r0,r11,ror#19 @ Sigma1(e)
1697 add r2,r2,r1 @ X[i]
1698 ldr r3,[r14],#4 @ *K256++
1699 add r6,r6,r2 @ h+=X[i]
1700 str r2,[sp,#13*4]
1701 eor r2,r4,r5
1702 add r6,r6,r0,ror#6 @ h+=Sigma1(e)
1703 and r2,r2,r11
1704 add r6,r6,r3 @ h+=K256[i]
1705 eor r2,r2,r5 @ Ch(e,f,g)
1706 eor r0,r7,r7,ror#11
1707 add r6,r6,r2 @ h+=Ch(e,f,g)
1708#if 29==31
1709 and r3,r3,#0xff
1710 cmp r3,#0xf2 @ done?
1711#endif
1712#if 29<15
1713# if __ARM_ARCH__>=7
1714 ldr r2,[r1],#4 @ prefetch
1715# else
1716 ldrb r2,[r1,#3]
1717# endif
1718 eor r3,r7,r8 @ a^b, b^c in next round
1719#else
1720 ldr r2,[sp,#15*4] @ from future BODY_16_xx
1721 eor r3,r7,r8 @ a^b, b^c in next round
1722 ldr r1,[sp,#12*4] @ from future BODY_16_xx
1723#endif
1724 eor r0,r0,r7,ror#20 @ Sigma0(a)
1725 and r12,r12,r3 @ (b^c)&=(a^b)
1726 add r10,r10,r6 @ d+=h
1727 eor r12,r12,r8 @ Maj(a,b,c)
1728 add r6,r6,r0,ror#2 @ h+=Sigma0(a)
1729 @ add r6,r6,r12 @ h+=Maj(a,b,c)
1730 @ ldr r2,[sp,#15*4] @ 30
1731 @ ldr r1,[sp,#12*4]
1732 mov r0,r2,ror#7
1733 add r6,r6,r12 @ h+=Maj(a,b,c) from the past
1734 mov r12,r1,ror#17
1735 eor r0,r0,r2,ror#18
1736 eor r12,r12,r1,ror#19
1737 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1738 ldr r2,[sp,#14*4]
1739 eor r12,r12,r1,lsr#10 @ sigma1(X[i+14])
1740 ldr r1,[sp,#7*4]
1741
1742 add r12,r12,r0
1743 eor r0,r10,r10,ror#5 @ from BODY_00_15
1744 add r2,r2,r12
1745 eor r0,r0,r10,ror#19 @ Sigma1(e)
1746 add r2,r2,r1 @ X[i]
1747 ldr r12,[r14],#4 @ *K256++
1748 add r5,r5,r2 @ h+=X[i]
1749 str r2,[sp,#14*4]
1750 eor r2,r11,r4
1751 add r5,r5,r0,ror#6 @ h+=Sigma1(e)
1752 and r2,r2,r10
1753 add r5,r5,r12 @ h+=K256[i]
1754 eor r2,r2,r4 @ Ch(e,f,g)
1755 eor r0,r6,r6,ror#11
1756 add r5,r5,r2 @ h+=Ch(e,f,g)
1757#if 30==31
1758 and r12,r12,#0xff
1759 cmp r12,#0xf2 @ done?
1760#endif
1761#if 30<15
1762# if __ARM_ARCH__>=7
1763 ldr r2,[r1],#4 @ prefetch
1764# else
1765 ldrb r2,[r1,#3]
1766# endif
1767 eor r12,r6,r7 @ a^b, b^c in next round
1768#else
1769 ldr r2,[sp,#0*4] @ from future BODY_16_xx
1770 eor r12,r6,r7 @ a^b, b^c in next round
1771 ldr r1,[sp,#13*4] @ from future BODY_16_xx
1772#endif
1773 eor r0,r0,r6,ror#20 @ Sigma0(a)
1774 and r3,r3,r12 @ (b^c)&=(a^b)
1775 add r9,r9,r5 @ d+=h
1776 eor r3,r3,r7 @ Maj(a,b,c)
1777 add r5,r5,r0,ror#2 @ h+=Sigma0(a)
1778 @ add r5,r5,r3 @ h+=Maj(a,b,c)
1779 @ ldr r2,[sp,#0*4] @ 31
1780 @ ldr r1,[sp,#13*4]
1781 mov r0,r2,ror#7
1782 add r5,r5,r3 @ h+=Maj(a,b,c) from the past
1783 mov r3,r1,ror#17
1784 eor r0,r0,r2,ror#18
1785 eor r3,r3,r1,ror#19
1786 eor r0,r0,r2,lsr#3 @ sigma0(X[i+1])
1787 ldr r2,[sp,#15*4]
1788 eor r3,r3,r1,lsr#10 @ sigma1(X[i+14])
1789 ldr r1,[sp,#8*4]
1790
1791 add r3,r3,r0
1792 eor r0,r9,r9,ror#5 @ from BODY_00_15
1793 add r2,r2,r3
1794 eor r0,r0,r9,ror#19 @ Sigma1(e)
1795 add r2,r2,r1 @ X[i]
1796 ldr r3,[r14],#4 @ *K256++
1797 add r4,r4,r2 @ h+=X[i]
1798 str r2,[sp,#15*4]
1799 eor r2,r10,r11
1800 add r4,r4,r0,ror#6 @ h+=Sigma1(e)
1801 and r2,r2,r9
1802 add r4,r4,r3 @ h+=K256[i]
1803 eor r2,r2,r11 @ Ch(e,f,g)
1804 eor r0,r5,r5,ror#11
1805 add r4,r4,r2 @ h+=Ch(e,f,g)
1806#if 31==31
1807 and r3,r3,#0xff
1808 cmp r3,#0xf2 @ done?
1809#endif
1810#if 31<15
1811# if __ARM_ARCH__>=7
1812 ldr r2,[r1],#4 @ prefetch
1813# else
1814 ldrb r2,[r1,#3]
1815# endif
1816 eor r3,r5,r6 @ a^b, b^c in next round
1817#else
1818 ldr r2,[sp,#1*4] @ from future BODY_16_xx
1819 eor r3,r5,r6 @ a^b, b^c in next round
1820 ldr r1,[sp,#14*4] @ from future BODY_16_xx
1821#endif
1822 eor r0,r0,r5,ror#20 @ Sigma0(a)
1823 and r12,r12,r3 @ (b^c)&=(a^b)
1824 add r8,r8,r4 @ d+=h
1825 eor r12,r12,r6 @ Maj(a,b,c)
1826 add r4,r4,r0,ror#2 @ h+=Sigma0(a)
1827 @ add r4,r4,r12 @ h+=Maj(a,b,c)
1828#if __ARM_ARCH__>=7
1829 ite eq @ Thumb2 thing, sanity check in ARM
1830#endif
1831 ldreq r3,[sp,#16*4] @ pull ctx
1832 bne .Lrounds_16_xx
1833
1834 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
1835 ldr r0,[r3,#0]
1836 ldr r2,[r3,#4]
1837 ldr r12,[r3,#8]
1838 add r4,r4,r0
1839 ldr r0,[r3,#12]
1840 add r5,r5,r2
1841 ldr r2,[r3,#16]
1842 add r6,r6,r12
1843 ldr r12,[r3,#20]
1844 add r7,r7,r0
1845 ldr r0,[r3,#24]
1846 add r8,r8,r2
1847 ldr r2,[r3,#28]
1848 add r9,r9,r12
1849 ldr r1,[sp,#17*4] @ pull inp
1850 ldr r12,[sp,#18*4] @ pull inp+len
1851 add r10,r10,r0
1852 add r11,r11,r2
1853 stmia r3,{r4,r5,r6,r7,r8,r9,r10,r11}
1854 cmp r1,r12
1855 sub r14,r14,#256 @ rewind Ktbl
1856 bne .Loop
1857
1858 add sp,sp,#19*4 @ destroy frame
1859#if __ARM_ARCH__>=5
1860 ldmia sp!,{r4-r11,pc}
1861#else
1862 ldmia sp!,{r4-r11,lr}
1863 tst lr,#1
1864 moveq pc,lr @ be binary compatible with V4, yet
1865 .word 0xe12fff1e @ interoperable with Thumb ISA:-)
1866#endif
1867.size sha256_block_data_order,.-sha256_block_data_order
1868#if __ARM_MAX_ARCH__>=7
1869.arch armv7-a
1870.fpu neon
1871
1872.global sha256_block_data_order_neon
1873.type sha256_block_data_order_neon,%function
1874.align 4
1875sha256_block_data_order_neon:
1876.LNEON:
1877 stmdb sp!,{r4-r12,lr}
1878
1879 sub r11,sp,#16*4+16
1880 adrl r14,K256
1881 bic r11,r11,#15 @ align for 128-bit stores
1882 mov r12,sp
1883 mov sp,r11 @ alloca
1884 add r2,r1,r2,lsl#6 @ len to point at the end of inp
1885
1886 vld1.8 {q0},[r1]!
1887 vld1.8 {q1},[r1]!
1888 vld1.8 {q2},[r1]!
1889 vld1.8 {q3},[r1]!
1890 vld1.32 {q8},[r14,:128]!
1891 vld1.32 {q9},[r14,:128]!
1892 vld1.32 {q10},[r14,:128]!
1893 vld1.32 {q11},[r14,:128]!
1894 vrev32.8 q0,q0 @ yes, even on
1895 str r0,[sp,#64]
1896 vrev32.8 q1,q1 @ big-endian
1897 str r1,[sp,#68]
1898 mov r1,sp
1899 vrev32.8 q2,q2
1900 str r2,[sp,#72]
1901 vrev32.8 q3,q3
1902 str r12,[sp,#76] @ save original sp
1903 vadd.i32 q8,q8,q0
1904 vadd.i32 q9,q9,q1
1905 vst1.32 {q8},[r1,:128]!
1906 vadd.i32 q10,q10,q2
1907 vst1.32 {q9},[r1,:128]!
1908 vadd.i32 q11,q11,q3
1909 vst1.32 {q10},[r1,:128]!
1910 vst1.32 {q11},[r1,:128]!
1911
1912 ldmia r0,{r4-r11}
1913 sub r1,r1,#64
1914 ldr r2,[sp,#0]
1915 eor r12,r12,r12
1916 eor r3,r5,r6
1917 b .L_00_48
1918
1919.align 4
1920.L_00_48:
1921 vext.8 q8,q0,q1,#4
1922 add r11,r11,r2
1923 eor r2,r9,r10
1924 eor r0,r8,r8,ror#5
1925 vext.8 q9,q2,q3,#4
1926 add r4,r4,r12
1927 and r2,r2,r8
1928 eor r12,r0,r8,ror#19
1929 vshr.u32 q10,q8,#7
1930 eor r0,r4,r4,ror#11
1931 eor r2,r2,r10
1932 vadd.i32 q0,q0,q9
1933 add r11,r11,r12,ror#6
1934 eor r12,r4,r5
1935 vshr.u32 q9,q8,#3
1936 eor r0,r0,r4,ror#20
1937 add r11,r11,r2
1938 vsli.32 q10,q8,#25
1939 ldr r2,[sp,#4]
1940 and r3,r3,r12
1941 vshr.u32 q11,q8,#18
1942 add r7,r7,r11
1943 add r11,r11,r0,ror#2
1944 eor r3,r3,r5
1945 veor q9,q9,q10
1946 add r10,r10,r2
1947 vsli.32 q11,q8,#14
1948 eor r2,r8,r9
1949 eor r0,r7,r7,ror#5
1950 vshr.u32 d24,d7,#17
1951 add r11,r11,r3
1952 and r2,r2,r7
1953 veor q9,q9,q11
1954 eor r3,r0,r7,ror#19
1955 eor r0,r11,r11,ror#11
1956 vsli.32 d24,d7,#15
1957 eor r2,r2,r9
1958 add r10,r10,r3,ror#6
1959 vshr.u32 d25,d7,#10
1960 eor r3,r11,r4
1961 eor r0,r0,r11,ror#20
1962 vadd.i32 q0,q0,q9
1963 add r10,r10,r2
1964 ldr r2,[sp,#8]
1965 veor d25,d25,d24
1966 and r12,r12,r3
1967 add r6,r6,r10
1968 vshr.u32 d24,d7,#19
1969 add r10,r10,r0,ror#2
1970 eor r12,r12,r4
1971 vsli.32 d24,d7,#13
1972 add r9,r9,r2
1973 eor r2,r7,r8
1974 veor d25,d25,d24
1975 eor r0,r6,r6,ror#5
1976 add r10,r10,r12
1977 vadd.i32 d0,d0,d25
1978 and r2,r2,r6
1979 eor r12,r0,r6,ror#19
1980 vshr.u32 d24,d0,#17
1981 eor r0,r10,r10,ror#11
1982 eor r2,r2,r8
1983 vsli.32 d24,d0,#15
1984 add r9,r9,r12,ror#6
1985 eor r12,r10,r11
1986 vshr.u32 d25,d0,#10
1987 eor r0,r0,r10,ror#20
1988 add r9,r9,r2
1989 veor d25,d25,d24
1990 ldr r2,[sp,#12]
1991 and r3,r3,r12
1992 vshr.u32 d24,d0,#19
1993 add r5,r5,r9
1994 add r9,r9,r0,ror#2
1995 eor r3,r3,r11
1996 vld1.32 {q8},[r14,:128]!
1997 add r8,r8,r2
1998 vsli.32 d24,d0,#13
1999 eor r2,r6,r7
2000 eor r0,r5,r5,ror#5
2001 veor d25,d25,d24
2002 add r9,r9,r3
2003 and r2,r2,r5
2004 vadd.i32 d1,d1,d25
2005 eor r3,r0,r5,ror#19
2006 eor r0,r9,r9,ror#11
2007 vadd.i32 q8,q8,q0
2008 eor r2,r2,r7
2009 add r8,r8,r3,ror#6
2010 eor r3,r9,r10
2011 eor r0,r0,r9,ror#20
2012 add r8,r8,r2
2013 ldr r2,[sp,#16]
2014 and r12,r12,r3
2015 add r4,r4,r8
2016 vst1.32 {q8},[r1,:128]!
2017 add r8,r8,r0,ror#2
2018 eor r12,r12,r10
2019 vext.8 q8,q1,q2,#4
2020 add r7,r7,r2
2021 eor r2,r5,r6
2022 eor r0,r4,r4,ror#5
2023 vext.8 q9,q3,q0,#4
2024 add r8,r8,r12
2025 and r2,r2,r4
2026 eor r12,r0,r4,ror#19
2027 vshr.u32 q10,q8,#7
2028 eor r0,r8,r8,ror#11
2029 eor r2,r2,r6
2030 vadd.i32 q1,q1,q9
2031 add r7,r7,r12,ror#6
2032 eor r12,r8,r9
2033 vshr.u32 q9,q8,#3
2034 eor r0,r0,r8,ror#20
2035 add r7,r7,r2
2036 vsli.32 q10,q8,#25
2037 ldr r2,[sp,#20]
2038 and r3,r3,r12
2039 vshr.u32 q11,q8,#18
2040 add r11,r11,r7
2041 add r7,r7,r0,ror#2
2042 eor r3,r3,r9
2043 veor q9,q9,q10
2044 add r6,r6,r2
2045 vsli.32 q11,q8,#14
2046 eor r2,r4,r5
2047 eor r0,r11,r11,ror#5
2048 vshr.u32 d24,d1,#17
2049 add r7,r7,r3
2050 and r2,r2,r11
2051 veor q9,q9,q11
2052 eor r3,r0,r11,ror#19
2053 eor r0,r7,r7,ror#11
2054 vsli.32 d24,d1,#15
2055 eor r2,r2,r5
2056 add r6,r6,r3,ror#6
2057 vshr.u32 d25,d1,#10
2058 eor r3,r7,r8
2059 eor r0,r0,r7,ror#20
2060 vadd.i32 q1,q1,q9
2061 add r6,r6,r2
2062 ldr r2,[sp,#24]
2063 veor d25,d25,d24
2064 and r12,r12,r3
2065 add r10,r10,r6
2066 vshr.u32 d24,d1,#19
2067 add r6,r6,r0,ror#2
2068 eor r12,r12,r8
2069 vsli.32 d24,d1,#13
2070 add r5,r5,r2
2071 eor r2,r11,r4
2072 veor d25,d25,d24
2073 eor r0,r10,r10,ror#5
2074 add r6,r6,r12
2075 vadd.i32 d2,d2,d25
2076 and r2,r2,r10
2077 eor r12,r0,r10,ror#19
2078 vshr.u32 d24,d2,#17
2079 eor r0,r6,r6,ror#11
2080 eor r2,r2,r4
2081 vsli.32 d24,d2,#15
2082 add r5,r5,r12,ror#6
2083 eor r12,r6,r7
2084 vshr.u32 d25,d2,#10
2085 eor r0,r0,r6,ror#20
2086 add r5,r5,r2
2087 veor d25,d25,d24
2088 ldr r2,[sp,#28]
2089 and r3,r3,r12
2090 vshr.u32 d24,d2,#19
2091 add r9,r9,r5
2092 add r5,r5,r0,ror#2
2093 eor r3,r3,r7
2094 vld1.32 {q8},[r14,:128]!
2095 add r4,r4,r2
2096 vsli.32 d24,d2,#13
2097 eor r2,r10,r11
2098 eor r0,r9,r9,ror#5
2099 veor d25,d25,d24
2100 add r5,r5,r3
2101 and r2,r2,r9
2102 vadd.i32 d3,d3,d25
2103 eor r3,r0,r9,ror#19
2104 eor r0,r5,r5,ror#11
2105 vadd.i32 q8,q8,q1
2106 eor r2,r2,r11
2107 add r4,r4,r3,ror#6
2108 eor r3,r5,r6
2109 eor r0,r0,r5,ror#20
2110 add r4,r4,r2
2111 ldr r2,[sp,#32]
2112 and r12,r12,r3
2113 add r8,r8,r4
2114 vst1.32 {q8},[r1,:128]!
2115 add r4,r4,r0,ror#2
2116 eor r12,r12,r6
2117 vext.8 q8,q2,q3,#4
2118 add r11,r11,r2
2119 eor r2,r9,r10
2120 eor r0,r8,r8,ror#5
2121 vext.8 q9,q0,q1,#4
2122 add r4,r4,r12
2123 and r2,r2,r8
2124 eor r12,r0,r8,ror#19
2125 vshr.u32 q10,q8,#7
2126 eor r0,r4,r4,ror#11
2127 eor r2,r2,r10
2128 vadd.i32 q2,q2,q9
2129 add r11,r11,r12,ror#6
2130 eor r12,r4,r5
2131 vshr.u32 q9,q8,#3
2132 eor r0,r0,r4,ror#20
2133 add r11,r11,r2
2134 vsli.32 q10,q8,#25
2135 ldr r2,[sp,#36]
2136 and r3,r3,r12
2137 vshr.u32 q11,q8,#18
2138 add r7,r7,r11
2139 add r11,r11,r0,ror#2
2140 eor r3,r3,r5
2141 veor q9,q9,q10
2142 add r10,r10,r2
2143 vsli.32 q11,q8,#14
2144 eor r2,r8,r9
2145 eor r0,r7,r7,ror#5
2146 vshr.u32 d24,d3,#17
2147 add r11,r11,r3
2148 and r2,r2,r7
2149 veor q9,q9,q11
2150 eor r3,r0,r7,ror#19
2151 eor r0,r11,r11,ror#11
2152 vsli.32 d24,d3,#15
2153 eor r2,r2,r9
2154 add r10,r10,r3,ror#6
2155 vshr.u32 d25,d3,#10
2156 eor r3,r11,r4
2157 eor r0,r0,r11,ror#20
2158 vadd.i32 q2,q2,q9
2159 add r10,r10,r2
2160 ldr r2,[sp,#40]
2161 veor d25,d25,d24
2162 and r12,r12,r3
2163 add r6,r6,r10
2164 vshr.u32 d24,d3,#19
2165 add r10,r10,r0,ror#2
2166 eor r12,r12,r4
2167 vsli.32 d24,d3,#13
2168 add r9,r9,r2
2169 eor r2,r7,r8
2170 veor d25,d25,d24
2171 eor r0,r6,r6,ror#5
2172 add r10,r10,r12
2173 vadd.i32 d4,d4,d25
2174 and r2,r2,r6
2175 eor r12,r0,r6,ror#19
2176 vshr.u32 d24,d4,#17
2177 eor r0,r10,r10,ror#11
2178 eor r2,r2,r8
2179 vsli.32 d24,d4,#15
2180 add r9,r9,r12,ror#6
2181 eor r12,r10,r11
2182 vshr.u32 d25,d4,#10
2183 eor r0,r0,r10,ror#20
2184 add r9,r9,r2
2185 veor d25,d25,d24
2186 ldr r2,[sp,#44]
2187 and r3,r3,r12
2188 vshr.u32 d24,d4,#19
2189 add r5,r5,r9
2190 add r9,r9,r0,ror#2
2191 eor r3,r3,r11
2192 vld1.32 {q8},[r14,:128]!
2193 add r8,r8,r2
2194 vsli.32 d24,d4,#13
2195 eor r2,r6,r7
2196 eor r0,r5,r5,ror#5
2197 veor d25,d25,d24
2198 add r9,r9,r3
2199 and r2,r2,r5
2200 vadd.i32 d5,d5,d25
2201 eor r3,r0,r5,ror#19
2202 eor r0,r9,r9,ror#11
2203 vadd.i32 q8,q8,q2
2204 eor r2,r2,r7
2205 add r8,r8,r3,ror#6
2206 eor r3,r9,r10
2207 eor r0,r0,r9,ror#20
2208 add r8,r8,r2
2209 ldr r2,[sp,#48]
2210 and r12,r12,r3
2211 add r4,r4,r8
2212 vst1.32 {q8},[r1,:128]!
2213 add r8,r8,r0,ror#2
2214 eor r12,r12,r10
2215 vext.8 q8,q3,q0,#4
2216 add r7,r7,r2
2217 eor r2,r5,r6
2218 eor r0,r4,r4,ror#5
2219 vext.8 q9,q1,q2,#4
2220 add r8,r8,r12
2221 and r2,r2,r4
2222 eor r12,r0,r4,ror#19
2223 vshr.u32 q10,q8,#7
2224 eor r0,r8,r8,ror#11
2225 eor r2,r2,r6
2226 vadd.i32 q3,q3,q9
2227 add r7,r7,r12,ror#6
2228 eor r12,r8,r9
2229 vshr.u32 q9,q8,#3
2230 eor r0,r0,r8,ror#20
2231 add r7,r7,r2
2232 vsli.32 q10,q8,#25
2233 ldr r2,[sp,#52]
2234 and r3,r3,r12
2235 vshr.u32 q11,q8,#18
2236 add r11,r11,r7
2237 add r7,r7,r0,ror#2
2238 eor r3,r3,r9
2239 veor q9,q9,q10
2240 add r6,r6,r2
2241 vsli.32 q11,q8,#14
2242 eor r2,r4,r5
2243 eor r0,r11,r11,ror#5
2244 vshr.u32 d24,d5,#17
2245 add r7,r7,r3
2246 and r2,r2,r11
2247 veor q9,q9,q11
2248 eor r3,r0,r11,ror#19
2249 eor r0,r7,r7,ror#11
2250 vsli.32 d24,d5,#15
2251 eor r2,r2,r5
2252 add r6,r6,r3,ror#6
2253 vshr.u32 d25,d5,#10
2254 eor r3,r7,r8
2255 eor r0,r0,r7,ror#20
2256 vadd.i32 q3,q3,q9
2257 add r6,r6,r2
2258 ldr r2,[sp,#56]
2259 veor d25,d25,d24
2260 and r12,r12,r3
2261 add r10,r10,r6
2262 vshr.u32 d24,d5,#19
2263 add r6,r6,r0,ror#2
2264 eor r12,r12,r8
2265 vsli.32 d24,d5,#13
2266 add r5,r5,r2
2267 eor r2,r11,r4
2268 veor d25,d25,d24
2269 eor r0,r10,r10,ror#5
2270 add r6,r6,r12
2271 vadd.i32 d6,d6,d25
2272 and r2,r2,r10
2273 eor r12,r0,r10,ror#19
2274 vshr.u32 d24,d6,#17
2275 eor r0,r6,r6,ror#11
2276 eor r2,r2,r4
2277 vsli.32 d24,d6,#15
2278 add r5,r5,r12,ror#6
2279 eor r12,r6,r7
2280 vshr.u32 d25,d6,#10
2281 eor r0,r0,r6,ror#20
2282 add r5,r5,r2
2283 veor d25,d25,d24
2284 ldr r2,[sp,#60]
2285 and r3,r3,r12
2286 vshr.u32 d24,d6,#19
2287 add r9,r9,r5
2288 add r5,r5,r0,ror#2
2289 eor r3,r3,r7
2290 vld1.32 {q8},[r14,:128]!
2291 add r4,r4,r2
2292 vsli.32 d24,d6,#13
2293 eor r2,r10,r11
2294 eor r0,r9,r9,ror#5
2295 veor d25,d25,d24
2296 add r5,r5,r3
2297 and r2,r2,r9
2298 vadd.i32 d7,d7,d25
2299 eor r3,r0,r9,ror#19
2300 eor r0,r5,r5,ror#11
2301 vadd.i32 q8,q8,q3
2302 eor r2,r2,r11
2303 add r4,r4,r3,ror#6
2304 eor r3,r5,r6
2305 eor r0,r0,r5,ror#20
2306 add r4,r4,r2
2307 ldr r2,[r14]
2308 and r12,r12,r3
2309 add r8,r8,r4
2310 vst1.32 {q8},[r1,:128]!
2311 add r4,r4,r0,ror#2
2312 eor r12,r12,r6
2313 teq r2,#0 @ check for K256 terminator
2314 ldr r2,[sp,#0]
2315 sub r1,r1,#64
2316 bne .L_00_48
2317
2318 ldr r1,[sp,#68]
2319 ldr r0,[sp,#72]
2320 sub r14,r14,#256 @ rewind r14
2321 teq r1,r0
2322 it eq
2323 subeq r1,r1,#64 @ avoid SEGV
2324 vld1.8 {q0},[r1]! @ load next input block
2325 vld1.8 {q1},[r1]!
2326 vld1.8 {q2},[r1]!
2327 vld1.8 {q3},[r1]!
2328 it ne
2329 strne r1,[sp,#68]
2330 mov r1,sp
2331 add r11,r11,r2
2332 eor r2,r9,r10
2333 eor r0,r8,r8,ror#5
2334 add r4,r4,r12
2335 vld1.32 {q8},[r14,:128]!
2336 and r2,r2,r8
2337 eor r12,r0,r8,ror#19
2338 eor r0,r4,r4,ror#11
2339 eor r2,r2,r10
2340 vrev32.8 q0,q0
2341 add r11,r11,r12,ror#6
2342 eor r12,r4,r5
2343 eor r0,r0,r4,ror#20
2344 add r11,r11,r2
2345 vadd.i32 q8,q8,q0
2346 ldr r2,[sp,#4]
2347 and r3,r3,r12
2348 add r7,r7,r11
2349 add r11,r11,r0,ror#2
2350 eor r3,r3,r5
2351 add r10,r10,r2
2352 eor r2,r8,r9
2353 eor r0,r7,r7,ror#5
2354 add r11,r11,r3
2355 and r2,r2,r7
2356 eor r3,r0,r7,ror#19
2357 eor r0,r11,r11,ror#11
2358 eor r2,r2,r9
2359 add r10,r10,r3,ror#6
2360 eor r3,r11,r4
2361 eor r0,r0,r11,ror#20
2362 add r10,r10,r2
2363 ldr r2,[sp,#8]
2364 and r12,r12,r3
2365 add r6,r6,r10
2366 add r10,r10,r0,ror#2
2367 eor r12,r12,r4
2368 add r9,r9,r2
2369 eor r2,r7,r8
2370 eor r0,r6,r6,ror#5
2371 add r10,r10,r12
2372 and r2,r2,r6
2373 eor r12,r0,r6,ror#19
2374 eor r0,r10,r10,ror#11
2375 eor r2,r2,r8
2376 add r9,r9,r12,ror#6
2377 eor r12,r10,r11
2378 eor r0,r0,r10,ror#20
2379 add r9,r9,r2
2380 ldr r2,[sp,#12]
2381 and r3,r3,r12
2382 add r5,r5,r9
2383 add r9,r9,r0,ror#2
2384 eor r3,r3,r11
2385 add r8,r8,r2
2386 eor r2,r6,r7
2387 eor r0,r5,r5,ror#5
2388 add r9,r9,r3
2389 and r2,r2,r5
2390 eor r3,r0,r5,ror#19
2391 eor r0,r9,r9,ror#11
2392 eor r2,r2,r7
2393 add r8,r8,r3,ror#6
2394 eor r3,r9,r10
2395 eor r0,r0,r9,ror#20
2396 add r8,r8,r2
2397 ldr r2,[sp,#16]
2398 and r12,r12,r3
2399 add r4,r4,r8
2400 add r8,r8,r0,ror#2
2401 eor r12,r12,r10
2402 vst1.32 {q8},[r1,:128]!
2403 add r7,r7,r2
2404 eor r2,r5,r6
2405 eor r0,r4,r4,ror#5
2406 add r8,r8,r12
2407 vld1.32 {q8},[r14,:128]!
2408 and r2,r2,r4
2409 eor r12,r0,r4,ror#19
2410 eor r0,r8,r8,ror#11
2411 eor r2,r2,r6
2412 vrev32.8 q1,q1
2413 add r7,r7,r12,ror#6
2414 eor r12,r8,r9
2415 eor r0,r0,r8,ror#20
2416 add r7,r7,r2
2417 vadd.i32 q8,q8,q1
2418 ldr r2,[sp,#20]
2419 and r3,r3,r12
2420 add r11,r11,r7
2421 add r7,r7,r0,ror#2
2422 eor r3,r3,r9
2423 add r6,r6,r2
2424 eor r2,r4,r5
2425 eor r0,r11,r11,ror#5
2426 add r7,r7,r3
2427 and r2,r2,r11
2428 eor r3,r0,r11,ror#19
2429 eor r0,r7,r7,ror#11
2430 eor r2,r2,r5
2431 add r6,r6,r3,ror#6
2432 eor r3,r7,r8
2433 eor r0,r0,r7,ror#20
2434 add r6,r6,r2
2435 ldr r2,[sp,#24]
2436 and r12,r12,r3
2437 add r10,r10,r6
2438 add r6,r6,r0,ror#2
2439 eor r12,r12,r8
2440 add r5,r5,r2
2441 eor r2,r11,r4
2442 eor r0,r10,r10,ror#5
2443 add r6,r6,r12
2444 and r2,r2,r10
2445 eor r12,r0,r10,ror#19
2446 eor r0,r6,r6,ror#11
2447 eor r2,r2,r4
2448 add r5,r5,r12,ror#6
2449 eor r12,r6,r7
2450 eor r0,r0,r6,ror#20
2451 add r5,r5,r2
2452 ldr r2,[sp,#28]
2453 and r3,r3,r12
2454 add r9,r9,r5
2455 add r5,r5,r0,ror#2
2456 eor r3,r3,r7
2457 add r4,r4,r2
2458 eor r2,r10,r11
2459 eor r0,r9,r9,ror#5
2460 add r5,r5,r3
2461 and r2,r2,r9
2462 eor r3,r0,r9,ror#19
2463 eor r0,r5,r5,ror#11
2464 eor r2,r2,r11
2465 add r4,r4,r3,ror#6
2466 eor r3,r5,r6
2467 eor r0,r0,r5,ror#20
2468 add r4,r4,r2
2469 ldr r2,[sp,#32]
2470 and r12,r12,r3
2471 add r8,r8,r4
2472 add r4,r4,r0,ror#2
2473 eor r12,r12,r6
2474 vst1.32 {q8},[r1,:128]!
2475 add r11,r11,r2
2476 eor r2,r9,r10
2477 eor r0,r8,r8,ror#5
2478 add r4,r4,r12
2479 vld1.32 {q8},[r14,:128]!
2480 and r2,r2,r8
2481 eor r12,r0,r8,ror#19
2482 eor r0,r4,r4,ror#11
2483 eor r2,r2,r10
2484 vrev32.8 q2,q2
2485 add r11,r11,r12,ror#6
2486 eor r12,r4,r5
2487 eor r0,r0,r4,ror#20
2488 add r11,r11,r2
2489 vadd.i32 q8,q8,q2
2490 ldr r2,[sp,#36]
2491 and r3,r3,r12
2492 add r7,r7,r11
2493 add r11,r11,r0,ror#2
2494 eor r3,r3,r5
2495 add r10,r10,r2
2496 eor r2,r8,r9
2497 eor r0,r7,r7,ror#5
2498 add r11,r11,r3
2499 and r2,r2,r7
2500 eor r3,r0,r7,ror#19
2501 eor r0,r11,r11,ror#11
2502 eor r2,r2,r9
2503 add r10,r10,r3,ror#6
2504 eor r3,r11,r4
2505 eor r0,r0,r11,ror#20
2506 add r10,r10,r2
2507 ldr r2,[sp,#40]
2508 and r12,r12,r3
2509 add r6,r6,r10
2510 add r10,r10,r0,ror#2
2511 eor r12,r12,r4
2512 add r9,r9,r2
2513 eor r2,r7,r8
2514 eor r0,r6,r6,ror#5
2515 add r10,r10,r12
2516 and r2,r2,r6
2517 eor r12,r0,r6,ror#19
2518 eor r0,r10,r10,ror#11
2519 eor r2,r2,r8
2520 add r9,r9,r12,ror#6
2521 eor r12,r10,r11
2522 eor r0,r0,r10,ror#20
2523 add r9,r9,r2
2524 ldr r2,[sp,#44]
2525 and r3,r3,r12
2526 add r5,r5,r9
2527 add r9,r9,r0,ror#2
2528 eor r3,r3,r11
2529 add r8,r8,r2
2530 eor r2,r6,r7
2531 eor r0,r5,r5,ror#5
2532 add r9,r9,r3
2533 and r2,r2,r5
2534 eor r3,r0,r5,ror#19
2535 eor r0,r9,r9,ror#11
2536 eor r2,r2,r7
2537 add r8,r8,r3,ror#6
2538 eor r3,r9,r10
2539 eor r0,r0,r9,ror#20
2540 add r8,r8,r2
2541 ldr r2,[sp,#48]
2542 and r12,r12,r3
2543 add r4,r4,r8
2544 add r8,r8,r0,ror#2
2545 eor r12,r12,r10
2546 vst1.32 {q8},[r1,:128]!
2547 add r7,r7,r2
2548 eor r2,r5,r6
2549 eor r0,r4,r4,ror#5
2550 add r8,r8,r12
2551 vld1.32 {q8},[r14,:128]!
2552 and r2,r2,r4
2553 eor r12,r0,r4,ror#19
2554 eor r0,r8,r8,ror#11
2555 eor r2,r2,r6
2556 vrev32.8 q3,q3
2557 add r7,r7,r12,ror#6
2558 eor r12,r8,r9
2559 eor r0,r0,r8,ror#20
2560 add r7,r7,r2
2561 vadd.i32 q8,q8,q3
2562 ldr r2,[sp,#52]
2563 and r3,r3,r12
2564 add r11,r11,r7
2565 add r7,r7,r0,ror#2
2566 eor r3,r3,r9
2567 add r6,r6,r2
2568 eor r2,r4,r5
2569 eor r0,r11,r11,ror#5
2570 add r7,r7,r3
2571 and r2,r2,r11
2572 eor r3,r0,r11,ror#19
2573 eor r0,r7,r7,ror#11
2574 eor r2,r2,r5
2575 add r6,r6,r3,ror#6
2576 eor r3,r7,r8
2577 eor r0,r0,r7,ror#20
2578 add r6,r6,r2
2579 ldr r2,[sp,#56]
2580 and r12,r12,r3
2581 add r10,r10,r6
2582 add r6,r6,r0,ror#2
2583 eor r12,r12,r8
2584 add r5,r5,r2
2585 eor r2,r11,r4
2586 eor r0,r10,r10,ror#5
2587 add r6,r6,r12
2588 and r2,r2,r10
2589 eor r12,r0,r10,ror#19
2590 eor r0,r6,r6,ror#11
2591 eor r2,r2,r4
2592 add r5,r5,r12,ror#6
2593 eor r12,r6,r7
2594 eor r0,r0,r6,ror#20
2595 add r5,r5,r2
2596 ldr r2,[sp,#60]
2597 and r3,r3,r12
2598 add r9,r9,r5
2599 add r5,r5,r0,ror#2
2600 eor r3,r3,r7
2601 add r4,r4,r2
2602 eor r2,r10,r11
2603 eor r0,r9,r9,ror#5
2604 add r5,r5,r3
2605 and r2,r2,r9
2606 eor r3,r0,r9,ror#19
2607 eor r0,r5,r5,ror#11
2608 eor r2,r2,r11
2609 add r4,r4,r3,ror#6
2610 eor r3,r5,r6
2611 eor r0,r0,r5,ror#20
2612 add r4,r4,r2
2613 ldr r2,[sp,#64]
2614 and r12,r12,r3
2615 add r8,r8,r4
2616 add r4,r4,r0,ror#2
2617 eor r12,r12,r6
2618 vst1.32 {q8},[r1,:128]!
2619 ldr r0,[r2,#0]
2620 add r4,r4,r12 @ h+=Maj(a,b,c) from the past
2621 ldr r12,[r2,#4]
2622 ldr r3,[r2,#8]
2623 ldr r1,[r2,#12]
2624 add r4,r4,r0 @ accumulate
2625 ldr r0,[r2,#16]
2626 add r5,r5,r12
2627 ldr r12,[r2,#20]
2628 add r6,r6,r3
2629 ldr r3,[r2,#24]
2630 add r7,r7,r1
2631 ldr r1,[r2,#28]
2632 add r8,r8,r0
2633 str r4,[r2],#4
2634 add r9,r9,r12
2635 str r5,[r2],#4
2636 add r10,r10,r3
2637 str r6,[r2],#4
2638 add r11,r11,r1
2639 str r7,[r2],#4
2640 stmia r2,{r8-r11}
2641
2642 ittte ne
2643 movne r1,sp
2644 ldrne r2,[sp,#0]
2645 eorne r12,r12,r12
2646 ldreq sp,[sp,#76] @ restore original sp
2647 itt ne
2648 eorne r3,r5,r6
2649 bne .L_00_48
2650
2651 ldmia sp!,{r4-r12,pc}
2652.size sha256_block_data_order_neon,.-sha256_block_data_order_neon
2653#endif
2654#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
2655
2656# ifdef __thumb2__
2657# define INST(a,b,c,d) .byte c,d|0xc,a,b
2658# else
2659# define INST(a,b,c,d) .byte a,b,c,d
2660# endif
2661
2662.type sha256_block_data_order_armv8,%function
2663.align 5
2664sha256_block_data_order_armv8:
2665.LARMv8:
2666 vld1.32 {q0,q1},[r0]
2667# ifdef __thumb2__
2668 adr r3,.LARMv8
2669 sub r3,r3,#.LARMv8-K256
2670# else
2671 adrl r3,K256
2672# endif
2673 add r2,r1,r2,lsl#6 @ len to point at the end of inp
2674
2675.Loop_v8:
2676 vld1.8 {q8-q9},[r1]!
2677 vld1.8 {q10-q11},[r1]!
2678 vld1.32 {q12},[r3]!
2679 vrev32.8 q8,q8
2680 vrev32.8 q9,q9
2681 vrev32.8 q10,q10
2682 vrev32.8 q11,q11
2683 vmov q14,q0 @ offload
2684 vmov q15,q1
2685 teq r1,r2
2686 vld1.32 {q13},[r3]!
2687 vadd.i32 q12,q12,q8
2688 INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9
2689 vmov q2,q0
2690 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2691 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2692 INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11
2693 vld1.32 {q12},[r3]!
2694 vadd.i32 q13,q13,q9
2695 INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10
2696 vmov q2,q0
2697 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2698 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2699 INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8
2700 vld1.32 {q13},[r3]!
2701 vadd.i32 q12,q12,q10
2702 INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11
2703 vmov q2,q0
2704 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2705 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2706 INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9
2707 vld1.32 {q12},[r3]!
2708 vadd.i32 q13,q13,q11
2709 INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8
2710 vmov q2,q0
2711 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2712 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2713 INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10
2714 vld1.32 {q13},[r3]!
2715 vadd.i32 q12,q12,q8
2716 INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9
2717 vmov q2,q0
2718 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2719 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2720 INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11
2721 vld1.32 {q12},[r3]!
2722 vadd.i32 q13,q13,q9
2723 INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10
2724 vmov q2,q0
2725 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2726 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2727 INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8
2728 vld1.32 {q13},[r3]!
2729 vadd.i32 q12,q12,q10
2730 INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11
2731 vmov q2,q0
2732 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2733 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2734 INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9
2735 vld1.32 {q12},[r3]!
2736 vadd.i32 q13,q13,q11
2737 INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8
2738 vmov q2,q0
2739 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2740 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2741 INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10
2742 vld1.32 {q13},[r3]!
2743 vadd.i32 q12,q12,q8
2744 INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9
2745 vmov q2,q0
2746 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2747 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2748 INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11
2749 vld1.32 {q12},[r3]!
2750 vadd.i32 q13,q13,q9
2751 INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10
2752 vmov q2,q0
2753 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2754 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2755 INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8
2756 vld1.32 {q13},[r3]!
2757 vadd.i32 q12,q12,q10
2758 INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11
2759 vmov q2,q0
2760 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2761 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2762 INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9
2763 vld1.32 {q12},[r3]!
2764 vadd.i32 q13,q13,q11
2765 INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8
2766 vmov q2,q0
2767 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2768 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2769 INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10
2770 vld1.32 {q13},[r3]!
2771 vadd.i32 q12,q12,q8
2772 vmov q2,q0
2773 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2774 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2775
2776 vld1.32 {q12},[r3]!
2777 vadd.i32 q13,q13,q9
2778 vmov q2,q0
2779 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2780 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2781
2782 vld1.32 {q13},[r3]
2783 vadd.i32 q12,q12,q10
2784 sub r3,r3,#256-16 @ rewind
2785 vmov q2,q0
2786 INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12
2787 INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12
2788
2789 vadd.i32 q13,q13,q11
2790 vmov q2,q0
2791 INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13
2792 INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13
2793
2794 vadd.i32 q0,q0,q14
2795 vadd.i32 q1,q1,q15
2796 it ne
2797 bne .Loop_v8
2798
2799 vst1.32 {q0,q1},[r0]
2800
2801 bx lr @ bx lr
2802.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
2803#endif
2804.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro@openssl.org>"
2805.align 2
2806#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
2807.comm OPENSSL_armcap_P,4,4
2808#endif
diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c
new file mode 100644
index 000000000000..a84e869ef900
--- /dev/null
+++ b/arch/arm/crypto/sha256_glue.c
@@ -0,0 +1,128 @@
1/*
2 * Glue code for the SHA256 Secure Hash Algorithm assembly implementation
3 * using optimized ARM assembler and NEON instructions.
4 *
5 * Copyright © 2015 Google Inc.
6 *
7 * This file is based on sha256_ssse3_glue.c:
8 * Copyright (C) 2013 Intel Corporation
9 * Author: Tim Chen <tim.c.chen@linux.intel.com>
10 *
11 * This program is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by the Free
13 * Software Foundation; either version 2 of the License, or (at your option)
14 * any later version.
15 *
16 */
17
18#include <crypto/internal/hash.h>
19#include <linux/crypto.h>
20#include <linux/init.h>
21#include <linux/module.h>
22#include <linux/mm.h>
23#include <linux/cryptohash.h>
24#include <linux/types.h>
25#include <linux/string.h>
26#include <crypto/sha.h>
27#include <crypto/sha256_base.h>
28#include <asm/simd.h>
29#include <asm/neon.h>
30
31#include "sha256_glue.h"
32
33asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
34 unsigned int num_blks);
35
36int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data,
37 unsigned int len)
38{
39 /* make sure casting to sha256_block_fn() is safe */
40 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
41
42 return sha256_base_do_update(desc, data, len,
43 (sha256_block_fn *)sha256_block_data_order);
44}
45EXPORT_SYMBOL(crypto_sha256_arm_update);
46
47static int sha256_final(struct shash_desc *desc, u8 *out)
48{
49 sha256_base_do_finalize(desc,
50 (sha256_block_fn *)sha256_block_data_order);
51 return sha256_base_finish(desc, out);
52}
53
54int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data,
55 unsigned int len, u8 *out)
56{
57 sha256_base_do_update(desc, data, len,
58 (sha256_block_fn *)sha256_block_data_order);
59 return sha256_final(desc, out);
60}
61EXPORT_SYMBOL(crypto_sha256_arm_finup);
62
63static struct shash_alg algs[] = { {
64 .digestsize = SHA256_DIGEST_SIZE,
65 .init = sha256_base_init,
66 .update = crypto_sha256_arm_update,
67 .final = sha256_final,
68 .finup = crypto_sha256_arm_finup,
69 .descsize = sizeof(struct sha256_state),
70 .base = {
71 .cra_name = "sha256",
72 .cra_driver_name = "sha256-asm",
73 .cra_priority = 150,
74 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
75 .cra_blocksize = SHA256_BLOCK_SIZE,
76 .cra_module = THIS_MODULE,
77 }
78}, {
79 .digestsize = SHA224_DIGEST_SIZE,
80 .init = sha224_base_init,
81 .update = crypto_sha256_arm_update,
82 .final = sha256_final,
83 .finup = crypto_sha256_arm_finup,
84 .descsize = sizeof(struct sha256_state),
85 .base = {
86 .cra_name = "sha224",
87 .cra_driver_name = "sha224-asm",
88 .cra_priority = 150,
89 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
90 .cra_blocksize = SHA224_BLOCK_SIZE,
91 .cra_module = THIS_MODULE,
92 }
93} };
94
95static int __init sha256_mod_init(void)
96{
97 int res = crypto_register_shashes(algs, ARRAY_SIZE(algs));
98
99 if (res < 0)
100 return res;
101
102 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) {
103 res = crypto_register_shashes(sha256_neon_algs,
104 ARRAY_SIZE(sha256_neon_algs));
105
106 if (res < 0)
107 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
108 }
109
110 return res;
111}
112
113static void __exit sha256_mod_fini(void)
114{
115 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
116
117 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon())
118 crypto_unregister_shashes(sha256_neon_algs,
119 ARRAY_SIZE(sha256_neon_algs));
120}
121
122module_init(sha256_mod_init);
123module_exit(sha256_mod_fini);
124
125MODULE_LICENSE("GPL");
126MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm (ARM), including NEON");
127
128MODULE_ALIAS_CRYPTO("sha256");
diff --git a/arch/arm/crypto/sha256_glue.h b/arch/arm/crypto/sha256_glue.h
new file mode 100644
index 000000000000..7cf0bf786ada
--- /dev/null
+++ b/arch/arm/crypto/sha256_glue.h
@@ -0,0 +1,14 @@
1#ifndef _CRYPTO_SHA256_GLUE_H
2#define _CRYPTO_SHA256_GLUE_H
3
4#include <linux/crypto.h>
5
6extern struct shash_alg sha256_neon_algs[2];
7
8int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data,
9 unsigned int len);
10
11int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data,
12 unsigned int len, u8 *hash);
13
14#endif /* _CRYPTO_SHA256_GLUE_H */
diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c
new file mode 100644
index 000000000000..39ccd658817e
--- /dev/null
+++ b/arch/arm/crypto/sha256_neon_glue.c
@@ -0,0 +1,101 @@
1/*
2 * Glue code for the SHA256 Secure Hash Algorithm assembly implementation
3 * using NEON instructions.
4 *
5 * Copyright © 2015 Google Inc.
6 *
7 * This file is based on sha512_neon_glue.c:
8 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 *
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 2 of the License, or (at your option)
13 * any later version.
14 *
15 */
16
17#include <crypto/internal/hash.h>
18#include <linux/cryptohash.h>
19#include <linux/types.h>
20#include <linux/string.h>
21#include <crypto/sha.h>
22#include <crypto/sha256_base.h>
23#include <asm/byteorder.h>
24#include <asm/simd.h>
25#include <asm/neon.h>
26
27#include "sha256_glue.h"
28
29asmlinkage void sha256_block_data_order_neon(u32 *digest, const void *data,
30 unsigned int num_blks);
31
32static int sha256_update(struct shash_desc *desc, const u8 *data,
33 unsigned int len)
34{
35 struct sha256_state *sctx = shash_desc_ctx(desc);
36
37 if (!may_use_simd() ||
38 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
39 return crypto_sha256_arm_update(desc, data, len);
40
41 kernel_neon_begin();
42 sha256_base_do_update(desc, data, len,
43 (sha256_block_fn *)sha256_block_data_order_neon);
44 kernel_neon_end();
45
46 return 0;
47}
48
49static int sha256_finup(struct shash_desc *desc, const u8 *data,
50 unsigned int len, u8 *out)
51{
52 if (!may_use_simd())
53 return crypto_sha256_arm_finup(desc, data, len, out);
54
55 kernel_neon_begin();
56 if (len)
57 sha256_base_do_update(desc, data, len,
58 (sha256_block_fn *)sha256_block_data_order_neon);
59 sha256_base_do_finalize(desc,
60 (sha256_block_fn *)sha256_block_data_order_neon);
61 kernel_neon_end();
62
63 return sha256_base_finish(desc, out);
64}
65
66static int sha256_final(struct shash_desc *desc, u8 *out)
67{
68 return sha256_finup(desc, NULL, 0, out);
69}
70
71struct shash_alg sha256_neon_algs[] = { {
72 .digestsize = SHA256_DIGEST_SIZE,
73 .init = sha256_base_init,
74 .update = sha256_update,
75 .final = sha256_final,
76 .finup = sha256_finup,
77 .descsize = sizeof(struct sha256_state),
78 .base = {
79 .cra_name = "sha256",
80 .cra_driver_name = "sha256-neon",
81 .cra_priority = 250,
82 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
83 .cra_blocksize = SHA256_BLOCK_SIZE,
84 .cra_module = THIS_MODULE,
85 }
86}, {
87 .digestsize = SHA224_DIGEST_SIZE,
88 .init = sha224_base_init,
89 .update = sha256_update,
90 .final = sha256_final,
91 .finup = sha256_finup,
92 .descsize = sizeof(struct sha256_state),
93 .base = {
94 .cra_name = "sha224",
95 .cra_driver_name = "sha224-neon",
96 .cra_priority = 250,
97 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
98 .cra_blocksize = SHA224_BLOCK_SIZE,
99 .cra_module = THIS_MODULE,
100 }
101} };
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index fe74c0d1e485..eb0f43f3e3f1 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -1,6 +1,5 @@
1 1
2 2
3generic-y += auxvec.h
4generic-y += bitsperlong.h 3generic-y += bitsperlong.h
5generic-y += cputime.h 4generic-y += cputime.h
6generic-y += current.h 5generic-y += current.h
diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
index f67fd3afebdf..186270b3e194 100644
--- a/arch/arm/include/asm/assembler.h
+++ b/arch/arm/include/asm/assembler.h
@@ -237,6 +237,9 @@
237 .pushsection ".alt.smp.init", "a" ;\ 237 .pushsection ".alt.smp.init", "a" ;\
238 .long 9998b ;\ 238 .long 9998b ;\
2399997: instr ;\ 2399997: instr ;\
240 .if . - 9997b == 2 ;\
241 nop ;\
242 .endif ;\
240 .if . - 9997b != 4 ;\ 243 .if . - 9997b != 4 ;\
241 .error "ALT_UP() content must assemble to exactly 4 bytes";\ 244 .error "ALT_UP() content must assemble to exactly 4 bytes";\
242 .endif ;\ 245 .endif ;\
diff --git a/arch/arm/include/asm/auxvec.h b/arch/arm/include/asm/auxvec.h
new file mode 100644
index 000000000000..fbd388c46299
--- /dev/null
+++ b/arch/arm/include/asm/auxvec.h
@@ -0,0 +1 @@
#include <uapi/asm/auxvec.h>
diff --git a/arch/arm/include/asm/cpuidle.h b/arch/arm/include/asm/cpuidle.h
index af319ac4960c..0f8424924902 100644
--- a/arch/arm/include/asm/cpuidle.h
+++ b/arch/arm/include/asm/cpuidle.h
@@ -1,6 +1,8 @@
1#ifndef __ASM_ARM_CPUIDLE_H 1#ifndef __ASM_ARM_CPUIDLE_H
2#define __ASM_ARM_CPUIDLE_H 2#define __ASM_ARM_CPUIDLE_H
3 3
4#include <asm/proc-fns.h>
5
4#ifdef CONFIG_CPU_IDLE 6#ifdef CONFIG_CPU_IDLE
5extern int arm_cpuidle_simple_enter(struct cpuidle_device *dev, 7extern int arm_cpuidle_simple_enter(struct cpuidle_device *dev,
6 struct cpuidle_driver *drv, int index); 8 struct cpuidle_driver *drv, int index);
@@ -25,4 +27,25 @@ static inline int arm_cpuidle_simple_enter(struct cpuidle_device *dev,
25 */ 27 */
26#define ARM_CPUIDLE_WFI_STATE ARM_CPUIDLE_WFI_STATE_PWR(UINT_MAX) 28#define ARM_CPUIDLE_WFI_STATE ARM_CPUIDLE_WFI_STATE_PWR(UINT_MAX)
27 29
30struct device_node;
31
32struct cpuidle_ops {
33 int (*suspend)(int cpu, unsigned long arg);
34 int (*init)(struct device_node *, int cpu);
35};
36
37struct of_cpuidle_method {
38 const char *method;
39 struct cpuidle_ops *ops;
40};
41
42#define CPUIDLE_METHOD_OF_DECLARE(name, _method, _ops) \
43 static const struct of_cpuidle_method __cpuidle_method_of_table_##name \
44 __used __section(__cpuidle_method_of_table) \
45 = { .method = _method, .ops = _ops }
46
47extern int arm_cpuidle_suspend(int index);
48
49extern int arm_cpuidle_init(int cpu);
50
28#endif 51#endif
diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h
index 819777d0e91f..85e374f873ac 100644
--- a/arch/arm/include/asm/cputype.h
+++ b/arch/arm/include/asm/cputype.h
@@ -253,4 +253,20 @@ static inline int cpu_is_pj4(void)
253#else 253#else
254#define cpu_is_pj4() 0 254#define cpu_is_pj4() 0
255#endif 255#endif
256
257static inline int __attribute_const__ cpuid_feature_extract_field(u32 features,
258 int field)
259{
260 int feature = (features >> field) & 15;
261
262 /* feature registers are signed values */
263 if (feature > 8)
264 feature -= 16;
265
266 return feature;
267}
268
269#define cpuid_feature_extract(reg, field) \
270 cpuid_feature_extract_field(read_cpuid_ext(reg), field)
271
256#endif 272#endif
diff --git a/arch/arm/include/asm/elf.h b/arch/arm/include/asm/elf.h
index afb9cafd3786..d2315ffd8f12 100644
--- a/arch/arm/include/asm/elf.h
+++ b/arch/arm/include/asm/elf.h
@@ -1,7 +1,9 @@
1#ifndef __ASMARM_ELF_H 1#ifndef __ASMARM_ELF_H
2#define __ASMARM_ELF_H 2#define __ASMARM_ELF_H
3 3
4#include <asm/auxvec.h>
4#include <asm/hwcap.h> 5#include <asm/hwcap.h>
6#include <asm/vdso_datapage.h>
5 7
6/* 8/*
7 * ELF register definitions.. 9 * ELF register definitions..
@@ -115,7 +117,7 @@ int dump_task_regs(struct task_struct *t, elf_gregset_t *elfregs);
115 the loader. We need to make sure that it is out of the way of the program 117 the loader. We need to make sure that it is out of the way of the program
116 that it will "exec", and that there is sufficient room for the brk. */ 118 that it will "exec", and that there is sufficient room for the brk. */
117 119
118#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3) 120#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2)
119 121
120/* When the program starts, a1 contains a pointer to a function to be 122/* When the program starts, a1 contains a pointer to a function to be
121 registered with atexit, as per the SVR4 ABI. A value of 0 means we 123 registered with atexit, as per the SVR4 ABI. A value of 0 means we
@@ -125,11 +127,14 @@ int dump_task_regs(struct task_struct *t, elf_gregset_t *elfregs);
125extern void elf_set_personality(const struct elf32_hdr *); 127extern void elf_set_personality(const struct elf32_hdr *);
126#define SET_PERSONALITY(ex) elf_set_personality(&(ex)) 128#define SET_PERSONALITY(ex) elf_set_personality(&(ex))
127 129
128struct mm_struct;
129extern unsigned long arch_randomize_brk(struct mm_struct *mm);
130#define arch_randomize_brk arch_randomize_brk
131
132#ifdef CONFIG_MMU 130#ifdef CONFIG_MMU
131#ifdef CONFIG_VDSO
132#define ARCH_DLINFO \
133do { \
134 NEW_AUX_ENT(AT_SYSINFO_EHDR, \
135 (elf_addr_t)current->mm->context.vdso); \
136} while (0)
137#endif
133#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 138#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
134struct linux_binprm; 139struct linux_binprm;
135int arch_setup_additional_pages(struct linux_binprm *, int); 140int arch_setup_additional_pages(struct linux_binprm *, int);
diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h
index 53e69dae796f..4e78065a16aa 100644
--- a/arch/arm/include/asm/futex.h
+++ b/arch/arm/include/asm/futex.h
@@ -13,7 +13,7 @@
13 " .align 3\n" \ 13 " .align 3\n" \
14 " .long 1b, 4f, 2b, 4f\n" \ 14 " .long 1b, 4f, 2b, 4f\n" \
15 " .popsection\n" \ 15 " .popsection\n" \
16 " .pushsection .fixup,\"ax\"\n" \ 16 " .pushsection .text.fixup,\"ax\"\n" \
17 " .align 2\n" \ 17 " .align 2\n" \
18 "4: mov %0, " err_reg "\n" \ 18 "4: mov %0, " err_reg "\n" \
19 " b 3b\n" \ 19 " b 3b\n" \
diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h
index 70f9b9bfb1f9..5f337dc5c108 100644
--- a/arch/arm/include/asm/jump_label.h
+++ b/arch/arm/include/asm/jump_label.h
@@ -1,7 +1,7 @@
1#ifndef _ASM_ARM_JUMP_LABEL_H 1#ifndef _ASM_ARM_JUMP_LABEL_H
2#define _ASM_ARM_JUMP_LABEL_H 2#define _ASM_ARM_JUMP_LABEL_H
3 3
4#ifdef __KERNEL__ 4#ifndef __ASSEMBLY__
5 5
6#include <linux/types.h> 6#include <linux/types.h>
7 7
@@ -27,8 +27,6 @@ l_yes:
27 return true; 27 return true;
28} 28}
29 29
30#endif /* __KERNEL__ */
31
32typedef u32 jump_label_t; 30typedef u32 jump_label_t;
33 31
34struct jump_entry { 32struct jump_entry {
@@ -37,4 +35,5 @@ struct jump_entry {
37 jump_label_t key; 35 jump_label_t key;
38}; 36};
39 37
38#endif /* __ASSEMBLY__ */
40#endif 39#endif
diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h
index 816db0bf2dd8..d995821f1698 100644
--- a/arch/arm/include/asm/kvm_arm.h
+++ b/arch/arm/include/asm/kvm_arm.h
@@ -185,6 +185,7 @@
185#define HSR_COND (0xfU << HSR_COND_SHIFT) 185#define HSR_COND (0xfU << HSR_COND_SHIFT)
186 186
187#define FSC_FAULT (0x04) 187#define FSC_FAULT (0x04)
188#define FSC_ACCESS (0x08)
188#define FSC_PERM (0x0c) 189#define FSC_PERM (0x0c)
189 190
190/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 191/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 41008cd7c53f..d71607c16601 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -27,6 +27,8 @@
27#include <asm/fpstate.h> 27#include <asm/fpstate.h>
28#include <kvm/arm_arch_timer.h> 28#include <kvm/arm_arch_timer.h>
29 29
30#define __KVM_HAVE_ARCH_INTC_INITIALIZED
31
30#if defined(CONFIG_KVM_ARM_MAX_VCPUS) 32#if defined(CONFIG_KVM_ARM_MAX_VCPUS)
31#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS 33#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
32#else 34#else
@@ -165,19 +167,10 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
165 167
166unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); 168unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
167int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); 169int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
170int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
171int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
168 172
169/* We do not have shadow page tables, hence the empty hooks */ 173/* We do not have shadow page tables, hence the empty hooks */
170static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
171 unsigned long end)
172{
173 return 0;
174}
175
176static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
177{
178 return 0;
179}
180
181static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, 174static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
182 unsigned long address) 175 unsigned long address)
183{ 176{
diff --git a/arch/arm/include/asm/kvm_mmio.h b/arch/arm/include/asm/kvm_mmio.h
index 3f83db2f6cf0..d8e90c8cb5fa 100644
--- a/arch/arm/include/asm/kvm_mmio.h
+++ b/arch/arm/include/asm/kvm_mmio.h
@@ -28,28 +28,6 @@ struct kvm_decode {
28 bool sign_extend; 28 bool sign_extend;
29}; 29};
30 30
31/*
32 * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
33 * which is an anonymous type. Use our own type instead.
34 */
35struct kvm_exit_mmio {
36 phys_addr_t phys_addr;
37 u8 data[8];
38 u32 len;
39 bool is_write;
40 void *private;
41};
42
43static inline void kvm_prepare_mmio(struct kvm_run *run,
44 struct kvm_exit_mmio *mmio)
45{
46 run->mmio.phys_addr = mmio->phys_addr;
47 run->mmio.len = mmio->len;
48 run->mmio.is_write = mmio->is_write;
49 memcpy(run->mmio.data, mmio->data, mmio->len);
50 run->exit_reason = KVM_EXIT_MMIO;
51}
52
53int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); 31int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
54int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 32int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
55 phys_addr_t fault_ipa); 33 phys_addr_t fault_ipa);
diff --git a/arch/arm/include/asm/mach/time.h b/arch/arm/include/asm/mach/time.h
index 90c12e1e695c..0f79e4dec7f9 100644
--- a/arch/arm/include/asm/mach/time.h
+++ b/arch/arm/include/asm/mach/time.h
@@ -12,8 +12,7 @@
12 12
13extern void timer_tick(void); 13extern void timer_tick(void);
14 14
15struct timespec; 15typedef void (*clock_access_fn)(struct timespec64 *);
16typedef void (*clock_access_fn)(struct timespec *);
17extern int register_persistent_clock(clock_access_fn read_boot, 16extern int register_persistent_clock(clock_access_fn read_boot,
18 clock_access_fn read_persistent); 17 clock_access_fn read_persistent);
19 18
diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h
index 64fd15159b7d..a5b47421059d 100644
--- a/arch/arm/include/asm/mmu.h
+++ b/arch/arm/include/asm/mmu.h
@@ -11,6 +11,9 @@ typedef struct {
11#endif 11#endif
12 unsigned int vmalloc_seq; 12 unsigned int vmalloc_seq;
13 unsigned long sigpage; 13 unsigned long sigpage;
14#ifdef CONFIG_VDSO
15 unsigned long vdso;
16#endif
14} mm_context_t; 17} mm_context_t;
15 18
16#ifdef CONFIG_CPU_HAS_ASID 19#ifdef CONFIG_CPU_HAS_ASID
diff --git a/arch/arm/include/asm/pmu.h b/arch/arm/include/asm/pmu.h
index b1596bd59129..675e4ab79f68 100644
--- a/arch/arm/include/asm/pmu.h
+++ b/arch/arm/include/asm/pmu.h
@@ -92,6 +92,7 @@ struct pmu_hw_events {
92struct arm_pmu { 92struct arm_pmu {
93 struct pmu pmu; 93 struct pmu pmu;
94 cpumask_t active_irqs; 94 cpumask_t active_irqs;
95 int *irq_affinity;
95 char *name; 96 char *name;
96 irqreturn_t (*handle_irq)(int irq_num, void *dev); 97 irqreturn_t (*handle_irq)(int irq_num, void *dev);
97 void (*enable)(struct perf_event *event); 98 void (*enable)(struct perf_event *event);
diff --git a/arch/arm/include/asm/smp_plat.h b/arch/arm/include/asm/smp_plat.h
index 0ad7d490ee6f..993e5224d8f7 100644
--- a/arch/arm/include/asm/smp_plat.h
+++ b/arch/arm/include/asm/smp_plat.h
@@ -104,6 +104,7 @@ static inline u32 mpidr_hash_size(void)
104 return 1 << mpidr_hash.bits; 104 return 1 << mpidr_hash.bits;
105} 105}
106 106
107extern int platform_can_secondary_boot(void);
107extern int platform_can_cpu_hotplug(void); 108extern int platform_can_cpu_hotplug(void);
108 109
109#endif 110#endif
diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
index 72812a1f3d1c..bd32eded3e50 100644
--- a/arch/arm/include/asm/thread_info.h
+++ b/arch/arm/include/asm/thread_info.h
@@ -23,7 +23,6 @@
23#ifndef __ASSEMBLY__ 23#ifndef __ASSEMBLY__
24 24
25struct task_struct; 25struct task_struct;
26struct exec_domain;
27 26
28#include <asm/types.h> 27#include <asm/types.h>
29#include <asm/domain.h> 28#include <asm/domain.h>
@@ -53,7 +52,6 @@ struct thread_info {
53 int preempt_count; /* 0 => preemptable, <0 => bug */ 52 int preempt_count; /* 0 => preemptable, <0 => bug */
54 mm_segment_t addr_limit; /* address limit */ 53 mm_segment_t addr_limit; /* address limit */
55 struct task_struct *task; /* main task structure */ 54 struct task_struct *task; /* main task structure */
56 struct exec_domain *exec_domain; /* execution domain */
57 __u32 cpu; /* cpu */ 55 __u32 cpu; /* cpu */
58 __u32 cpu_domain; /* cpu domain */ 56 __u32 cpu_domain; /* cpu domain */
59 struct cpu_context_save cpu_context; /* cpu context */ 57 struct cpu_context_save cpu_context; /* cpu context */
@@ -73,7 +71,6 @@ struct thread_info {
73#define INIT_THREAD_INFO(tsk) \ 71#define INIT_THREAD_INFO(tsk) \
74{ \ 72{ \
75 .task = &tsk, \ 73 .task = &tsk, \
76 .exec_domain = &default_exec_domain, \
77 .flags = 0, \ 74 .flags = 0, \
78 .preempt_count = INIT_PREEMPT_COUNT, \ 75 .preempt_count = INIT_PREEMPT_COUNT, \
79 .addr_limit = KERNEL_DS, \ 76 .addr_limit = KERNEL_DS, \
diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index ce0786efd26c..74b17d09ef7a 100644
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -315,7 +315,7 @@ do { \
315 __asm__ __volatile__( \ 315 __asm__ __volatile__( \
316 "1: " TUSER(ldrb) " %1,[%2],#0\n" \ 316 "1: " TUSER(ldrb) " %1,[%2],#0\n" \
317 "2:\n" \ 317 "2:\n" \
318 " .pushsection .fixup,\"ax\"\n" \ 318 " .pushsection .text.fixup,\"ax\"\n" \
319 " .align 2\n" \ 319 " .align 2\n" \
320 "3: mov %0, %3\n" \ 320 "3: mov %0, %3\n" \
321 " mov %1, #0\n" \ 321 " mov %1, #0\n" \
@@ -351,7 +351,7 @@ do { \
351 __asm__ __volatile__( \ 351 __asm__ __volatile__( \
352 "1: " TUSER(ldr) " %1,[%2],#0\n" \ 352 "1: " TUSER(ldr) " %1,[%2],#0\n" \
353 "2:\n" \ 353 "2:\n" \
354 " .pushsection .fixup,\"ax\"\n" \ 354 " .pushsection .text.fixup,\"ax\"\n" \
355 " .align 2\n" \ 355 " .align 2\n" \
356 "3: mov %0, %3\n" \ 356 "3: mov %0, %3\n" \
357 " mov %1, #0\n" \ 357 " mov %1, #0\n" \
@@ -397,7 +397,7 @@ do { \
397 __asm__ __volatile__( \ 397 __asm__ __volatile__( \
398 "1: " TUSER(strb) " %1,[%2],#0\n" \ 398 "1: " TUSER(strb) " %1,[%2],#0\n" \
399 "2:\n" \ 399 "2:\n" \
400 " .pushsection .fixup,\"ax\"\n" \ 400 " .pushsection .text.fixup,\"ax\"\n" \
401 " .align 2\n" \ 401 " .align 2\n" \
402 "3: mov %0, %3\n" \ 402 "3: mov %0, %3\n" \
403 " b 2b\n" \ 403 " b 2b\n" \
@@ -430,7 +430,7 @@ do { \
430 __asm__ __volatile__( \ 430 __asm__ __volatile__( \
431 "1: " TUSER(str) " %1,[%2],#0\n" \ 431 "1: " TUSER(str) " %1,[%2],#0\n" \
432 "2:\n" \ 432 "2:\n" \
433 " .pushsection .fixup,\"ax\"\n" \ 433 " .pushsection .text.fixup,\"ax\"\n" \
434 " .align 2\n" \ 434 " .align 2\n" \
435 "3: mov %0, %3\n" \ 435 "3: mov %0, %3\n" \
436 " b 2b\n" \ 436 " b 2b\n" \
@@ -458,7 +458,7 @@ do { \
458 THUMB( "1: " TUSER(str) " " __reg_oper1 ", [%1]\n" ) \ 458 THUMB( "1: " TUSER(str) " " __reg_oper1 ", [%1]\n" ) \
459 THUMB( "2: " TUSER(str) " " __reg_oper0 ", [%1, #4]\n" ) \ 459 THUMB( "2: " TUSER(str) " " __reg_oper0 ", [%1, #4]\n" ) \
460 "3:\n" \ 460 "3:\n" \
461 " .pushsection .fixup,\"ax\"\n" \ 461 " .pushsection .text.fixup,\"ax\"\n" \
462 " .align 2\n" \ 462 " .align 2\n" \
463 "4: mov %0, %3\n" \ 463 "4: mov %0, %3\n" \
464 " b 3b\n" \ 464 " b 3b\n" \
diff --git a/arch/arm/include/asm/unified.h b/arch/arm/include/asm/unified.h
index b88beaba6b4a..200f9a7cd623 100644
--- a/arch/arm/include/asm/unified.h
+++ b/arch/arm/include/asm/unified.h
@@ -24,6 +24,14 @@
24 .syntax unified 24 .syntax unified
25#endif 25#endif
26 26
27#ifdef CONFIG_CPU_V7M
28#define AR_CLASS(x...)
29#define M_CLASS(x...) x
30#else
31#define AR_CLASS(x...) x
32#define M_CLASS(x...)
33#endif
34
27#ifdef CONFIG_THUMB2_KERNEL 35#ifdef CONFIG_THUMB2_KERNEL
28 36
29#if __GNUC__ < 4 37#if __GNUC__ < 4
diff --git a/arch/arm/include/asm/vdso.h b/arch/arm/include/asm/vdso.h
new file mode 100644
index 000000000000..d0295f1dd1a3
--- /dev/null
+++ b/arch/arm/include/asm/vdso.h
@@ -0,0 +1,32 @@
1#ifndef __ASM_VDSO_H
2#define __ASM_VDSO_H
3
4#ifdef __KERNEL__
5
6#ifndef __ASSEMBLY__
7
8struct mm_struct;
9
10#ifdef CONFIG_VDSO
11
12void arm_install_vdso(struct mm_struct *mm, unsigned long addr);
13
14extern char vdso_start, vdso_end;
15
16extern unsigned int vdso_total_pages;
17
18#else /* CONFIG_VDSO */
19
20static inline void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
21{
22}
23
24#define vdso_total_pages 0
25
26#endif /* CONFIG_VDSO */
27
28#endif /* __ASSEMBLY__ */
29
30#endif /* __KERNEL__ */
31
32#endif /* __ASM_VDSO_H */
diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h
new file mode 100644
index 000000000000..9be259442fca
--- /dev/null
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -0,0 +1,60 @@
1/*
2 * Adapted from arm64 version.
3 *
4 * Copyright (C) 2012 ARM Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18#ifndef __ASM_VDSO_DATAPAGE_H
19#define __ASM_VDSO_DATAPAGE_H
20
21#ifdef __KERNEL__
22
23#ifndef __ASSEMBLY__
24
25#include <asm/page.h>
26
27/* Try to be cache-friendly on systems that don't implement the
28 * generic timer: fit the unconditionally updated fields in the first
29 * 32 bytes.
30 */
31struct vdso_data {
32 u32 seq_count; /* sequence count - odd during updates */
33 u16 tk_is_cntvct; /* fall back to syscall if false */
34 u16 cs_shift; /* clocksource shift */
35 u32 xtime_coarse_sec; /* coarse time */
36 u32 xtime_coarse_nsec;
37
38 u32 wtm_clock_sec; /* wall to monotonic offset */
39 u32 wtm_clock_nsec;
40 u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */
41 u32 cs_mult; /* clocksource multiplier */
42
43 u64 cs_cycle_last; /* last cycle value */
44 u64 cs_mask; /* clocksource mask */
45
46 u64 xtime_clock_snsec; /* CLOCK_REALTIME sub-ns base */
47 u32 tz_minuteswest; /* timezone info for gettimeofday(2) */
48 u32 tz_dsttime;
49};
50
51union vdso_data_store {
52 struct vdso_data data;
53 u8 page[PAGE_SIZE];
54};
55
56#endif /* !__ASSEMBLY__ */
57
58#endif /* __KERNEL__ */
59
60#endif /* __ASM_VDSO_DATAPAGE_H */
diff --git a/arch/arm/include/asm/word-at-a-time.h b/arch/arm/include/asm/word-at-a-time.h
index a6d0a29861e7..5831dce4b51c 100644
--- a/arch/arm/include/asm/word-at-a-time.h
+++ b/arch/arm/include/asm/word-at-a-time.h
@@ -71,7 +71,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
71 asm( 71 asm(
72 "1: ldr %0, [%2]\n" 72 "1: ldr %0, [%2]\n"
73 "2:\n" 73 "2:\n"
74 " .pushsection .fixup,\"ax\"\n" 74 " .pushsection .text.fixup,\"ax\"\n"
75 " .align 2\n" 75 " .align 2\n"
76 "3: and %1, %2, #0x3\n" 76 "3: and %1, %2, #0x3\n"
77 " bic %2, %2, #0x3\n" 77 " bic %2, %2, #0x3\n"
diff --git a/arch/arm/include/uapi/asm/Kbuild b/arch/arm/include/uapi/asm/Kbuild
index 70a1c9da30ca..a1c05f93d920 100644
--- a/arch/arm/include/uapi/asm/Kbuild
+++ b/arch/arm/include/uapi/asm/Kbuild
@@ -1,6 +1,7 @@
1# UAPI Header export list 1# UAPI Header export list
2include include/uapi/asm-generic/Kbuild.asm 2include include/uapi/asm-generic/Kbuild.asm
3 3
4header-y += auxvec.h
4header-y += byteorder.h 5header-y += byteorder.h
5header-y += fcntl.h 6header-y += fcntl.h
6header-y += hwcap.h 7header-y += hwcap.h
diff --git a/arch/arm/include/uapi/asm/auxvec.h b/arch/arm/include/uapi/asm/auxvec.h
new file mode 100644
index 000000000000..cb02a767a500
--- /dev/null
+++ b/arch/arm/include/uapi/asm/auxvec.h
@@ -0,0 +1,7 @@
1#ifndef __ASM_AUXVEC_H
2#define __ASM_AUXVEC_H
3
4/* VDSO location */
5#define AT_SYSINFO_EHDR 33
6
7#endif
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index 0db25bc32864..2499867dd0d8 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -198,6 +198,9 @@ struct kvm_arch_memory_slot {
198/* Highest supported SPI, from VGIC_NR_IRQS */ 198/* Highest supported SPI, from VGIC_NR_IRQS */
199#define KVM_ARM_IRQ_GIC_MAX 127 199#define KVM_ARM_IRQ_GIC_MAX 127
200 200
201/* One single KVM irqchip, ie. the VGIC */
202#define KVM_NR_IRQCHIPS 1
203
201/* PSCI interface */ 204/* PSCI interface */
202#define KVM_PSCI_FN_BASE 0x95c1ba5e 205#define KVM_PSCI_FN_BASE 0x95c1ba5e
203#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n)) 206#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n))
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 902397dd1000..752725dcbf42 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -16,7 +16,7 @@ CFLAGS_REMOVE_return_address.o = -pg
16# Object file lists. 16# Object file lists.
17 17
18obj-y := elf.o entry-common.o irq.o opcodes.o \ 18obj-y := elf.o entry-common.o irq.o opcodes.o \
19 process.o ptrace.o return_address.o \ 19 process.o ptrace.o reboot.o return_address.o \
20 setup.o signal.o sigreturn_codes.o \ 20 setup.o signal.o sigreturn_codes.o \
21 stacktrace.o sys_arm.o time.o traps.o 21 stacktrace.o sys_arm.o time.o traps.o
22 22
@@ -34,7 +34,6 @@ obj-$(CONFIG_CPU_IDLE) += cpuidle.o
34obj-$(CONFIG_ISA_DMA_API) += dma.o 34obj-$(CONFIG_ISA_DMA_API) += dma.o
35obj-$(CONFIG_FIQ) += fiq.o fiqasm.o 35obj-$(CONFIG_FIQ) += fiq.o fiqasm.o
36obj-$(CONFIG_MODULES) += armksyms.o module.o 36obj-$(CONFIG_MODULES) += armksyms.o module.o
37obj-$(CONFIG_ARTHUR) += arthur.o
38obj-$(CONFIG_ISA_DMA) += dma-isa.o 37obj-$(CONFIG_ISA_DMA) += dma-isa.o
39obj-$(CONFIG_PCI) += bios32.o isa.o 38obj-$(CONFIG_PCI) += bios32.o isa.o
40obj-$(CONFIG_ARM_CPU_SUSPEND) += sleep.o suspend.o 39obj-$(CONFIG_ARM_CPU_SUSPEND) += sleep.o suspend.o
@@ -75,6 +74,7 @@ obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o perf_event_cpu.o
75CFLAGS_pj4-cp0.o := -marm 74CFLAGS_pj4-cp0.o := -marm
76AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt 75AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt
77obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o 76obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o
77obj-$(CONFIG_VDSO) += vdso.o
78 78
79ifneq ($(CONFIG_ARCH_EBSA110),y) 79ifneq ($(CONFIG_ARCH_EBSA110),y)
80 obj-y += io.o 80 obj-y += io.o
@@ -86,7 +86,7 @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
86 86
87obj-$(CONFIG_ARM_VIRT_EXT) += hyp-stub.o 87obj-$(CONFIG_ARM_VIRT_EXT) += hyp-stub.o
88ifeq ($(CONFIG_ARM_PSCI),y) 88ifeq ($(CONFIG_ARM_PSCI),y)
89obj-y += psci.o 89obj-y += psci.o psci-call.o
90obj-$(CONFIG_SMP) += psci_smp.o 90obj-$(CONFIG_SMP) += psci_smp.o
91endif 91endif
92 92
diff --git a/arch/arm/kernel/arthur.c b/arch/arm/kernel/arthur.c
deleted file mode 100644
index 321c5291d05f..000000000000
--- a/arch/arm/kernel/arthur.c
+++ /dev/null
@@ -1,94 +0,0 @@
1/*
2 * linux/arch/arm/kernel/arthur.c
3 *
4 * Copyright (C) 1998, 1999, 2000, 2001 Philip Blundell
5 *
6 * Arthur personality
7 */
8
9/*
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 */
15
16#include <linux/module.h>
17#include <linux/personality.h>
18#include <linux/stddef.h>
19#include <linux/signal.h>
20#include <linux/init.h>
21#include <linux/sched.h>
22
23#include <asm/ptrace.h>
24
25/* Arthur doesn't have many signals, and a lot of those that it does
26 have don't map easily to any Linux equivalent. Never mind. */
27
28#define ARTHUR_SIGABRT 1
29#define ARTHUR_SIGFPE 2
30#define ARTHUR_SIGILL 3
31#define ARTHUR_SIGINT 4
32#define ARTHUR_SIGSEGV 5
33#define ARTHUR_SIGTERM 6
34#define ARTHUR_SIGSTAK 7
35#define ARTHUR_SIGUSR1 8
36#define ARTHUR_SIGUSR2 9
37#define ARTHUR_SIGOSERROR 10
38
39static unsigned long arthur_to_linux_signals[32] = {
40 0, 1, 2, 3, 4, 5, 6, 7,
41 8, 9, 10, 11, 12, 13, 14, 15,
42 16, 17, 18, 19, 20, 21, 22, 23,
43 24, 25, 26, 27, 28, 29, 30, 31
44};
45
46static unsigned long linux_to_arthur_signals[32] = {
47 0, -1, ARTHUR_SIGINT, -1,
48 ARTHUR_SIGILL, 5, ARTHUR_SIGABRT, 7,
49 ARTHUR_SIGFPE, 9, ARTHUR_SIGUSR1, ARTHUR_SIGSEGV,
50 ARTHUR_SIGUSR2, 13, 14, ARTHUR_SIGTERM,
51 16, 17, 18, 19,
52 20, 21, 22, 23,
53 24, 25, 26, 27,
54 28, 29, 30, 31
55};
56
57static void arthur_lcall7(int nr, struct pt_regs *regs)
58{
59 struct siginfo info;
60 info.si_signo = SIGSWI;
61 info.si_errno = nr;
62 /* Bounce it to the emulator */
63 send_sig_info(SIGSWI, &info, current);
64}
65
66static struct exec_domain arthur_exec_domain = {
67 .name = "Arthur",
68 .handler = arthur_lcall7,
69 .pers_low = PER_RISCOS,
70 .pers_high = PER_RISCOS,
71 .signal_map = arthur_to_linux_signals,
72 .signal_invmap = linux_to_arthur_signals,
73 .module = THIS_MODULE,
74};
75
76/*
77 * We could do with some locking to stop Arthur being removed while
78 * processes are using it.
79 */
80
81static int __init arthur_init(void)
82{
83 return register_exec_domain(&arthur_exec_domain);
84}
85
86static void __exit arthur_exit(void)
87{
88 unregister_exec_domain(&arthur_exec_domain);
89}
90
91module_init(arthur_init);
92module_exit(arthur_exit);
93
94MODULE_LICENSE("GPL");
diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index 2d2d6087b9b1..871b8267d211 100644
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -25,6 +25,7 @@
25#include <asm/memory.h> 25#include <asm/memory.h>
26#include <asm/procinfo.h> 26#include <asm/procinfo.h>
27#include <asm/suspend.h> 27#include <asm/suspend.h>
28#include <asm/vdso_datapage.h>
28#include <asm/hardware/cache-l2x0.h> 29#include <asm/hardware/cache-l2x0.h>
29#include <linux/kbuild.h> 30#include <linux/kbuild.h>
30 31
@@ -66,7 +67,6 @@ int main(void)
66 DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); 67 DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
67 DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); 68 DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit));
68 DEFINE(TI_TASK, offsetof(struct thread_info, task)); 69 DEFINE(TI_TASK, offsetof(struct thread_info, task));
69 DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain));
70 DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); 70 DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
71 DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); 71 DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain));
72 DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); 72 DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context));
@@ -190,7 +190,6 @@ int main(void)
190 DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar)); 190 DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar));
191 DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar)); 191 DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar));
192 DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc)); 192 DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc));
193#ifdef CONFIG_KVM_ARM_VGIC
194 DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); 193 DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu));
195 DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr)); 194 DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr));
196 DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr)); 195 DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr));
@@ -200,15 +199,16 @@ int main(void)
200 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr)); 199 DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
201 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr)); 200 DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
202 DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr)); 201 DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
203#ifdef CONFIG_KVM_ARM_TIMER
204 DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl)); 202 DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
205 DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval)); 203 DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
206 DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff)); 204 DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff));
207 DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled)); 205 DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled));
208#endif
209 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); 206 DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
210#endif
211 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); 207 DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
212#endif 208#endif
209 BLANK();
210#ifdef CONFIG_VDSO
211 DEFINE(VDSO_DATA_SIZE, sizeof(union vdso_data_store));
212#endif
213 return 0; 213 return 0;
214} 214}
diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
index ab19b7c03423..fcbbbb1b9e95 100644
--- a/arch/arm/kernel/bios32.c
+++ b/arch/arm/kernel/bios32.c
@@ -618,21 +618,15 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
618int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, 618int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma,
619 enum pci_mmap_state mmap_state, int write_combine) 619 enum pci_mmap_state mmap_state, int write_combine)
620{ 620{
621 struct pci_sys_data *root = dev->sysdata; 621 if (mmap_state == pci_mmap_io)
622 unsigned long phys;
623
624 if (mmap_state == pci_mmap_io) {
625 return -EINVAL; 622 return -EINVAL;
626 } else {
627 phys = vma->vm_pgoff + (root->mem_offset >> PAGE_SHIFT);
628 }
629 623
630 /* 624 /*
631 * Mark this as IO 625 * Mark this as IO
632 */ 626 */
633 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 627 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
634 628
635 if (remap_pfn_range(vma, vma->vm_start, phys, 629 if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
636 vma->vm_end - vma->vm_start, 630 vma->vm_end - vma->vm_start,
637 vma->vm_page_prot)) 631 vma->vm_page_prot))
638 return -EAGAIN; 632 return -EAGAIN;
diff --git a/arch/arm/kernel/cpuidle.c b/arch/arm/kernel/cpuidle.c
index 89545f6c8403..318da33465f4 100644
--- a/arch/arm/kernel/cpuidle.c
+++ b/arch/arm/kernel/cpuidle.c
@@ -10,8 +10,28 @@
10 */ 10 */
11 11
12#include <linux/cpuidle.h> 12#include <linux/cpuidle.h>
13#include <asm/proc-fns.h> 13#include <linux/of.h>
14#include <linux/of_device.h>
15#include <asm/cpuidle.h>
14 16
17extern struct of_cpuidle_method __cpuidle_method_of_table[];
18
19static const struct of_cpuidle_method __cpuidle_method_of_table_sentinel
20 __used __section(__cpuidle_method_of_table_end);
21
22static struct cpuidle_ops cpuidle_ops[NR_CPUS];
23
24/**
25 * arm_cpuidle_simple_enter() - a wrapper to cpu_do_idle()
26 * @dev: not used
27 * @drv: not used
28 * @index: not used
29 *
30 * A trivial wrapper to allow the cpu_do_idle function to be assigned as a
31 * cpuidle callback by matching the function signature.
32 *
33 * Returns the index passed as parameter
34 */
15int arm_cpuidle_simple_enter(struct cpuidle_device *dev, 35int arm_cpuidle_simple_enter(struct cpuidle_device *dev,
16 struct cpuidle_driver *drv, int index) 36 struct cpuidle_driver *drv, int index)
17{ 37{
@@ -19,3 +39,114 @@ int arm_cpuidle_simple_enter(struct cpuidle_device *dev,
19 39
20 return index; 40 return index;
21} 41}
42
43/**
44 * arm_cpuidle_suspend() - function to enter low power idle states
45 * @index: an integer used as an identifier for the low level PM callbacks
46 *
47 * This function calls the underlying arch specific low level PM code as
48 * registered at the init time.
49 *
50 * Returns -EOPNOTSUPP if no suspend callback is defined, the result of the
51 * callback otherwise.
52 */
53int arm_cpuidle_suspend(int index)
54{
55 int ret = -EOPNOTSUPP;
56 int cpu = smp_processor_id();
57
58 if (cpuidle_ops[cpu].suspend)
59 ret = cpuidle_ops[cpu].suspend(cpu, index);
60
61 return ret;
62}
63
64/**
65 * arm_cpuidle_get_ops() - find a registered cpuidle_ops by name
66 * @method: the method name
67 *
68 * Search in the __cpuidle_method_of_table array the cpuidle ops matching the
69 * method name.
70 *
71 * Returns a struct cpuidle_ops pointer, NULL if not found.
72 */
73static struct cpuidle_ops *__init arm_cpuidle_get_ops(const char *method)
74{
75 struct of_cpuidle_method *m = __cpuidle_method_of_table;
76
77 for (; m->method; m++)
78 if (!strcmp(m->method, method))
79 return m->ops;
80
81 return NULL;
82}
83
84/**
85 * arm_cpuidle_read_ops() - Initialize the cpuidle ops with the device tree
86 * @dn: a pointer to a struct device node corresponding to a cpu node
87 * @cpu: the cpu identifier
88 *
89 * Get the method name defined in the 'enable-method' property, retrieve the
90 * associated cpuidle_ops and do a struct copy. This copy is needed because all
91 * cpuidle_ops are tagged __initdata and will be unloaded after the init
92 * process.
93 *
94 * Return 0 on sucess, -ENOENT if no 'enable-method' is defined, -EOPNOTSUPP if
95 * no cpuidle_ops is registered for the 'enable-method'.
96 */
97static int __init arm_cpuidle_read_ops(struct device_node *dn, int cpu)
98{
99 const char *enable_method;
100 struct cpuidle_ops *ops;
101
102 enable_method = of_get_property(dn, "enable-method", NULL);
103 if (!enable_method)
104 return -ENOENT;
105
106 ops = arm_cpuidle_get_ops(enable_method);
107 if (!ops) {
108 pr_warn("%s: unsupported enable-method property: %s\n",
109 dn->full_name, enable_method);
110 return -EOPNOTSUPP;
111 }
112
113 cpuidle_ops[cpu] = *ops; /* structure copy */
114
115 pr_notice("cpuidle: enable-method property '%s'"
116 " found operations\n", enable_method);
117
118 return 0;
119}
120
121/**
122 * arm_cpuidle_init() - Initialize cpuidle_ops for a specific cpu
123 * @cpu: the cpu to be initialized
124 *
125 * Initialize the cpuidle ops with the device for the cpu and then call
126 * the cpu's idle initialization callback. This may fail if the underlying HW
127 * is not operational.
128 *
129 * Returns:
130 * 0 on success,
131 * -ENODEV if it fails to find the cpu node in the device tree,
132 * -EOPNOTSUPP if it does not find a registered cpuidle_ops for this cpu,
133 * -ENOENT if it fails to find an 'enable-method' property,
134 * -ENXIO if the HW reports a failure or a misconfiguration,
135 * -ENOMEM if the HW report an memory allocation failure
136 */
137int __init arm_cpuidle_init(int cpu)
138{
139 struct device_node *cpu_node = of_cpu_device_node_get(cpu);
140 int ret;
141
142 if (!cpu_node)
143 return -ENODEV;
144
145 ret = arm_cpuidle_read_ops(cpu_node, cpu);
146 if (!ret && cpuidle_ops[cpu].init)
147 ret = cpuidle_ops[cpu].init(cpu_node, cpu);
148
149 of_node_put(cpu_node);
150
151 return ret;
152}
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index 672b21942fff..570306c49406 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -545,7 +545,7 @@ ENDPROC(__und_usr)
545/* 545/*
546 * The out of line fixup for the ldrt instructions above. 546 * The out of line fixup for the ldrt instructions above.
547 */ 547 */
548 .pushsection .fixup, "ax" 548 .pushsection .text.fixup, "ax"
549 .align 2 549 .align 2
5504: str r4, [sp, #S_PC] @ retry current instruction 5504: str r4, [sp, #S_PC] @ retry current instruction
551 ret r9 551 ret r9
diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index 01963273c07a..3637973a9708 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -138,9 +138,9 @@ ENTRY(stext)
138 @ mmu has been enabled 138 @ mmu has been enabled
139 adr lr, BSYM(1f) @ return (PIC) address 139 adr lr, BSYM(1f) @ return (PIC) address
140 mov r8, r4 @ set TTBR1 to swapper_pg_dir 140 mov r8, r4 @ set TTBR1 to swapper_pg_dir
141 ARM( add pc, r10, #PROCINFO_INITFUNC ) 141 ldr r12, [r10, #PROCINFO_INITFUNC]
142 THUMB( add r12, r10, #PROCINFO_INITFUNC ) 142 add r12, r12, r10
143 THUMB( ret r12 ) 143 ret r12
1441: b __enable_mmu 1441: b __enable_mmu
145ENDPROC(stext) 145ENDPROC(stext)
146 .ltorg 146 .ltorg
@@ -386,10 +386,10 @@ ENTRY(secondary_startup)
386 ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir 386 ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir
387 adr lr, BSYM(__enable_mmu) @ return address 387 adr lr, BSYM(__enable_mmu) @ return address
388 mov r13, r12 @ __secondary_switched address 388 mov r13, r12 @ __secondary_switched address
389 ARM( add pc, r10, #PROCINFO_INITFUNC ) @ initialise processor 389 ldr r12, [r10, #PROCINFO_INITFUNC]
390 @ (return control reg) 390 add r12, r12, r10 @ initialise processor
391 THUMB( add r12, r10, #PROCINFO_INITFUNC ) 391 @ (return control reg)
392 THUMB( ret r12 ) 392 ret r12
393ENDPROC(secondary_startup) 393ENDPROC(secondary_startup)
394ENDPROC(secondary_startup_arm) 394ENDPROC(secondary_startup_arm)
395 395
diff --git a/arch/arm/kernel/hibernate.c b/arch/arm/kernel/hibernate.c
index c4cc50e58c13..a71501ff6f18 100644
--- a/arch/arm/kernel/hibernate.c
+++ b/arch/arm/kernel/hibernate.c
@@ -22,6 +22,7 @@
22#include <asm/suspend.h> 22#include <asm/suspend.h>
23#include <asm/memory.h> 23#include <asm/memory.h>
24#include <asm/sections.h> 24#include <asm/sections.h>
25#include "reboot.h"
25 26
26int pfn_is_nosave(unsigned long pfn) 27int pfn_is_nosave(unsigned long pfn)
27{ 28{
@@ -61,7 +62,7 @@ static int notrace arch_save_image(unsigned long unused)
61 62
62 ret = swsusp_save(); 63 ret = swsusp_save();
63 if (ret == 0) 64 if (ret == 0)
64 soft_restart(virt_to_phys(cpu_resume)); 65 _soft_restart(virt_to_phys(cpu_resume), false);
65 return ret; 66 return ret;
66} 67}
67 68
@@ -86,7 +87,7 @@ static void notrace arch_restore_image(void *unused)
86 for (pbe = restore_pblist; pbe; pbe = pbe->next) 87 for (pbe = restore_pblist; pbe; pbe = pbe->next)
87 copy_page(pbe->orig_address, pbe->address); 88 copy_page(pbe->orig_address, pbe->address);
88 89
89 soft_restart(virt_to_phys(cpu_resume)); 90 _soft_restart(virt_to_phys(cpu_resume), false);
90} 91}
91 92
92static u64 resume_stack[PAGE_SIZE/2/sizeof(u64)] __nosavedata; 93static u64 resume_stack[PAGE_SIZE/2/sizeof(u64)] __nosavedata;
@@ -99,7 +100,6 @@ static u64 resume_stack[PAGE_SIZE/2/sizeof(u64)] __nosavedata;
99 */ 100 */
100int swsusp_arch_resume(void) 101int swsusp_arch_resume(void)
101{ 102{
102 extern void call_with_stack(void (*fn)(void *), void *arg, void *sp);
103 call_with_stack(arch_restore_image, 0, 103 call_with_stack(arch_restore_image, 0,
104 resume_stack + ARRAY_SIZE(resume_stack)); 104 resume_stack + ARRAY_SIZE(resume_stack));
105 return 0; 105 return 0;
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index 7fc70ae21185..dc7d0a95bd36 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -648,7 +648,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp)
648 * Per-cpu breakpoints are not supported by our stepping 648 * Per-cpu breakpoints are not supported by our stepping
649 * mechanism. 649 * mechanism.
650 */ 650 */
651 if (!bp->hw.bp_target) 651 if (!bp->hw.target)
652 return -EINVAL; 652 return -EINVAL;
653 653
654 /* 654 /*
diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c
index de2b085ad753..8bf3b7c09888 100644
--- a/arch/arm/kernel/machine_kexec.c
+++ b/arch/arm/kernel/machine_kexec.c
@@ -46,7 +46,8 @@ int machine_kexec_prepare(struct kimage *image)
46 * and implements CPU hotplug for the current HW. If not, we won't be 46 * and implements CPU hotplug for the current HW. If not, we won't be
47 * able to kexec reliably, so fail the prepare operation. 47 * able to kexec reliably, so fail the prepare operation.
48 */ 48 */
49 if (num_possible_cpus() > 1 && !platform_can_cpu_hotplug()) 49 if (num_possible_cpus() > 1 && platform_can_secondary_boot() &&
50 !platform_can_cpu_hotplug())
50 return -EINVAL; 51 return -EINVAL;
51 52
52 /* 53 /*
diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c
index 2e11961f65ae..af791f4a6205 100644
--- a/arch/arm/kernel/module.c
+++ b/arch/arm/kernel/module.c
@@ -98,14 +98,19 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
98 case R_ARM_PC24: 98 case R_ARM_PC24:
99 case R_ARM_CALL: 99 case R_ARM_CALL:
100 case R_ARM_JUMP24: 100 case R_ARM_JUMP24:
101 if (sym->st_value & 3) {
102 pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (ARM -> Thumb)\n",
103 module->name, relindex, i, symname);
104 return -ENOEXEC;
105 }
106
101 offset = __mem_to_opcode_arm(*(u32 *)loc); 107 offset = __mem_to_opcode_arm(*(u32 *)loc);
102 offset = (offset & 0x00ffffff) << 2; 108 offset = (offset & 0x00ffffff) << 2;
103 if (offset & 0x02000000) 109 if (offset & 0x02000000)
104 offset -= 0x04000000; 110 offset -= 0x04000000;
105 111
106 offset += sym->st_value - loc; 112 offset += sym->st_value - loc;
107 if (offset & 3 || 113 if (offset <= (s32)0xfe000000 ||
108 offset <= (s32)0xfe000000 ||
109 offset >= (s32)0x02000000) { 114 offset >= (s32)0x02000000) {
110 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", 115 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
111 module->name, relindex, i, symname, 116 module->name, relindex, i, symname,
@@ -155,6 +160,22 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
155#ifdef CONFIG_THUMB2_KERNEL 160#ifdef CONFIG_THUMB2_KERNEL
156 case R_ARM_THM_CALL: 161 case R_ARM_THM_CALL:
157 case R_ARM_THM_JUMP24: 162 case R_ARM_THM_JUMP24:
163 /*
164 * For function symbols, only Thumb addresses are
165 * allowed (no interworking).
166 *
167 * For non-function symbols, the destination
168 * has no specific ARM/Thumb disposition, so
169 * the branch is resolved under the assumption
170 * that interworking is not required.
171 */
172 if (ELF32_ST_TYPE(sym->st_info) == STT_FUNC &&
173 !(sym->st_value & 1)) {
174 pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (Thumb -> ARM)\n",
175 module->name, relindex, i, symname);
176 return -ENOEXEC;
177 }
178
158 upper = __mem_to_opcode_thumb16(*(u16 *)loc); 179 upper = __mem_to_opcode_thumb16(*(u16 *)loc);
159 lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2)); 180 lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2));
160 181
@@ -182,18 +203,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
182 offset -= 0x02000000; 203 offset -= 0x02000000;
183 offset += sym->st_value - loc; 204 offset += sym->st_value - loc;
184 205
185 /* 206 if (offset <= (s32)0xff000000 ||
186 * For function symbols, only Thumb addresses are
187 * allowed (no interworking).
188 *
189 * For non-function symbols, the destination
190 * has no specific ARM/Thumb disposition, so
191 * the branch is resolved under the assumption
192 * that interworking is not required.
193 */
194 if ((ELF32_ST_TYPE(sym->st_info) == STT_FUNC &&
195 !(offset & 1)) ||
196 offset <= (s32)0xff000000 ||
197 offset >= (s32)0x01000000) { 207 offset >= (s32)0x01000000) {
198 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", 208 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
199 module->name, relindex, i, symname, 209 module->name, relindex, i, symname,
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index 557e128e4df0..4a86a0133ac3 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -259,20 +259,29 @@ out:
259} 259}
260 260
261static int 261static int
262validate_event(struct pmu_hw_events *hw_events, 262validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events,
263 struct perf_event *event) 263 struct perf_event *event)
264{ 264{
265 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 265 struct arm_pmu *armpmu;
266 266
267 if (is_software_event(event)) 267 if (is_software_event(event))
268 return 1; 268 return 1;
269 269
270 /*
271 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
272 * core perf code won't check that the pmu->ctx == leader->ctx
273 * until after pmu->event_init(event).
274 */
275 if (event->pmu != pmu)
276 return 0;
277
270 if (event->state < PERF_EVENT_STATE_OFF) 278 if (event->state < PERF_EVENT_STATE_OFF)
271 return 1; 279 return 1;
272 280
273 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 281 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
274 return 1; 282 return 1;
275 283
284 armpmu = to_arm_pmu(event->pmu);
276 return armpmu->get_event_idx(hw_events, event) >= 0; 285 return armpmu->get_event_idx(hw_events, event) >= 0;
277} 286}
278 287
@@ -288,15 +297,15 @@ validate_group(struct perf_event *event)
288 */ 297 */
289 memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask)); 298 memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask));
290 299
291 if (!validate_event(&fake_pmu, leader)) 300 if (!validate_event(event->pmu, &fake_pmu, leader))
292 return -EINVAL; 301 return -EINVAL;
293 302
294 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 303 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
295 if (!validate_event(&fake_pmu, sibling)) 304 if (!validate_event(event->pmu, &fake_pmu, sibling))
296 return -EINVAL; 305 return -EINVAL;
297 } 306 }
298 307
299 if (!validate_event(&fake_pmu, event)) 308 if (!validate_event(event->pmu, &fake_pmu, event))
300 return -EINVAL; 309 return -EINVAL;
301 310
302 return 0; 311 return 0;
diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c
index 61b53c46edfa..91c7ba182dcd 100644
--- a/arch/arm/kernel/perf_event_cpu.c
+++ b/arch/arm/kernel/perf_event_cpu.c
@@ -92,11 +92,16 @@ static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)
92 free_percpu_irq(irq, &hw_events->percpu_pmu); 92 free_percpu_irq(irq, &hw_events->percpu_pmu);
93 } else { 93 } else {
94 for (i = 0; i < irqs; ++i) { 94 for (i = 0; i < irqs; ++i) {
95 if (!cpumask_test_and_clear_cpu(i, &cpu_pmu->active_irqs)) 95 int cpu = i;
96
97 if (cpu_pmu->irq_affinity)
98 cpu = cpu_pmu->irq_affinity[i];
99
100 if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs))
96 continue; 101 continue;
97 irq = platform_get_irq(pmu_device, i); 102 irq = platform_get_irq(pmu_device, i);
98 if (irq >= 0) 103 if (irq >= 0)
99 free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, i)); 104 free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
100 } 105 }
101 } 106 }
102} 107}
@@ -128,32 +133,37 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
128 on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1); 133 on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1);
129 } else { 134 } else {
130 for (i = 0; i < irqs; ++i) { 135 for (i = 0; i < irqs; ++i) {
136 int cpu = i;
137
131 err = 0; 138 err = 0;
132 irq = platform_get_irq(pmu_device, i); 139 irq = platform_get_irq(pmu_device, i);
133 if (irq < 0) 140 if (irq < 0)
134 continue; 141 continue;
135 142
143 if (cpu_pmu->irq_affinity)
144 cpu = cpu_pmu->irq_affinity[i];
145
136 /* 146 /*
137 * If we have a single PMU interrupt that we can't shift, 147 * If we have a single PMU interrupt that we can't shift,
138 * assume that we're running on a uniprocessor machine and 148 * assume that we're running on a uniprocessor machine and
139 * continue. Otherwise, continue without this interrupt. 149 * continue. Otherwise, continue without this interrupt.
140 */ 150 */
141 if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { 151 if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {
142 pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n", 152 pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n",
143 irq, i); 153 irq, cpu);
144 continue; 154 continue;
145 } 155 }
146 156
147 err = request_irq(irq, handler, 157 err = request_irq(irq, handler,
148 IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", 158 IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",
149 per_cpu_ptr(&hw_events->percpu_pmu, i)); 159 per_cpu_ptr(&hw_events->percpu_pmu, cpu));
150 if (err) { 160 if (err) {
151 pr_err("unable to request IRQ%d for ARM PMU counters\n", 161 pr_err("unable to request IRQ%d for ARM PMU counters\n",
152 irq); 162 irq);
153 return err; 163 return err;
154 } 164 }
155 165
156 cpumask_set_cpu(i, &cpu_pmu->active_irqs); 166 cpumask_set_cpu(cpu, &cpu_pmu->active_irqs);
157 } 167 }
158 } 168 }
159 169
@@ -243,6 +253,8 @@ static const struct of_device_id cpu_pmu_of_device_ids[] = {
243 {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init}, 253 {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init},
244 {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init}, 254 {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init},
245 {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, 255 {.compatible = "qcom,krait-pmu", .data = krait_pmu_init},
256 {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init},
257 {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init},
246 {}, 258 {},
247}; 259};
248 260
@@ -289,6 +301,48 @@ static int probe_current_pmu(struct arm_pmu *pmu)
289 return ret; 301 return ret;
290} 302}
291 303
304static int of_pmu_irq_cfg(struct platform_device *pdev)
305{
306 int i;
307 int *irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
308
309 if (!irqs)
310 return -ENOMEM;
311
312 for (i = 0; i < pdev->num_resources; ++i) {
313 struct device_node *dn;
314 int cpu;
315
316 dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",
317 i);
318 if (!dn) {
319 pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",
320 of_node_full_name(dn), i);
321 break;
322 }
323
324 for_each_possible_cpu(cpu)
325 if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))
326 break;
327
328 of_node_put(dn);
329 if (cpu >= nr_cpu_ids) {
330 pr_warn("Failed to find logical CPU for %s\n",
331 dn->name);
332 break;
333 }
334
335 irqs[i] = cpu;
336 }
337
338 if (i == pdev->num_resources)
339 cpu_pmu->irq_affinity = irqs;
340 else
341 kfree(irqs);
342
343 return 0;
344}
345
292static int cpu_pmu_device_probe(struct platform_device *pdev) 346static int cpu_pmu_device_probe(struct platform_device *pdev)
293{ 347{
294 const struct of_device_id *of_id; 348 const struct of_device_id *of_id;
@@ -313,7 +367,10 @@ static int cpu_pmu_device_probe(struct platform_device *pdev)
313 367
314 if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) { 368 if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) {
315 init_fn = of_id->data; 369 init_fn = of_id->data;
316 ret = init_fn(pmu); 370
371 ret = of_pmu_irq_cfg(pdev);
372 if (!ret)
373 ret = init_fn(pmu);
317 } else { 374 } else {
318 ret = probe_current_pmu(pmu); 375 ret = probe_current_pmu(pmu);
319 } 376 }
diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c
index 8993770c47de..f4207a4dcb01 100644
--- a/arch/arm/kernel/perf_event_v7.c
+++ b/arch/arm/kernel/perf_event_v7.c
@@ -140,6 +140,23 @@ enum krait_perf_types {
140 KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210, 140 KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210,
141}; 141};
142 142
143/* ARMv7 Scorpion specific event types */
144enum scorpion_perf_types {
145 SCORPION_LPM0_GROUP0 = 0x4c,
146 SCORPION_LPM1_GROUP0 = 0x50,
147 SCORPION_LPM2_GROUP0 = 0x54,
148 SCORPION_L2LPM_GROUP0 = 0x58,
149 SCORPION_VLPM_GROUP0 = 0x5c,
150
151 SCORPION_ICACHE_ACCESS = 0x10053,
152 SCORPION_ICACHE_MISS = 0x10052,
153
154 SCORPION_DTLB_ACCESS = 0x12013,
155 SCORPION_DTLB_MISS = 0x12012,
156
157 SCORPION_ITLB_MISS = 0x12021,
158};
159
143/* 160/*
144 * Cortex-A8 HW events mapping 161 * Cortex-A8 HW events mapping
145 * 162 *
@@ -482,6 +499,49 @@ static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
482}; 499};
483 500
484/* 501/*
502 * Scorpion HW events mapping
503 */
504static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
505 PERF_MAP_ALL_UNSUPPORTED,
506 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
507 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
508 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
509 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
510 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
511};
512
513static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
514 [PERF_COUNT_HW_CACHE_OP_MAX]
515 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
516 PERF_CACHE_MAP_ALL_UNSUPPORTED,
517 /*
518 * The performance counters don't differentiate between read and write
519 * accesses/misses so this isn't strictly correct, but it's the best we
520 * can do. Writes and reads get combined.
521 */
522 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
523 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
524 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
525 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
526 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
527 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
528 /*
529 * Only ITLB misses and DTLB refills are supported. If users want the
530 * DTLB refills misses a raw counter must be used.
531 */
532 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
533 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
534 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
535 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
536 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
537 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
538 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
539 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
540 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
541 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
542};
543
544/*
485 * Perf Events' indices 545 * Perf Events' indices
486 */ 546 */
487#define ARMV7_IDX_CYCLE_COUNTER 0 547#define ARMV7_IDX_CYCLE_COUNTER 0
@@ -976,6 +1036,12 @@ static int krait_map_event_no_branch(struct perf_event *event)
976 &krait_perf_cache_map, 0xFFFFF); 1036 &krait_perf_cache_map, 0xFFFFF);
977} 1037}
978 1038
1039static int scorpion_map_event(struct perf_event *event)
1040{
1041 return armpmu_map_event(event, &scorpion_perf_map,
1042 &scorpion_perf_cache_map, 0xFFFFF);
1043}
1044
979static void armv7pmu_init(struct arm_pmu *cpu_pmu) 1045static void armv7pmu_init(struct arm_pmu *cpu_pmu)
980{ 1046{
981 cpu_pmu->handle_irq = armv7pmu_handle_irq; 1047 cpu_pmu->handle_irq = armv7pmu_handle_irq;
@@ -1103,6 +1169,12 @@ static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1103#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) 1169#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT)
1104#define PMRESRn_EN BIT(31) 1170#define PMRESRn_EN BIT(31)
1105 1171
1172#define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */
1173#define EVENT_GROUP(event) ((event) & 0xf) /* G */
1174#define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */
1175#define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */
1176#define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */
1177
1106static u32 krait_read_pmresrn(int n) 1178static u32 krait_read_pmresrn(int n)
1107{ 1179{
1108 u32 val; 1180 u32 val;
@@ -1141,19 +1213,19 @@ static void krait_write_pmresrn(int n, u32 val)
1141 } 1213 }
1142} 1214}
1143 1215
1144static u32 krait_read_vpmresr0(void) 1216static u32 venum_read_pmresr(void)
1145{ 1217{
1146 u32 val; 1218 u32 val;
1147 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); 1219 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1148 return val; 1220 return val;
1149} 1221}
1150 1222
1151static void krait_write_vpmresr0(u32 val) 1223static void venum_write_pmresr(u32 val)
1152{ 1224{
1153 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); 1225 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1154} 1226}
1155 1227
1156static void krait_pre_vpmresr0(u32 *venum_orig_val, u32 *fp_orig_val) 1228static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1157{ 1229{
1158 u32 venum_new_val; 1230 u32 venum_new_val;
1159 u32 fp_new_val; 1231 u32 fp_new_val;
@@ -1170,7 +1242,7 @@ static void krait_pre_vpmresr0(u32 *venum_orig_val, u32 *fp_orig_val)
1170 fmxr(FPEXC, fp_new_val); 1242 fmxr(FPEXC, fp_new_val);
1171} 1243}
1172 1244
1173static void krait_post_vpmresr0(u32 venum_orig_val, u32 fp_orig_val) 1245static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1174{ 1246{
1175 BUG_ON(preemptible()); 1247 BUG_ON(preemptible());
1176 /* Restore FPEXC */ 1248 /* Restore FPEXC */
@@ -1193,16 +1265,11 @@ static void krait_evt_setup(int idx, u32 config_base)
1193 u32 val; 1265 u32 val;
1194 u32 mask; 1266 u32 mask;
1195 u32 vval, fval; 1267 u32 vval, fval;
1196 unsigned int region; 1268 unsigned int region = EVENT_REGION(config_base);
1197 unsigned int group; 1269 unsigned int group = EVENT_GROUP(config_base);
1198 unsigned int code; 1270 unsigned int code = EVENT_CODE(config_base);
1199 unsigned int group_shift; 1271 unsigned int group_shift;
1200 bool venum_event; 1272 bool venum_event = EVENT_VENUM(config_base);
1201
1202 venum_event = !!(config_base & VENUM_EVENT);
1203 region = (config_base >> 12) & 0xf;
1204 code = (config_base >> 4) & 0xff;
1205 group = (config_base >> 0) & 0xf;
1206 1273
1207 group_shift = group * 8; 1274 group_shift = group * 8;
1208 mask = 0xff << group_shift; 1275 mask = 0xff << group_shift;
@@ -1217,16 +1284,14 @@ static void krait_evt_setup(int idx, u32 config_base)
1217 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); 1284 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1218 armv7_pmnc_write_evtsel(idx, val); 1285 armv7_pmnc_write_evtsel(idx, val);
1219 1286
1220 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1221
1222 if (venum_event) { 1287 if (venum_event) {
1223 krait_pre_vpmresr0(&vval, &fval); 1288 venum_pre_pmresr(&vval, &fval);
1224 val = krait_read_vpmresr0(); 1289 val = venum_read_pmresr();
1225 val &= ~mask; 1290 val &= ~mask;
1226 val |= code << group_shift; 1291 val |= code << group_shift;
1227 val |= PMRESRn_EN; 1292 val |= PMRESRn_EN;
1228 krait_write_vpmresr0(val); 1293 venum_write_pmresr(val);
1229 krait_post_vpmresr0(vval, fval); 1294 venum_post_pmresr(vval, fval);
1230 } else { 1295 } else {
1231 val = krait_read_pmresrn(region); 1296 val = krait_read_pmresrn(region);
1232 val &= ~mask; 1297 val &= ~mask;
@@ -1236,7 +1301,7 @@ static void krait_evt_setup(int idx, u32 config_base)
1236 } 1301 }
1237} 1302}
1238 1303
1239static u32 krait_clear_pmresrn_group(u32 val, int group) 1304static u32 clear_pmresrn_group(u32 val, int group)
1240{ 1305{
1241 u32 mask; 1306 u32 mask;
1242 int group_shift; 1307 int group_shift;
@@ -1256,23 +1321,19 @@ static void krait_clearpmu(u32 config_base)
1256{ 1321{
1257 u32 val; 1322 u32 val;
1258 u32 vval, fval; 1323 u32 vval, fval;
1259 unsigned int region; 1324 unsigned int region = EVENT_REGION(config_base);
1260 unsigned int group; 1325 unsigned int group = EVENT_GROUP(config_base);
1261 bool venum_event; 1326 bool venum_event = EVENT_VENUM(config_base);
1262
1263 venum_event = !!(config_base & VENUM_EVENT);
1264 region = (config_base >> 12) & 0xf;
1265 group = (config_base >> 0) & 0xf;
1266 1327
1267 if (venum_event) { 1328 if (venum_event) {
1268 krait_pre_vpmresr0(&vval, &fval); 1329 venum_pre_pmresr(&vval, &fval);
1269 val = krait_read_vpmresr0(); 1330 val = venum_read_pmresr();
1270 val = krait_clear_pmresrn_group(val, group); 1331 val = clear_pmresrn_group(val, group);
1271 krait_write_vpmresr0(val); 1332 venum_write_pmresr(val);
1272 krait_post_vpmresr0(vval, fval); 1333 venum_post_pmresr(vval, fval);
1273 } else { 1334 } else {
1274 val = krait_read_pmresrn(region); 1335 val = krait_read_pmresrn(region);
1275 val = krait_clear_pmresrn_group(val, group); 1336 val = clear_pmresrn_group(val, group);
1276 krait_write_pmresrn(region, val); 1337 krait_write_pmresrn(region, val);
1277 } 1338 }
1278} 1339}
@@ -1342,6 +1403,8 @@ static void krait_pmu_enable_event(struct perf_event *event)
1342static void krait_pmu_reset(void *info) 1403static void krait_pmu_reset(void *info)
1343{ 1404{
1344 u32 vval, fval; 1405 u32 vval, fval;
1406 struct arm_pmu *cpu_pmu = info;
1407 u32 idx, nb_cnt = cpu_pmu->num_events;
1345 1408
1346 armv7pmu_reset(info); 1409 armv7pmu_reset(info);
1347 1410
@@ -1350,9 +1413,16 @@ static void krait_pmu_reset(void *info)
1350 krait_write_pmresrn(1, 0); 1413 krait_write_pmresrn(1, 0);
1351 krait_write_pmresrn(2, 0); 1414 krait_write_pmresrn(2, 0);
1352 1415
1353 krait_pre_vpmresr0(&vval, &fval); 1416 venum_pre_pmresr(&vval, &fval);
1354 krait_write_vpmresr0(0); 1417 venum_write_pmresr(0);
1355 krait_post_vpmresr0(vval, fval); 1418 venum_post_pmresr(vval, fval);
1419
1420 /* Reset PMxEVNCTCR to sane default */
1421 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1422 armv7_pmnc_select_counter(idx);
1423 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1424 }
1425
1356} 1426}
1357 1427
1358static int krait_event_to_bit(struct perf_event *event, unsigned int region, 1428static int krait_event_to_bit(struct perf_event *event, unsigned int region,
@@ -1386,26 +1456,18 @@ static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1386{ 1456{
1387 int idx; 1457 int idx;
1388 int bit = -1; 1458 int bit = -1;
1389 unsigned int prefix;
1390 unsigned int region;
1391 unsigned int code;
1392 unsigned int group;
1393 bool krait_event;
1394 struct hw_perf_event *hwc = &event->hw; 1459 struct hw_perf_event *hwc = &event->hw;
1460 unsigned int region = EVENT_REGION(hwc->config_base);
1461 unsigned int code = EVENT_CODE(hwc->config_base);
1462 unsigned int group = EVENT_GROUP(hwc->config_base);
1463 bool venum_event = EVENT_VENUM(hwc->config_base);
1464 bool krait_event = EVENT_CPU(hwc->config_base);
1395 1465
1396 region = (hwc->config_base >> 12) & 0xf; 1466 if (venum_event || krait_event) {
1397 code = (hwc->config_base >> 4) & 0xff;
1398 group = (hwc->config_base >> 0) & 0xf;
1399 krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK);
1400
1401 if (krait_event) {
1402 /* Ignore invalid events */ 1467 /* Ignore invalid events */
1403 if (group > 3 || region > 2) 1468 if (group > 3 || region > 2)
1404 return -EINVAL; 1469 return -EINVAL;
1405 prefix = hwc->config_base & KRAIT_EVENT_MASK; 1470 if (venum_event && (code & 0xe0))
1406 if (prefix != KRAIT_EVENT && prefix != VENUM_EVENT)
1407 return -EINVAL;
1408 if (prefix == VENUM_EVENT && (code & 0xe0))
1409 return -EINVAL; 1471 return -EINVAL;
1410 1472
1411 bit = krait_event_to_bit(event, region, group); 1473 bit = krait_event_to_bit(event, region, group);
@@ -1425,15 +1487,12 @@ static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1425{ 1487{
1426 int bit; 1488 int bit;
1427 struct hw_perf_event *hwc = &event->hw; 1489 struct hw_perf_event *hwc = &event->hw;
1428 unsigned int region; 1490 unsigned int region = EVENT_REGION(hwc->config_base);
1429 unsigned int group; 1491 unsigned int group = EVENT_GROUP(hwc->config_base);
1430 bool krait_event; 1492 bool venum_event = EVENT_VENUM(hwc->config_base);
1493 bool krait_event = EVENT_CPU(hwc->config_base);
1431 1494
1432 region = (hwc->config_base >> 12) & 0xf; 1495 if (venum_event || krait_event) {
1433 group = (hwc->config_base >> 0) & 0xf;
1434 krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK);
1435
1436 if (krait_event) {
1437 bit = krait_event_to_bit(event, region, group); 1496 bit = krait_event_to_bit(event, region, group);
1438 clear_bit(bit, cpuc->used_mask); 1497 clear_bit(bit, cpuc->used_mask);
1439 } 1498 }
@@ -1458,6 +1517,344 @@ static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1458 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; 1517 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1459 return 0; 1518 return 0;
1460} 1519}
1520
1521/*
1522 * Scorpion Local Performance Monitor Register (LPMn)
1523 *
1524 * 31 30 24 16 8 0
1525 * +--------------------------------+
1526 * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0
1527 * +--------------------------------+
1528 * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1
1529 * +--------------------------------+
1530 * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2
1531 * +--------------------------------+
1532 * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3
1533 * +--------------------------------+
1534 * VLPM | EN | CC | CC | CC | CC | N = 2, R = ?
1535 * +--------------------------------+
1536 * EN | G=3 | G=2 | G=1 | G=0
1537 *
1538 *
1539 * Event Encoding:
1540 *
1541 * hwc->config_base = 0xNRCCG
1542 *
1543 * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1544 * R = region register
1545 * CC = class of events the group G is choosing from
1546 * G = group or particular event
1547 *
1548 * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1549 *
1550 * A region (R) corresponds to a piece of the CPU (execution unit, instruction
1551 * unit, etc.) while the event code (CC) corresponds to a particular class of
1552 * events (interrupts for example). An event code is broken down into
1553 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1554 * example).
1555 */
1556
1557static u32 scorpion_read_pmresrn(int n)
1558{
1559 u32 val;
1560
1561 switch (n) {
1562 case 0:
1563 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1564 break;
1565 case 1:
1566 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1567 break;
1568 case 2:
1569 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1570 break;
1571 case 3:
1572 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1573 break;
1574 default:
1575 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1576 }
1577
1578 return val;
1579}
1580
1581static void scorpion_write_pmresrn(int n, u32 val)
1582{
1583 switch (n) {
1584 case 0:
1585 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1586 break;
1587 case 1:
1588 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1589 break;
1590 case 2:
1591 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1592 break;
1593 case 3:
1594 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1595 break;
1596 default:
1597 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1598 }
1599}
1600
1601static u32 scorpion_get_pmresrn_event(unsigned int region)
1602{
1603 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1604 SCORPION_LPM1_GROUP0,
1605 SCORPION_LPM2_GROUP0,
1606 SCORPION_L2LPM_GROUP0 };
1607 return pmresrn_table[region];
1608}
1609
1610static void scorpion_evt_setup(int idx, u32 config_base)
1611{
1612 u32 val;
1613 u32 mask;
1614 u32 vval, fval;
1615 unsigned int region = EVENT_REGION(config_base);
1616 unsigned int group = EVENT_GROUP(config_base);
1617 unsigned int code = EVENT_CODE(config_base);
1618 unsigned int group_shift;
1619 bool venum_event = EVENT_VENUM(config_base);
1620
1621 group_shift = group * 8;
1622 mask = 0xff << group_shift;
1623
1624 /* Configure evtsel for the region and group */
1625 if (venum_event)
1626 val = SCORPION_VLPM_GROUP0;
1627 else
1628 val = scorpion_get_pmresrn_event(region);
1629 val += group;
1630 /* Mix in mode-exclusion bits */
1631 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1632 armv7_pmnc_write_evtsel(idx, val);
1633
1634 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1635
1636 if (venum_event) {
1637 venum_pre_pmresr(&vval, &fval);
1638 val = venum_read_pmresr();
1639 val &= ~mask;
1640 val |= code << group_shift;
1641 val |= PMRESRn_EN;
1642 venum_write_pmresr(val);
1643 venum_post_pmresr(vval, fval);
1644 } else {
1645 val = scorpion_read_pmresrn(region);
1646 val &= ~mask;
1647 val |= code << group_shift;
1648 val |= PMRESRn_EN;
1649 scorpion_write_pmresrn(region, val);
1650 }
1651}
1652
1653static void scorpion_clearpmu(u32 config_base)
1654{
1655 u32 val;
1656 u32 vval, fval;
1657 unsigned int region = EVENT_REGION(config_base);
1658 unsigned int group = EVENT_GROUP(config_base);
1659 bool venum_event = EVENT_VENUM(config_base);
1660
1661 if (venum_event) {
1662 venum_pre_pmresr(&vval, &fval);
1663 val = venum_read_pmresr();
1664 val = clear_pmresrn_group(val, group);
1665 venum_write_pmresr(val);
1666 venum_post_pmresr(vval, fval);
1667 } else {
1668 val = scorpion_read_pmresrn(region);
1669 val = clear_pmresrn_group(val, group);
1670 scorpion_write_pmresrn(region, val);
1671 }
1672}
1673
1674static void scorpion_pmu_disable_event(struct perf_event *event)
1675{
1676 unsigned long flags;
1677 struct hw_perf_event *hwc = &event->hw;
1678 int idx = hwc->idx;
1679 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1680 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1681
1682 /* Disable counter and interrupt */
1683 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1684
1685 /* Disable counter */
1686 armv7_pmnc_disable_counter(idx);
1687
1688 /*
1689 * Clear pmresr code (if destined for PMNx counters)
1690 */
1691 if (hwc->config_base & KRAIT_EVENT_MASK)
1692 scorpion_clearpmu(hwc->config_base);
1693
1694 /* Disable interrupt for this counter */
1695 armv7_pmnc_disable_intens(idx);
1696
1697 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1698}
1699
1700static void scorpion_pmu_enable_event(struct perf_event *event)
1701{
1702 unsigned long flags;
1703 struct hw_perf_event *hwc = &event->hw;
1704 int idx = hwc->idx;
1705 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1706 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1707
1708 /*
1709 * Enable counter and interrupt, and set the counter to count
1710 * the event that we're interested in.
1711 */
1712 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1713
1714 /* Disable counter */
1715 armv7_pmnc_disable_counter(idx);
1716
1717 /*
1718 * Set event (if destined for PMNx counters)
1719 * We don't set the event for the cycle counter because we
1720 * don't have the ability to perform event filtering.
1721 */
1722 if (hwc->config_base & KRAIT_EVENT_MASK)
1723 scorpion_evt_setup(idx, hwc->config_base);
1724 else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1725 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1726
1727 /* Enable interrupt for this counter */
1728 armv7_pmnc_enable_intens(idx);
1729
1730 /* Enable counter */
1731 armv7_pmnc_enable_counter(idx);
1732
1733 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1734}
1735
1736static void scorpion_pmu_reset(void *info)
1737{
1738 u32 vval, fval;
1739 struct arm_pmu *cpu_pmu = info;
1740 u32 idx, nb_cnt = cpu_pmu->num_events;
1741
1742 armv7pmu_reset(info);
1743
1744 /* Clear all pmresrs */
1745 scorpion_write_pmresrn(0, 0);
1746 scorpion_write_pmresrn(1, 0);
1747 scorpion_write_pmresrn(2, 0);
1748 scorpion_write_pmresrn(3, 0);
1749
1750 venum_pre_pmresr(&vval, &fval);
1751 venum_write_pmresr(0);
1752 venum_post_pmresr(vval, fval);
1753
1754 /* Reset PMxEVNCTCR to sane default */
1755 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1756 armv7_pmnc_select_counter(idx);
1757 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1758 }
1759}
1760
1761static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1762 unsigned int group)
1763{
1764 int bit;
1765 struct hw_perf_event *hwc = &event->hw;
1766 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1767
1768 if (hwc->config_base & VENUM_EVENT)
1769 bit = SCORPION_VLPM_GROUP0;
1770 else
1771 bit = scorpion_get_pmresrn_event(region);
1772 bit -= scorpion_get_pmresrn_event(0);
1773 bit += group;
1774 /*
1775 * Lower bits are reserved for use by the counters (see
1776 * armv7pmu_get_event_idx() for more info)
1777 */
1778 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1779
1780 return bit;
1781}
1782
1783/*
1784 * We check for column exclusion constraints here.
1785 * Two events cant use the same group within a pmresr register.
1786 */
1787static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1788 struct perf_event *event)
1789{
1790 int idx;
1791 int bit = -1;
1792 struct hw_perf_event *hwc = &event->hw;
1793 unsigned int region = EVENT_REGION(hwc->config_base);
1794 unsigned int group = EVENT_GROUP(hwc->config_base);
1795 bool venum_event = EVENT_VENUM(hwc->config_base);
1796 bool scorpion_event = EVENT_CPU(hwc->config_base);
1797
1798 if (venum_event || scorpion_event) {
1799 /* Ignore invalid events */
1800 if (group > 3 || region > 3)
1801 return -EINVAL;
1802
1803 bit = scorpion_event_to_bit(event, region, group);
1804 if (test_and_set_bit(bit, cpuc->used_mask))
1805 return -EAGAIN;
1806 }
1807
1808 idx = armv7pmu_get_event_idx(cpuc, event);
1809 if (idx < 0 && bit >= 0)
1810 clear_bit(bit, cpuc->used_mask);
1811
1812 return idx;
1813}
1814
1815static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1816 struct perf_event *event)
1817{
1818 int bit;
1819 struct hw_perf_event *hwc = &event->hw;
1820 unsigned int region = EVENT_REGION(hwc->config_base);
1821 unsigned int group = EVENT_GROUP(hwc->config_base);
1822 bool venum_event = EVENT_VENUM(hwc->config_base);
1823 bool scorpion_event = EVENT_CPU(hwc->config_base);
1824
1825 if (venum_event || scorpion_event) {
1826 bit = scorpion_event_to_bit(event, region, group);
1827 clear_bit(bit, cpuc->used_mask);
1828 }
1829}
1830
1831static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1832{
1833 armv7pmu_init(cpu_pmu);
1834 cpu_pmu->name = "armv7_scorpion";
1835 cpu_pmu->map_event = scorpion_map_event;
1836 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1837 cpu_pmu->reset = scorpion_pmu_reset;
1838 cpu_pmu->enable = scorpion_pmu_enable_event;
1839 cpu_pmu->disable = scorpion_pmu_disable_event;
1840 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1841 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1842 return 0;
1843}
1844
1845static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1846{
1847 armv7pmu_init(cpu_pmu);
1848 cpu_pmu->name = "armv7_scorpion_mp";
1849 cpu_pmu->map_event = scorpion_map_event;
1850 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1851 cpu_pmu->reset = scorpion_pmu_reset;
1852 cpu_pmu->enable = scorpion_pmu_enable_event;
1853 cpu_pmu->disable = scorpion_pmu_disable_event;
1854 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1855 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1856 return 0;
1857}
1461#else 1858#else
1462static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) 1859static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1463{ 1860{
@@ -1498,4 +1895,14 @@ static inline int krait_pmu_init(struct arm_pmu *cpu_pmu)
1498{ 1895{
1499 return -ENODEV; 1896 return -ENODEV;
1500} 1897}
1898
1899static inline int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1900{
1901 return -ENODEV;
1902}
1903
1904static inline int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1905{
1906 return -ENODEV;
1907}
1501#endif /* CONFIG_CPU_V7 */ 1908#endif /* CONFIG_CPU_V7 */
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index fdfa3a78ec8c..f192a2a41719 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -17,12 +17,9 @@
17#include <linux/stddef.h> 17#include <linux/stddef.h>
18#include <linux/unistd.h> 18#include <linux/unistd.h>
19#include <linux/user.h> 19#include <linux/user.h>
20#include <linux/delay.h>
21#include <linux/reboot.h>
22#include <linux/interrupt.h> 20#include <linux/interrupt.h>
23#include <linux/kallsyms.h> 21#include <linux/kallsyms.h>
24#include <linux/init.h> 22#include <linux/init.h>
25#include <linux/cpu.h>
26#include <linux/elfcore.h> 23#include <linux/elfcore.h>
27#include <linux/pm.h> 24#include <linux/pm.h>
28#include <linux/tick.h> 25#include <linux/tick.h>
@@ -31,16 +28,14 @@
31#include <linux/random.h> 28#include <linux/random.h>
32#include <linux/hw_breakpoint.h> 29#include <linux/hw_breakpoint.h>
33#include <linux/leds.h> 30#include <linux/leds.h>
34#include <linux/reboot.h>
35 31
36#include <asm/cacheflush.h>
37#include <asm/idmap.h>
38#include <asm/processor.h> 32#include <asm/processor.h>
39#include <asm/thread_notify.h> 33#include <asm/thread_notify.h>
40#include <asm/stacktrace.h> 34#include <asm/stacktrace.h>
41#include <asm/system_misc.h> 35#include <asm/system_misc.h>
42#include <asm/mach/time.h> 36#include <asm/mach/time.h>
43#include <asm/tls.h> 37#include <asm/tls.h>
38#include <asm/vdso.h>
44 39
45#ifdef CONFIG_CC_STACKPROTECTOR 40#ifdef CONFIG_CC_STACKPROTECTOR
46#include <linux/stackprotector.h> 41#include <linux/stackprotector.h>
@@ -59,69 +54,6 @@ static const char *isa_modes[] __maybe_unused = {
59 "ARM" , "Thumb" , "Jazelle", "ThumbEE" 54 "ARM" , "Thumb" , "Jazelle", "ThumbEE"
60}; 55};
61 56
62extern void call_with_stack(void (*fn)(void *), void *arg, void *sp);
63typedef void (*phys_reset_t)(unsigned long);
64
65/*
66 * A temporary stack to use for CPU reset. This is static so that we
67 * don't clobber it with the identity mapping. When running with this
68 * stack, any references to the current task *will not work* so you
69 * should really do as little as possible before jumping to your reset
70 * code.
71 */
72static u64 soft_restart_stack[16];
73
74static void __soft_restart(void *addr)
75{
76 phys_reset_t phys_reset;
77
78 /* Take out a flat memory mapping. */
79 setup_mm_for_reboot();
80
81 /* Clean and invalidate caches */
82 flush_cache_all();
83
84 /* Turn off caching */
85 cpu_proc_fin();
86
87 /* Push out any further dirty data, and ensure cache is empty */
88 flush_cache_all();
89
90 /* Switch to the identity mapping. */
91 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
92 phys_reset((unsigned long)addr);
93
94 /* Should never get here. */
95 BUG();
96}
97
98void soft_restart(unsigned long addr)
99{
100 u64 *stack = soft_restart_stack + ARRAY_SIZE(soft_restart_stack);
101
102 /* Disable interrupts first */
103 raw_local_irq_disable();
104 local_fiq_disable();
105
106 /* Disable the L2 if we're the last man standing. */
107 if (num_online_cpus() == 1)
108 outer_disable();
109
110 /* Change to the new stack and continue with the reset. */
111 call_with_stack(__soft_restart, (void *)addr, (void *)stack);
112
113 /* Should never get here. */
114 BUG();
115}
116
117/*
118 * Function pointers to optional machine specific functions
119 */
120void (*pm_power_off)(void);
121EXPORT_SYMBOL(pm_power_off);
122
123void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd);
124
125/* 57/*
126 * This is our default idle handler. 58 * This is our default idle handler.
127 */ 59 */
@@ -166,79 +98,6 @@ void arch_cpu_idle_dead(void)
166} 98}
167#endif 99#endif
168 100
169/*
170 * Called by kexec, immediately prior to machine_kexec().
171 *
172 * This must completely disable all secondary CPUs; simply causing those CPUs
173 * to execute e.g. a RAM-based pin loop is not sufficient. This allows the
174 * kexec'd kernel to use any and all RAM as it sees fit, without having to
175 * avoid any code or data used by any SW CPU pin loop. The CPU hotplug
176 * functionality embodied in disable_nonboot_cpus() to achieve this.
177 */
178void machine_shutdown(void)
179{
180 disable_nonboot_cpus();
181}
182
183/*
184 * Halting simply requires that the secondary CPUs stop performing any
185 * activity (executing tasks, handling interrupts). smp_send_stop()
186 * achieves this.
187 */
188void machine_halt(void)
189{
190 local_irq_disable();
191 smp_send_stop();
192
193 local_irq_disable();
194 while (1);
195}
196
197/*
198 * Power-off simply requires that the secondary CPUs stop performing any
199 * activity (executing tasks, handling interrupts). smp_send_stop()
200 * achieves this. When the system power is turned off, it will take all CPUs
201 * with it.
202 */
203void machine_power_off(void)
204{
205 local_irq_disable();
206 smp_send_stop();
207
208 if (pm_power_off)
209 pm_power_off();
210}
211
212/*
213 * Restart requires that the secondary CPUs stop performing any activity
214 * while the primary CPU resets the system. Systems with a single CPU can
215 * use soft_restart() as their machine descriptor's .restart hook, since that
216 * will cause the only available CPU to reset. Systems with multiple CPUs must
217 * provide a HW restart implementation, to ensure that all CPUs reset at once.
218 * This is required so that any code running after reset on the primary CPU
219 * doesn't have to co-ordinate with other CPUs to ensure they aren't still
220 * executing pre-reset code, and using RAM that the primary CPU's code wishes
221 * to use. Implementing such co-ordination would be essentially impossible.
222 */
223void machine_restart(char *cmd)
224{
225 local_irq_disable();
226 smp_send_stop();
227
228 if (arm_pm_restart)
229 arm_pm_restart(reboot_mode, cmd);
230 else
231 do_kernel_restart(cmd);
232
233 /* Give a grace period for failure to restart of 1s */
234 mdelay(1000);
235
236 /* Whoops - the platform was unable to reboot. Tell the user! */
237 printk("Reboot failed -- System halted\n");
238 local_irq_disable();
239 while (1);
240}
241
242void __show_regs(struct pt_regs *regs) 101void __show_regs(struct pt_regs *regs)
243{ 102{
244 unsigned long flags; 103 unsigned long flags;
@@ -475,7 +334,7 @@ const char *arch_vma_name(struct vm_area_struct *vma)
475} 334}
476 335
477/* If possible, provide a placement hint at a random offset from the 336/* If possible, provide a placement hint at a random offset from the
478 * stack for the signal page. 337 * stack for the sigpage and vdso pages.
479 */ 338 */
480static unsigned long sigpage_addr(const struct mm_struct *mm, 339static unsigned long sigpage_addr(const struct mm_struct *mm,
481 unsigned int npages) 340 unsigned int npages)
@@ -519,6 +378,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
519{ 378{
520 struct mm_struct *mm = current->mm; 379 struct mm_struct *mm = current->mm;
521 struct vm_area_struct *vma; 380 struct vm_area_struct *vma;
381 unsigned long npages;
522 unsigned long addr; 382 unsigned long addr;
523 unsigned long hint; 383 unsigned long hint;
524 int ret = 0; 384 int ret = 0;
@@ -528,9 +388,12 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
528 if (!signal_page) 388 if (!signal_page)
529 return -ENOMEM; 389 return -ENOMEM;
530 390
391 npages = 1; /* for sigpage */
392 npages += vdso_total_pages;
393
531 down_write(&mm->mmap_sem); 394 down_write(&mm->mmap_sem);
532 hint = sigpage_addr(mm, 1); 395 hint = sigpage_addr(mm, npages);
533 addr = get_unmapped_area(NULL, hint, PAGE_SIZE, 0, 0); 396 addr = get_unmapped_area(NULL, hint, npages << PAGE_SHIFT, 0, 0);
534 if (IS_ERR_VALUE(addr)) { 397 if (IS_ERR_VALUE(addr)) {
535 ret = addr; 398 ret = addr;
536 goto up_fail; 399 goto up_fail;
@@ -547,6 +410,12 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
547 410
548 mm->context.sigpage = addr; 411 mm->context.sigpage = addr;
549 412
413 /* Unlike the sigpage, failure to install the vdso is unlikely
414 * to be fatal to the process, so no error check needed
415 * here.
416 */
417 arm_install_vdso(mm, addr + PAGE_SIZE);
418
550 up_fail: 419 up_fail:
551 up_write(&mm->mmap_sem); 420 up_write(&mm->mmap_sem);
552 return ret; 421 return ret;
diff --git a/arch/arm/kernel/psci-call.S b/arch/arm/kernel/psci-call.S
new file mode 100644
index 000000000000..a78e9e1e206d
--- /dev/null
+++ b/arch/arm/kernel/psci-call.S
@@ -0,0 +1,31 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License version 2 as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * Copyright (C) 2015 ARM Limited
12 *
13 * Author: Mark Rutland <mark.rutland@arm.com>
14 */
15
16#include <linux/linkage.h>
17
18#include <asm/opcodes-sec.h>
19#include <asm/opcodes-virt.h>
20
21/* int __invoke_psci_fn_hvc(u32 function_id, u32 arg0, u32 arg1, u32 arg2) */
22ENTRY(__invoke_psci_fn_hvc)
23 __HVC(0)
24 bx lr
25ENDPROC(__invoke_psci_fn_hvc)
26
27/* int __invoke_psci_fn_smc(u32 function_id, u32 arg0, u32 arg1, u32 arg2) */
28ENTRY(__invoke_psci_fn_smc)
29 __SMC(0)
30 bx lr
31ENDPROC(__invoke_psci_fn_smc)
diff --git a/arch/arm/kernel/psci.c b/arch/arm/kernel/psci.c
index f73891b6b730..f90fdf4ce7c7 100644
--- a/arch/arm/kernel/psci.c
+++ b/arch/arm/kernel/psci.c
@@ -23,8 +23,6 @@
23 23
24#include <asm/compiler.h> 24#include <asm/compiler.h>
25#include <asm/errno.h> 25#include <asm/errno.h>
26#include <asm/opcodes-sec.h>
27#include <asm/opcodes-virt.h>
28#include <asm/psci.h> 26#include <asm/psci.h>
29#include <asm/system_misc.h> 27#include <asm/system_misc.h>
30 28
@@ -33,6 +31,9 @@ struct psci_operations psci_ops;
33static int (*invoke_psci_fn)(u32, u32, u32, u32); 31static int (*invoke_psci_fn)(u32, u32, u32, u32);
34typedef int (*psci_initcall_t)(const struct device_node *); 32typedef int (*psci_initcall_t)(const struct device_node *);
35 33
34asmlinkage int __invoke_psci_fn_hvc(u32, u32, u32, u32);
35asmlinkage int __invoke_psci_fn_smc(u32, u32, u32, u32);
36
36enum psci_function { 37enum psci_function {
37 PSCI_FN_CPU_SUSPEND, 38 PSCI_FN_CPU_SUSPEND,
38 PSCI_FN_CPU_ON, 39 PSCI_FN_CPU_ON,
@@ -71,40 +72,6 @@ static u32 psci_power_state_pack(struct psci_power_state state)
71 & PSCI_0_2_POWER_STATE_AFFL_MASK); 72 & PSCI_0_2_POWER_STATE_AFFL_MASK);
72} 73}
73 74
74/*
75 * The following two functions are invoked via the invoke_psci_fn pointer
76 * and will not be inlined, allowing us to piggyback on the AAPCS.
77 */
78static noinline int __invoke_psci_fn_hvc(u32 function_id, u32 arg0, u32 arg1,
79 u32 arg2)
80{
81 asm volatile(
82 __asmeq("%0", "r0")
83 __asmeq("%1", "r1")
84 __asmeq("%2", "r2")
85 __asmeq("%3", "r3")
86 __HVC(0)
87 : "+r" (function_id)
88 : "r" (arg0), "r" (arg1), "r" (arg2));
89
90 return function_id;
91}
92
93static noinline int __invoke_psci_fn_smc(u32 function_id, u32 arg0, u32 arg1,
94 u32 arg2)
95{
96 asm volatile(
97 __asmeq("%0", "r0")
98 __asmeq("%1", "r1")
99 __asmeq("%2", "r2")
100 __asmeq("%3", "r3")
101 __SMC(0)
102 : "+r" (function_id)
103 : "r" (arg0), "r" (arg1), "r" (arg2));
104
105 return function_id;
106}
107
108static int psci_get_version(void) 75static int psci_get_version(void)
109{ 76{
110 int err; 77 int err;
diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c
new file mode 100644
index 000000000000..1a4d232796be
--- /dev/null
+++ b/arch/arm/kernel/reboot.c
@@ -0,0 +1,155 @@
1/*
2 * Copyright (C) 1996-2000 Russell King - Converted to ARM.
3 * Original Copyright (C) 1995 Linus Torvalds
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 */
9#include <linux/cpu.h>
10#include <linux/delay.h>
11#include <linux/reboot.h>
12
13#include <asm/cacheflush.h>
14#include <asm/idmap.h>
15
16#include "reboot.h"
17
18typedef void (*phys_reset_t)(unsigned long);
19
20/*
21 * Function pointers to optional machine specific functions
22 */
23void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd);
24void (*pm_power_off)(void);
25EXPORT_SYMBOL(pm_power_off);
26
27/*
28 * A temporary stack to use for CPU reset. This is static so that we
29 * don't clobber it with the identity mapping. When running with this
30 * stack, any references to the current task *will not work* so you
31 * should really do as little as possible before jumping to your reset
32 * code.
33 */
34static u64 soft_restart_stack[16];
35
36static void __soft_restart(void *addr)
37{
38 phys_reset_t phys_reset;
39
40 /* Take out a flat memory mapping. */
41 setup_mm_for_reboot();
42
43 /* Clean and invalidate caches */
44 flush_cache_all();
45
46 /* Turn off caching */
47 cpu_proc_fin();
48
49 /* Push out any further dirty data, and ensure cache is empty */
50 flush_cache_all();
51
52 /* Switch to the identity mapping. */
53 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
54 phys_reset((unsigned long)addr);
55
56 /* Should never get here. */
57 BUG();
58}
59
60void _soft_restart(unsigned long addr, bool disable_l2)
61{
62 u64 *stack = soft_restart_stack + ARRAY_SIZE(soft_restart_stack);
63
64 /* Disable interrupts first */
65 raw_local_irq_disable();
66 local_fiq_disable();
67
68 /* Disable the L2 if we're the last man standing. */
69 if (disable_l2)
70 outer_disable();
71
72 /* Change to the new stack and continue with the reset. */
73 call_with_stack(__soft_restart, (void *)addr, (void *)stack);
74
75 /* Should never get here. */
76 BUG();
77}
78
79void soft_restart(unsigned long addr)
80{
81 _soft_restart(addr, num_online_cpus() == 1);
82}
83
84/*
85 * Called by kexec, immediately prior to machine_kexec().
86 *
87 * This must completely disable all secondary CPUs; simply causing those CPUs
88 * to execute e.g. a RAM-based pin loop is not sufficient. This allows the
89 * kexec'd kernel to use any and all RAM as it sees fit, without having to
90 * avoid any code or data used by any SW CPU pin loop. The CPU hotplug
91 * functionality embodied in disable_nonboot_cpus() to achieve this.
92 */
93void machine_shutdown(void)
94{
95 disable_nonboot_cpus();
96}
97
98/*
99 * Halting simply requires that the secondary CPUs stop performing any
100 * activity (executing tasks, handling interrupts). smp_send_stop()
101 * achieves this.
102 */
103void machine_halt(void)
104{
105 local_irq_disable();
106 smp_send_stop();
107
108 local_irq_disable();
109 while (1);
110}
111
112/*
113 * Power-off simply requires that the secondary CPUs stop performing any
114 * activity (executing tasks, handling interrupts). smp_send_stop()
115 * achieves this. When the system power is turned off, it will take all CPUs
116 * with it.
117 */
118void machine_power_off(void)
119{
120 local_irq_disable();
121 smp_send_stop();
122
123 if (pm_power_off)
124 pm_power_off();
125}
126
127/*
128 * Restart requires that the secondary CPUs stop performing any activity
129 * while the primary CPU resets the system. Systems with a single CPU can
130 * use soft_restart() as their machine descriptor's .restart hook, since that
131 * will cause the only available CPU to reset. Systems with multiple CPUs must
132 * provide a HW restart implementation, to ensure that all CPUs reset at once.
133 * This is required so that any code running after reset on the primary CPU
134 * doesn't have to co-ordinate with other CPUs to ensure they aren't still
135 * executing pre-reset code, and using RAM that the primary CPU's code wishes
136 * to use. Implementing such co-ordination would be essentially impossible.
137 */
138void machine_restart(char *cmd)
139{
140 local_irq_disable();
141 smp_send_stop();
142
143 if (arm_pm_restart)
144 arm_pm_restart(reboot_mode, cmd);
145 else
146 do_kernel_restart(cmd);
147
148 /* Give a grace period for failure to restart of 1s */
149 mdelay(1000);
150
151 /* Whoops - the platform was unable to reboot. Tell the user! */
152 printk("Reboot failed -- System halted\n");
153 local_irq_disable();
154 while (1);
155}
diff --git a/arch/arm/kernel/reboot.h b/arch/arm/kernel/reboot.h
new file mode 100644
index 000000000000..bf7a0b1f076e
--- /dev/null
+++ b/arch/arm/kernel/reboot.h
@@ -0,0 +1,7 @@
1#ifndef REBOOT_H
2#define REBOOT_H
3
4extern void call_with_stack(void (*fn)(void *), void *arg, void *sp);
5extern void _soft_restart(unsigned long addr, bool disable_l2);
6
7#endif
diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c
index 24b4a04846eb..36ed35073289 100644
--- a/arch/arm/kernel/return_address.c
+++ b/arch/arm/kernel/return_address.c
@@ -56,8 +56,6 @@ void *return_address(unsigned int level)
56 return NULL; 56 return NULL;
57} 57}
58 58
59#else /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) */ 59#endif /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) */
60
61#endif /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) / else */
62 60
63EXPORT_SYMBOL_GPL(return_address); 61EXPORT_SYMBOL_GPL(return_address);
diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
index 1d60bebea4b8..6c777e908a24 100644
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -372,30 +372,48 @@ void __init early_print(const char *str, ...)
372 372
373static void __init cpuid_init_hwcaps(void) 373static void __init cpuid_init_hwcaps(void)
374{ 374{
375 unsigned int divide_instrs, vmsa; 375 int block;
376 u32 isar5;
376 377
377 if (cpu_architecture() < CPU_ARCH_ARMv7) 378 if (cpu_architecture() < CPU_ARCH_ARMv7)
378 return; 379 return;
379 380
380 divide_instrs = (read_cpuid_ext(CPUID_EXT_ISAR0) & 0x0f000000) >> 24; 381 block = cpuid_feature_extract(CPUID_EXT_ISAR0, 24);
381 382 if (block >= 2)
382 switch (divide_instrs) {
383 case 2:
384 elf_hwcap |= HWCAP_IDIVA; 383 elf_hwcap |= HWCAP_IDIVA;
385 case 1: 384 if (block >= 1)
386 elf_hwcap |= HWCAP_IDIVT; 385 elf_hwcap |= HWCAP_IDIVT;
387 }
388 386
389 /* LPAE implies atomic ldrd/strd instructions */ 387 /* LPAE implies atomic ldrd/strd instructions */
390 vmsa = (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xf) >> 0; 388 block = cpuid_feature_extract(CPUID_EXT_MMFR0, 0);
391 if (vmsa >= 5) 389 if (block >= 5)
392 elf_hwcap |= HWCAP_LPAE; 390 elf_hwcap |= HWCAP_LPAE;
391
392 /* check for supported v8 Crypto instructions */
393 isar5 = read_cpuid_ext(CPUID_EXT_ISAR5);
394
395 block = cpuid_feature_extract_field(isar5, 4);
396 if (block >= 2)
397 elf_hwcap2 |= HWCAP2_PMULL;
398 if (block >= 1)
399 elf_hwcap2 |= HWCAP2_AES;
400
401 block = cpuid_feature_extract_field(isar5, 8);
402 if (block >= 1)
403 elf_hwcap2 |= HWCAP2_SHA1;
404
405 block = cpuid_feature_extract_field(isar5, 12);
406 if (block >= 1)
407 elf_hwcap2 |= HWCAP2_SHA2;
408
409 block = cpuid_feature_extract_field(isar5, 16);
410 if (block >= 1)
411 elf_hwcap2 |= HWCAP2_CRC32;
393} 412}
394 413
395static void __init elf_hwcap_fixup(void) 414static void __init elf_hwcap_fixup(void)
396{ 415{
397 unsigned id = read_cpuid_id(); 416 unsigned id = read_cpuid_id();
398 unsigned sync_prim;
399 417
400 /* 418 /*
401 * HWCAP_TLS is available only on 1136 r1p0 and later, 419 * HWCAP_TLS is available only on 1136 r1p0 and later,
@@ -416,9 +434,9 @@ static void __init elf_hwcap_fixup(void)
416 * avoid advertising SWP; it may not be atomic with 434 * avoid advertising SWP; it may not be atomic with
417 * multiprocessing cores. 435 * multiprocessing cores.
418 */ 436 */
419 sync_prim = ((read_cpuid_ext(CPUID_EXT_ISAR3) >> 8) & 0xf0) | 437 if (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) > 1 ||
420 ((read_cpuid_ext(CPUID_EXT_ISAR4) >> 20) & 0x0f); 438 (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) == 1 &&
421 if (sync_prim >= 0x13) 439 cpuid_feature_extract(CPUID_EXT_ISAR3, 20) >= 3))
422 elf_hwcap &= ~HWCAP_SWP; 440 elf_hwcap &= ~HWCAP_SWP;
423} 441}
424 442
diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c
index 023ac905e4c3..423663e23791 100644
--- a/arch/arm/kernel/signal.c
+++ b/arch/arm/kernel/signal.c
@@ -318,17 +318,6 @@ get_sigframe(struct ksignal *ksig, struct pt_regs *regs, int framesize)
318 return frame; 318 return frame;
319} 319}
320 320
321/*
322 * translate the signal
323 */
324static inline int map_sig(int sig)
325{
326 struct thread_info *thread = current_thread_info();
327 if (sig < 32 && thread->exec_domain && thread->exec_domain->signal_invmap)
328 sig = thread->exec_domain->signal_invmap[sig];
329 return sig;
330}
331
332static int 321static int
333setup_return(struct pt_regs *regs, struct ksignal *ksig, 322setup_return(struct pt_regs *regs, struct ksignal *ksig,
334 unsigned long __user *rc, void __user *frame) 323 unsigned long __user *rc, void __user *frame)
@@ -412,7 +401,7 @@ setup_return(struct pt_regs *regs, struct ksignal *ksig,
412 } 401 }
413 } 402 }
414 403
415 regs->ARM_r0 = map_sig(ksig->sig); 404 regs->ARM_r0 = ksig->sig;
416 regs->ARM_sp = (unsigned long)frame; 405 regs->ARM_sp = (unsigned long)frame;
417 regs->ARM_lr = retcode; 406 regs->ARM_lr = retcode;
418 regs->ARM_pc = handler; 407 regs->ARM_pc = handler;
diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S
index e1e60e5a7a27..7d37bfc50830 100644
--- a/arch/arm/kernel/sleep.S
+++ b/arch/arm/kernel/sleep.S
@@ -116,14 +116,7 @@ cpu_resume_after_mmu:
116 ldmfd sp!, {r4 - r11, pc} 116 ldmfd sp!, {r4 - r11, pc}
117ENDPROC(cpu_resume_after_mmu) 117ENDPROC(cpu_resume_after_mmu)
118 118
119/* 119 .text
120 * Note: Yes, part of the following code is located into the .data section.
121 * This is to allow sleep_save_sp to be accessed with a relative load
122 * while we can't rely on any MMU translation. We could have put
123 * sleep_save_sp in the .text section as well, but some setups might
124 * insist on it to be truly read-only.
125 */
126 .data
127 .align 120 .align
128ENTRY(cpu_resume) 121ENTRY(cpu_resume)
129ARM_BE8(setend be) @ ensure we are in BE mode 122ARM_BE8(setend be) @ ensure we are in BE mode
@@ -145,6 +138,8 @@ ARM_BE8(setend be) @ ensure we are in BE mode
145 compute_mpidr_hash r1, r4, r5, r6, r0, r3 138 compute_mpidr_hash r1, r4, r5, r6, r0, r3
1461: 1391:
147 adr r0, _sleep_save_sp 140 adr r0, _sleep_save_sp
141 ldr r2, [r0]
142 add r0, r0, r2
148 ldr r0, [r0, #SLEEP_SAVE_SP_PHYS] 143 ldr r0, [r0, #SLEEP_SAVE_SP_PHYS]
149 ldr r0, [r0, r1, lsl #2] 144 ldr r0, [r0, r1, lsl #2]
150 145
@@ -156,10 +151,12 @@ THUMB( bx r3 )
156ENDPROC(cpu_resume) 151ENDPROC(cpu_resume)
157 152
158 .align 2 153 .align 2
154_sleep_save_sp:
155 .long sleep_save_sp - .
159mpidr_hash_ptr: 156mpidr_hash_ptr:
160 .long mpidr_hash - . @ mpidr_hash struct offset 157 .long mpidr_hash - . @ mpidr_hash struct offset
161 158
159 .data
162 .type sleep_save_sp, #object 160 .type sleep_save_sp, #object
163ENTRY(sleep_save_sp) 161ENTRY(sleep_save_sp)
164_sleep_save_sp:
165 .space SLEEP_SAVE_SP_SZ @ struct sleep_save_sp 162 .space SLEEP_SAVE_SP_SZ @ struct sleep_save_sp
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 86ef244c5a24..cca5b8758185 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -145,6 +145,11 @@ void __init smp_init_cpus(void)
145 smp_ops.smp_init_cpus(); 145 smp_ops.smp_init_cpus();
146} 146}
147 147
148int platform_can_secondary_boot(void)
149{
150 return !!smp_ops.smp_boot_secondary;
151}
152
148int platform_can_cpu_hotplug(void) 153int platform_can_cpu_hotplug(void)
149{ 154{
150#ifdef CONFIG_HOTPLUG_CPU 155#ifdef CONFIG_HOTPLUG_CPU
diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c
index afdd51e30bec..1361756782c7 100644
--- a/arch/arm/kernel/swp_emulate.c
+++ b/arch/arm/kernel/swp_emulate.c
@@ -42,7 +42,7 @@
42 " cmp %0, #0\n" \ 42 " cmp %0, #0\n" \
43 " movne %0, %4\n" \ 43 " movne %0, %4\n" \
44 "2:\n" \ 44 "2:\n" \
45 " .section .fixup,\"ax\"\n" \ 45 " .section .text.fixup,\"ax\"\n" \
46 " .align 2\n" \ 46 " .align 2\n" \
47 "3: mov %0, %5\n" \ 47 "3: mov %0, %5\n" \
48 " b 2b\n" \ 48 " b 2b\n" \
diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c
index 0cc7e58c47cc..a66e37e211a9 100644
--- a/arch/arm/kernel/time.c
+++ b/arch/arm/kernel/time.c
@@ -76,7 +76,7 @@ void timer_tick(void)
76} 76}
77#endif 77#endif
78 78
79static void dummy_clock_access(struct timespec *ts) 79static void dummy_clock_access(struct timespec64 *ts)
80{ 80{
81 ts->tv_sec = 0; 81 ts->tv_sec = 0;
82 ts->tv_nsec = 0; 82 ts->tv_nsec = 0;
@@ -85,12 +85,12 @@ static void dummy_clock_access(struct timespec *ts)
85static clock_access_fn __read_persistent_clock = dummy_clock_access; 85static clock_access_fn __read_persistent_clock = dummy_clock_access;
86static clock_access_fn __read_boot_clock = dummy_clock_access;; 86static clock_access_fn __read_boot_clock = dummy_clock_access;;
87 87
88void read_persistent_clock(struct timespec *ts) 88void read_persistent_clock64(struct timespec64 *ts)
89{ 89{
90 __read_persistent_clock(ts); 90 __read_persistent_clock(ts);
91} 91}
92 92
93void read_boot_clock(struct timespec *ts) 93void read_boot_clock64(struct timespec64 *ts)
94{ 94{
95 __read_boot_clock(ts); 95 __read_boot_clock(ts);
96} 96}
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index 788e23fe64d8..3dce1a342030 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -505,12 +505,10 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason)
505 505
506static int bad_syscall(int n, struct pt_regs *regs) 506static int bad_syscall(int n, struct pt_regs *regs)
507{ 507{
508 struct thread_info *thread = current_thread_info();
509 siginfo_t info; 508 siginfo_t info;
510 509
511 if ((current->personality & PER_MASK) != PER_LINUX && 510 if ((current->personality & PER_MASK) != PER_LINUX) {
512 thread->exec_domain->handler) { 511 send_sig(SIGSEGV, current, 1);
513 thread->exec_domain->handler(n, regs);
514 return regs->ARM_r0; 512 return regs->ARM_r0;
515 } 513 }
516 514
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
new file mode 100644
index 000000000000..efe17dd9b921
--- /dev/null
+++ b/arch/arm/kernel/vdso.c
@@ -0,0 +1,337 @@
1/*
2 * Adapted from arm64 version.
3 *
4 * Copyright (C) 2012 ARM Limited
5 * Copyright (C) 2015 Mentor Graphics Corporation.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/elf.h>
21#include <linux/err.h>
22#include <linux/kernel.h>
23#include <linux/mm.h>
24#include <linux/of.h>
25#include <linux/printk.h>
26#include <linux/slab.h>
27#include <linux/timekeeper_internal.h>
28#include <linux/vmalloc.h>
29#include <asm/arch_timer.h>
30#include <asm/barrier.h>
31#include <asm/cacheflush.h>
32#include <asm/page.h>
33#include <asm/vdso.h>
34#include <asm/vdso_datapage.h>
35#include <clocksource/arm_arch_timer.h>
36
37#define MAX_SYMNAME 64
38
39static struct page **vdso_text_pagelist;
40
41/* Total number of pages needed for the data and text portions of the VDSO. */
42unsigned int vdso_total_pages __read_mostly;
43
44/*
45 * The VDSO data page.
46 */
47static union vdso_data_store vdso_data_store __page_aligned_data;
48static struct vdso_data *vdso_data = &vdso_data_store.data;
49
50static struct page *vdso_data_page;
51static struct vm_special_mapping vdso_data_mapping = {
52 .name = "[vvar]",
53 .pages = &vdso_data_page,
54};
55
56static struct vm_special_mapping vdso_text_mapping = {
57 .name = "[vdso]",
58};
59
60struct elfinfo {
61 Elf32_Ehdr *hdr; /* ptr to ELF */
62 Elf32_Sym *dynsym; /* ptr to .dynsym section */
63 unsigned long dynsymsize; /* size of .dynsym section */
64 char *dynstr; /* ptr to .dynstr section */
65};
66
67/* Cached result of boot-time check for whether the arch timer exists,
68 * and if so, whether the virtual counter is useable.
69 */
70static bool cntvct_ok __read_mostly;
71
72static bool __init cntvct_functional(void)
73{
74 struct device_node *np;
75 bool ret = false;
76
77 if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER))
78 goto out;
79
80 /* The arm_arch_timer core should export
81 * arch_timer_use_virtual or similar so we don't have to do
82 * this.
83 */
84 np = of_find_compatible_node(NULL, NULL, "arm,armv7-timer");
85 if (!np)
86 goto out_put;
87
88 if (of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
89 goto out_put;
90
91 ret = true;
92
93out_put:
94 of_node_put(np);
95out:
96 return ret;
97}
98
99static void * __init find_section(Elf32_Ehdr *ehdr, const char *name,
100 unsigned long *size)
101{
102 Elf32_Shdr *sechdrs;
103 unsigned int i;
104 char *secnames;
105
106 /* Grab section headers and strings so we can tell who is who */
107 sechdrs = (void *)ehdr + ehdr->e_shoff;
108 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
109
110 /* Find the section they want */
111 for (i = 1; i < ehdr->e_shnum; i++) {
112 if (strcmp(secnames + sechdrs[i].sh_name, name) == 0) {
113 if (size)
114 *size = sechdrs[i].sh_size;
115 return (void *)ehdr + sechdrs[i].sh_offset;
116 }
117 }
118
119 if (size)
120 *size = 0;
121 return NULL;
122}
123
124static Elf32_Sym * __init find_symbol(struct elfinfo *lib, const char *symname)
125{
126 unsigned int i;
127
128 for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) {
129 char name[MAX_SYMNAME], *c;
130
131 if (lib->dynsym[i].st_name == 0)
132 continue;
133 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
134 MAX_SYMNAME);
135 c = strchr(name, '@');
136 if (c)
137 *c = 0;
138 if (strcmp(symname, name) == 0)
139 return &lib->dynsym[i];
140 }
141 return NULL;
142}
143
144static void __init vdso_nullpatch_one(struct elfinfo *lib, const char *symname)
145{
146 Elf32_Sym *sym;
147
148 sym = find_symbol(lib, symname);
149 if (!sym)
150 return;
151
152 sym->st_name = 0;
153}
154
155static void __init patch_vdso(void *ehdr)
156{
157 struct elfinfo einfo;
158
159 einfo = (struct elfinfo) {
160 .hdr = ehdr,
161 };
162
163 einfo.dynsym = find_section(einfo.hdr, ".dynsym", &einfo.dynsymsize);
164 einfo.dynstr = find_section(einfo.hdr, ".dynstr", NULL);
165
166 /* If the virtual counter is absent or non-functional we don't
167 * want programs to incur the slight additional overhead of
168 * dispatching through the VDSO only to fall back to syscalls.
169 */
170 if (!cntvct_ok) {
171 vdso_nullpatch_one(&einfo, "__vdso_gettimeofday");
172 vdso_nullpatch_one(&einfo, "__vdso_clock_gettime");
173 }
174}
175
176static int __init vdso_init(void)
177{
178 unsigned int text_pages;
179 int i;
180
181 if (memcmp(&vdso_start, "\177ELF", 4)) {
182 pr_err("VDSO is not a valid ELF object!\n");
183 return -ENOEXEC;
184 }
185
186 text_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT;
187 pr_debug("vdso: %i text pages at base %p\n", text_pages, &vdso_start);
188
189 /* Allocate the VDSO text pagelist */
190 vdso_text_pagelist = kcalloc(text_pages, sizeof(struct page *),
191 GFP_KERNEL);
192 if (vdso_text_pagelist == NULL)
193 return -ENOMEM;
194
195 /* Grab the VDSO data page. */
196 vdso_data_page = virt_to_page(vdso_data);
197
198 /* Grab the VDSO text pages. */
199 for (i = 0; i < text_pages; i++) {
200 struct page *page;
201
202 page = virt_to_page(&vdso_start + i * PAGE_SIZE);
203 vdso_text_pagelist[i] = page;
204 }
205
206 vdso_text_mapping.pages = vdso_text_pagelist;
207
208 vdso_total_pages = 1; /* for the data/vvar page */
209 vdso_total_pages += text_pages;
210
211 cntvct_ok = cntvct_functional();
212
213 patch_vdso(&vdso_start);
214
215 return 0;
216}
217arch_initcall(vdso_init);
218
219static int install_vvar(struct mm_struct *mm, unsigned long addr)
220{
221 struct vm_area_struct *vma;
222
223 vma = _install_special_mapping(mm, addr, PAGE_SIZE,
224 VM_READ | VM_MAYREAD,
225 &vdso_data_mapping);
226
227 return IS_ERR(vma) ? PTR_ERR(vma) : 0;
228}
229
230/* assumes mmap_sem is write-locked */
231void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
232{
233 struct vm_area_struct *vma;
234 unsigned long len;
235
236 mm->context.vdso = 0;
237
238 if (vdso_text_pagelist == NULL)
239 return;
240
241 if (install_vvar(mm, addr))
242 return;
243
244 /* Account for vvar page. */
245 addr += PAGE_SIZE;
246 len = (vdso_total_pages - 1) << PAGE_SHIFT;
247
248 vma = _install_special_mapping(mm, addr, len,
249 VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
250 &vdso_text_mapping);
251
252 if (!IS_ERR(vma))
253 mm->context.vdso = addr;
254}
255
256static void vdso_write_begin(struct vdso_data *vdata)
257{
258 ++vdso_data->seq_count;
259 smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
260}
261
262static void vdso_write_end(struct vdso_data *vdata)
263{
264 smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
265 ++vdso_data->seq_count;
266}
267
268static bool tk_is_cntvct(const struct timekeeper *tk)
269{
270 if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER))
271 return false;
272
273 if (strcmp(tk->tkr_mono.clock->name, "arch_sys_counter") != 0)
274 return false;
275
276 return true;
277}
278
279/**
280 * update_vsyscall - update the vdso data page
281 *
282 * Increment the sequence counter, making it odd, indicating to
283 * userspace that an update is in progress. Update the fields used
284 * for coarse clocks and, if the architected system timer is in use,
285 * the fields used for high precision clocks. Increment the sequence
286 * counter again, making it even, indicating to userspace that the
287 * update is finished.
288 *
289 * Userspace is expected to sample seq_count before reading any other
290 * fields from the data page. If seq_count is odd, userspace is
291 * expected to wait until it becomes even. After copying data from
292 * the page, userspace must sample seq_count again; if it has changed
293 * from its previous value, userspace must retry the whole sequence.
294 *
295 * Calls to update_vsyscall are serialized by the timekeeping core.
296 */
297void update_vsyscall(struct timekeeper *tk)
298{
299 struct timespec xtime_coarse;
300 struct timespec64 *wtm = &tk->wall_to_monotonic;
301
302 if (!cntvct_ok) {
303 /* The entry points have been zeroed, so there is no
304 * point in updating the data page.
305 */
306 return;
307 }
308
309 vdso_write_begin(vdso_data);
310
311 xtime_coarse = __current_kernel_time();
312 vdso_data->tk_is_cntvct = tk_is_cntvct(tk);
313 vdso_data->xtime_coarse_sec = xtime_coarse.tv_sec;
314 vdso_data->xtime_coarse_nsec = xtime_coarse.tv_nsec;
315 vdso_data->wtm_clock_sec = wtm->tv_sec;
316 vdso_data->wtm_clock_nsec = wtm->tv_nsec;
317
318 if (vdso_data->tk_is_cntvct) {
319 vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
320 vdso_data->xtime_clock_sec = tk->xtime_sec;
321 vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec;
322 vdso_data->cs_mult = tk->tkr_mono.mult;
323 vdso_data->cs_shift = tk->tkr_mono.shift;
324 vdso_data->cs_mask = tk->tkr_mono.mask;
325 }
326
327 vdso_write_end(vdso_data);
328
329 flush_dcache_page(virt_to_page(vdso_data));
330}
331
332void update_vsyscall_tz(void)
333{
334 vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
335 vdso_data->tz_dsttime = sys_tz.tz_dsttime;
336 flush_dcache_page(virt_to_page(vdso_data));
337}
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index b31aa73e8076..7a301be9ac67 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -74,7 +74,7 @@ SECTIONS
74 ARM_EXIT_DISCARD(EXIT_DATA) 74 ARM_EXIT_DISCARD(EXIT_DATA)
75 EXIT_CALL 75 EXIT_CALL
76#ifndef CONFIG_MMU 76#ifndef CONFIG_MMU
77 *(.fixup) 77 *(.text.fixup)
78 *(__ex_table) 78 *(__ex_table)
79#endif 79#endif
80#ifndef CONFIG_SMP_ON_UP 80#ifndef CONFIG_SMP_ON_UP
@@ -100,6 +100,7 @@ SECTIONS
100 100
101 .text : { /* Real text segment */ 101 .text : { /* Real text segment */
102 _stext = .; /* Text and read-only data */ 102 _stext = .; /* Text and read-only data */
103 IDMAP_TEXT
103 __exception_text_start = .; 104 __exception_text_start = .;
104 *(.exception.text) 105 *(.exception.text)
105 __exception_text_end = .; 106 __exception_text_end = .;
@@ -108,10 +109,6 @@ SECTIONS
108 SCHED_TEXT 109 SCHED_TEXT
109 LOCK_TEXT 110 LOCK_TEXT
110 KPROBES_TEXT 111 KPROBES_TEXT
111 IDMAP_TEXT
112#ifdef CONFIG_MMU
113 *(.fixup)
114#endif
115 *(.gnu.warning) 112 *(.gnu.warning)
116 *(.glue_7) 113 *(.glue_7)
117 *(.glue_7t) 114 *(.glue_7t)
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 338ace78ed18..f1f79d104309 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -18,6 +18,7 @@ if VIRTUALIZATION
18 18
19config KVM 19config KVM
20 bool "Kernel-based Virtual Machine (KVM) support" 20 bool "Kernel-based Virtual Machine (KVM) support"
21 depends on MMU && OF
21 select PREEMPT_NOTIFIERS 22 select PREEMPT_NOTIFIERS
22 select ANON_INODES 23 select ANON_INODES
23 select HAVE_KVM_CPU_RELAX_INTERCEPT 24 select HAVE_KVM_CPU_RELAX_INTERCEPT
@@ -26,10 +27,12 @@ config KVM
26 select KVM_ARM_HOST 27 select KVM_ARM_HOST
27 select KVM_GENERIC_DIRTYLOG_READ_PROTECT 28 select KVM_GENERIC_DIRTYLOG_READ_PROTECT
28 select SRCU 29 select SRCU
29 depends on ARM_VIRT_EXT && ARM_LPAE 30 select MMU_NOTIFIER
31 select HAVE_KVM_EVENTFD
32 select HAVE_KVM_IRQFD
33 depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER
30 ---help--- 34 ---help---
31 Support hosting virtualized guest machines. You will also 35 Support hosting virtualized guest machines.
32 need to select one or more of the processor modules below.
33 36
34 This module provides access to the hardware capabilities through 37 This module provides access to the hardware capabilities through
35 a character device node named /dev/kvm. 38 a character device node named /dev/kvm.
@@ -37,10 +40,7 @@ config KVM
37 If unsure, say N. 40 If unsure, say N.
38 41
39config KVM_ARM_HOST 42config KVM_ARM_HOST
40 bool "KVM host support for ARM cpus." 43 bool
41 depends on KVM
42 depends on MMU
43 select MMU_NOTIFIER
44 ---help--- 44 ---help---
45 Provides host support for ARM processors. 45 Provides host support for ARM processors.
46 46
@@ -55,20 +55,4 @@ config KVM_ARM_MAX_VCPUS
55 large, so only choose a reasonable number that you expect to 55 large, so only choose a reasonable number that you expect to
56 actually use. 56 actually use.
57 57
58config KVM_ARM_VGIC
59 bool "KVM support for Virtual GIC"
60 depends on KVM_ARM_HOST && OF
61 select HAVE_KVM_IRQCHIP
62 default y
63 ---help---
64 Adds support for a hardware assisted, in-kernel GIC emulation.
65
66config KVM_ARM_TIMER
67 bool "KVM support for Architected Timers"
68 depends on KVM_ARM_VGIC && ARM_ARCH_TIMER
69 select HAVE_KVM_IRQCHIP
70 default y
71 ---help---
72 Adds support for the Architected Timers in virtual machines
73
74endif # VIRTUALIZATION 58endif # VIRTUALIZATION
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index 443b8bea43e9..139e46c08b6e 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -7,7 +7,7 @@ ifeq ($(plus_virt),+virt)
7 plus_virt_def := -DREQUIRES_VIRT=1 7 plus_virt_def := -DREQUIRES_VIRT=1
8endif 8endif
9 9
10ccflags-y += -Ivirt/kvm -Iarch/arm/kvm 10ccflags-y += -Iarch/arm/kvm
11CFLAGS_arm.o := -I. $(plus_virt_def) 11CFLAGS_arm.o := -I. $(plus_virt_def)
12CFLAGS_mmu.o := -I. 12CFLAGS_mmu.o := -I.
13 13
@@ -15,12 +15,12 @@ AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt)
15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt) 15AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt)
16 16
17KVM := ../../../virt/kvm 17KVM := ../../../virt/kvm
18kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o 18kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
19 19
20obj-y += kvm-arm.o init.o interrupts.o 20obj-y += kvm-arm.o init.o interrupts.o
21obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o 21obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
22obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o 22obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
23obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o 23obj-y += $(KVM)/arm/vgic.o
24obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o 24obj-y += $(KVM)/arm/vgic-v2.o
25obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o 25obj-y += $(KVM)/arm/vgic-v2-emul.o
26obj-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o 26obj-y += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 5560f74f9eee..6f536451ab78 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -61,8 +61,6 @@ static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
61static u8 kvm_next_vmid; 61static u8 kvm_next_vmid;
62static DEFINE_SPINLOCK(kvm_vmid_lock); 62static DEFINE_SPINLOCK(kvm_vmid_lock);
63 63
64static bool vgic_present;
65
66static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu) 64static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
67{ 65{
68 BUG_ON(preemptible()); 66 BUG_ON(preemptible());
@@ -173,8 +171,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
173 int r; 171 int r;
174 switch (ext) { 172 switch (ext) {
175 case KVM_CAP_IRQCHIP: 173 case KVM_CAP_IRQCHIP:
176 r = vgic_present; 174 case KVM_CAP_IRQFD:
177 break; 175 case KVM_CAP_IOEVENTFD:
178 case KVM_CAP_DEVICE_CTRL: 176 case KVM_CAP_DEVICE_CTRL:
179 case KVM_CAP_USER_MEMORY: 177 case KVM_CAP_USER_MEMORY:
180 case KVM_CAP_SYNC_MMU: 178 case KVM_CAP_SYNC_MMU:
@@ -183,6 +181,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
183 case KVM_CAP_ARM_PSCI: 181 case KVM_CAP_ARM_PSCI:
184 case KVM_CAP_ARM_PSCI_0_2: 182 case KVM_CAP_ARM_PSCI_0_2:
185 case KVM_CAP_READONLY_MEM: 183 case KVM_CAP_READONLY_MEM:
184 case KVM_CAP_MP_STATE:
186 r = 1; 185 r = 1;
187 break; 186 break;
188 case KVM_CAP_COALESCED_MMIO: 187 case KVM_CAP_COALESCED_MMIO:
@@ -268,7 +267,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
268 267
269int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 268int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
270{ 269{
271 return 0; 270 return kvm_timer_should_fire(vcpu);
272} 271}
273 272
274int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 273int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
@@ -313,13 +312,29 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
313int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 312int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
314 struct kvm_mp_state *mp_state) 313 struct kvm_mp_state *mp_state)
315{ 314{
316 return -EINVAL; 315 if (vcpu->arch.pause)
316 mp_state->mp_state = KVM_MP_STATE_STOPPED;
317 else
318 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
319
320 return 0;
317} 321}
318 322
319int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 323int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
320 struct kvm_mp_state *mp_state) 324 struct kvm_mp_state *mp_state)
321{ 325{
322 return -EINVAL; 326 switch (mp_state->mp_state) {
327 case KVM_MP_STATE_RUNNABLE:
328 vcpu->arch.pause = false;
329 break;
330 case KVM_MP_STATE_STOPPED:
331 vcpu->arch.pause = true;
332 break;
333 default:
334 return -EINVAL;
335 }
336
337 return 0;
323} 338}
324 339
325/** 340/**
@@ -452,6 +467,11 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
452 return 0; 467 return 0;
453} 468}
454 469
470bool kvm_arch_intc_initialized(struct kvm *kvm)
471{
472 return vgic_initialized(kvm);
473}
474
455static void vcpu_pause(struct kvm_vcpu *vcpu) 475static void vcpu_pause(struct kvm_vcpu *vcpu)
456{ 476{
457 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu); 477 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
@@ -831,8 +851,6 @@ static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
831 851
832 switch (dev_id) { 852 switch (dev_id) {
833 case KVM_ARM_DEVICE_VGIC_V2: 853 case KVM_ARM_DEVICE_VGIC_V2:
834 if (!vgic_present)
835 return -ENXIO;
836 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true); 854 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
837 default: 855 default:
838 return -ENODEV; 856 return -ENODEV;
@@ -847,10 +865,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
847 865
848 switch (ioctl) { 866 switch (ioctl) {
849 case KVM_CREATE_IRQCHIP: { 867 case KVM_CREATE_IRQCHIP: {
850 if (vgic_present) 868 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
851 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
852 else
853 return -ENXIO;
854 } 869 }
855 case KVM_ARM_SET_DEVICE_ADDR: { 870 case KVM_ARM_SET_DEVICE_ADDR: {
856 struct kvm_arm_device_addr dev_addr; 871 struct kvm_arm_device_addr dev_addr;
@@ -1035,10 +1050,6 @@ static int init_hyp_mode(void)
1035 if (err) 1050 if (err)
1036 goto out_free_context; 1051 goto out_free_context;
1037 1052
1038#ifdef CONFIG_KVM_ARM_VGIC
1039 vgic_present = true;
1040#endif
1041
1042 /* 1053 /*
1043 * Init HYP architected timer support 1054 * Init HYP architected timer support
1044 */ 1055 */
diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c
index 384bab67c462..d503fbb787d3 100644
--- a/arch/arm/kvm/guest.c
+++ b/arch/arm/kvm/guest.c
@@ -109,22 +109,6 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
109 return -EINVAL; 109 return -EINVAL;
110} 110}
111 111
112#ifndef CONFIG_KVM_ARM_TIMER
113
114#define NUM_TIMER_REGS 0
115
116static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
117{
118 return 0;
119}
120
121static bool is_timer_reg(u64 index)
122{
123 return false;
124}
125
126#else
127
128#define NUM_TIMER_REGS 3 112#define NUM_TIMER_REGS 3
129 113
130static bool is_timer_reg(u64 index) 114static bool is_timer_reg(u64 index)
@@ -152,8 +136,6 @@ static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
152 return 0; 136 return 0;
153} 137}
154 138
155#endif
156
157static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 139static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
158{ 140{
159 void __user *uaddr = (void __user *)(long)reg->addr; 141 void __user *uaddr = (void __user *)(long)reg->addr;
diff --git a/arch/arm/kvm/interrupts_head.S b/arch/arm/kvm/interrupts_head.S
index 14d488388480..35e4a3a0c476 100644
--- a/arch/arm/kvm/interrupts_head.S
+++ b/arch/arm/kvm/interrupts_head.S
@@ -402,7 +402,6 @@ vcpu .req r0 @ vcpu pointer always in r0
402 * Assumes vcpu pointer in vcpu reg 402 * Assumes vcpu pointer in vcpu reg
403 */ 403 */
404.macro save_vgic_state 404.macro save_vgic_state
405#ifdef CONFIG_KVM_ARM_VGIC
406 /* Get VGIC VCTRL base into r2 */ 405 /* Get VGIC VCTRL base into r2 */
407 ldr r2, [vcpu, #VCPU_KVM] 406 ldr r2, [vcpu, #VCPU_KVM]
408 ldr r2, [r2, #KVM_VGIC_VCTRL] 407 ldr r2, [r2, #KVM_VGIC_VCTRL]
@@ -460,7 +459,6 @@ ARM_BE8(rev r6, r6 )
460 subs r4, r4, #1 459 subs r4, r4, #1
461 bne 1b 460 bne 1b
4622: 4612:
463#endif
464.endm 462.endm
465 463
466/* 464/*
@@ -469,7 +467,6 @@ ARM_BE8(rev r6, r6 )
469 * Assumes vcpu pointer in vcpu reg 467 * Assumes vcpu pointer in vcpu reg
470 */ 468 */
471.macro restore_vgic_state 469.macro restore_vgic_state
472#ifdef CONFIG_KVM_ARM_VGIC
473 /* Get VGIC VCTRL base into r2 */ 470 /* Get VGIC VCTRL base into r2 */
474 ldr r2, [vcpu, #VCPU_KVM] 471 ldr r2, [vcpu, #VCPU_KVM]
475 ldr r2, [r2, #KVM_VGIC_VCTRL] 472 ldr r2, [r2, #KVM_VGIC_VCTRL]
@@ -501,7 +498,6 @@ ARM_BE8(rev r6, r6 )
501 subs r4, r4, #1 498 subs r4, r4, #1
502 bne 1b 499 bne 1b
5032: 5002:
504#endif
505.endm 501.endm
506 502
507#define CNTHCTL_PL1PCTEN (1 << 0) 503#define CNTHCTL_PL1PCTEN (1 << 0)
@@ -515,7 +511,6 @@ ARM_BE8(rev r6, r6 )
515 * Clobbers r2-r5 511 * Clobbers r2-r5
516 */ 512 */
517.macro save_timer_state 513.macro save_timer_state
518#ifdef CONFIG_KVM_ARM_TIMER
519 ldr r4, [vcpu, #VCPU_KVM] 514 ldr r4, [vcpu, #VCPU_KVM]
520 ldr r2, [r4, #KVM_TIMER_ENABLED] 515 ldr r2, [r4, #KVM_TIMER_ENABLED]
521 cmp r2, #0 516 cmp r2, #0
@@ -537,7 +532,6 @@ ARM_BE8(rev r6, r6 )
537 mcrr p15, 4, r2, r2, c14 @ CNTVOFF 532 mcrr p15, 4, r2, r2, c14 @ CNTVOFF
538 533
5391: 5341:
540#endif
541 @ Allow physical timer/counter access for the host 535 @ Allow physical timer/counter access for the host
542 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL 536 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL
543 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN) 537 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN)
@@ -559,7 +553,6 @@ ARM_BE8(rev r6, r6 )
559 bic r2, r2, #CNTHCTL_PL1PCEN 553 bic r2, r2, #CNTHCTL_PL1PCEN
560 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL 554 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL
561 555
562#ifdef CONFIG_KVM_ARM_TIMER
563 ldr r4, [vcpu, #VCPU_KVM] 556 ldr r4, [vcpu, #VCPU_KVM]
564 ldr r2, [r4, #KVM_TIMER_ENABLED] 557 ldr r2, [r4, #KVM_TIMER_ENABLED]
565 cmp r2, #0 558 cmp r2, #0
@@ -579,7 +572,6 @@ ARM_BE8(rev r6, r6 )
579 and r2, r2, #3 572 and r2, r2, #3
580 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL 573 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
5811: 5741:
582#endif
583.endm 575.endm
584 576
585.equ vmentry, 0 577.equ vmentry, 0
diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c
index 5d3bfc0eb3f0..974b1c606d04 100644
--- a/arch/arm/kvm/mmio.c
+++ b/arch/arm/kvm/mmio.c
@@ -121,12 +121,11 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
121 return 0; 121 return 0;
122} 122}
123 123
124static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 124static int decode_hsr(struct kvm_vcpu *vcpu, bool *is_write, int *len)
125 struct kvm_exit_mmio *mmio)
126{ 125{
127 unsigned long rt; 126 unsigned long rt;
128 int len; 127 int access_size;
129 bool is_write, sign_extend; 128 bool sign_extend;
130 129
131 if (kvm_vcpu_dabt_isextabt(vcpu)) { 130 if (kvm_vcpu_dabt_isextabt(vcpu)) {
132 /* cache operation on I/O addr, tell guest unsupported */ 131 /* cache operation on I/O addr, tell guest unsupported */
@@ -140,17 +139,15 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
140 return 1; 139 return 1;
141 } 140 }
142 141
143 len = kvm_vcpu_dabt_get_as(vcpu); 142 access_size = kvm_vcpu_dabt_get_as(vcpu);
144 if (unlikely(len < 0)) 143 if (unlikely(access_size < 0))
145 return len; 144 return access_size;
146 145
147 is_write = kvm_vcpu_dabt_iswrite(vcpu); 146 *is_write = kvm_vcpu_dabt_iswrite(vcpu);
148 sign_extend = kvm_vcpu_dabt_issext(vcpu); 147 sign_extend = kvm_vcpu_dabt_issext(vcpu);
149 rt = kvm_vcpu_dabt_get_rd(vcpu); 148 rt = kvm_vcpu_dabt_get_rd(vcpu);
150 149
151 mmio->is_write = is_write; 150 *len = access_size;
152 mmio->phys_addr = fault_ipa;
153 mmio->len = len;
154 vcpu->arch.mmio_decode.sign_extend = sign_extend; 151 vcpu->arch.mmio_decode.sign_extend = sign_extend;
155 vcpu->arch.mmio_decode.rt = rt; 152 vcpu->arch.mmio_decode.rt = rt;
156 153
@@ -165,20 +162,20 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
165int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, 162int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
166 phys_addr_t fault_ipa) 163 phys_addr_t fault_ipa)
167{ 164{
168 struct kvm_exit_mmio mmio;
169 unsigned long data; 165 unsigned long data;
170 unsigned long rt; 166 unsigned long rt;
171 int ret; 167 int ret;
168 bool is_write;
169 int len;
170 u8 data_buf[8];
172 171
173 /* 172 /*
174 * Prepare MMIO operation. First stash it in a private 173 * Prepare MMIO operation. First decode the syndrome data we get
175 * structure that we can use for in-kernel emulation. If the 174 * from the CPU. Then try if some in-kernel emulation feels
176 * kernel can't handle it, copy it into run->mmio and let user 175 * responsible, otherwise let user space do its magic.
177 * space do its magic.
178 */ 176 */
179
180 if (kvm_vcpu_dabt_isvalid(vcpu)) { 177 if (kvm_vcpu_dabt_isvalid(vcpu)) {
181 ret = decode_hsr(vcpu, fault_ipa, &mmio); 178 ret = decode_hsr(vcpu, &is_write, &len);
182 if (ret) 179 if (ret)
183 return ret; 180 return ret;
184 } else { 181 } else {
@@ -188,21 +185,34 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
188 185
189 rt = vcpu->arch.mmio_decode.rt; 186 rt = vcpu->arch.mmio_decode.rt;
190 187
191 if (mmio.is_write) { 188 if (is_write) {
192 data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), 189 data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
193 mmio.len); 190
191 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, fault_ipa, data);
192 mmio_write_buf(data_buf, len, data);
194 193
195 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, mmio.len, 194 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, fault_ipa, len,
196 fault_ipa, data); 195 data_buf);
197 mmio_write_buf(mmio.data, mmio.len, data);
198 } else { 196 } else {
199 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, mmio.len, 197 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len,
200 fault_ipa, 0); 198 fault_ipa, 0);
199
200 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_ipa, len,
201 data_buf);
201 } 202 }
202 203
203 if (vgic_handle_mmio(vcpu, run, &mmio)) 204 /* Now prepare kvm_run for the potential return to userland. */
205 run->mmio.is_write = is_write;
206 run->mmio.phys_addr = fault_ipa;
207 run->mmio.len = len;
208 memcpy(run->mmio.data, data_buf, len);
209
210 if (!ret) {
211 /* We handled the access successfully in the kernel. */
212 kvm_handle_mmio_return(vcpu, run);
204 return 1; 213 return 1;
214 }
205 215
206 kvm_prepare_mmio(run, &mmio); 216 run->exit_reason = KVM_EXIT_MMIO;
207 return 0; 217 return 0;
208} 218}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 5656d79c5a44..15b050d46fc9 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1330,10 +1330,51 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1330 1330
1331out_unlock: 1331out_unlock:
1332 spin_unlock(&kvm->mmu_lock); 1332 spin_unlock(&kvm->mmu_lock);
1333 kvm_set_pfn_accessed(pfn);
1333 kvm_release_pfn_clean(pfn); 1334 kvm_release_pfn_clean(pfn);
1334 return ret; 1335 return ret;
1335} 1336}
1336 1337
1338/*
1339 * Resolve the access fault by making the page young again.
1340 * Note that because the faulting entry is guaranteed not to be
1341 * cached in the TLB, we don't need to invalidate anything.
1342 */
1343static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
1344{
1345 pmd_t *pmd;
1346 pte_t *pte;
1347 pfn_t pfn;
1348 bool pfn_valid = false;
1349
1350 trace_kvm_access_fault(fault_ipa);
1351
1352 spin_lock(&vcpu->kvm->mmu_lock);
1353
1354 pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
1355 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1356 goto out;
1357
1358 if (kvm_pmd_huge(*pmd)) { /* THP, HugeTLB */
1359 *pmd = pmd_mkyoung(*pmd);
1360 pfn = pmd_pfn(*pmd);
1361 pfn_valid = true;
1362 goto out;
1363 }
1364
1365 pte = pte_offset_kernel(pmd, fault_ipa);
1366 if (pte_none(*pte)) /* Nothing there either */
1367 goto out;
1368
1369 *pte = pte_mkyoung(*pte); /* Just a page... */
1370 pfn = pte_pfn(*pte);
1371 pfn_valid = true;
1372out:
1373 spin_unlock(&vcpu->kvm->mmu_lock);
1374 if (pfn_valid)
1375 kvm_set_pfn_accessed(pfn);
1376}
1377
1337/** 1378/**
1338 * kvm_handle_guest_abort - handles all 2nd stage aborts 1379 * kvm_handle_guest_abort - handles all 2nd stage aborts
1339 * @vcpu: the VCPU pointer 1380 * @vcpu: the VCPU pointer
@@ -1364,7 +1405,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1364 1405
1365 /* Check the stage-2 fault is trans. fault or write fault */ 1406 /* Check the stage-2 fault is trans. fault or write fault */
1366 fault_status = kvm_vcpu_trap_get_fault_type(vcpu); 1407 fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
1367 if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { 1408 if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
1409 fault_status != FSC_ACCESS) {
1368 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", 1410 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
1369 kvm_vcpu_trap_get_class(vcpu), 1411 kvm_vcpu_trap_get_class(vcpu),
1370 (unsigned long)kvm_vcpu_trap_get_fault(vcpu), 1412 (unsigned long)kvm_vcpu_trap_get_fault(vcpu),
@@ -1400,6 +1442,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1400 /* Userspace should not be able to register out-of-bounds IPAs */ 1442 /* Userspace should not be able to register out-of-bounds IPAs */
1401 VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE); 1443 VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
1402 1444
1445 if (fault_status == FSC_ACCESS) {
1446 handle_access_fault(vcpu, fault_ipa);
1447 ret = 1;
1448 goto out_unlock;
1449 }
1450
1403 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); 1451 ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1404 if (ret == 0) 1452 if (ret == 0)
1405 ret = 1; 1453 ret = 1;
@@ -1408,15 +1456,16 @@ out_unlock:
1408 return ret; 1456 return ret;
1409} 1457}
1410 1458
1411static void handle_hva_to_gpa(struct kvm *kvm, 1459static int handle_hva_to_gpa(struct kvm *kvm,
1412 unsigned long start, 1460 unsigned long start,
1413 unsigned long end, 1461 unsigned long end,
1414 void (*handler)(struct kvm *kvm, 1462 int (*handler)(struct kvm *kvm,
1415 gpa_t gpa, void *data), 1463 gpa_t gpa, void *data),
1416 void *data) 1464 void *data)
1417{ 1465{
1418 struct kvm_memslots *slots; 1466 struct kvm_memslots *slots;
1419 struct kvm_memory_slot *memslot; 1467 struct kvm_memory_slot *memslot;
1468 int ret = 0;
1420 1469
1421 slots = kvm_memslots(kvm); 1470 slots = kvm_memslots(kvm);
1422 1471
@@ -1440,14 +1489,17 @@ static void handle_hva_to_gpa(struct kvm *kvm,
1440 1489
1441 for (; gfn < gfn_end; ++gfn) { 1490 for (; gfn < gfn_end; ++gfn) {
1442 gpa_t gpa = gfn << PAGE_SHIFT; 1491 gpa_t gpa = gfn << PAGE_SHIFT;
1443 handler(kvm, gpa, data); 1492 ret |= handler(kvm, gpa, data);
1444 } 1493 }
1445 } 1494 }
1495
1496 return ret;
1446} 1497}
1447 1498
1448static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) 1499static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1449{ 1500{
1450 unmap_stage2_range(kvm, gpa, PAGE_SIZE); 1501 unmap_stage2_range(kvm, gpa, PAGE_SIZE);
1502 return 0;
1451} 1503}
1452 1504
1453int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) 1505int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
@@ -1473,7 +1525,7 @@ int kvm_unmap_hva_range(struct kvm *kvm,
1473 return 0; 1525 return 0;
1474} 1526}
1475 1527
1476static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data) 1528static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
1477{ 1529{
1478 pte_t *pte = (pte_t *)data; 1530 pte_t *pte = (pte_t *)data;
1479 1531
@@ -1485,6 +1537,7 @@ static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
1485 * through this calling path. 1537 * through this calling path.
1486 */ 1538 */
1487 stage2_set_pte(kvm, NULL, gpa, pte, 0); 1539 stage2_set_pte(kvm, NULL, gpa, pte, 0);
1540 return 0;
1488} 1541}
1489 1542
1490 1543
@@ -1501,6 +1554,67 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1501 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte); 1554 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
1502} 1555}
1503 1556
1557static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1558{
1559 pmd_t *pmd;
1560 pte_t *pte;
1561
1562 pmd = stage2_get_pmd(kvm, NULL, gpa);
1563 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1564 return 0;
1565
1566 if (kvm_pmd_huge(*pmd)) { /* THP, HugeTLB */
1567 if (pmd_young(*pmd)) {
1568 *pmd = pmd_mkold(*pmd);
1569 return 1;
1570 }
1571
1572 return 0;
1573 }
1574
1575 pte = pte_offset_kernel(pmd, gpa);
1576 if (pte_none(*pte))
1577 return 0;
1578
1579 if (pte_young(*pte)) {
1580 *pte = pte_mkold(*pte); /* Just a page... */
1581 return 1;
1582 }
1583
1584 return 0;
1585}
1586
1587static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
1588{
1589 pmd_t *pmd;
1590 pte_t *pte;
1591
1592 pmd = stage2_get_pmd(kvm, NULL, gpa);
1593 if (!pmd || pmd_none(*pmd)) /* Nothing there */
1594 return 0;
1595
1596 if (kvm_pmd_huge(*pmd)) /* THP, HugeTLB */
1597 return pmd_young(*pmd);
1598
1599 pte = pte_offset_kernel(pmd, gpa);
1600 if (!pte_none(*pte)) /* Just a page... */
1601 return pte_young(*pte);
1602
1603 return 0;
1604}
1605
1606int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1607{
1608 trace_kvm_age_hva(start, end);
1609 return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
1610}
1611
1612int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1613{
1614 trace_kvm_test_age_hva(hva);
1615 return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
1616}
1617
1504void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) 1618void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
1505{ 1619{
1506 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1620 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h
index 6817664b46b8..0ec35392d208 100644
--- a/arch/arm/kvm/trace.h
+++ b/arch/arm/kvm/trace.h
@@ -68,6 +68,21 @@ TRACE_EVENT(kvm_guest_fault,
68 __entry->hxfar, __entry->vcpu_pc) 68 __entry->hxfar, __entry->vcpu_pc)
69); 69);
70 70
71TRACE_EVENT(kvm_access_fault,
72 TP_PROTO(unsigned long ipa),
73 TP_ARGS(ipa),
74
75 TP_STRUCT__entry(
76 __field( unsigned long, ipa )
77 ),
78
79 TP_fast_assign(
80 __entry->ipa = ipa;
81 ),
82
83 TP_printk("IPA: %lx", __entry->ipa)
84);
85
71TRACE_EVENT(kvm_irq_line, 86TRACE_EVENT(kvm_irq_line,
72 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level), 87 TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
73 TP_ARGS(type, vcpu_idx, irq_num, level), 88 TP_ARGS(type, vcpu_idx, irq_num, level),
@@ -210,6 +225,39 @@ TRACE_EVENT(kvm_set_spte_hva,
210 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva) 225 TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
211); 226);
212 227
228TRACE_EVENT(kvm_age_hva,
229 TP_PROTO(unsigned long start, unsigned long end),
230 TP_ARGS(start, end),
231
232 TP_STRUCT__entry(
233 __field( unsigned long, start )
234 __field( unsigned long, end )
235 ),
236
237 TP_fast_assign(
238 __entry->start = start;
239 __entry->end = end;
240 ),
241
242 TP_printk("mmu notifier age hva: %#08lx -- %#08lx",
243 __entry->start, __entry->end)
244);
245
246TRACE_EVENT(kvm_test_age_hva,
247 TP_PROTO(unsigned long hva),
248 TP_ARGS(hva),
249
250 TP_STRUCT__entry(
251 __field( unsigned long, hva )
252 ),
253
254 TP_fast_assign(
255 __entry->hva = hva;
256 ),
257
258 TP_printk("mmu notifier test age hva: %#08lx", __entry->hva)
259);
260
213TRACE_EVENT(kvm_hvc, 261TRACE_EVENT(kvm_hvc,
214 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), 262 TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
215 TP_ARGS(vcpu_pc, r0, imm), 263 TP_ARGS(vcpu_pc, r0, imm),
diff --git a/arch/arm/lib/clear_user.S b/arch/arm/lib/clear_user.S
index 14a0d988c82c..1710fd7db2d5 100644
--- a/arch/arm/lib/clear_user.S
+++ b/arch/arm/lib/clear_user.S
@@ -47,7 +47,7 @@ USER( strnebt r2, [r0])
47ENDPROC(__clear_user) 47ENDPROC(__clear_user)
48ENDPROC(__clear_user_std) 48ENDPROC(__clear_user_std)
49 49
50 .pushsection .fixup,"ax" 50 .pushsection .text.fixup,"ax"
51 .align 0 51 .align 0
529001: ldmfd sp!, {r0, pc} 529001: ldmfd sp!, {r0, pc}
53 .popsection 53 .popsection
diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S
index a9d3db16ecb5..9648b0675a3e 100644
--- a/arch/arm/lib/copy_to_user.S
+++ b/arch/arm/lib/copy_to_user.S
@@ -100,7 +100,7 @@ WEAK(__copy_to_user)
100ENDPROC(__copy_to_user) 100ENDPROC(__copy_to_user)
101ENDPROC(__copy_to_user_std) 101ENDPROC(__copy_to_user_std)
102 102
103 .pushsection .fixup,"ax" 103 .pushsection .text.fixup,"ax"
104 .align 0 104 .align 0
105 copy_abort_preamble 105 copy_abort_preamble
106 ldmfd sp!, {r1, r2, r3} 106 ldmfd sp!, {r1, r2, r3}
diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S
index 7d08b43d2c0e..1d0957e61f89 100644
--- a/arch/arm/lib/csumpartialcopyuser.S
+++ b/arch/arm/lib/csumpartialcopyuser.S
@@ -68,7 +68,7 @@
68 * so properly, we would have to add in whatever registers were loaded before 68 * so properly, we would have to add in whatever registers were loaded before
69 * the fault, which, with the current asm above is not predictable. 69 * the fault, which, with the current asm above is not predictable.
70 */ 70 */
71 .pushsection .fixup,"ax" 71 .pushsection .text.fixup,"ax"
72 .align 4 72 .align 4
739001: mov r4, #-EFAULT 739001: mov r4, #-EFAULT
74 ldr r5, [sp, #8*4] @ *err_ptr 74 ldr r5, [sp, #8*4] @ *err_ptr
diff --git a/arch/arm/lib/delay.c b/arch/arm/lib/delay.c
index 312d43eb686a..8044591dca72 100644
--- a/arch/arm/lib/delay.c
+++ b/arch/arm/lib/delay.c
@@ -83,6 +83,12 @@ void __init register_current_timer_delay(const struct delay_timer *timer)
83 NSEC_PER_SEC, 3600); 83 NSEC_PER_SEC, 3600);
84 res = cyc_to_ns(1ULL, new_mult, new_shift); 84 res = cyc_to_ns(1ULL, new_mult, new_shift);
85 85
86 if (res > 1000) {
87 pr_err("Ignoring delay timer %ps, which has insufficient resolution of %lluns\n",
88 timer, res);
89 return;
90 }
91
86 if (!delay_calibrated && (!delay_res || (res < delay_res))) { 92 if (!delay_calibrated && (!delay_res || (res < delay_res))) {
87 pr_info("Switching to timer-based delay loop, resolution %lluns\n", res); 93 pr_info("Switching to timer-based delay loop, resolution %lluns\n", res);
88 delay_timer = timer; 94 delay_timer = timer;
diff --git a/arch/arm/mach-davinci/cpuidle.c b/arch/arm/mach-davinci/cpuidle.c
index e365c1bb1265..306ebc51599a 100644
--- a/arch/arm/mach-davinci/cpuidle.c
+++ b/arch/arm/mach-davinci/cpuidle.c
@@ -17,7 +17,6 @@
17#include <linux/cpuidle.h> 17#include <linux/cpuidle.h>
18#include <linux/io.h> 18#include <linux/io.h>
19#include <linux/export.h> 19#include <linux/export.h>
20#include <asm/proc-fns.h>
21#include <asm/cpuidle.h> 20#include <asm/cpuidle.h>
22 21
23#include <mach/cpuidle.h> 22#include <mach/cpuidle.h>
diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
index 8a275f297522..91fe97144570 100644
--- a/arch/arm/mach-dove/pcie.c
+++ b/arch/arm/mach-dove/pcie.c
@@ -155,17 +155,13 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MARVELL, PCI_ANY_ID, rc_pci_fixup);
155static struct pci_bus __init * 155static struct pci_bus __init *
156dove_pcie_scan_bus(int nr, struct pci_sys_data *sys) 156dove_pcie_scan_bus(int nr, struct pci_sys_data *sys)
157{ 157{
158 struct pci_bus *bus; 158 if (nr >= num_pcie_ports) {
159
160 if (nr < num_pcie_ports) {
161 bus = pci_scan_root_bus(NULL, sys->busnr, &pcie_ops, sys,
162 &sys->resources);
163 } else {
164 bus = NULL;
165 BUG(); 159 BUG();
160 return NULL;
166 } 161 }
167 162
168 return bus; 163 return pci_scan_root_bus(NULL, sys->busnr, &pcie_ops, sys,
164 &sys->resources);
169} 165}
170 166
171static int __init dove_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) 167static int __init dove_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
diff --git a/arch/arm/mach-exynos/exynos.c b/arch/arm/mach-exynos/exynos.c
index 9e9dfdfad9d7..f44c2e05c82e 100644
--- a/arch/arm/mach-exynos/exynos.c
+++ b/arch/arm/mach-exynos/exynos.c
@@ -166,16 +166,14 @@ static void __init exynos_init_io(void)
166 exynos_map_io(); 166 exynos_map_io();
167} 167}
168 168
169/*
170 * Apparently, these SoCs are not able to wake-up from suspend using
171 * the PMU. Too bad. Should they suddenly become capable of such a
172 * feat, the matches below should be moved to suspend.c.
173 */
169static const struct of_device_id exynos_dt_pmu_match[] = { 174static const struct of_device_id exynos_dt_pmu_match[] = {
170 { .compatible = "samsung,exynos3250-pmu" },
171 { .compatible = "samsung,exynos4210-pmu" },
172 { .compatible = "samsung,exynos4212-pmu" },
173 { .compatible = "samsung,exynos4412-pmu" },
174 { .compatible = "samsung,exynos4415-pmu" },
175 { .compatible = "samsung,exynos5250-pmu" },
176 { .compatible = "samsung,exynos5260-pmu" }, 175 { .compatible = "samsung,exynos5260-pmu" },
177 { .compatible = "samsung,exynos5410-pmu" }, 176 { .compatible = "samsung,exynos5410-pmu" },
178 { .compatible = "samsung,exynos5420-pmu" },
179 { /*sentinel*/ }, 177 { /*sentinel*/ },
180}; 178};
181 179
@@ -186,9 +184,6 @@ static void exynos_map_pmu(void)
186 np = of_find_matching_node(NULL, exynos_dt_pmu_match); 184 np = of_find_matching_node(NULL, exynos_dt_pmu_match);
187 if (np) 185 if (np)
188 pmu_base_addr = of_iomap(np, 0); 186 pmu_base_addr = of_iomap(np, 0);
189
190 if (!pmu_base_addr)
191 panic("failed to find exynos pmu register\n");
192} 187}
193 188
194static void __init exynos_init_irq(void) 189static void __init exynos_init_irq(void)
diff --git a/arch/arm/mach-exynos/sleep.S b/arch/arm/mach-exynos/sleep.S
index 31d25834b9c4..cf950790fbdc 100644
--- a/arch/arm/mach-exynos/sleep.S
+++ b/arch/arm/mach-exynos/sleep.S
@@ -23,14 +23,7 @@
23#define CPU_MASK 0xff0ffff0 23#define CPU_MASK 0xff0ffff0
24#define CPU_CORTEX_A9 0x410fc090 24#define CPU_CORTEX_A9 0x410fc090
25 25
26 /* 26 .text
27 * The following code is located into the .data section. This is to
28 * allow l2x0_regs_phys to be accessed with a relative load while we
29 * can't rely on any MMU translation. We could have put l2x0_regs_phys
30 * in the .text section as well, but some setups might insist on it to
31 * be truly read-only. (Reference from: arch/arm/kernel/sleep.S)
32 */
33 .data
34 .align 27 .align
35 28
36 /* 29 /*
@@ -69,10 +62,12 @@ ENTRY(exynos_cpu_resume_ns)
69 cmp r0, r1 62 cmp r0, r1
70 bne skip_cp15 63 bne skip_cp15
71 64
72 adr r0, cp15_save_power 65 adr r0, _cp15_save_power
73 ldr r1, [r0] 66 ldr r1, [r0]
74 adr r0, cp15_save_diag 67 ldr r1, [r0, r1]
68 adr r0, _cp15_save_diag
75 ldr r2, [r0] 69 ldr r2, [r0]
70 ldr r2, [r0, r2]
76 mov r0, #SMC_CMD_C15RESUME 71 mov r0, #SMC_CMD_C15RESUME
77 dsb 72 dsb
78 smc #0 73 smc #0
@@ -118,14 +113,20 @@ skip_l2x0:
118skip_cp15: 113skip_cp15:
119 b cpu_resume 114 b cpu_resume
120ENDPROC(exynos_cpu_resume_ns) 115ENDPROC(exynos_cpu_resume_ns)
116
117 .align
118_cp15_save_power:
119 .long cp15_save_power - .
120_cp15_save_diag:
121 .long cp15_save_diag - .
122#ifdef CONFIG_CACHE_L2X0
1231: .long l2x0_saved_regs - .
124#endif /* CONFIG_CACHE_L2X0 */
125
126 .data
121 .globl cp15_save_diag 127 .globl cp15_save_diag
122cp15_save_diag: 128cp15_save_diag:
123 .long 0 @ cp15 diagnostic 129 .long 0 @ cp15 diagnostic
124 .globl cp15_save_power 130 .globl cp15_save_power
125cp15_save_power: 131cp15_save_power:
126 .long 0 @ cp15 power control 132 .long 0 @ cp15 power control
127
128#ifdef CONFIG_CACHE_L2X0
129 .align
1301: .long l2x0_saved_regs - .
131#endif /* CONFIG_CACHE_L2X0 */
diff --git a/arch/arm/mach-exynos/suspend.c b/arch/arm/mach-exynos/suspend.c
index 318d127df147..2146d918aedd 100644
--- a/arch/arm/mach-exynos/suspend.c
+++ b/arch/arm/mach-exynos/suspend.c
@@ -18,7 +18,9 @@
18#include <linux/syscore_ops.h> 18#include <linux/syscore_ops.h>
19#include <linux/cpu_pm.h> 19#include <linux/cpu_pm.h>
20#include <linux/io.h> 20#include <linux/io.h>
21#include <linux/irqchip/arm-gic.h> 21#include <linux/irq.h>
22#include <linux/irqdomain.h>
23#include <linux/of_address.h>
22#include <linux/err.h> 24#include <linux/err.h>
23#include <linux/regulator/machine.h> 25#include <linux/regulator/machine.h>
24 26
@@ -43,8 +45,8 @@
43#define EXYNOS5420_CPU_STATE 0x28 45#define EXYNOS5420_CPU_STATE 0x28
44 46
45/** 47/**
46 * struct exynos_wkup_irq - Exynos GIC to PMU IRQ mapping 48 * struct exynos_wkup_irq - PMU IRQ to mask mapping
47 * @hwirq: Hardware IRQ signal of the GIC 49 * @hwirq: Hardware IRQ signal of the PMU
48 * @mask: Mask in PMU wake-up mask register 50 * @mask: Mask in PMU wake-up mask register
49 */ 51 */
50struct exynos_wkup_irq { 52struct exynos_wkup_irq {
@@ -93,14 +95,14 @@ static const struct exynos_wkup_irq exynos3250_wkup_irq[] = {
93}; 95};
94 96
95static const struct exynos_wkup_irq exynos4_wkup_irq[] = { 97static const struct exynos_wkup_irq exynos4_wkup_irq[] = {
96 { 76, BIT(1) }, /* RTC alarm */ 98 { 44, BIT(1) }, /* RTC alarm */
97 { 77, BIT(2) }, /* RTC tick */ 99 { 45, BIT(2) }, /* RTC tick */
98 { /* sentinel */ }, 100 { /* sentinel */ },
99}; 101};
100 102
101static const struct exynos_wkup_irq exynos5250_wkup_irq[] = { 103static const struct exynos_wkup_irq exynos5250_wkup_irq[] = {
102 { 75, BIT(1) }, /* RTC alarm */ 104 { 43, BIT(1) }, /* RTC alarm */
103 { 76, BIT(2) }, /* RTC tick */ 105 { 44, BIT(2) }, /* RTC tick */
104 { /* sentinel */ }, 106 { /* sentinel */ },
105}; 107};
106 108
@@ -167,6 +169,113 @@ static int exynos_irq_set_wake(struct irq_data *data, unsigned int state)
167 return -ENOENT; 169 return -ENOENT;
168} 170}
169 171
172static struct irq_chip exynos_pmu_chip = {
173 .name = "PMU",
174 .irq_eoi = irq_chip_eoi_parent,
175 .irq_mask = irq_chip_mask_parent,
176 .irq_unmask = irq_chip_unmask_parent,
177 .irq_retrigger = irq_chip_retrigger_hierarchy,
178 .irq_set_wake = exynos_irq_set_wake,
179#ifdef CONFIG_SMP
180 .irq_set_affinity = irq_chip_set_affinity_parent,
181#endif
182};
183
184static int exynos_pmu_domain_xlate(struct irq_domain *domain,
185 struct device_node *controller,
186 const u32 *intspec,
187 unsigned int intsize,
188 unsigned long *out_hwirq,
189 unsigned int *out_type)
190{
191 if (domain->of_node != controller)
192 return -EINVAL; /* Shouldn't happen, really... */
193 if (intsize != 3)
194 return -EINVAL; /* Not GIC compliant */
195 if (intspec[0] != 0)
196 return -EINVAL; /* No PPI should point to this domain */
197
198 *out_hwirq = intspec[1];
199 *out_type = intspec[2];
200 return 0;
201}
202
203static int exynos_pmu_domain_alloc(struct irq_domain *domain,
204 unsigned int virq,
205 unsigned int nr_irqs, void *data)
206{
207 struct of_phandle_args *args = data;
208 struct of_phandle_args parent_args;
209 irq_hw_number_t hwirq;
210 int i;
211
212 if (args->args_count != 3)
213 return -EINVAL; /* Not GIC compliant */
214 if (args->args[0] != 0)
215 return -EINVAL; /* No PPI should point to this domain */
216
217 hwirq = args->args[1];
218
219 for (i = 0; i < nr_irqs; i++)
220 irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
221 &exynos_pmu_chip, NULL);
222
223 parent_args = *args;
224 parent_args.np = domain->parent->of_node;
225 return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &parent_args);
226}
227
228static struct irq_domain_ops exynos_pmu_domain_ops = {
229 .xlate = exynos_pmu_domain_xlate,
230 .alloc = exynos_pmu_domain_alloc,
231 .free = irq_domain_free_irqs_common,
232};
233
234static int __init exynos_pmu_irq_init(struct device_node *node,
235 struct device_node *parent)
236{
237 struct irq_domain *parent_domain, *domain;
238
239 if (!parent) {
240 pr_err("%s: no parent, giving up\n", node->full_name);
241 return -ENODEV;
242 }
243
244 parent_domain = irq_find_host(parent);
245 if (!parent_domain) {
246 pr_err("%s: unable to obtain parent domain\n", node->full_name);
247 return -ENXIO;
248 }
249
250 pmu_base_addr = of_iomap(node, 0);
251
252 if (!pmu_base_addr) {
253 pr_err("%s: failed to find exynos pmu register\n",
254 node->full_name);
255 return -ENOMEM;
256 }
257
258 domain = irq_domain_add_hierarchy(parent_domain, 0, 0,
259 node, &exynos_pmu_domain_ops,
260 NULL);
261 if (!domain) {
262 iounmap(pmu_base_addr);
263 return -ENOMEM;
264 }
265
266 return 0;
267}
268
269#define EXYNOS_PMU_IRQ(symbol, name) OF_DECLARE_2(irqchip, symbol, name, exynos_pmu_irq_init)
270
271EXYNOS_PMU_IRQ(exynos3250_pmu_irq, "samsung,exynos3250-pmu");
272EXYNOS_PMU_IRQ(exynos4210_pmu_irq, "samsung,exynos4210-pmu");
273EXYNOS_PMU_IRQ(exynos4212_pmu_irq, "samsung,exynos4212-pmu");
274EXYNOS_PMU_IRQ(exynos4412_pmu_irq, "samsung,exynos4412-pmu");
275EXYNOS_PMU_IRQ(exynos4415_pmu_irq, "samsung,exynos4415-pmu");
276EXYNOS_PMU_IRQ(exynos5250_pmu_irq, "samsung,exynos5250-pmu");
277EXYNOS_PMU_IRQ(exynos5420_pmu_irq, "samsung,exynos5420-pmu");
278
170static int exynos_cpu_do_idle(void) 279static int exynos_cpu_do_idle(void)
171{ 280{
172 /* issue the standby signal into the pm unit. */ 281 /* issue the standby signal into the pm unit. */
@@ -615,17 +724,19 @@ static struct syscore_ops exynos_pm_syscore_ops;
615void __init exynos_pm_init(void) 724void __init exynos_pm_init(void)
616{ 725{
617 const struct of_device_id *match; 726 const struct of_device_id *match;
727 struct device_node *np;
618 u32 tmp; 728 u32 tmp;
619 729
620 of_find_matching_node_and_match(NULL, exynos_pmu_of_device_ids, &match); 730 np = of_find_matching_node_and_match(NULL, exynos_pmu_of_device_ids, &match);
621 if (!match) { 731 if (!np) {
622 pr_err("Failed to find PMU node\n"); 732 pr_err("Failed to find PMU node\n");
623 return; 733 return;
624 } 734 }
625 pm_data = (struct exynos_pm_data *) match->data;
626 735
627 /* Platform-specific GIC callback */ 736 if (WARN_ON(!of_find_property(np, "interrupt-controller", NULL)))
628 gic_arch_extn.irq_set_wake = exynos_irq_set_wake; 737 pr_warn("Outdated DT detected, suspend/resume will NOT work\n");
738
739 pm_data = (struct exynos_pm_data *) match->data;
629 740
630 /* All wakeup disable */ 741 /* All wakeup disable */
631 tmp = pmu_raw_readl(S5P_WAKEUP_MASK); 742 tmp = pmu_raw_readl(S5P_WAKEUP_MASK);
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig
index e8627e04e1e6..c8dffcee9736 100644
--- a/arch/arm/mach-imx/Kconfig
+++ b/arch/arm/mach-imx/Kconfig
@@ -631,6 +631,7 @@ config SOC_IMX6SX
631 631
632config SOC_VF610 632config SOC_VF610
633 bool "Vybrid Family VF610 support" 633 bool "Vybrid Family VF610 support"
634 select IRQ_DOMAIN_HIERARCHY
634 select ARM_GIC 635 select ARM_GIC
635 select PINCTRL_VF610 636 select PINCTRL_VF610
636 select PL310_ERRATA_769419 if CACHE_L2X0 637 select PL310_ERRATA_769419 if CACHE_L2X0
diff --git a/arch/arm/mach-imx/cpuidle-imx6q.c b/arch/arm/mach-imx/cpuidle-imx6q.c
index d76d08623f9f..8e21ccc1eda2 100644
--- a/arch/arm/mach-imx/cpuidle-imx6q.c
+++ b/arch/arm/mach-imx/cpuidle-imx6q.c
@@ -9,7 +9,6 @@
9#include <linux/cpuidle.h> 9#include <linux/cpuidle.h>
10#include <linux/module.h> 10#include <linux/module.h>
11#include <asm/cpuidle.h> 11#include <asm/cpuidle.h>
12#include <asm/proc-fns.h>
13 12
14#include "common.h" 13#include "common.h"
15#include "cpuidle.h" 14#include "cpuidle.h"
diff --git a/arch/arm/mach-imx/cpuidle-imx6sl.c b/arch/arm/mach-imx/cpuidle-imx6sl.c
index 7d92e6584551..5742a9fd1ef2 100644
--- a/arch/arm/mach-imx/cpuidle-imx6sl.c
+++ b/arch/arm/mach-imx/cpuidle-imx6sl.c
@@ -9,7 +9,6 @@
9#include <linux/cpuidle.h> 9#include <linux/cpuidle.h>
10#include <linux/module.h> 10#include <linux/module.h>
11#include <asm/cpuidle.h> 11#include <asm/cpuidle.h>
12#include <asm/proc-fns.h>
13 12
14#include "common.h" 13#include "common.h"
15#include "cpuidle.h" 14#include "cpuidle.h"
diff --git a/arch/arm/mach-imx/cpuidle-imx6sx.c b/arch/arm/mach-imx/cpuidle-imx6sx.c
index 5a36722b089d..2c9f1a8bf245 100644
--- a/arch/arm/mach-imx/cpuidle-imx6sx.c
+++ b/arch/arm/mach-imx/cpuidle-imx6sx.c
@@ -10,7 +10,6 @@
10#include <linux/cpu_pm.h> 10#include <linux/cpu_pm.h>
11#include <linux/module.h> 11#include <linux/module.h>
12#include <asm/cpuidle.h> 12#include <asm/cpuidle.h>
13#include <asm/proc-fns.h>
14#include <asm/suspend.h> 13#include <asm/suspend.h>
15 14
16#include "common.h" 15#include "common.h"
diff --git a/arch/arm/mach-mv78xx0/pcie.c b/arch/arm/mach-mv78xx0/pcie.c
index 445e553f4a28..097ea4cb1136 100644
--- a/arch/arm/mach-mv78xx0/pcie.c
+++ b/arch/arm/mach-mv78xx0/pcie.c
@@ -197,17 +197,13 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MARVELL, PCI_ANY_ID, rc_pci_fixup);
197static struct pci_bus __init * 197static struct pci_bus __init *
198mv78xx0_pcie_scan_bus(int nr, struct pci_sys_data *sys) 198mv78xx0_pcie_scan_bus(int nr, struct pci_sys_data *sys)
199{ 199{
200 struct pci_bus *bus; 200 if (nr >= num_pcie_ports) {
201
202 if (nr < num_pcie_ports) {
203 bus = pci_scan_root_bus(NULL, sys->busnr, &pcie_ops, sys,
204 &sys->resources);
205 } else {
206 bus = NULL;
207 BUG(); 201 BUG();
202 return NULL;
208 } 203 }
209 204
210 return bus; 205 return pci_scan_root_bus(NULL, sys->busnr, &pcie_ops, sys,
206 &sys->resources);
211} 207}
212 208
213static int __init mv78xx0_pcie_map_irq(const struct pci_dev *dev, u8 slot, 209static int __init mv78xx0_pcie_map_irq(const struct pci_dev *dev, u8 slot,
diff --git a/arch/arm/mach-omap2/cpuidle44xx.c b/arch/arm/mach-omap2/cpuidle44xx.c
index 01e398a868bc..4b8e9f4d59ea 100644
--- a/arch/arm/mach-omap2/cpuidle44xx.c
+++ b/arch/arm/mach-omap2/cpuidle44xx.c
@@ -14,10 +14,9 @@
14#include <linux/cpuidle.h> 14#include <linux/cpuidle.h>
15#include <linux/cpu_pm.h> 15#include <linux/cpu_pm.h>
16#include <linux/export.h> 16#include <linux/export.h>
17#include <linux/clockchips.h> 17#include <linux/tick.h>
18 18
19#include <asm/cpuidle.h> 19#include <asm/cpuidle.h>
20#include <asm/proc-fns.h>
21 20
22#include "common.h" 21#include "common.h"
23#include "pm.h" 22#include "pm.h"
@@ -84,7 +83,6 @@ static int omap_enter_idle_coupled(struct cpuidle_device *dev,
84{ 83{
85 struct idle_statedata *cx = state_ptr + index; 84 struct idle_statedata *cx = state_ptr + index;
86 u32 mpuss_can_lose_context = 0; 85 u32 mpuss_can_lose_context = 0;
87 int cpu_id = smp_processor_id();
88 86
89 /* 87 /*
90 * CPU0 has to wait and stay ON until CPU1 is OFF state. 88 * CPU0 has to wait and stay ON until CPU1 is OFF state.
@@ -112,7 +110,7 @@ static int omap_enter_idle_coupled(struct cpuidle_device *dev,
112 mpuss_can_lose_context = (cx->mpu_state == PWRDM_POWER_RET) && 110 mpuss_can_lose_context = (cx->mpu_state == PWRDM_POWER_RET) &&
113 (cx->mpu_logic_state == PWRDM_POWER_OFF); 111 (cx->mpu_logic_state == PWRDM_POWER_OFF);
114 112
115 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu_id); 113 tick_broadcast_enter();
116 114
117 /* 115 /*
118 * Call idle CPU PM enter notifier chain so that 116 * Call idle CPU PM enter notifier chain so that
@@ -169,7 +167,7 @@ static int omap_enter_idle_coupled(struct cpuidle_device *dev,
169 if (dev->cpu == 0 && mpuss_can_lose_context) 167 if (dev->cpu == 0 && mpuss_can_lose_context)
170 cpu_cluster_pm_exit(); 168 cpu_cluster_pm_exit();
171 169
172 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu_id); 170 tick_broadcast_exit();
173 171
174fail: 172fail:
175 cpuidle_coupled_parallel_barrier(dev, &abort_barrier); 173 cpuidle_coupled_parallel_barrier(dev, &abort_barrier);
@@ -184,8 +182,7 @@ fail:
184 */ 182 */
185static void omap_setup_broadcast_timer(void *arg) 183static void omap_setup_broadcast_timer(void *arg)
186{ 184{
187 int cpu = smp_processor_id(); 185 tick_broadcast_enable();
188 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ON, &cpu);
189} 186}
190 187
191static struct cpuidle_driver omap4_idle_driver = { 188static struct cpuidle_driver omap4_idle_driver = {
diff --git a/arch/arm/mach-omap2/hsmmc.c b/arch/arm/mach-omap2/hsmmc.c
index dc6e79c4484a..9a8611ab5dfa 100644
--- a/arch/arm/mach-omap2/hsmmc.c
+++ b/arch/arm/mach-omap2/hsmmc.c
@@ -150,9 +150,13 @@ static int nop_mmc_set_power(struct device *dev, int power_on, int vdd)
150static inline void omap_hsmmc_mux(struct omap_hsmmc_platform_data 150static inline void omap_hsmmc_mux(struct omap_hsmmc_platform_data
151 *mmc_controller, int controller_nr) 151 *mmc_controller, int controller_nr)
152{ 152{
153 if (gpio_is_valid(mmc_controller->switch_pin) && 153 if (gpio_is_valid(mmc_controller->gpio_cd) &&
154 (mmc_controller->switch_pin < OMAP_MAX_GPIO_LINES)) 154 (mmc_controller->gpio_cd < OMAP_MAX_GPIO_LINES))
155 omap_mux_init_gpio(mmc_controller->switch_pin, 155 omap_mux_init_gpio(mmc_controller->gpio_cd,
156 OMAP_PIN_INPUT_PULLUP);
157 if (gpio_is_valid(mmc_controller->gpio_cod) &&
158 (mmc_controller->gpio_cod < OMAP_MAX_GPIO_LINES))
159 omap_mux_init_gpio(mmc_controller->gpio_cod,
156 OMAP_PIN_INPUT_PULLUP); 160 OMAP_PIN_INPUT_PULLUP);
157 if (gpio_is_valid(mmc_controller->gpio_wp) && 161 if (gpio_is_valid(mmc_controller->gpio_wp) &&
158 (mmc_controller->gpio_wp < OMAP_MAX_GPIO_LINES)) 162 (mmc_controller->gpio_wp < OMAP_MAX_GPIO_LINES))
@@ -250,15 +254,20 @@ static int __init omap_hsmmc_pdata_init(struct omap2_hsmmc_info *c,
250 mmc->internal_clock = !c->ext_clock; 254 mmc->internal_clock = !c->ext_clock;
251 mmc->reg_offset = 0; 255 mmc->reg_offset = 0;
252 256
253 mmc->switch_pin = c->gpio_cd; 257 if (c->cover_only) {
258 /* detect if mobile phone cover removed */
259 mmc->gpio_cd = -EINVAL;
260 mmc->gpio_cod = c->gpio_cd;
261 } else {
262 /* card detect pin on the mmc socket itself */
263 mmc->gpio_cd = c->gpio_cd;
264 mmc->gpio_cod = -EINVAL;
265 }
254 mmc->gpio_wp = c->gpio_wp; 266 mmc->gpio_wp = c->gpio_wp;
255 267
256 mmc->remux = c->remux; 268 mmc->remux = c->remux;
257 mmc->init_card = c->init_card; 269 mmc->init_card = c->init_card;
258 270
259 if (c->cover_only)
260 mmc->cover = 1;
261
262 if (c->nonremovable) 271 if (c->nonremovable)
263 mmc->nonremovable = 1; 272 mmc->nonremovable = 1;
264 273
@@ -358,7 +367,15 @@ void omap_hsmmc_late_init(struct omap2_hsmmc_info *c)
358 if (!mmc_pdata) 367 if (!mmc_pdata)
359 continue; 368 continue;
360 369
361 mmc_pdata->switch_pin = c->gpio_cd; 370 if (c->cover_only) {
371 /* detect if mobile phone cover removed */
372 mmc_pdata->gpio_cd = -EINVAL;
373 mmc_pdata->gpio_cod = c->gpio_cd;
374 } else {
375 /* card detect pin on the mmc socket itself */
376 mmc_pdata->gpio_cd = c->gpio_cd;
377 mmc_pdata->gpio_cod = -EINVAL;
378 }
362 mmc_pdata->gpio_wp = c->gpio_wp; 379 mmc_pdata->gpio_wp = c->gpio_wp;
363 380
364 res = omap_device_register(pdev); 381 res = omap_device_register(pdev);
diff --git a/arch/arm/mach-omap2/omap-wakeupgen.c b/arch/arm/mach-omap2/omap-wakeupgen.c
index f961c46453b9..3b56722dfd8a 100644
--- a/arch/arm/mach-omap2/omap-wakeupgen.c
+++ b/arch/arm/mach-omap2/omap-wakeupgen.c
@@ -20,11 +20,12 @@
20#include <linux/init.h> 20#include <linux/init.h>
21#include <linux/io.h> 21#include <linux/io.h>
22#include <linux/irq.h> 22#include <linux/irq.h>
23#include <linux/irqdomain.h>
24#include <linux/of_address.h>
23#include <linux/platform_device.h> 25#include <linux/platform_device.h>
24#include <linux/cpu.h> 26#include <linux/cpu.h>
25#include <linux/notifier.h> 27#include <linux/notifier.h>
26#include <linux/cpu_pm.h> 28#include <linux/cpu_pm.h>
27#include <linux/irqchip/arm-gic.h>
28 29
29#include "omap-wakeupgen.h" 30#include "omap-wakeupgen.h"
30#include "omap-secure.h" 31#include "omap-secure.h"
@@ -78,29 +79,12 @@ static inline void sar_writel(u32 val, u32 offset, u8 idx)
78 79
79static inline int _wakeupgen_get_irq_info(u32 irq, u32 *bit_posn, u8 *reg_index) 80static inline int _wakeupgen_get_irq_info(u32 irq, u32 *bit_posn, u8 *reg_index)
80{ 81{
81 unsigned int spi_irq;
82
83 /*
84 * PPIs and SGIs are not supported.
85 */
86 if (irq < OMAP44XX_IRQ_GIC_START)
87 return -EINVAL;
88
89 /*
90 * Subtract the GIC offset.
91 */
92 spi_irq = irq - OMAP44XX_IRQ_GIC_START;
93 if (spi_irq > MAX_IRQS) {
94 pr_err("omap wakeupGen: Invalid IRQ%d\n", irq);
95 return -EINVAL;
96 }
97
98 /* 82 /*
99 * Each WakeupGen register controls 32 interrupt. 83 * Each WakeupGen register controls 32 interrupt.
100 * i.e. 1 bit per SPI IRQ 84 * i.e. 1 bit per SPI IRQ
101 */ 85 */
102 *reg_index = spi_irq >> 5; 86 *reg_index = irq >> 5;
103 *bit_posn = spi_irq %= 32; 87 *bit_posn = irq %= 32;
104 88
105 return 0; 89 return 0;
106} 90}
@@ -141,6 +125,7 @@ static void wakeupgen_mask(struct irq_data *d)
141 raw_spin_lock_irqsave(&wakeupgen_lock, flags); 125 raw_spin_lock_irqsave(&wakeupgen_lock, flags);
142 _wakeupgen_clear(d->hwirq, irq_target_cpu[d->hwirq]); 126 _wakeupgen_clear(d->hwirq, irq_target_cpu[d->hwirq]);
143 raw_spin_unlock_irqrestore(&wakeupgen_lock, flags); 127 raw_spin_unlock_irqrestore(&wakeupgen_lock, flags);
128 irq_chip_mask_parent(d);
144} 129}
145 130
146/* 131/*
@@ -153,6 +138,7 @@ static void wakeupgen_unmask(struct irq_data *d)
153 raw_spin_lock_irqsave(&wakeupgen_lock, flags); 138 raw_spin_lock_irqsave(&wakeupgen_lock, flags);
154 _wakeupgen_set(d->hwirq, irq_target_cpu[d->hwirq]); 139 _wakeupgen_set(d->hwirq, irq_target_cpu[d->hwirq]);
155 raw_spin_unlock_irqrestore(&wakeupgen_lock, flags); 140 raw_spin_unlock_irqrestore(&wakeupgen_lock, flags);
141 irq_chip_unmask_parent(d);
156} 142}
157 143
158#ifdef CONFIG_HOTPLUG_CPU 144#ifdef CONFIG_HOTPLUG_CPU
@@ -400,15 +386,91 @@ int omap_secure_apis_support(void)
400 return omap_secure_apis; 386 return omap_secure_apis;
401} 387}
402 388
389static struct irq_chip wakeupgen_chip = {
390 .name = "WUGEN",
391 .irq_eoi = irq_chip_eoi_parent,
392 .irq_mask = wakeupgen_mask,
393 .irq_unmask = wakeupgen_unmask,
394 .irq_retrigger = irq_chip_retrigger_hierarchy,
395 .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
396#ifdef CONFIG_SMP
397 .irq_set_affinity = irq_chip_set_affinity_parent,
398#endif
399};
400
401static int wakeupgen_domain_xlate(struct irq_domain *domain,
402 struct device_node *controller,
403 const u32 *intspec,
404 unsigned int intsize,
405 unsigned long *out_hwirq,
406 unsigned int *out_type)
407{
408 if (domain->of_node != controller)
409 return -EINVAL; /* Shouldn't happen, really... */
410 if (intsize != 3)
411 return -EINVAL; /* Not GIC compliant */
412 if (intspec[0] != 0)
413 return -EINVAL; /* No PPI should point to this domain */
414
415 *out_hwirq = intspec[1];
416 *out_type = intspec[2];
417 return 0;
418}
419
420static int wakeupgen_domain_alloc(struct irq_domain *domain,
421 unsigned int virq,
422 unsigned int nr_irqs, void *data)
423{
424 struct of_phandle_args *args = data;
425 struct of_phandle_args parent_args;
426 irq_hw_number_t hwirq;
427 int i;
428
429 if (args->args_count != 3)
430 return -EINVAL; /* Not GIC compliant */
431 if (args->args[0] != 0)
432 return -EINVAL; /* No PPI should point to this domain */
433
434 hwirq = args->args[1];
435 if (hwirq >= MAX_IRQS)
436 return -EINVAL; /* Can't deal with this */
437
438 for (i = 0; i < nr_irqs; i++)
439 irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
440 &wakeupgen_chip, NULL);
441
442 parent_args = *args;
443 parent_args.np = domain->parent->of_node;
444 return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &parent_args);
445}
446
447static struct irq_domain_ops wakeupgen_domain_ops = {
448 .xlate = wakeupgen_domain_xlate,
449 .alloc = wakeupgen_domain_alloc,
450 .free = irq_domain_free_irqs_common,
451};
452
403/* 453/*
404 * Initialise the wakeupgen module. 454 * Initialise the wakeupgen module.
405 */ 455 */
406int __init omap_wakeupgen_init(void) 456static int __init wakeupgen_init(struct device_node *node,
457 struct device_node *parent)
407{ 458{
459 struct irq_domain *parent_domain, *domain;
408 int i; 460 int i;
409 unsigned int boot_cpu = smp_processor_id(); 461 unsigned int boot_cpu = smp_processor_id();
410 u32 val; 462 u32 val;
411 463
464 if (!parent) {
465 pr_err("%s: no parent, giving up\n", node->full_name);
466 return -ENODEV;
467 }
468
469 parent_domain = irq_find_host(parent);
470 if (!parent_domain) {
471 pr_err("%s: unable to obtain parent domain\n", node->full_name);
472 return -ENXIO;
473 }
412 /* Not supported on OMAP4 ES1.0 silicon */ 474 /* Not supported on OMAP4 ES1.0 silicon */
413 if (omap_rev() == OMAP4430_REV_ES1_0) { 475 if (omap_rev() == OMAP4430_REV_ES1_0) {
414 WARN(1, "WakeupGen: Not supported on OMAP4430 ES1.0\n"); 476 WARN(1, "WakeupGen: Not supported on OMAP4430 ES1.0\n");
@@ -416,7 +478,7 @@ int __init omap_wakeupgen_init(void)
416 } 478 }
417 479
418 /* Static mapping, never released */ 480 /* Static mapping, never released */
419 wakeupgen_base = ioremap(OMAP_WKUPGEN_BASE, SZ_4K); 481 wakeupgen_base = of_iomap(node, 0);
420 if (WARN_ON(!wakeupgen_base)) 482 if (WARN_ON(!wakeupgen_base))
421 return -ENOMEM; 483 return -ENOMEM;
422 484
@@ -429,6 +491,14 @@ int __init omap_wakeupgen_init(void)
429 max_irqs = AM43XX_IRQS; 491 max_irqs = AM43XX_IRQS;
430 } 492 }
431 493
494 domain = irq_domain_add_hierarchy(parent_domain, 0, max_irqs,
495 node, &wakeupgen_domain_ops,
496 NULL);
497 if (!domain) {
498 iounmap(wakeupgen_base);
499 return -ENOMEM;
500 }
501
432 /* Clear all IRQ bitmasks at wakeupGen level */ 502 /* Clear all IRQ bitmasks at wakeupGen level */
433 for (i = 0; i < irq_banks; i++) { 503 for (i = 0; i < irq_banks; i++) {
434 wakeupgen_writel(0, i, CPU0_ID); 504 wakeupgen_writel(0, i, CPU0_ID);
@@ -437,14 +507,6 @@ int __init omap_wakeupgen_init(void)
437 } 507 }
438 508
439 /* 509 /*
440 * Override GIC architecture specific functions to add
441 * OMAP WakeupGen interrupt controller along with GIC
442 */
443 gic_arch_extn.irq_mask = wakeupgen_mask;
444 gic_arch_extn.irq_unmask = wakeupgen_unmask;
445 gic_arch_extn.flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SKIP_SET_WAKE;
446
447 /*
448 * FIXME: Add support to set_smp_affinity() once the core 510 * FIXME: Add support to set_smp_affinity() once the core
449 * GIC code has necessary hooks in place. 511 * GIC code has necessary hooks in place.
450 */ 512 */
@@ -474,3 +536,9 @@ int __init omap_wakeupgen_init(void)
474 536
475 return 0; 537 return 0;
476} 538}
539
540/*
541 * We cannot use the IRQCHIP_DECLARE macro that lives in
542 * drivers/irqchip, so we're forced to roll our own. Not very nice.
543 */
544OF_DECLARE_2(irqchip, ti_wakeupgen, "ti,omap4-wugen-mpu", wakeupgen_init);
diff --git a/arch/arm/mach-omap2/omap-wakeupgen.h b/arch/arm/mach-omap2/omap-wakeupgen.h
index b3c8eccfae79..a3491ad12368 100644
--- a/arch/arm/mach-omap2/omap-wakeupgen.h
+++ b/arch/arm/mach-omap2/omap-wakeupgen.h
@@ -33,7 +33,6 @@
33#define OMAP_TIMESTAMPCYCLELO 0xc08 33#define OMAP_TIMESTAMPCYCLELO 0xc08
34#define OMAP_TIMESTAMPCYCLEHI 0xc0c 34#define OMAP_TIMESTAMPCYCLEHI 0xc0c
35 35
36extern int __init omap_wakeupgen_init(void);
37extern void __iomem *omap_get_wakeupgen_base(void); 36extern void __iomem *omap_get_wakeupgen_base(void);
38extern int omap_secure_apis_support(void); 37extern int omap_secure_apis_support(void);
39#endif 38#endif
diff --git a/arch/arm/mach-omap2/omap4-common.c b/arch/arm/mach-omap2/omap4-common.c
index cee0fe1ee6ff..7bb116a6f86f 100644
--- a/arch/arm/mach-omap2/omap4-common.c
+++ b/arch/arm/mach-omap2/omap4-common.c
@@ -22,7 +22,6 @@
22#include <linux/of_platform.h> 22#include <linux/of_platform.h>
23#include <linux/export.h> 23#include <linux/export.h>
24#include <linux/irqchip/arm-gic.h> 24#include <linux/irqchip/arm-gic.h>
25#include <linux/irqchip/irq-crossbar.h>
26#include <linux/of_address.h> 25#include <linux/of_address.h>
27#include <linux/reboot.h> 26#include <linux/reboot.h>
28#include <linux/genalloc.h> 27#include <linux/genalloc.h>
@@ -242,26 +241,26 @@ static int __init omap4_sar_ram_init(void)
242} 241}
243omap_early_initcall(omap4_sar_ram_init); 242omap_early_initcall(omap4_sar_ram_init);
244 243
245static const struct of_device_id gic_match[] = { 244static const struct of_device_id intc_match[] = {
246 { .compatible = "arm,cortex-a9-gic", }, 245 { .compatible = "ti,omap4-wugen-mpu", },
247 { .compatible = "arm,cortex-a15-gic", }, 246 { .compatible = "ti,omap5-wugen-mpu", },
248 { }, 247 { },
249}; 248};
250 249
251static struct device_node *gic_node; 250static struct device_node *intc_node;
252 251
253unsigned int omap4_xlate_irq(unsigned int hwirq) 252unsigned int omap4_xlate_irq(unsigned int hwirq)
254{ 253{
255 struct of_phandle_args irq_data; 254 struct of_phandle_args irq_data;
256 unsigned int irq; 255 unsigned int irq;
257 256
258 if (!gic_node) 257 if (!intc_node)
259 gic_node = of_find_matching_node(NULL, gic_match); 258 intc_node = of_find_matching_node(NULL, intc_match);
260 259
261 if (WARN_ON(!gic_node)) 260 if (WARN_ON(!intc_node))
262 return hwirq; 261 return hwirq;
263 262
264 irq_data.np = gic_node; 263 irq_data.np = intc_node;
265 irq_data.args_count = 3; 264 irq_data.args_count = 3;
266 irq_data.args[0] = 0; 265 irq_data.args[0] = 0;
267 irq_data.args[1] = hwirq - OMAP44XX_IRQ_GIC_START; 266 irq_data.args[1] = hwirq - OMAP44XX_IRQ_GIC_START;
@@ -278,6 +277,12 @@ void __init omap_gic_of_init(void)
278{ 277{
279 struct device_node *np; 278 struct device_node *np;
280 279
280 intc_node = of_find_matching_node(NULL, intc_match);
281 if (WARN_ON(!intc_node)) {
282 pr_err("No WUGEN found in DT, system will misbehave.\n");
283 pr_err("UPDATE YOUR DEVICE TREE!\n");
284 }
285
281 /* Extract GIC distributor and TWD bases for OMAP4460 ROM Errata WA */ 286 /* Extract GIC distributor and TWD bases for OMAP4460 ROM Errata WA */
282 if (!cpu_is_omap446x()) 287 if (!cpu_is_omap446x())
283 goto skip_errata_init; 288 goto skip_errata_init;
@@ -291,9 +296,5 @@ void __init omap_gic_of_init(void)
291 WARN_ON(!twd_base); 296 WARN_ON(!twd_base);
292 297
293skip_errata_init: 298skip_errata_init:
294 omap_wakeupgen_init();
295#ifdef CONFIG_IRQ_CROSSBAR
296 irqcrossbar_init();
297#endif
298 irqchip_init(); 299 irqchip_init();
299} 300}
diff --git a/arch/arm/mach-orion5x/pci.c b/arch/arm/mach-orion5x/pci.c
index 87a12d6930ff..b02f3947be51 100644
--- a/arch/arm/mach-orion5x/pci.c
+++ b/arch/arm/mach-orion5x/pci.c
@@ -540,37 +540,33 @@ void __init orion5x_pci_set_cardbus_mode(void)
540 540
541int __init orion5x_pci_sys_setup(int nr, struct pci_sys_data *sys) 541int __init orion5x_pci_sys_setup(int nr, struct pci_sys_data *sys)
542{ 542{
543 int ret = 0;
544
545 vga_base = ORION5X_PCIE_MEM_PHYS_BASE; 543 vga_base = ORION5X_PCIE_MEM_PHYS_BASE;
546 544
547 if (nr == 0) { 545 if (nr == 0) {
548 orion_pcie_set_local_bus_nr(PCIE_BASE, sys->busnr); 546 orion_pcie_set_local_bus_nr(PCIE_BASE, sys->busnr);
549 ret = pcie_setup(sys); 547 return pcie_setup(sys);
550 } else if (nr == 1 && !orion5x_pci_disabled) { 548 }
549
550 if (nr == 1 && !orion5x_pci_disabled) {
551 orion5x_pci_set_bus_nr(sys->busnr); 551 orion5x_pci_set_bus_nr(sys->busnr);
552 ret = pci_setup(sys); 552 return pci_setup(sys);
553 } 553 }
554 554
555 return ret; 555 return 0;
556} 556}
557 557
558struct pci_bus __init *orion5x_pci_sys_scan_bus(int nr, struct pci_sys_data *sys) 558struct pci_bus __init *orion5x_pci_sys_scan_bus(int nr, struct pci_sys_data *sys)
559{ 559{
560 struct pci_bus *bus; 560 if (nr == 0)
561 return pci_scan_root_bus(NULL, sys->busnr, &pcie_ops, sys,
562 &sys->resources);
561 563
562 if (nr == 0) { 564 if (nr == 1 && !orion5x_pci_disabled)
563 bus = pci_scan_root_bus(NULL, sys->busnr, &pcie_ops, sys, 565 return pci_scan_root_bus(NULL, sys->busnr, &pci_ops, sys,
564 &sys->resources); 566 &sys->resources);
565 } else if (nr == 1 && !orion5x_pci_disabled) {
566 bus = pci_scan_root_bus(NULL, sys->busnr, &pci_ops, sys,
567 &sys->resources);
568 } else {
569 bus = NULL;
570 BUG();
571 }
572 567
573 return bus; 568 BUG();
569 return NULL;
574} 570}
575 571
576int __init orion5x_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) 572int __init orion5x_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
diff --git a/arch/arm/mach-pxa/raumfeld.c b/arch/arm/mach-pxa/raumfeld.c
index a762b23ac830..6dc4f025e674 100644
--- a/arch/arm/mach-pxa/raumfeld.c
+++ b/arch/arm/mach-pxa/raumfeld.c
@@ -758,8 +758,10 @@ static void raumfeld_power_signal_charged(void)
758 struct power_supply *psy = 758 struct power_supply *psy =
759 power_supply_get_by_name(raumfeld_power_supplicants[0]); 759 power_supply_get_by_name(raumfeld_power_supplicants[0]);
760 760
761 if (psy) 761 if (psy) {
762 power_supply_set_battery_charged(psy); 762 power_supply_set_battery_charged(psy);
763 power_supply_put(psy);
764 }
763} 765}
764 766
765static int raumfeld_power_resume(void) 767static int raumfeld_power_resume(void)
diff --git a/arch/arm/mach-s3c64xx/cpuidle.c b/arch/arm/mach-s3c64xx/cpuidle.c
index 2eb072440dfa..93aa8cb70195 100644
--- a/arch/arm/mach-s3c64xx/cpuidle.c
+++ b/arch/arm/mach-s3c64xx/cpuidle.c
@@ -16,7 +16,7 @@
16#include <linux/export.h> 16#include <linux/export.h>
17#include <linux/time.h> 17#include <linux/time.h>
18 18
19#include <asm/proc-fns.h> 19#include <asm/cpuidle.h>
20 20
21#include <mach/map.h> 21#include <mach/map.h>
22 22
diff --git a/arch/arm/mach-s5pv210/sleep.S b/arch/arm/mach-s5pv210/sleep.S
index 7c43ddd33ba8..dfbfc0f7f8b8 100644
--- a/arch/arm/mach-s5pv210/sleep.S
+++ b/arch/arm/mach-s5pv210/sleep.S
@@ -14,7 +14,7 @@
14 14
15#include <linux/linkage.h> 15#include <linux/linkage.h>
16 16
17 .data 17 .text
18 .align 18 .align
19 19
20 /* 20 /*
diff --git a/arch/arm/mach-shmobile/intc-sh73a0.c b/arch/arm/mach-shmobile/intc-sh73a0.c
index 9e3618028acc..fd63ae6532fc 100644
--- a/arch/arm/mach-shmobile/intc-sh73a0.c
+++ b/arch/arm/mach-shmobile/intc-sh73a0.c
@@ -252,11 +252,6 @@ static irqreturn_t sh73a0_intcs_demux(int irq, void *dev_id)
252 return IRQ_HANDLED; 252 return IRQ_HANDLED;
253} 253}
254 254
255static int sh73a0_set_wake(struct irq_data *data, unsigned int on)
256{
257 return 0; /* always allow wakeup */
258}
259
260#define PINTER0_PHYS 0xe69000a0 255#define PINTER0_PHYS 0xe69000a0
261#define PINTER1_PHYS 0xe69000a4 256#define PINTER1_PHYS 0xe69000a4
262#define PINTER0_VIRT IOMEM(0xe69000a0) 257#define PINTER0_VIRT IOMEM(0xe69000a0)
@@ -318,8 +313,8 @@ void __init sh73a0_init_irq(void)
318 void __iomem *gic_cpu_base = IOMEM(0xf0000100); 313 void __iomem *gic_cpu_base = IOMEM(0xf0000100);
319 void __iomem *intevtsa = ioremap_nocache(0xffd20100, PAGE_SIZE); 314 void __iomem *intevtsa = ioremap_nocache(0xffd20100, PAGE_SIZE);
320 315
316 gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE);
321 gic_init(0, 29, gic_dist_base, gic_cpu_base); 317 gic_init(0, 29, gic_dist_base, gic_cpu_base);
322 gic_arch_extn.irq_set_wake = sh73a0_set_wake;
323 318
324 register_intc_controller(&intcs_desc); 319 register_intc_controller(&intcs_desc);
325 register_intc_controller(&intc_pint0_desc); 320 register_intc_controller(&intc_pint0_desc);
diff --git a/arch/arm/mach-shmobile/setup-r8a7779.c b/arch/arm/mach-shmobile/setup-r8a7779.c
index 27dceaf9e688..c03e562be12b 100644
--- a/arch/arm/mach-shmobile/setup-r8a7779.c
+++ b/arch/arm/mach-shmobile/setup-r8a7779.c
@@ -713,18 +713,13 @@ void __init r8a7779_init_late(void)
713} 713}
714 714
715#ifdef CONFIG_USE_OF 715#ifdef CONFIG_USE_OF
716static int r8a7779_set_wake(struct irq_data *data, unsigned int on)
717{
718 return 0; /* always allow wakeup */
719}
720
721void __init r8a7779_init_irq_dt(void) 716void __init r8a7779_init_irq_dt(void)
722{ 717{
723#ifdef CONFIG_ARCH_SHMOBILE_LEGACY 718#ifdef CONFIG_ARCH_SHMOBILE_LEGACY
724 void __iomem *gic_dist_base = ioremap_nocache(0xf0001000, 0x1000); 719 void __iomem *gic_dist_base = ioremap_nocache(0xf0001000, 0x1000);
725 void __iomem *gic_cpu_base = ioremap_nocache(0xf0000100, 0x1000); 720 void __iomem *gic_cpu_base = ioremap_nocache(0xf0000100, 0x1000);
726#endif 721#endif
727 gic_arch_extn.irq_set_wake = r8a7779_set_wake; 722 gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE);
728 723
729#ifdef CONFIG_ARCH_SHMOBILE_LEGACY 724#ifdef CONFIG_ARCH_SHMOBILE_LEGACY
730 gic_init(0, 29, gic_dist_base, gic_cpu_base); 725 gic_init(0, 29, gic_dist_base, gic_cpu_base);
diff --git a/arch/arm/mach-tegra/cpuidle-tegra114.c b/arch/arm/mach-tegra/cpuidle-tegra114.c
index f2b586d7b15d..155807fa6fdd 100644
--- a/arch/arm/mach-tegra/cpuidle-tegra114.c
+++ b/arch/arm/mach-tegra/cpuidle-tegra114.c
@@ -15,7 +15,7 @@
15 */ 15 */
16 16
17#include <asm/firmware.h> 17#include <asm/firmware.h>
18#include <linux/clockchips.h> 18#include <linux/tick.h>
19#include <linux/cpuidle.h> 19#include <linux/cpuidle.h>
20#include <linux/cpu_pm.h> 20#include <linux/cpu_pm.h>
21#include <linux/kernel.h> 21#include <linux/kernel.h>
@@ -44,7 +44,7 @@ static int tegra114_idle_power_down(struct cpuidle_device *dev,
44 tegra_set_cpu_in_lp2(); 44 tegra_set_cpu_in_lp2();
45 cpu_pm_enter(); 45 cpu_pm_enter();
46 46
47 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu); 47 tick_broadcast_enter();
48 48
49 call_firmware_op(prepare_idle); 49 call_firmware_op(prepare_idle);
50 50
@@ -52,7 +52,7 @@ static int tegra114_idle_power_down(struct cpuidle_device *dev,
52 if (call_firmware_op(do_idle, 0) == -ENOSYS) 52 if (call_firmware_op(do_idle, 0) == -ENOSYS)
53 cpu_suspend(0, tegra30_sleep_cpu_secondary_finish); 53 cpu_suspend(0, tegra30_sleep_cpu_secondary_finish);
54 54
55 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu); 55 tick_broadcast_exit();
56 56
57 cpu_pm_exit(); 57 cpu_pm_exit();
58 tegra_clear_cpu_in_lp2(); 58 tegra_clear_cpu_in_lp2();
diff --git a/arch/arm/mach-tegra/cpuidle-tegra20.c b/arch/arm/mach-tegra/cpuidle-tegra20.c
index 4f25a7c7ca0f..88de2dce2e87 100644
--- a/arch/arm/mach-tegra/cpuidle-tegra20.c
+++ b/arch/arm/mach-tegra/cpuidle-tegra20.c
@@ -20,14 +20,13 @@
20 */ 20 */
21 21
22#include <linux/clk/tegra.h> 22#include <linux/clk/tegra.h>
23#include <linux/clockchips.h> 23#include <linux/tick.h>
24#include <linux/cpuidle.h> 24#include <linux/cpuidle.h>
25#include <linux/cpu_pm.h> 25#include <linux/cpu_pm.h>
26#include <linux/kernel.h> 26#include <linux/kernel.h>
27#include <linux/module.h> 27#include <linux/module.h>
28 28
29#include <asm/cpuidle.h> 29#include <asm/cpuidle.h>
30#include <asm/proc-fns.h>
31#include <asm/smp_plat.h> 30#include <asm/smp_plat.h>
32#include <asm/suspend.h> 31#include <asm/suspend.h>
33 32
@@ -136,11 +135,11 @@ static bool tegra20_cpu_cluster_power_down(struct cpuidle_device *dev,
136 if (tegra20_reset_cpu_1() || !tegra_cpu_rail_off_ready()) 135 if (tegra20_reset_cpu_1() || !tegra_cpu_rail_off_ready())
137 return false; 136 return false;
138 137
139 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu); 138 tick_broadcast_enter();
140 139
141 tegra_idle_lp2_last(); 140 tegra_idle_lp2_last();
142 141
143 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu); 142 tick_broadcast_exit();
144 143
145 if (cpu_online(1)) 144 if (cpu_online(1))
146 tegra20_wake_cpu1_from_reset(); 145 tegra20_wake_cpu1_from_reset();
@@ -153,13 +152,13 @@ static bool tegra20_idle_enter_lp2_cpu_1(struct cpuidle_device *dev,
153 struct cpuidle_driver *drv, 152 struct cpuidle_driver *drv,
154 int index) 153 int index)
155{ 154{
156 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu); 155 tick_broadcast_enter();
157 156
158 cpu_suspend(0, tegra20_sleep_cpu_secondary_finish); 157 cpu_suspend(0, tegra20_sleep_cpu_secondary_finish);
159 158
160 tegra20_cpu_clear_resettable(); 159 tegra20_cpu_clear_resettable();
161 160
162 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu); 161 tick_broadcast_exit();
163 162
164 return true; 163 return true;
165} 164}
diff --git a/arch/arm/mach-tegra/cpuidle-tegra30.c b/arch/arm/mach-tegra/cpuidle-tegra30.c
index f8815ed65d9d..4dbe1dae937c 100644
--- a/arch/arm/mach-tegra/cpuidle-tegra30.c
+++ b/arch/arm/mach-tegra/cpuidle-tegra30.c
@@ -20,14 +20,13 @@
20 */ 20 */
21 21
22#include <linux/clk/tegra.h> 22#include <linux/clk/tegra.h>
23#include <linux/clockchips.h> 23#include <linux/tick.h>
24#include <linux/cpuidle.h> 24#include <linux/cpuidle.h>
25#include <linux/cpu_pm.h> 25#include <linux/cpu_pm.h>
26#include <linux/kernel.h> 26#include <linux/kernel.h>
27#include <linux/module.h> 27#include <linux/module.h>
28 28
29#include <asm/cpuidle.h> 29#include <asm/cpuidle.h>
30#include <asm/proc-fns.h>
31#include <asm/smp_plat.h> 30#include <asm/smp_plat.h>
32#include <asm/suspend.h> 31#include <asm/suspend.h>
33 32
@@ -76,11 +75,11 @@ static bool tegra30_cpu_cluster_power_down(struct cpuidle_device *dev,
76 return false; 75 return false;
77 } 76 }
78 77
79 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu); 78 tick_broadcast_enter();
80 79
81 tegra_idle_lp2_last(); 80 tegra_idle_lp2_last();
82 81
83 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu); 82 tick_broadcast_exit();
84 83
85 return true; 84 return true;
86} 85}
@@ -90,13 +89,13 @@ static bool tegra30_cpu_core_power_down(struct cpuidle_device *dev,
90 struct cpuidle_driver *drv, 89 struct cpuidle_driver *drv,
91 int index) 90 int index)
92{ 91{
93 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu); 92 tick_broadcast_enter();
94 93
95 smp_wmb(); 94 smp_wmb();
96 95
97 cpu_suspend(0, tegra30_sleep_cpu_secondary_finish); 96 cpu_suspend(0, tegra30_sleep_cpu_secondary_finish);
98 97
99 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu); 98 tick_broadcast_exit();
100 99
101 return true; 100 return true;
102} 101}
diff --git a/arch/arm/mach-tegra/iomap.h b/arch/arm/mach-tegra/iomap.h
index ee79808e93a3..81dc950b4881 100644
--- a/arch/arm/mach-tegra/iomap.h
+++ b/arch/arm/mach-tegra/iomap.h
@@ -31,21 +31,6 @@
31#define TEGRA_ARM_INT_DIST_BASE 0x50041000 31#define TEGRA_ARM_INT_DIST_BASE 0x50041000
32#define TEGRA_ARM_INT_DIST_SIZE SZ_4K 32#define TEGRA_ARM_INT_DIST_SIZE SZ_4K
33 33
34#define TEGRA_PRIMARY_ICTLR_BASE 0x60004000
35#define TEGRA_PRIMARY_ICTLR_SIZE SZ_64
36
37#define TEGRA_SECONDARY_ICTLR_BASE 0x60004100
38#define TEGRA_SECONDARY_ICTLR_SIZE SZ_64
39
40#define TEGRA_TERTIARY_ICTLR_BASE 0x60004200
41#define TEGRA_TERTIARY_ICTLR_SIZE SZ_64
42
43#define TEGRA_QUATERNARY_ICTLR_BASE 0x60004300
44#define TEGRA_QUATERNARY_ICTLR_SIZE SZ_64
45
46#define TEGRA_QUINARY_ICTLR_BASE 0x60004400
47#define TEGRA_QUINARY_ICTLR_SIZE SZ_64
48
49#define TEGRA_TMR1_BASE 0x60005000 34#define TEGRA_TMR1_BASE 0x60005000
50#define TEGRA_TMR1_SIZE SZ_8 35#define TEGRA_TMR1_SIZE SZ_8
51 36
diff --git a/arch/arm/mach-tegra/irq.c b/arch/arm/mach-tegra/irq.c
index ab95f5391a2b..3b9098d27ea5 100644
--- a/arch/arm/mach-tegra/irq.c
+++ b/arch/arm/mach-tegra/irq.c
@@ -30,43 +30,9 @@
30#include "board.h" 30#include "board.h"
31#include "iomap.h" 31#include "iomap.h"
32 32
33#define ICTLR_CPU_IEP_VFIQ 0x08
34#define ICTLR_CPU_IEP_FIR 0x14
35#define ICTLR_CPU_IEP_FIR_SET 0x18
36#define ICTLR_CPU_IEP_FIR_CLR 0x1c
37
38#define ICTLR_CPU_IER 0x20
39#define ICTLR_CPU_IER_SET 0x24
40#define ICTLR_CPU_IER_CLR 0x28
41#define ICTLR_CPU_IEP_CLASS 0x2C
42
43#define ICTLR_COP_IER 0x30
44#define ICTLR_COP_IER_SET 0x34
45#define ICTLR_COP_IER_CLR 0x38
46#define ICTLR_COP_IEP_CLASS 0x3c
47
48#define FIRST_LEGACY_IRQ 32
49#define TEGRA_MAX_NUM_ICTLRS 5
50
51#define SGI_MASK 0xFFFF 33#define SGI_MASK 0xFFFF
52 34
53static int num_ictlrs;
54
55static void __iomem *ictlr_reg_base[] = {
56 IO_ADDRESS(TEGRA_PRIMARY_ICTLR_BASE),
57 IO_ADDRESS(TEGRA_SECONDARY_ICTLR_BASE),
58 IO_ADDRESS(TEGRA_TERTIARY_ICTLR_BASE),
59 IO_ADDRESS(TEGRA_QUATERNARY_ICTLR_BASE),
60 IO_ADDRESS(TEGRA_QUINARY_ICTLR_BASE),
61};
62
63#ifdef CONFIG_PM_SLEEP 35#ifdef CONFIG_PM_SLEEP
64static u32 cop_ier[TEGRA_MAX_NUM_ICTLRS];
65static u32 cop_iep[TEGRA_MAX_NUM_ICTLRS];
66static u32 cpu_ier[TEGRA_MAX_NUM_ICTLRS];
67static u32 cpu_iep[TEGRA_MAX_NUM_ICTLRS];
68
69static u32 ictlr_wake_mask[TEGRA_MAX_NUM_ICTLRS];
70static void __iomem *tegra_gic_cpu_base; 36static void __iomem *tegra_gic_cpu_base;
71#endif 37#endif
72 38
@@ -83,140 +49,7 @@ bool tegra_pending_sgi(void)
83 return false; 49 return false;
84} 50}
85 51
86static inline void tegra_irq_write_mask(unsigned int irq, unsigned long reg)
87{
88 void __iomem *base;
89 u32 mask;
90
91 BUG_ON(irq < FIRST_LEGACY_IRQ ||
92 irq >= FIRST_LEGACY_IRQ + num_ictlrs * 32);
93
94 base = ictlr_reg_base[(irq - FIRST_LEGACY_IRQ) / 32];
95 mask = BIT((irq - FIRST_LEGACY_IRQ) % 32);
96
97 __raw_writel(mask, base + reg);
98}
99
100static void tegra_mask(struct irq_data *d)
101{
102 if (d->hwirq < FIRST_LEGACY_IRQ)
103 return;
104
105 tegra_irq_write_mask(d->hwirq, ICTLR_CPU_IER_CLR);
106}
107
108static void tegra_unmask(struct irq_data *d)
109{
110 if (d->hwirq < FIRST_LEGACY_IRQ)
111 return;
112
113 tegra_irq_write_mask(d->hwirq, ICTLR_CPU_IER_SET);
114}
115
116static void tegra_ack(struct irq_data *d)
117{
118 if (d->hwirq < FIRST_LEGACY_IRQ)
119 return;
120
121 tegra_irq_write_mask(d->hwirq, ICTLR_CPU_IEP_FIR_CLR);
122}
123
124static void tegra_eoi(struct irq_data *d)
125{
126 if (d->hwirq < FIRST_LEGACY_IRQ)
127 return;
128
129 tegra_irq_write_mask(d->hwirq, ICTLR_CPU_IEP_FIR_CLR);
130}
131
132static int tegra_retrigger(struct irq_data *d)
133{
134 if (d->hwirq < FIRST_LEGACY_IRQ)
135 return 0;
136
137 tegra_irq_write_mask(d->hwirq, ICTLR_CPU_IEP_FIR_SET);
138
139 return 1;
140}
141
142#ifdef CONFIG_PM_SLEEP 52#ifdef CONFIG_PM_SLEEP
143static int tegra_set_wake(struct irq_data *d, unsigned int enable)
144{
145 u32 irq = d->hwirq;
146 u32 index, mask;
147
148 if (irq < FIRST_LEGACY_IRQ ||
149 irq >= FIRST_LEGACY_IRQ + num_ictlrs * 32)
150 return -EINVAL;
151
152 index = ((irq - FIRST_LEGACY_IRQ) / 32);
153 mask = BIT((irq - FIRST_LEGACY_IRQ) % 32);
154 if (enable)
155 ictlr_wake_mask[index] |= mask;
156 else
157 ictlr_wake_mask[index] &= ~mask;
158
159 return 0;
160}
161
162static int tegra_legacy_irq_suspend(void)
163{
164 unsigned long flags;
165 int i;
166
167 local_irq_save(flags);
168 for (i = 0; i < num_ictlrs; i++) {
169 void __iomem *ictlr = ictlr_reg_base[i];
170 /* Save interrupt state */
171 cpu_ier[i] = readl_relaxed(ictlr + ICTLR_CPU_IER);
172 cpu_iep[i] = readl_relaxed(ictlr + ICTLR_CPU_IEP_CLASS);
173 cop_ier[i] = readl_relaxed(ictlr + ICTLR_COP_IER);
174 cop_iep[i] = readl_relaxed(ictlr + ICTLR_COP_IEP_CLASS);
175
176 /* Disable COP interrupts */
177 writel_relaxed(~0ul, ictlr + ICTLR_COP_IER_CLR);
178
179 /* Disable CPU interrupts */
180 writel_relaxed(~0ul, ictlr + ICTLR_CPU_IER_CLR);
181
182 /* Enable the wakeup sources of ictlr */
183 writel_relaxed(ictlr_wake_mask[i], ictlr + ICTLR_CPU_IER_SET);
184 }
185 local_irq_restore(flags);
186
187 return 0;
188}
189
190static void tegra_legacy_irq_resume(void)
191{
192 unsigned long flags;
193 int i;
194
195 local_irq_save(flags);
196 for (i = 0; i < num_ictlrs; i++) {
197 void __iomem *ictlr = ictlr_reg_base[i];
198 writel_relaxed(cpu_iep[i], ictlr + ICTLR_CPU_IEP_CLASS);
199 writel_relaxed(~0ul, ictlr + ICTLR_CPU_IER_CLR);
200 writel_relaxed(cpu_ier[i], ictlr + ICTLR_CPU_IER_SET);
201 writel_relaxed(cop_iep[i], ictlr + ICTLR_COP_IEP_CLASS);
202 writel_relaxed(~0ul, ictlr + ICTLR_COP_IER_CLR);
203 writel_relaxed(cop_ier[i], ictlr + ICTLR_COP_IER_SET);
204 }
205 local_irq_restore(flags);
206}
207
208static struct syscore_ops tegra_legacy_irq_syscore_ops = {
209 .suspend = tegra_legacy_irq_suspend,
210 .resume = tegra_legacy_irq_resume,
211};
212
213int tegra_legacy_irq_syscore_init(void)
214{
215 register_syscore_ops(&tegra_legacy_irq_syscore_ops);
216
217 return 0;
218}
219
220static int tegra_gic_notifier(struct notifier_block *self, 53static int tegra_gic_notifier(struct notifier_block *self,
221 unsigned long cmd, void *v) 54 unsigned long cmd, void *v)
222{ 55{
@@ -251,45 +84,19 @@ static void tegra114_gic_cpu_pm_registration(void)
251 cpu_pm_register_notifier(&tegra_gic_notifier_block); 84 cpu_pm_register_notifier(&tegra_gic_notifier_block);
252} 85}
253#else 86#else
254#define tegra_set_wake NULL
255static void tegra114_gic_cpu_pm_registration(void) { } 87static void tegra114_gic_cpu_pm_registration(void) { }
256#endif 88#endif
257 89
90static const struct of_device_id tegra_ictlr_match[] __initconst = {
91 { .compatible = "nvidia,tegra20-ictlr" },
92 { .compatible = "nvidia,tegra30-ictlr" },
93 { }
94};
95
258void __init tegra_init_irq(void) 96void __init tegra_init_irq(void)
259{ 97{
260 int i; 98 if (WARN_ON(!of_find_matching_node(NULL, tegra_ictlr_match)))
261 void __iomem *distbase; 99 pr_warn("Outdated DT detected, suspend/resume will NOT work\n");
262
263 distbase = IO_ADDRESS(TEGRA_ARM_INT_DIST_BASE);
264 num_ictlrs = readl_relaxed(distbase + GIC_DIST_CTR) & 0x1f;
265
266 if (num_ictlrs > ARRAY_SIZE(ictlr_reg_base)) {
267 WARN(1, "Too many (%d) interrupt controllers found. Maximum is %d.",
268 num_ictlrs, ARRAY_SIZE(ictlr_reg_base));
269 num_ictlrs = ARRAY_SIZE(ictlr_reg_base);
270 }
271
272 for (i = 0; i < num_ictlrs; i++) {
273 void __iomem *ictlr = ictlr_reg_base[i];
274 writel(~0, ictlr + ICTLR_CPU_IER_CLR);
275 writel(0, ictlr + ICTLR_CPU_IEP_CLASS);
276 }
277
278 gic_arch_extn.irq_ack = tegra_ack;
279 gic_arch_extn.irq_eoi = tegra_eoi;
280 gic_arch_extn.irq_mask = tegra_mask;
281 gic_arch_extn.irq_unmask = tegra_unmask;
282 gic_arch_extn.irq_retrigger = tegra_retrigger;
283 gic_arch_extn.irq_set_wake = tegra_set_wake;
284 gic_arch_extn.flags = IRQCHIP_MASK_ON_SUSPEND;
285
286 /*
287 * Check if there is a devicetree present, since the GIC will be
288 * initialized elsewhere under DT.
289 */
290 if (!of_have_populated_dt())
291 gic_init(0, 29, distbase,
292 IO_ADDRESS(TEGRA_ARM_PERIF_BASE + 0x100));
293 100
294 tegra114_gic_cpu_pm_registration(); 101 tegra114_gic_cpu_pm_registration();
295} 102}
diff --git a/arch/arm/mach-tegra/irq.h b/arch/arm/mach-tegra/irq.h
index bc05ce5613fb..5142649bba05 100644
--- a/arch/arm/mach-tegra/irq.h
+++ b/arch/arm/mach-tegra/irq.h
@@ -19,10 +19,4 @@
19 19
20bool tegra_pending_sgi(void); 20bool tegra_pending_sgi(void);
21 21
22#ifdef CONFIG_PM_SLEEP
23int tegra_legacy_irq_syscore_init(void);
24#else
25static inline int tegra_legacy_irq_syscore_init(void) { return 0; }
26#endif
27
28#endif 22#endif
diff --git a/arch/arm/mach-tegra/tegra.c b/arch/arm/mach-tegra/tegra.c
index 914341bcef25..861d88486dbe 100644
--- a/arch/arm/mach-tegra/tegra.c
+++ b/arch/arm/mach-tegra/tegra.c
@@ -82,7 +82,6 @@ static void __init tegra_dt_init_irq(void)
82{ 82{
83 tegra_init_irq(); 83 tegra_init_irq();
84 irqchip_init(); 84 irqchip_init();
85 tegra_legacy_irq_syscore_init();
86} 85}
87 86
88static void __init tegra_dt_init(void) 87static void __init tegra_dt_init(void)
diff --git a/arch/arm/mach-ux500/cpu.c b/arch/arm/mach-ux500/cpu.c
index dbb2970ee7da..6ced0f680262 100644
--- a/arch/arm/mach-ux500/cpu.c
+++ b/arch/arm/mach-ux500/cpu.c
@@ -52,7 +52,7 @@ void ux500_restart(enum reboot_mode mode, const char *cmd)
52*/ 52*/
53void __init ux500_init_irq(void) 53void __init ux500_init_irq(void)
54{ 54{
55 gic_arch_extn.flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND; 55 gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND);
56 irqchip_init(); 56 irqchip_init();
57 57
58 /* 58 /*
diff --git a/arch/arm/mach-vexpress/Kconfig b/arch/arm/mach-vexpress/Kconfig
index 3c2509b4b694..4be537977040 100644
--- a/arch/arm/mach-vexpress/Kconfig
+++ b/arch/arm/mach-vexpress/Kconfig
@@ -42,6 +42,7 @@ if ARCH_VEXPRESS
42config ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA 42config ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA
43 bool "Enable A5 and A9 only errata work-arounds" 43 bool "Enable A5 and A9 only errata work-arounds"
44 default y 44 default y
45 select ARM_ERRATA_643719 if SMP
45 select ARM_ERRATA_720789 46 select ARM_ERRATA_720789
46 select PL310_ERRATA_753970 if CACHE_L2X0 47 select PL310_ERRATA_753970 if CACHE_L2X0
47 help 48 help
diff --git a/arch/arm/mach-zynq/common.c b/arch/arm/mach-zynq/common.c
index c887196cfdbe..58ef2a700414 100644
--- a/arch/arm/mach-zynq/common.c
+++ b/arch/arm/mach-zynq/common.c
@@ -186,7 +186,7 @@ static void __init zynq_map_io(void)
186 186
187static void __init zynq_irq_init(void) 187static void __init zynq_irq_init(void)
188{ 188{
189 gic_arch_extn.flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND; 189 gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND);
190 irqchip_init(); 190 irqchip_init();
191} 191}
192 192
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 9b4f29e595a4..b7644310236b 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -738,7 +738,7 @@ config CPU_ICACHE_DISABLE
738 738
739config CPU_DCACHE_DISABLE 739config CPU_DCACHE_DISABLE
740 bool "Disable D-Cache (C-bit)" 740 bool "Disable D-Cache (C-bit)"
741 depends on CPU_CP15 741 depends on CPU_CP15 && !SMP
742 help 742 help
743 Say Y here to disable the processor data cache. Unless 743 Say Y here to disable the processor data cache. Unless
744 you have a reason not to or are unsure, say N. 744 you have a reason not to or are unsure, say N.
@@ -825,6 +825,20 @@ config KUSER_HELPERS
825 Say N here only if you are absolutely certain that you do not 825 Say N here only if you are absolutely certain that you do not
826 need these helpers; otherwise, the safe option is to say Y. 826 need these helpers; otherwise, the safe option is to say Y.
827 827
828config VDSO
829 bool "Enable VDSO for acceleration of some system calls"
830 depends on AEABI && MMU
831 default y if ARM_ARCH_TIMER
832 select GENERIC_TIME_VSYSCALL
833 help
834 Place in the process address space an ELF shared object
835 providing fast implementations of gettimeofday and
836 clock_gettime. Systems that implement the ARM architected
837 timer will receive maximum benefit.
838
839 You must have glibc 2.22 or later for programs to seamlessly
840 take advantage of this.
841
828config DMA_CACHE_RWFO 842config DMA_CACHE_RWFO
829 bool "Enable read/write for ownership DMA cache maintenance" 843 bool "Enable read/write for ownership DMA cache maintenance"
830 depends on CPU_V6K && SMP 844 depends on CPU_V6K && SMP
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 2c0c541c60ca..9769f1eefe3b 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -201,7 +201,7 @@ union offset_union {
201 THUMB( "1: "ins" %1, [%2]\n" ) \ 201 THUMB( "1: "ins" %1, [%2]\n" ) \
202 THUMB( " add %2, %2, #1\n" ) \ 202 THUMB( " add %2, %2, #1\n" ) \
203 "2:\n" \ 203 "2:\n" \
204 " .pushsection .fixup,\"ax\"\n" \ 204 " .pushsection .text.fixup,\"ax\"\n" \
205 " .align 2\n" \ 205 " .align 2\n" \
206 "3: mov %0, #1\n" \ 206 "3: mov %0, #1\n" \
207 " b 2b\n" \ 207 " b 2b\n" \
@@ -261,7 +261,7 @@ union offset_union {
261 " mov %1, %1, "NEXT_BYTE"\n" \ 261 " mov %1, %1, "NEXT_BYTE"\n" \
262 "2: "ins" %1, [%2]\n" \ 262 "2: "ins" %1, [%2]\n" \
263 "3:\n" \ 263 "3:\n" \
264 " .pushsection .fixup,\"ax\"\n" \ 264 " .pushsection .text.fixup,\"ax\"\n" \
265 " .align 2\n" \ 265 " .align 2\n" \
266 "4: mov %0, #1\n" \ 266 "4: mov %0, #1\n" \
267 " b 3b\n" \ 267 " b 3b\n" \
@@ -301,7 +301,7 @@ union offset_union {
301 " mov %1, %1, "NEXT_BYTE"\n" \ 301 " mov %1, %1, "NEXT_BYTE"\n" \
302 "4: "ins" %1, [%2]\n" \ 302 "4: "ins" %1, [%2]\n" \
303 "5:\n" \ 303 "5:\n" \
304 " .pushsection .fixup,\"ax\"\n" \ 304 " .pushsection .text.fixup,\"ax\"\n" \
305 " .align 2\n" \ 305 " .align 2\n" \
306 "6: mov %0, #1\n" \ 306 "6: mov %0, #1\n" \
307 " b 5b\n" \ 307 " b 5b\n" \
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 8f15f70622a6..e309c8f35af5 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -1647,6 +1647,7 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
1647 struct device_node *np; 1647 struct device_node *np;
1648 struct resource res; 1648 struct resource res;
1649 u32 cache_id, old_aux; 1649 u32 cache_id, old_aux;
1650 u32 cache_level = 2;
1650 1651
1651 np = of_find_matching_node(NULL, l2x0_ids); 1652 np = of_find_matching_node(NULL, l2x0_ids);
1652 if (!np) 1653 if (!np)
@@ -1679,6 +1680,12 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
1679 if (!of_property_read_bool(np, "cache-unified")) 1680 if (!of_property_read_bool(np, "cache-unified"))
1680 pr_err("L2C: device tree omits to specify unified cache\n"); 1681 pr_err("L2C: device tree omits to specify unified cache\n");
1681 1682
1683 if (of_property_read_u32(np, "cache-level", &cache_level))
1684 pr_err("L2C: device tree omits to specify cache-level\n");
1685
1686 if (cache_level != 2)
1687 pr_err("L2C: device tree specifies invalid cache level\n");
1688
1682 /* Read back current (default) hardware configuration */ 1689 /* Read back current (default) hardware configuration */
1683 if (data->save) 1690 if (data->save)
1684 data->save(l2x0_base); 1691 data->save(l2x0_base);
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index b966656d2c2d..a134d8a13d00 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -36,10 +36,10 @@ ENTRY(v7_invalidate_l1)
36 mcr p15, 2, r0, c0, c0, 0 36 mcr p15, 2, r0, c0, c0, 0
37 mrc p15, 1, r0, c0, c0, 0 37 mrc p15, 1, r0, c0, c0, 0
38 38
39 ldr r1, =0x7fff 39 movw r1, #0x7fff
40 and r2, r1, r0, lsr #13 40 and r2, r1, r0, lsr #13
41 41
42 ldr r1, =0x3ff 42 movw r1, #0x3ff
43 43
44 and r3, r1, r0, lsr #3 @ NumWays - 1 44 and r3, r1, r0, lsr #3 @ NumWays - 1
45 add r2, r2, #1 @ NumSets 45 add r2, r2, #1 @ NumSets
@@ -90,21 +90,20 @@ ENDPROC(v7_flush_icache_all)
90ENTRY(v7_flush_dcache_louis) 90ENTRY(v7_flush_dcache_louis)
91 dmb @ ensure ordering with previous memory accesses 91 dmb @ ensure ordering with previous memory accesses
92 mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr 92 mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr
93 ALT_SMP(ands r3, r0, #(7 << 21)) @ extract LoUIS from clidr 93ALT_SMP(mov r3, r0, lsr #20) @ move LoUIS into position
94 ALT_UP(ands r3, r0, #(7 << 27)) @ extract LoUU from clidr 94ALT_UP( mov r3, r0, lsr #26) @ move LoUU into position
95 ands r3, r3, #7 << 1 @ extract LoU*2 field from clidr
96 bne start_flush_levels @ LoU != 0, start flushing
95#ifdef CONFIG_ARM_ERRATA_643719 97#ifdef CONFIG_ARM_ERRATA_643719
96 ALT_SMP(mrceq p15, 0, r2, c0, c0, 0) @ read main ID register 98ALT_SMP(mrc p15, 0, r2, c0, c0, 0) @ read main ID register
97 ALT_UP(reteq lr) @ LoUU is zero, so nothing to do 99ALT_UP( ret lr) @ LoUU is zero, so nothing to do
98 ldreq r1, =0x410fc090 @ ID of ARM Cortex A9 r0p? 100 movw r1, #:lower16:(0x410fc090 >> 4) @ ID of ARM Cortex A9 r0p?
99 biceq r2, r2, #0x0000000f @ clear minor revision number 101 movt r1, #:upper16:(0x410fc090 >> 4)
100 teqeq r2, r1 @ test for errata affected core and if so... 102 teq r1, r2, lsr #4 @ test for errata affected core and if so...
101 orreqs r3, #(1 << 21) @ fix LoUIS value (and set flags state to 'ne') 103 moveq r3, #1 << 1 @ fix LoUIS value
104 beq start_flush_levels @ start flushing cache levels
102#endif 105#endif
103 ALT_SMP(mov r3, r3, lsr #20) @ r3 = LoUIS * 2 106 ret lr
104 ALT_UP(mov r3, r3, lsr #26) @ r3 = LoUU * 2
105 reteq lr @ return if level == 0
106 mov r10, #0 @ r10 (starting level) = 0
107 b flush_levels @ start flushing cache levels
108ENDPROC(v7_flush_dcache_louis) 107ENDPROC(v7_flush_dcache_louis)
109 108
110/* 109/*
@@ -119,9 +118,10 @@ ENDPROC(v7_flush_dcache_louis)
119ENTRY(v7_flush_dcache_all) 118ENTRY(v7_flush_dcache_all)
120 dmb @ ensure ordering with previous memory accesses 119 dmb @ ensure ordering with previous memory accesses
121 mrc p15, 1, r0, c0, c0, 1 @ read clidr 120 mrc p15, 1, r0, c0, c0, 1 @ read clidr
122 ands r3, r0, #0x7000000 @ extract loc from clidr 121 mov r3, r0, lsr #23 @ move LoC into position
123 mov r3, r3, lsr #23 @ left align loc bit field 122 ands r3, r3, #7 << 1 @ extract LoC*2 from clidr
124 beq finished @ if loc is 0, then no need to clean 123 beq finished @ if loc is 0, then no need to clean
124start_flush_levels:
125 mov r10, #0 @ start clean at cache level 0 125 mov r10, #0 @ start clean at cache level 0
126flush_levels: 126flush_levels:
127 add r2, r10, r10, lsr #1 @ work out 3x current cache level 127 add r2, r10, r10, lsr #1 @ work out 3x current cache level
@@ -140,10 +140,10 @@ flush_levels:
140#endif 140#endif
141 and r2, r1, #7 @ extract the length of the cache lines 141 and r2, r1, #7 @ extract the length of the cache lines
142 add r2, r2, #4 @ add 4 (line length offset) 142 add r2, r2, #4 @ add 4 (line length offset)
143 ldr r4, =0x3ff 143 movw r4, #0x3ff
144 ands r4, r4, r1, lsr #3 @ find maximum number on the way size 144 ands r4, r4, r1, lsr #3 @ find maximum number on the way size
145 clz r5, r4 @ find bit position of way size increment 145 clz r5, r4 @ find bit position of way size increment
146 ldr r7, =0x7fff 146 movw r7, #0x7fff
147 ands r7, r7, r1, lsr #13 @ extract max number of the index size 147 ands r7, r7, r1, lsr #13 @ extract max number of the index size
148loop1: 148loop1:
149 mov r9, r7 @ create working copy of max index 149 mov r9, r7 @ create working copy of max index
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index c27447653903..09c5fe3d30c2 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -289,11 +289,11 @@ static void __dma_free_buffer(struct page *page, size_t size)
289 289
290static void *__alloc_from_contiguous(struct device *dev, size_t size, 290static void *__alloc_from_contiguous(struct device *dev, size_t size,
291 pgprot_t prot, struct page **ret_page, 291 pgprot_t prot, struct page **ret_page,
292 const void *caller); 292 const void *caller, bool want_vaddr);
293 293
294static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 294static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
295 pgprot_t prot, struct page **ret_page, 295 pgprot_t prot, struct page **ret_page,
296 const void *caller); 296 const void *caller, bool want_vaddr);
297 297
298static void * 298static void *
299__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 299__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
@@ -357,10 +357,10 @@ static int __init atomic_pool_init(void)
357 357
358 if (dev_get_cma_area(NULL)) 358 if (dev_get_cma_area(NULL))
359 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 359 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
360 &page, atomic_pool_init); 360 &page, atomic_pool_init, true);
361 else 361 else
362 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 362 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
363 &page, atomic_pool_init); 363 &page, atomic_pool_init, true);
364 if (ptr) { 364 if (ptr) {
365 int ret; 365 int ret;
366 366
@@ -467,13 +467,15 @@ static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
467 467
468static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 468static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
469 pgprot_t prot, struct page **ret_page, 469 pgprot_t prot, struct page **ret_page,
470 const void *caller) 470 const void *caller, bool want_vaddr)
471{ 471{
472 struct page *page; 472 struct page *page;
473 void *ptr; 473 void *ptr = NULL;
474 page = __dma_alloc_buffer(dev, size, gfp); 474 page = __dma_alloc_buffer(dev, size, gfp);
475 if (!page) 475 if (!page)
476 return NULL; 476 return NULL;
477 if (!want_vaddr)
478 goto out;
477 479
478 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 480 ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
479 if (!ptr) { 481 if (!ptr) {
@@ -481,6 +483,7 @@ static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
481 return NULL; 483 return NULL;
482 } 484 }
483 485
486 out:
484 *ret_page = page; 487 *ret_page = page;
485 return ptr; 488 return ptr;
486} 489}
@@ -523,12 +526,12 @@ static int __free_from_pool(void *start, size_t size)
523 526
524static void *__alloc_from_contiguous(struct device *dev, size_t size, 527static void *__alloc_from_contiguous(struct device *dev, size_t size,
525 pgprot_t prot, struct page **ret_page, 528 pgprot_t prot, struct page **ret_page,
526 const void *caller) 529 const void *caller, bool want_vaddr)
527{ 530{
528 unsigned long order = get_order(size); 531 unsigned long order = get_order(size);
529 size_t count = size >> PAGE_SHIFT; 532 size_t count = size >> PAGE_SHIFT;
530 struct page *page; 533 struct page *page;
531 void *ptr; 534 void *ptr = NULL;
532 535
533 page = dma_alloc_from_contiguous(dev, count, order); 536 page = dma_alloc_from_contiguous(dev, count, order);
534 if (!page) 537 if (!page)
@@ -536,6 +539,9 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size,
536 539
537 __dma_clear_buffer(page, size); 540 __dma_clear_buffer(page, size);
538 541
542 if (!want_vaddr)
543 goto out;
544
539 if (PageHighMem(page)) { 545 if (PageHighMem(page)) {
540 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); 546 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
541 if (!ptr) { 547 if (!ptr) {
@@ -546,17 +552,21 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size,
546 __dma_remap(page, size, prot); 552 __dma_remap(page, size, prot);
547 ptr = page_address(page); 553 ptr = page_address(page);
548 } 554 }
555
556 out:
549 *ret_page = page; 557 *ret_page = page;
550 return ptr; 558 return ptr;
551} 559}
552 560
553static void __free_from_contiguous(struct device *dev, struct page *page, 561static void __free_from_contiguous(struct device *dev, struct page *page,
554 void *cpu_addr, size_t size) 562 void *cpu_addr, size_t size, bool want_vaddr)
555{ 563{
556 if (PageHighMem(page)) 564 if (want_vaddr) {
557 __dma_free_remap(cpu_addr, size); 565 if (PageHighMem(page))
558 else 566 __dma_free_remap(cpu_addr, size);
559 __dma_remap(page, size, PAGE_KERNEL); 567 else
568 __dma_remap(page, size, PAGE_KERNEL);
569 }
560 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 570 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
561} 571}
562 572
@@ -574,12 +584,12 @@ static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
574 584
575#define nommu() 1 585#define nommu() 1
576 586
577#define __get_dma_pgprot(attrs, prot) __pgprot(0) 587#define __get_dma_pgprot(attrs, prot) __pgprot(0)
578#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL 588#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL
579#define __alloc_from_pool(size, ret_page) NULL 589#define __alloc_from_pool(size, ret_page) NULL
580#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL 590#define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL
581#define __free_from_pool(cpu_addr, size) 0 591#define __free_from_pool(cpu_addr, size) 0
582#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0) 592#define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0)
583#define __dma_free_remap(cpu_addr, size) do { } while (0) 593#define __dma_free_remap(cpu_addr, size) do { } while (0)
584 594
585#endif /* CONFIG_MMU */ 595#endif /* CONFIG_MMU */
@@ -599,11 +609,13 @@ static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
599 609
600 610
601static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 611static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
602 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller) 612 gfp_t gfp, pgprot_t prot, bool is_coherent,
613 struct dma_attrs *attrs, const void *caller)
603{ 614{
604 u64 mask = get_coherent_dma_mask(dev); 615 u64 mask = get_coherent_dma_mask(dev);
605 struct page *page = NULL; 616 struct page *page = NULL;
606 void *addr; 617 void *addr;
618 bool want_vaddr;
607 619
608#ifdef CONFIG_DMA_API_DEBUG 620#ifdef CONFIG_DMA_API_DEBUG
609 u64 limit = (mask + 1) & ~mask; 621 u64 limit = (mask + 1) & ~mask;
@@ -631,20 +643,21 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
631 643
632 *handle = DMA_ERROR_CODE; 644 *handle = DMA_ERROR_CODE;
633 size = PAGE_ALIGN(size); 645 size = PAGE_ALIGN(size);
646 want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
634 647
635 if (is_coherent || nommu()) 648 if (is_coherent || nommu())
636 addr = __alloc_simple_buffer(dev, size, gfp, &page); 649 addr = __alloc_simple_buffer(dev, size, gfp, &page);
637 else if (!(gfp & __GFP_WAIT)) 650 else if (!(gfp & __GFP_WAIT))
638 addr = __alloc_from_pool(size, &page); 651 addr = __alloc_from_pool(size, &page);
639 else if (!dev_get_cma_area(dev)) 652 else if (!dev_get_cma_area(dev))
640 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); 653 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr);
641 else 654 else
642 addr = __alloc_from_contiguous(dev, size, prot, &page, caller); 655 addr = __alloc_from_contiguous(dev, size, prot, &page, caller, want_vaddr);
643 656
644 if (addr) 657 if (page)
645 *handle = pfn_to_dma(dev, page_to_pfn(page)); 658 *handle = pfn_to_dma(dev, page_to_pfn(page));
646 659
647 return addr; 660 return want_vaddr ? addr : page;
648} 661}
649 662
650/* 663/*
@@ -661,7 +674,7 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
661 return memory; 674 return memory;
662 675
663 return __dma_alloc(dev, size, handle, gfp, prot, false, 676 return __dma_alloc(dev, size, handle, gfp, prot, false,
664 __builtin_return_address(0)); 677 attrs, __builtin_return_address(0));
665} 678}
666 679
667static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 680static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
@@ -674,7 +687,7 @@ static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
674 return memory; 687 return memory;
675 688
676 return __dma_alloc(dev, size, handle, gfp, prot, true, 689 return __dma_alloc(dev, size, handle, gfp, prot, true,
677 __builtin_return_address(0)); 690 attrs, __builtin_return_address(0));
678} 691}
679 692
680/* 693/*
@@ -715,6 +728,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
715 bool is_coherent) 728 bool is_coherent)
716{ 729{
717 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 730 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
731 bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
718 732
719 if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) 733 if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
720 return; 734 return;
@@ -726,14 +740,15 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
726 } else if (__free_from_pool(cpu_addr, size)) { 740 } else if (__free_from_pool(cpu_addr, size)) {
727 return; 741 return;
728 } else if (!dev_get_cma_area(dev)) { 742 } else if (!dev_get_cma_area(dev)) {
729 __dma_free_remap(cpu_addr, size); 743 if (want_vaddr)
744 __dma_free_remap(cpu_addr, size);
730 __dma_free_buffer(page, size); 745 __dma_free_buffer(page, size);
731 } else { 746 } else {
732 /* 747 /*
733 * Non-atomic allocations cannot be freed with IRQs disabled 748 * Non-atomic allocations cannot be freed with IRQs disabled
734 */ 749 */
735 WARN_ON(irqs_disabled()); 750 WARN_ON(irqs_disabled());
736 __free_from_contiguous(dev, page, cpu_addr, size); 751 __free_from_contiguous(dev, page, cpu_addr, size, want_vaddr);
737 } 752 }
738} 753}
739 754
@@ -1135,13 +1150,28 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1135 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 1150 gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1136 1151
1137 while (count) { 1152 while (count) {
1138 int j, order = __fls(count); 1153 int j, order;
1154
1155 for (order = __fls(count); order > 0; --order) {
1156 /*
1157 * We do not want OOM killer to be invoked as long
1158 * as we can fall back to single pages, so we force
1159 * __GFP_NORETRY for orders higher than zero.
1160 */
1161 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
1162 if (pages[i])
1163 break;
1164 }
1139 1165
1140 pages[i] = alloc_pages(gfp, order); 1166 if (!pages[i]) {
1141 while (!pages[i] && order) 1167 /*
1142 pages[i] = alloc_pages(gfp, --order); 1168 * Fall back to single page allocation.
1143 if (!pages[i]) 1169 * Might invoke OOM killer as last resort.
1144 goto error; 1170 */
1171 pages[i] = alloc_pages(gfp, 0);
1172 if (!pages[i])
1173 goto error;
1174 }
1145 1175
1146 if (order) { 1176 if (order) {
1147 split_page(pages[i], order); 1177 split_page(pages[i], order);
@@ -1206,7 +1236,7 @@ __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
1206static dma_addr_t 1236static dma_addr_t
1207__iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1237__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
1208{ 1238{
1209 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1239 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1210 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1240 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1211 dma_addr_t dma_addr, iova; 1241 dma_addr_t dma_addr, iova;
1212 int i, ret = DMA_ERROR_CODE; 1242 int i, ret = DMA_ERROR_CODE;
@@ -1242,7 +1272,7 @@ fail:
1242 1272
1243static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1273static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1244{ 1274{
1245 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1275 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1246 1276
1247 /* 1277 /*
1248 * add optional in-page offset from iova to size and align 1278 * add optional in-page offset from iova to size and align
@@ -1457,7 +1487,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1457 enum dma_data_direction dir, struct dma_attrs *attrs, 1487 enum dma_data_direction dir, struct dma_attrs *attrs,
1458 bool is_coherent) 1488 bool is_coherent)
1459{ 1489{
1460 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1490 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1461 dma_addr_t iova, iova_base; 1491 dma_addr_t iova, iova_base;
1462 int ret = 0; 1492 int ret = 0;
1463 unsigned int count; 1493 unsigned int count;
@@ -1678,7 +1708,7 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p
1678 unsigned long offset, size_t size, enum dma_data_direction dir, 1708 unsigned long offset, size_t size, enum dma_data_direction dir,
1679 struct dma_attrs *attrs) 1709 struct dma_attrs *attrs)
1680{ 1710{
1681 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1711 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1682 dma_addr_t dma_addr; 1712 dma_addr_t dma_addr;
1683 int ret, prot, len = PAGE_ALIGN(size + offset); 1713 int ret, prot, len = PAGE_ALIGN(size + offset);
1684 1714
@@ -1731,7 +1761,7 @@ static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1731 size_t size, enum dma_data_direction dir, 1761 size_t size, enum dma_data_direction dir,
1732 struct dma_attrs *attrs) 1762 struct dma_attrs *attrs)
1733{ 1763{
1734 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1764 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1735 dma_addr_t iova = handle & PAGE_MASK; 1765 dma_addr_t iova = handle & PAGE_MASK;
1736 int offset = handle & ~PAGE_MASK; 1766 int offset = handle & ~PAGE_MASK;
1737 int len = PAGE_ALIGN(size + offset); 1767 int len = PAGE_ALIGN(size + offset);
@@ -1756,7 +1786,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1756 size_t size, enum dma_data_direction dir, 1786 size_t size, enum dma_data_direction dir,
1757 struct dma_attrs *attrs) 1787 struct dma_attrs *attrs)
1758{ 1788{
1759 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1789 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1760 dma_addr_t iova = handle & PAGE_MASK; 1790 dma_addr_t iova = handle & PAGE_MASK;
1761 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1791 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1762 int offset = handle & ~PAGE_MASK; 1792 int offset = handle & ~PAGE_MASK;
@@ -1775,7 +1805,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1775static void arm_iommu_sync_single_for_cpu(struct device *dev, 1805static void arm_iommu_sync_single_for_cpu(struct device *dev,
1776 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1806 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1777{ 1807{
1778 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1808 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1779 dma_addr_t iova = handle & PAGE_MASK; 1809 dma_addr_t iova = handle & PAGE_MASK;
1780 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1810 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1781 unsigned int offset = handle & ~PAGE_MASK; 1811 unsigned int offset = handle & ~PAGE_MASK;
@@ -1789,7 +1819,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
1789static void arm_iommu_sync_single_for_device(struct device *dev, 1819static void arm_iommu_sync_single_for_device(struct device *dev,
1790 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1820 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1791{ 1821{
1792 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 1822 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1793 dma_addr_t iova = handle & PAGE_MASK; 1823 dma_addr_t iova = handle & PAGE_MASK;
1794 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1824 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1795 unsigned int offset = handle & ~PAGE_MASK; 1825 unsigned int offset = handle & ~PAGE_MASK;
@@ -1950,7 +1980,7 @@ static int __arm_iommu_attach_device(struct device *dev,
1950 return err; 1980 return err;
1951 1981
1952 kref_get(&mapping->kref); 1982 kref_get(&mapping->kref);
1953 dev->archdata.mapping = mapping; 1983 to_dma_iommu_mapping(dev) = mapping;
1954 1984
1955 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 1985 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
1956 return 0; 1986 return 0;
@@ -1995,7 +2025,7 @@ static void __arm_iommu_detach_device(struct device *dev)
1995 2025
1996 iommu_detach_device(mapping->domain, dev); 2026 iommu_detach_device(mapping->domain, dev);
1997 kref_put(&mapping->kref, release_iommu_mapping); 2027 kref_put(&mapping->kref, release_iommu_mapping);
1998 dev->archdata.mapping = NULL; 2028 to_dma_iommu_mapping(dev) = NULL;
1999 2029
2000 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 2030 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2001} 2031}
@@ -2027,6 +2057,13 @@ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2027 if (!iommu) 2057 if (!iommu)
2028 return false; 2058 return false;
2029 2059
2060 /*
2061 * currently arm_iommu_create_mapping() takes a max of size_t
2062 * for size param. So check this limit for now.
2063 */
2064 if (size > SIZE_MAX)
2065 return false;
2066
2030 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 2067 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
2031 if (IS_ERR(mapping)) { 2068 if (IS_ERR(mapping)) {
2032 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 2069 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
@@ -2046,7 +2083,7 @@ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2046 2083
2047static void arm_teardown_iommu_dma_ops(struct device *dev) 2084static void arm_teardown_iommu_dma_ops(struct device *dev)
2048{ 2085{
2049 struct dma_iommu_mapping *mapping = dev->archdata.mapping; 2086 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2050 2087
2051 if (!mapping) 2088 if (!mapping)
2052 return; 2089 return;
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 1609b022a72f..be92fa0f2f35 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -86,55 +86,6 @@ static int __init parse_tag_initrd2(const struct tag *tag)
86 86
87__tagtable(ATAG_INITRD2, parse_tag_initrd2); 87__tagtable(ATAG_INITRD2, parse_tag_initrd2);
88 88
89/*
90 * This keeps memory configuration data used by a couple memory
91 * initialization functions, as well as show_mem() for the skipping
92 * of holes in the memory map. It is populated by arm_add_memory().
93 */
94void show_mem(unsigned int filter)
95{
96 int free = 0, total = 0, reserved = 0;
97 int shared = 0, cached = 0, slab = 0;
98 struct memblock_region *reg;
99
100 printk("Mem-info:\n");
101 show_free_areas(filter);
102
103 for_each_memblock (memory, reg) {
104 unsigned int pfn1, pfn2;
105 struct page *page, *end;
106
107 pfn1 = memblock_region_memory_base_pfn(reg);
108 pfn2 = memblock_region_memory_end_pfn(reg);
109
110 page = pfn_to_page(pfn1);
111 end = pfn_to_page(pfn2 - 1) + 1;
112
113 do {
114 total++;
115 if (PageReserved(page))
116 reserved++;
117 else if (PageSwapCache(page))
118 cached++;
119 else if (PageSlab(page))
120 slab++;
121 else if (!page_count(page))
122 free++;
123 else
124 shared += page_count(page) - 1;
125 pfn1++;
126 page = pfn_to_page(pfn1);
127 } while (pfn1 < pfn2);
128 }
129
130 printk("%d pages of RAM\n", total);
131 printk("%d free pages\n", free);
132 printk("%d reserved pages\n", reserved);
133 printk("%d slab pages\n", slab);
134 printk("%d pages shared\n", shared);
135 printk("%d pages swap cached\n", cached);
136}
137
138static void __init find_limits(unsigned long *min, unsigned long *max_low, 89static void __init find_limits(unsigned long *min, unsigned long *max_low,
139 unsigned long *max_high) 90 unsigned long *max_high)
140{ 91{
@@ -335,6 +286,9 @@ void __init bootmem_init(void)
335 286
336 find_limits(&min, &max_low, &max_high); 287 find_limits(&min, &max_low, &max_high);
337 288
289 early_memtest((phys_addr_t)min << PAGE_SHIFT,
290 (phys_addr_t)max_low << PAGE_SHIFT);
291
338 /* 292 /*
339 * Sparsemem tries to allocate bootmem in memory_present(), 293 * Sparsemem tries to allocate bootmem in memory_present(),
340 * so must be done after the fixed reservations 294 * so must be done after the fixed reservations
diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c
index 5e85ed371364..407dc786583a 100644
--- a/arch/arm/mm/mmap.c
+++ b/arch/arm/mm/mmap.c
@@ -169,14 +169,22 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
169 return addr; 169 return addr;
170} 170}
171 171
172unsigned long arch_mmap_rnd(void)
173{
174 unsigned long rnd;
175
176 /* 8 bits of randomness in 20 address space bits */
177 rnd = (unsigned long)get_random_int() % (1 << 8);
178
179 return rnd << PAGE_SHIFT;
180}
181
172void arch_pick_mmap_layout(struct mm_struct *mm) 182void arch_pick_mmap_layout(struct mm_struct *mm)
173{ 183{
174 unsigned long random_factor = 0UL; 184 unsigned long random_factor = 0UL;
175 185
176 /* 8 bits of randomness in 20 address space bits */ 186 if (current->flags & PF_RANDOMIZE)
177 if ((current->flags & PF_RANDOMIZE) && 187 random_factor = arch_mmap_rnd();
178 !(current->personality & ADDR_NO_RANDOMIZE))
179 random_factor = (get_random_int() % (1 << 8)) << PAGE_SHIFT;
180 188
181 if (mmap_is_legacy()) { 189 if (mmap_is_legacy()) {
182 mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; 190 mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index 86ee5d47ce3c..aa0519eed698 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -507,7 +507,7 @@ cpu_arm1020_name:
507 507
508 .align 508 .align
509 509
510 .section ".proc.info.init", #alloc, #execinstr 510 .section ".proc.info.init", #alloc
511 511
512 .type __arm1020_proc_info,#object 512 .type __arm1020_proc_info,#object
513__arm1020_proc_info: 513__arm1020_proc_info:
@@ -519,7 +519,7 @@ __arm1020_proc_info:
519 .long PMD_TYPE_SECT | \ 519 .long PMD_TYPE_SECT | \
520 PMD_SECT_AP_WRITE | \ 520 PMD_SECT_AP_WRITE | \
521 PMD_SECT_AP_READ 521 PMD_SECT_AP_READ
522 b __arm1020_setup 522 initfn __arm1020_setup, __arm1020_proc_info
523 .long cpu_arch_name 523 .long cpu_arch_name
524 .long cpu_elf_name 524 .long cpu_elf_name
525 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB 525 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB
diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S
index a6331d78601f..bff4c7f70fd6 100644
--- a/arch/arm/mm/proc-arm1020e.S
+++ b/arch/arm/mm/proc-arm1020e.S
@@ -465,7 +465,7 @@ arm1020e_crval:
465 465
466 .align 466 .align
467 467
468 .section ".proc.info.init", #alloc, #execinstr 468 .section ".proc.info.init", #alloc
469 469
470 .type __arm1020e_proc_info,#object 470 .type __arm1020e_proc_info,#object
471__arm1020e_proc_info: 471__arm1020e_proc_info:
@@ -479,7 +479,7 @@ __arm1020e_proc_info:
479 PMD_BIT4 | \ 479 PMD_BIT4 | \
480 PMD_SECT_AP_WRITE | \ 480 PMD_SECT_AP_WRITE | \
481 PMD_SECT_AP_READ 481 PMD_SECT_AP_READ
482 b __arm1020e_setup 482 initfn __arm1020e_setup, __arm1020e_proc_info
483 .long cpu_arch_name 483 .long cpu_arch_name
484 .long cpu_elf_name 484 .long cpu_elf_name
485 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_EDSP 485 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_EDSP
diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S
index a126b7a59928..dbb2413fe04d 100644
--- a/arch/arm/mm/proc-arm1022.S
+++ b/arch/arm/mm/proc-arm1022.S
@@ -448,7 +448,7 @@ arm1022_crval:
448 448
449 .align 449 .align
450 450
451 .section ".proc.info.init", #alloc, #execinstr 451 .section ".proc.info.init", #alloc
452 452
453 .type __arm1022_proc_info,#object 453 .type __arm1022_proc_info,#object
454__arm1022_proc_info: 454__arm1022_proc_info:
@@ -462,7 +462,7 @@ __arm1022_proc_info:
462 PMD_BIT4 | \ 462 PMD_BIT4 | \
463 PMD_SECT_AP_WRITE | \ 463 PMD_SECT_AP_WRITE | \
464 PMD_SECT_AP_READ 464 PMD_SECT_AP_READ
465 b __arm1022_setup 465 initfn __arm1022_setup, __arm1022_proc_info
466 .long cpu_arch_name 466 .long cpu_arch_name
467 .long cpu_elf_name 467 .long cpu_elf_name
468 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_EDSP 468 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_EDSP
diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S
index fc294067e977..0b37b2cef9d3 100644
--- a/arch/arm/mm/proc-arm1026.S
+++ b/arch/arm/mm/proc-arm1026.S
@@ -442,7 +442,7 @@ arm1026_crval:
442 string cpu_arm1026_name, "ARM1026EJ-S" 442 string cpu_arm1026_name, "ARM1026EJ-S"
443 .align 443 .align
444 444
445 .section ".proc.info.init", #alloc, #execinstr 445 .section ".proc.info.init", #alloc
446 446
447 .type __arm1026_proc_info,#object 447 .type __arm1026_proc_info,#object
448__arm1026_proc_info: 448__arm1026_proc_info:
@@ -456,7 +456,7 @@ __arm1026_proc_info:
456 PMD_BIT4 | \ 456 PMD_BIT4 | \
457 PMD_SECT_AP_WRITE | \ 457 PMD_SECT_AP_WRITE | \
458 PMD_SECT_AP_READ 458 PMD_SECT_AP_READ
459 b __arm1026_setup 459 initfn __arm1026_setup, __arm1026_proc_info
460 .long cpu_arch_name 460 .long cpu_arch_name
461 .long cpu_elf_name 461 .long cpu_elf_name
462 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_JAVA 462 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_JAVA
diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S
index 2baa66b3ac9b..3651cd70e418 100644
--- a/arch/arm/mm/proc-arm720.S
+++ b/arch/arm/mm/proc-arm720.S
@@ -186,7 +186,7 @@ arm720_crval:
186 * See <asm/procinfo.h> for a definition of this structure. 186 * See <asm/procinfo.h> for a definition of this structure.
187 */ 187 */
188 188
189 .section ".proc.info.init", #alloc, #execinstr 189 .section ".proc.info.init", #alloc
190 190
191.macro arm720_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cpu_flush:req 191.macro arm720_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cpu_flush:req
192 .type __\name\()_proc_info,#object 192 .type __\name\()_proc_info,#object
@@ -203,7 +203,7 @@ __\name\()_proc_info:
203 PMD_BIT4 | \ 203 PMD_BIT4 | \
204 PMD_SECT_AP_WRITE | \ 204 PMD_SECT_AP_WRITE | \
205 PMD_SECT_AP_READ 205 PMD_SECT_AP_READ
206 b \cpu_flush @ cpu_flush 206 initfn \cpu_flush, __\name\()_proc_info @ cpu_flush
207 .long cpu_arch_name @ arch_name 207 .long cpu_arch_name @ arch_name
208 .long cpu_elf_name @ elf_name 208 .long cpu_elf_name @ elf_name
209 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB @ elf_hwcap 209 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB @ elf_hwcap
diff --git a/arch/arm/mm/proc-arm740.S b/arch/arm/mm/proc-arm740.S
index ac1ea6b3bce4..024fb7732407 100644
--- a/arch/arm/mm/proc-arm740.S
+++ b/arch/arm/mm/proc-arm740.S
@@ -132,14 +132,14 @@ __arm740_setup:
132 132
133 .align 133 .align
134 134
135 .section ".proc.info.init", #alloc, #execinstr 135 .section ".proc.info.init", #alloc
136 .type __arm740_proc_info,#object 136 .type __arm740_proc_info,#object
137__arm740_proc_info: 137__arm740_proc_info:
138 .long 0x41807400 138 .long 0x41807400
139 .long 0xfffffff0 139 .long 0xfffffff0
140 .long 0 140 .long 0
141 .long 0 141 .long 0
142 b __arm740_setup 142 initfn __arm740_setup, __arm740_proc_info
143 .long cpu_arch_name 143 .long cpu_arch_name
144 .long cpu_elf_name 144 .long cpu_elf_name
145 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_26BIT 145 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_26BIT
diff --git a/arch/arm/mm/proc-arm7tdmi.S b/arch/arm/mm/proc-arm7tdmi.S
index bf6ba4bc30ff..25472d94426d 100644
--- a/arch/arm/mm/proc-arm7tdmi.S
+++ b/arch/arm/mm/proc-arm7tdmi.S
@@ -76,7 +76,7 @@ __arm7tdmi_setup:
76 76
77 .align 77 .align
78 78
79 .section ".proc.info.init", #alloc, #execinstr 79 .section ".proc.info.init", #alloc
80 80
81.macro arm7tdmi_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, \ 81.macro arm7tdmi_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, \
82 extra_hwcaps=0 82 extra_hwcaps=0
@@ -86,7 +86,7 @@ __\name\()_proc_info:
86 .long \cpu_mask 86 .long \cpu_mask
87 .long 0 87 .long 0
88 .long 0 88 .long 0
89 b __arm7tdmi_setup 89 initfn __arm7tdmi_setup, __\name\()_proc_info
90 .long cpu_arch_name 90 .long cpu_arch_name
91 .long cpu_elf_name 91 .long cpu_elf_name
92 .long HWCAP_SWP | HWCAP_26BIT | ( \extra_hwcaps ) 92 .long HWCAP_SWP | HWCAP_26BIT | ( \extra_hwcaps )
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 22bf8dde4f84..7a14bd4414c9 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -448,7 +448,7 @@ arm920_crval:
448 448
449 .align 449 .align
450 450
451 .section ".proc.info.init", #alloc, #execinstr 451 .section ".proc.info.init", #alloc
452 452
453 .type __arm920_proc_info,#object 453 .type __arm920_proc_info,#object
454__arm920_proc_info: 454__arm920_proc_info:
@@ -464,7 +464,7 @@ __arm920_proc_info:
464 PMD_BIT4 | \ 464 PMD_BIT4 | \
465 PMD_SECT_AP_WRITE | \ 465 PMD_SECT_AP_WRITE | \
466 PMD_SECT_AP_READ 466 PMD_SECT_AP_READ
467 b __arm920_setup 467 initfn __arm920_setup, __arm920_proc_info
468 .long cpu_arch_name 468 .long cpu_arch_name
469 .long cpu_elf_name 469 .long cpu_elf_name
470 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB 470 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB
diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S
index 0c6d5ac5a6d4..edccfcdcd551 100644
--- a/arch/arm/mm/proc-arm922.S
+++ b/arch/arm/mm/proc-arm922.S
@@ -426,7 +426,7 @@ arm922_crval:
426 426
427 .align 427 .align
428 428
429 .section ".proc.info.init", #alloc, #execinstr 429 .section ".proc.info.init", #alloc
430 430
431 .type __arm922_proc_info,#object 431 .type __arm922_proc_info,#object
432__arm922_proc_info: 432__arm922_proc_info:
@@ -442,7 +442,7 @@ __arm922_proc_info:
442 PMD_BIT4 | \ 442 PMD_BIT4 | \
443 PMD_SECT_AP_WRITE | \ 443 PMD_SECT_AP_WRITE | \
444 PMD_SECT_AP_READ 444 PMD_SECT_AP_READ
445 b __arm922_setup 445 initfn __arm922_setup, __arm922_proc_info
446 .long cpu_arch_name 446 .long cpu_arch_name
447 .long cpu_elf_name 447 .long cpu_elf_name
448 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB 448 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB
diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S
index c32d073282ea..ede8c54ab4aa 100644
--- a/arch/arm/mm/proc-arm925.S
+++ b/arch/arm/mm/proc-arm925.S
@@ -494,7 +494,7 @@ arm925_crval:
494 494
495 .align 495 .align
496 496
497 .section ".proc.info.init", #alloc, #execinstr 497 .section ".proc.info.init", #alloc
498 498
499.macro arm925_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache 499.macro arm925_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache
500 .type __\name\()_proc_info,#object 500 .type __\name\()_proc_info,#object
@@ -510,7 +510,7 @@ __\name\()_proc_info:
510 PMD_BIT4 | \ 510 PMD_BIT4 | \
511 PMD_SECT_AP_WRITE | \ 511 PMD_SECT_AP_WRITE | \
512 PMD_SECT_AP_READ 512 PMD_SECT_AP_READ
513 b __arm925_setup 513 initfn __arm925_setup, __\name\()_proc_info
514 .long cpu_arch_name 514 .long cpu_arch_name
515 .long cpu_elf_name 515 .long cpu_elf_name
516 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB 516 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 252b2503038d..fb827c633693 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -474,7 +474,7 @@ arm926_crval:
474 474
475 .align 475 .align
476 476
477 .section ".proc.info.init", #alloc, #execinstr 477 .section ".proc.info.init", #alloc
478 478
479 .type __arm926_proc_info,#object 479 .type __arm926_proc_info,#object
480__arm926_proc_info: 480__arm926_proc_info:
@@ -490,7 +490,7 @@ __arm926_proc_info:
490 PMD_BIT4 | \ 490 PMD_BIT4 | \
491 PMD_SECT_AP_WRITE | \ 491 PMD_SECT_AP_WRITE | \
492 PMD_SECT_AP_READ 492 PMD_SECT_AP_READ
493 b __arm926_setup 493 initfn __arm926_setup, __arm926_proc_info
494 .long cpu_arch_name 494 .long cpu_arch_name
495 .long cpu_elf_name 495 .long cpu_elf_name
496 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_JAVA 496 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_JAVA
diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S
index e5212d489377..ee5b66f847c4 100644
--- a/arch/arm/mm/proc-arm940.S
+++ b/arch/arm/mm/proc-arm940.S
@@ -297,26 +297,16 @@ __arm940_setup:
297 mcr p15, 0, r0, c6, c0, 1 297 mcr p15, 0, r0, c6, c0, 1
298 298
299 ldr r0, =(CONFIG_DRAM_BASE & 0xFFFFF000) @ base[31:12] of RAM 299 ldr r0, =(CONFIG_DRAM_BASE & 0xFFFFF000) @ base[31:12] of RAM
300 ldr r1, =(CONFIG_DRAM_SIZE >> 12) @ size of RAM (must be >= 4KB) 300 ldr r7, =CONFIG_DRAM_SIZE >> 12 @ size of RAM (must be >= 4KB)
301 mov r2, #10 @ 11 is the minimum (4KB) 301 pr_val r3, r0, r7, #1
3021: add r2, r2, #1 @ area size *= 2 302 mcr p15, 0, r3, c6, c1, 0 @ set area 1, RAM
303 mov r1, r1, lsr #1 303 mcr p15, 0, r3, c6, c1, 1
304 bne 1b @ count not zero r-shift
305 orr r0, r0, r2, lsl #1 @ the area register value
306 orr r0, r0, #1 @ set enable bit
307 mcr p15, 0, r0, c6, c1, 0 @ set area 1, RAM
308 mcr p15, 0, r0, c6, c1, 1
309 304
310 ldr r0, =(CONFIG_FLASH_MEM_BASE & 0xFFFFF000) @ base[31:12] of FLASH 305 ldr r0, =(CONFIG_FLASH_MEM_BASE & 0xFFFFF000) @ base[31:12] of FLASH
311 ldr r1, =(CONFIG_FLASH_SIZE >> 12) @ size of FLASH (must be >= 4KB) 306 ldr r7, =CONFIG_FLASH_SIZE @ size of FLASH (must be >= 4KB)
312 mov r2, #10 @ 11 is the minimum (4KB) 307 pr_val r3, r0, r6, #1
3131: add r2, r2, #1 @ area size *= 2 308 mcr p15, 0, r3, c6, c2, 0 @ set area 2, ROM/FLASH
314 mov r1, r1, lsr #1 309 mcr p15, 0, r3, c6, c2, 1
315 bne 1b @ count not zero r-shift
316 orr r0, r0, r2, lsl #1 @ the area register value
317 orr r0, r0, #1 @ set enable bit
318 mcr p15, 0, r0, c6, c2, 0 @ set area 2, ROM/FLASH
319 mcr p15, 0, r0, c6, c2, 1
320 310
321 mov r0, #0x06 311 mov r0, #0x06
322 mcr p15, 0, r0, c2, c0, 0 @ Region 1&2 cacheable 312 mcr p15, 0, r0, c2, c0, 0 @ Region 1&2 cacheable
@@ -354,14 +344,14 @@ __arm940_setup:
354 344
355 .align 345 .align
356 346
357 .section ".proc.info.init", #alloc, #execinstr 347 .section ".proc.info.init", #alloc
358 348
359 .type __arm940_proc_info,#object 349 .type __arm940_proc_info,#object
360__arm940_proc_info: 350__arm940_proc_info:
361 .long 0x41009400 351 .long 0x41009400
362 .long 0xff00fff0 352 .long 0xff00fff0
363 .long 0 353 .long 0
364 b __arm940_setup 354 initfn __arm940_setup, __arm940_proc_info
365 .long cpu_arch_name 355 .long cpu_arch_name
366 .long cpu_elf_name 356 .long cpu_elf_name
367 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB 357 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB
diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S
index b3dd9b2d0b8e..7361837edc31 100644
--- a/arch/arm/mm/proc-arm946.S
+++ b/arch/arm/mm/proc-arm946.S
@@ -343,24 +343,14 @@ __arm946_setup:
343 mcr p15, 0, r0, c6, c0, 0 @ set region 0, default 343 mcr p15, 0, r0, c6, c0, 0 @ set region 0, default
344 344
345 ldr r0, =(CONFIG_DRAM_BASE & 0xFFFFF000) @ base[31:12] of RAM 345 ldr r0, =(CONFIG_DRAM_BASE & 0xFFFFF000) @ base[31:12] of RAM
346 ldr r1, =(CONFIG_DRAM_SIZE >> 12) @ size of RAM (must be >= 4KB) 346 ldr r7, =CONFIG_DRAM_SIZE @ size of RAM (must be >= 4KB)
347 mov r2, #10 @ 11 is the minimum (4KB) 347 pr_val r3, r0, r7, #1
3481: add r2, r2, #1 @ area size *= 2 348 mcr p15, 0, r3, c6, c1, 0
349 mov r1, r1, lsr #1
350 bne 1b @ count not zero r-shift
351 orr r0, r0, r2, lsl #1 @ the region register value
352 orr r0, r0, #1 @ set enable bit
353 mcr p15, 0, r0, c6, c1, 0 @ set region 1, RAM
354 349
355 ldr r0, =(CONFIG_FLASH_MEM_BASE & 0xFFFFF000) @ base[31:12] of FLASH 350 ldr r0, =(CONFIG_FLASH_MEM_BASE & 0xFFFFF000) @ base[31:12] of FLASH
356 ldr r1, =(CONFIG_FLASH_SIZE >> 12) @ size of FLASH (must be >= 4KB) 351 ldr r7, =CONFIG_FLASH_SIZE @ size of FLASH (must be >= 4KB)
357 mov r2, #10 @ 11 is the minimum (4KB) 352 pr_val r3, r0, r7, #1
3581: add r2, r2, #1 @ area size *= 2 353 mcr p15, 0, r3, c6, c2, 0
359 mov r1, r1, lsr #1
360 bne 1b @ count not zero r-shift
361 orr r0, r0, r2, lsl #1 @ the region register value
362 orr r0, r0, #1 @ set enable bit
363 mcr p15, 0, r0, c6, c2, 0 @ set region 2, ROM/FLASH
364 354
365 mov r0, #0x06 355 mov r0, #0x06
366 mcr p15, 0, r0, c2, c0, 0 @ region 1,2 d-cacheable 356 mcr p15, 0, r0, c2, c0, 0 @ region 1,2 d-cacheable
@@ -409,14 +399,14 @@ __arm946_setup:
409 399
410 .align 400 .align
411 401
412 .section ".proc.info.init", #alloc, #execinstr 402 .section ".proc.info.init", #alloc
413 .type __arm946_proc_info,#object 403 .type __arm946_proc_info,#object
414__arm946_proc_info: 404__arm946_proc_info:
415 .long 0x41009460 405 .long 0x41009460
416 .long 0xff00fff0 406 .long 0xff00fff0
417 .long 0 407 .long 0
418 .long 0 408 .long 0
419 b __arm946_setup 409 initfn __arm946_setup, __arm946_proc_info
420 .long cpu_arch_name 410 .long cpu_arch_name
421 .long cpu_elf_name 411 .long cpu_elf_name
422 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB 412 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB
diff --git a/arch/arm/mm/proc-arm9tdmi.S b/arch/arm/mm/proc-arm9tdmi.S
index 8227322bbb8f..7fac8c612134 100644
--- a/arch/arm/mm/proc-arm9tdmi.S
+++ b/arch/arm/mm/proc-arm9tdmi.S
@@ -70,7 +70,7 @@ __arm9tdmi_setup:
70 70
71 .align 71 .align
72 72
73 .section ".proc.info.init", #alloc, #execinstr 73 .section ".proc.info.init", #alloc
74 74
75.macro arm9tdmi_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req 75.macro arm9tdmi_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req
76 .type __\name\()_proc_info, #object 76 .type __\name\()_proc_info, #object
@@ -79,7 +79,7 @@ __\name\()_proc_info:
79 .long \cpu_mask 79 .long \cpu_mask
80 .long 0 80 .long 0
81 .long 0 81 .long 0
82 b __arm9tdmi_setup 82 initfn __arm9tdmi_setup, __\name\()_proc_info
83 .long cpu_arch_name 83 .long cpu_arch_name
84 .long cpu_elf_name 84 .long cpu_elf_name
85 .long HWCAP_SWP | HWCAP_THUMB | HWCAP_26BIT 85 .long HWCAP_SWP | HWCAP_THUMB | HWCAP_26BIT
diff --git a/arch/arm/mm/proc-fa526.S b/arch/arm/mm/proc-fa526.S
index c494886892ba..4001b73af4ee 100644
--- a/arch/arm/mm/proc-fa526.S
+++ b/arch/arm/mm/proc-fa526.S
@@ -190,7 +190,7 @@ fa526_cr1_set:
190 190
191 .align 191 .align
192 192
193 .section ".proc.info.init", #alloc, #execinstr 193 .section ".proc.info.init", #alloc
194 194
195 .type __fa526_proc_info,#object 195 .type __fa526_proc_info,#object
196__fa526_proc_info: 196__fa526_proc_info:
@@ -206,7 +206,7 @@ __fa526_proc_info:
206 PMD_BIT4 | \ 206 PMD_BIT4 | \
207 PMD_SECT_AP_WRITE | \ 207 PMD_SECT_AP_WRITE | \
208 PMD_SECT_AP_READ 208 PMD_SECT_AP_READ
209 b __fa526_setup 209 initfn __fa526_setup, __fa526_proc_info
210 .long cpu_arch_name 210 .long cpu_arch_name
211 .long cpu_elf_name 211 .long cpu_elf_name
212 .long HWCAP_SWP | HWCAP_HALF 212 .long HWCAP_SWP | HWCAP_HALF
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index 03a1b75f2e16..e494d6d6acbe 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -584,7 +584,7 @@ feroceon_crval:
584 584
585 .align 585 .align
586 586
587 .section ".proc.info.init", #alloc, #execinstr 587 .section ".proc.info.init", #alloc
588 588
589.macro feroceon_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache:req 589.macro feroceon_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache:req
590 .type __\name\()_proc_info,#object 590 .type __\name\()_proc_info,#object
@@ -601,7 +601,8 @@ __\name\()_proc_info:
601 PMD_BIT4 | \ 601 PMD_BIT4 | \
602 PMD_SECT_AP_WRITE | \ 602 PMD_SECT_AP_WRITE | \
603 PMD_SECT_AP_READ 603 PMD_SECT_AP_READ
604 b __feroceon_setup 604 initfn __feroceon_setup, __\name\()_proc_info
605 .long __feroceon_setup
605 .long cpu_arch_name 606 .long cpu_arch_name
606 .long cpu_elf_name 607 .long cpu_elf_name
607 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP 608 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 082b9f2f7e90..c671f345266a 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -331,3 +331,31 @@ ENTRY(\name\()_tlb_fns)
331 .globl \x 331 .globl \x
332 .equ \x, \y 332 .equ \x, \y
333.endm 333.endm
334
335.macro initfn, func, base
336 .long \func - \base
337.endm
338
339 /*
340 * Macro to calculate the log2 size for the protection region
341 * registers. This calculates rd = log2(size) - 1. tmp must
342 * not be the same register as rd.
343 */
344.macro pr_sz, rd, size, tmp
345 mov \tmp, \size, lsr #12
346 mov \rd, #11
3471: movs \tmp, \tmp, lsr #1
348 addne \rd, \rd, #1
349 bne 1b
350.endm
351
352 /*
353 * Macro to generate a protection region register value
354 * given a pre-masked address, size, and enable bit.
355 * Corrupts size.
356 */
357.macro pr_val, dest, addr, size, enable
358 pr_sz \dest, \size, \size @ calculate log2(size) - 1
359 orr \dest, \addr, \dest, lsl #1 @ mask in the region size
360 orr \dest, \dest, \enable
361.endm
diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S
index 53d393455f13..d65edf717bf7 100644
--- a/arch/arm/mm/proc-mohawk.S
+++ b/arch/arm/mm/proc-mohawk.S
@@ -427,7 +427,7 @@ mohawk_crval:
427 427
428 .align 428 .align
429 429
430 .section ".proc.info.init", #alloc, #execinstr 430 .section ".proc.info.init", #alloc
431 431
432 .type __88sv331x_proc_info,#object 432 .type __88sv331x_proc_info,#object
433__88sv331x_proc_info: 433__88sv331x_proc_info:
@@ -443,7 +443,7 @@ __88sv331x_proc_info:
443 PMD_BIT4 | \ 443 PMD_BIT4 | \
444 PMD_SECT_AP_WRITE | \ 444 PMD_SECT_AP_WRITE | \
445 PMD_SECT_AP_READ 445 PMD_SECT_AP_READ
446 b __mohawk_setup 446 initfn __mohawk_setup, __88sv331x_proc_info
447 .long cpu_arch_name 447 .long cpu_arch_name
448 .long cpu_elf_name 448 .long cpu_elf_name
449 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP 449 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP
diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S
index 8008a0461cf5..ee2ce496239f 100644
--- a/arch/arm/mm/proc-sa110.S
+++ b/arch/arm/mm/proc-sa110.S
@@ -199,7 +199,7 @@ sa110_crval:
199 199
200 .align 200 .align
201 201
202 .section ".proc.info.init", #alloc, #execinstr 202 .section ".proc.info.init", #alloc
203 203
204 .type __sa110_proc_info,#object 204 .type __sa110_proc_info,#object
205__sa110_proc_info: 205__sa110_proc_info:
@@ -213,7 +213,7 @@ __sa110_proc_info:
213 .long PMD_TYPE_SECT | \ 213 .long PMD_TYPE_SECT | \
214 PMD_SECT_AP_WRITE | \ 214 PMD_SECT_AP_WRITE | \
215 PMD_SECT_AP_READ 215 PMD_SECT_AP_READ
216 b __sa110_setup 216 initfn __sa110_setup, __sa110_proc_info
217 .long cpu_arch_name 217 .long cpu_arch_name
218 .long cpu_elf_name 218 .long cpu_elf_name
219 .long HWCAP_SWP | HWCAP_HALF | HWCAP_26BIT | HWCAP_FAST_MULT 219 .long HWCAP_SWP | HWCAP_HALF | HWCAP_26BIT | HWCAP_FAST_MULT
diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S
index 89f97ac648a9..222d5836f666 100644
--- a/arch/arm/mm/proc-sa1100.S
+++ b/arch/arm/mm/proc-sa1100.S
@@ -242,7 +242,7 @@ sa1100_crval:
242 242
243 .align 243 .align
244 244
245 .section ".proc.info.init", #alloc, #execinstr 245 .section ".proc.info.init", #alloc
246 246
247.macro sa1100_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req 247.macro sa1100_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req
248 .type __\name\()_proc_info,#object 248 .type __\name\()_proc_info,#object
@@ -257,7 +257,7 @@ __\name\()_proc_info:
257 .long PMD_TYPE_SECT | \ 257 .long PMD_TYPE_SECT | \
258 PMD_SECT_AP_WRITE | \ 258 PMD_SECT_AP_WRITE | \
259 PMD_SECT_AP_READ 259 PMD_SECT_AP_READ
260 b __sa1100_setup 260 initfn __sa1100_setup, __\name\()_proc_info
261 .long cpu_arch_name 261 .long cpu_arch_name
262 .long cpu_elf_name 262 .long cpu_elf_name
263 .long HWCAP_SWP | HWCAP_HALF | HWCAP_26BIT | HWCAP_FAST_MULT 263 .long HWCAP_SWP | HWCAP_HALF | HWCAP_26BIT | HWCAP_FAST_MULT
diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S
index d0390f4b3f18..06d890a2342b 100644
--- a/arch/arm/mm/proc-v6.S
+++ b/arch/arm/mm/proc-v6.S
@@ -264,7 +264,7 @@ v6_crval:
264 string cpu_elf_name, "v6" 264 string cpu_elf_name, "v6"
265 .align 265 .align
266 266
267 .section ".proc.info.init", #alloc, #execinstr 267 .section ".proc.info.init", #alloc
268 268
269 /* 269 /*
270 * Match any ARMv6 processor core. 270 * Match any ARMv6 processor core.
@@ -287,7 +287,7 @@ __v6_proc_info:
287 PMD_SECT_XN | \ 287 PMD_SECT_XN | \
288 PMD_SECT_AP_WRITE | \ 288 PMD_SECT_AP_WRITE | \
289 PMD_SECT_AP_READ 289 PMD_SECT_AP_READ
290 b __v6_setup 290 initfn __v6_setup, __v6_proc_info
291 .long cpu_arch_name 291 .long cpu_arch_name
292 .long cpu_elf_name 292 .long cpu_elf_name
293 /* See also feat_v6_fixup() for HWCAP_TLS */ 293 /* See also feat_v6_fixup() for HWCAP_TLS */
diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S
index ed448d8a596b..10405b8d31af 100644
--- a/arch/arm/mm/proc-v7-2level.S
+++ b/arch/arm/mm/proc-v7-2level.S
@@ -37,15 +37,18 @@
37 * It is assumed that: 37 * It is assumed that:
38 * - we are not using split page tables 38 * - we are not using split page tables
39 */ 39 */
40ENTRY(cpu_v7_switch_mm) 40ENTRY(cpu_ca8_switch_mm)
41#ifdef CONFIG_MMU 41#ifdef CONFIG_MMU
42 mov r2, #0 42 mov r2, #0
43 mmid r1, r1 @ get mm->context.id
44 ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP)
45 ALT_UP(orr r0, r0, #TTB_FLAGS_UP)
46#ifdef CONFIG_ARM_ERRATA_430973 43#ifdef CONFIG_ARM_ERRATA_430973
47 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB 44 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
48#endif 45#endif
46#endif
47ENTRY(cpu_v7_switch_mm)
48#ifdef CONFIG_MMU
49 mmid r1, r1 @ get mm->context.id
50 ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP)
51 ALT_UP(orr r0, r0, #TTB_FLAGS_UP)
49#ifdef CONFIG_PID_IN_CONTEXTIDR 52#ifdef CONFIG_PID_IN_CONTEXTIDR
50 mrc p15, 0, r2, c13, c0, 1 @ read current context ID 53 mrc p15, 0, r2, c13, c0, 1 @ read current context ID
51 lsr r2, r2, #8 @ extract the PID 54 lsr r2, r2, #8 @ extract the PID
@@ -61,6 +64,7 @@ ENTRY(cpu_v7_switch_mm)
61#endif 64#endif
62 bx lr 65 bx lr
63ENDPROC(cpu_v7_switch_mm) 66ENDPROC(cpu_v7_switch_mm)
67ENDPROC(cpu_ca8_switch_mm)
64 68
65/* 69/*
66 * cpu_v7_set_pte_ext(ptep, pte) 70 * cpu_v7_set_pte_ext(ptep, pte)
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 8b4ee5e81c14..3d1054f11a8a 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -153,6 +153,21 @@ ENDPROC(cpu_v7_do_resume)
153#endif 153#endif
154 154
155/* 155/*
156 * Cortex-A8
157 */
158 globl_equ cpu_ca8_proc_init, cpu_v7_proc_init
159 globl_equ cpu_ca8_proc_fin, cpu_v7_proc_fin
160 globl_equ cpu_ca8_reset, cpu_v7_reset
161 globl_equ cpu_ca8_do_idle, cpu_v7_do_idle
162 globl_equ cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area
163 globl_equ cpu_ca8_set_pte_ext, cpu_v7_set_pte_ext
164 globl_equ cpu_ca8_suspend_size, cpu_v7_suspend_size
165#ifdef CONFIG_ARM_CPU_SUSPEND
166 globl_equ cpu_ca8_do_suspend, cpu_v7_do_suspend
167 globl_equ cpu_ca8_do_resume, cpu_v7_do_resume
168#endif
169
170/*
156 * Cortex-A9 processor functions 171 * Cortex-A9 processor functions
157 */ 172 */
158 globl_equ cpu_ca9mp_proc_init, cpu_v7_proc_init 173 globl_equ cpu_ca9mp_proc_init, cpu_v7_proc_init
@@ -451,7 +466,10 @@ __v7_setup_stack:
451 466
452 @ define struct processor (see <asm/proc-fns.h> and proc-macros.S) 467 @ define struct processor (see <asm/proc-fns.h> and proc-macros.S)
453 define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 468 define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
469#ifndef CONFIG_ARM_LPAE
470 define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
454 define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 471 define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
472#endif
455#ifdef CONFIG_CPU_PJ4B 473#ifdef CONFIG_CPU_PJ4B
456 define_processor_functions pj4b, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 474 define_processor_functions pj4b, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
457#endif 475#endif
@@ -462,19 +480,19 @@ __v7_setup_stack:
462 string cpu_elf_name, "v7" 480 string cpu_elf_name, "v7"
463 .align 481 .align
464 482
465 .section ".proc.info.init", #alloc, #execinstr 483 .section ".proc.info.init", #alloc
466 484
467 /* 485 /*
468 * Standard v7 proc info content 486 * Standard v7 proc info content
469 */ 487 */
470.macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions 488.macro __v7_proc name, initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions
471 ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ 489 ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
472 PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags) 490 PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags)
473 ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ 491 ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
474 PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags) 492 PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags)
475 .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \ 493 .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \
476 PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags 494 PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags
477 W(b) \initfunc 495 initfn \initfunc, \name
478 .long cpu_arch_name 496 .long cpu_arch_name
479 .long cpu_elf_name 497 .long cpu_elf_name
480 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_FAST_MULT | \ 498 .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_FAST_MULT | \
@@ -494,7 +512,7 @@ __v7_setup_stack:
494__v7_ca5mp_proc_info: 512__v7_ca5mp_proc_info:
495 .long 0x410fc050 513 .long 0x410fc050
496 .long 0xff0ffff0 514 .long 0xff0ffff0
497 __v7_proc __v7_ca5mp_setup 515 __v7_proc __v7_ca5mp_proc_info, __v7_ca5mp_setup
498 .size __v7_ca5mp_proc_info, . - __v7_ca5mp_proc_info 516 .size __v7_ca5mp_proc_info, . - __v7_ca5mp_proc_info
499 517
500 /* 518 /*
@@ -504,9 +522,19 @@ __v7_ca5mp_proc_info:
504__v7_ca9mp_proc_info: 522__v7_ca9mp_proc_info:
505 .long 0x410fc090 523 .long 0x410fc090
506 .long 0xff0ffff0 524 .long 0xff0ffff0
507 __v7_proc __v7_ca9mp_setup, proc_fns = ca9mp_processor_functions 525 __v7_proc __v7_ca9mp_proc_info, __v7_ca9mp_setup, proc_fns = ca9mp_processor_functions
508 .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info 526 .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info
509 527
528 /*
529 * ARM Ltd. Cortex A8 processor.
530 */
531 .type __v7_ca8_proc_info, #object
532__v7_ca8_proc_info:
533 .long 0x410fc080
534 .long 0xff0ffff0
535 __v7_proc __v7_ca8_proc_info, __v7_setup, proc_fns = ca8_processor_functions
536 .size __v7_ca8_proc_info, . - __v7_ca8_proc_info
537
510#endif /* CONFIG_ARM_LPAE */ 538#endif /* CONFIG_ARM_LPAE */
511 539
512 /* 540 /*
@@ -517,7 +545,7 @@ __v7_ca9mp_proc_info:
517__v7_pj4b_proc_info: 545__v7_pj4b_proc_info:
518 .long 0x560f5800 546 .long 0x560f5800
519 .long 0xff0fff00 547 .long 0xff0fff00
520 __v7_proc __v7_pj4b_setup, proc_fns = pj4b_processor_functions 548 __v7_proc __v7_pj4b_proc_info, __v7_pj4b_setup, proc_fns = pj4b_processor_functions
521 .size __v7_pj4b_proc_info, . - __v7_pj4b_proc_info 549 .size __v7_pj4b_proc_info, . - __v7_pj4b_proc_info
522#endif 550#endif
523 551
@@ -528,7 +556,7 @@ __v7_pj4b_proc_info:
528__v7_cr7mp_proc_info: 556__v7_cr7mp_proc_info:
529 .long 0x410fc170 557 .long 0x410fc170
530 .long 0xff0ffff0 558 .long 0xff0ffff0
531 __v7_proc __v7_cr7mp_setup 559 __v7_proc __v7_cr7mp_proc_info, __v7_cr7mp_setup
532 .size __v7_cr7mp_proc_info, . - __v7_cr7mp_proc_info 560 .size __v7_cr7mp_proc_info, . - __v7_cr7mp_proc_info
533 561
534 /* 562 /*
@@ -538,7 +566,7 @@ __v7_cr7mp_proc_info:
538__v7_ca7mp_proc_info: 566__v7_ca7mp_proc_info:
539 .long 0x410fc070 567 .long 0x410fc070
540 .long 0xff0ffff0 568 .long 0xff0ffff0
541 __v7_proc __v7_ca7mp_setup 569 __v7_proc __v7_ca7mp_proc_info, __v7_ca7mp_setup
542 .size __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info 570 .size __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info
543 571
544 /* 572 /*
@@ -548,7 +576,7 @@ __v7_ca7mp_proc_info:
548__v7_ca12mp_proc_info: 576__v7_ca12mp_proc_info:
549 .long 0x410fc0d0 577 .long 0x410fc0d0
550 .long 0xff0ffff0 578 .long 0xff0ffff0
551 __v7_proc __v7_ca12mp_setup 579 __v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup
552 .size __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info 580 .size __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info
553 581
554 /* 582 /*
@@ -558,7 +586,7 @@ __v7_ca12mp_proc_info:
558__v7_ca15mp_proc_info: 586__v7_ca15mp_proc_info:
559 .long 0x410fc0f0 587 .long 0x410fc0f0
560 .long 0xff0ffff0 588 .long 0xff0ffff0
561 __v7_proc __v7_ca15mp_setup 589 __v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup
562 .size __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info 590 .size __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info
563 591
564 /* 592 /*
@@ -568,7 +596,7 @@ __v7_ca15mp_proc_info:
568__v7_b15mp_proc_info: 596__v7_b15mp_proc_info:
569 .long 0x420f00f0 597 .long 0x420f00f0
570 .long 0xff0ffff0 598 .long 0xff0ffff0
571 __v7_proc __v7_b15mp_setup 599 __v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup
572 .size __v7_b15mp_proc_info, . - __v7_b15mp_proc_info 600 .size __v7_b15mp_proc_info, . - __v7_b15mp_proc_info
573 601
574 /* 602 /*
@@ -578,7 +606,7 @@ __v7_b15mp_proc_info:
578__v7_ca17mp_proc_info: 606__v7_ca17mp_proc_info:
579 .long 0x410fc0e0 607 .long 0x410fc0e0
580 .long 0xff0ffff0 608 .long 0xff0ffff0
581 __v7_proc __v7_ca17mp_setup 609 __v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup
582 .size __v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info 610 .size __v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info
583 611
584 /* 612 /*
@@ -594,7 +622,7 @@ __krait_proc_info:
594 * do support them. They also don't indicate support for fused multiply 622 * do support them. They also don't indicate support for fused multiply
595 * instructions even though they actually do support them. 623 * instructions even though they actually do support them.
596 */ 624 */
597 __v7_proc __v7_setup, hwcaps = HWCAP_IDIV | HWCAP_VFPv4 625 __v7_proc __krait_proc_info, __v7_setup, hwcaps = HWCAP_IDIV | HWCAP_VFPv4
598 .size __krait_proc_info, . - __krait_proc_info 626 .size __krait_proc_info, . - __krait_proc_info
599 627
600 /* 628 /*
@@ -604,5 +632,5 @@ __krait_proc_info:
604__v7_proc_info: 632__v7_proc_info:
605 .long 0x000f0000 @ Required ID value 633 .long 0x000f0000 @ Required ID value
606 .long 0x000f0000 @ Mask for ID 634 .long 0x000f0000 @ Mask for ID
607 __v7_proc __v7_setup 635 __v7_proc __v7_proc_info, __v7_setup
608 .size __v7_proc_info, . - __v7_proc_info 636 .size __v7_proc_info, . - __v7_proc_info
diff --git a/arch/arm/mm/proc-v7m.S b/arch/arm/mm/proc-v7m.S
index d1e68b553d3b..e08e1f2bab76 100644
--- a/arch/arm/mm/proc-v7m.S
+++ b/arch/arm/mm/proc-v7m.S
@@ -135,7 +135,7 @@ __v7m_setup_stack_top:
135 string cpu_elf_name "v7m" 135 string cpu_elf_name "v7m"
136 string cpu_v7m_name "ARMv7-M" 136 string cpu_v7m_name "ARMv7-M"
137 137
138 .section ".proc.info.init", #alloc, #execinstr 138 .section ".proc.info.init", #alloc
139 139
140 /* 140 /*
141 * Match any ARMv7-M processor core. 141 * Match any ARMv7-M processor core.
@@ -146,7 +146,7 @@ __v7m_proc_info:
146 .long 0x000f0000 @ Mask for ID 146 .long 0x000f0000 @ Mask for ID
147 .long 0 @ proc_info_list.__cpu_mm_mmu_flags 147 .long 0 @ proc_info_list.__cpu_mm_mmu_flags
148 .long 0 @ proc_info_list.__cpu_io_mmu_flags 148 .long 0 @ proc_info_list.__cpu_io_mmu_flags
149 b __v7m_setup @ proc_info_list.__cpu_flush 149 initfn __v7m_setup, __v7m_proc_info @ proc_info_list.__cpu_flush
150 .long cpu_arch_name 150 .long cpu_arch_name
151 .long cpu_elf_name 151 .long cpu_elf_name
152 .long HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT 152 .long HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index f8acdfece036..293dcc2c441f 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -499,7 +499,7 @@ xsc3_crval:
499 499
500 .align 500 .align
501 501
502 .section ".proc.info.init", #alloc, #execinstr 502 .section ".proc.info.init", #alloc
503 503
504.macro xsc3_proc_info name:req, cpu_val:req, cpu_mask:req 504.macro xsc3_proc_info name:req, cpu_val:req, cpu_mask:req
505 .type __\name\()_proc_info,#object 505 .type __\name\()_proc_info,#object
@@ -514,7 +514,7 @@ __\name\()_proc_info:
514 .long PMD_TYPE_SECT | \ 514 .long PMD_TYPE_SECT | \
515 PMD_SECT_AP_WRITE | \ 515 PMD_SECT_AP_WRITE | \
516 PMD_SECT_AP_READ 516 PMD_SECT_AP_READ
517 b __xsc3_setup 517 initfn __xsc3_setup, __\name\()_proc_info
518 .long cpu_arch_name 518 .long cpu_arch_name
519 .long cpu_elf_name 519 .long cpu_elf_name
520 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP 520 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index afa2b3c4df4a..b6bbfdb6dfdc 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -612,7 +612,7 @@ xscale_crval:
612 612
613 .align 613 .align
614 614
615 .section ".proc.info.init", #alloc, #execinstr 615 .section ".proc.info.init", #alloc
616 616
617.macro xscale_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache 617.macro xscale_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache
618 .type __\name\()_proc_info,#object 618 .type __\name\()_proc_info,#object
@@ -627,7 +627,7 @@ __\name\()_proc_info:
627 .long PMD_TYPE_SECT | \ 627 .long PMD_TYPE_SECT | \
628 PMD_SECT_AP_WRITE | \ 628 PMD_SECT_AP_WRITE | \
629 PMD_SECT_AP_READ 629 PMD_SECT_AP_READ
630 b __xscale_setup 630 initfn __xscale_setup, __\name\()_proc_info
631 .long cpu_arch_name 631 .long cpu_arch_name
632 .long cpu_elf_name 632 .long cpu_elf_name
633 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP 633 .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP
diff --git a/arch/arm/nwfpe/entry.S b/arch/arm/nwfpe/entry.S
index 5d65be1f1e8a..71df43547659 100644
--- a/arch/arm/nwfpe/entry.S
+++ b/arch/arm/nwfpe/entry.S
@@ -113,7 +113,7 @@ next:
113 @ to fault. Emit the appropriate exception gunk to fix things up. 113 @ to fault. Emit the appropriate exception gunk to fix things up.
114 @ ??? For some reason, faults can happen at .Lx2 even with a 114 @ ??? For some reason, faults can happen at .Lx2 even with a
115 @ plain LDR instruction. Weird, but it seems harmless. 115 @ plain LDR instruction. Weird, but it seems harmless.
116 .pushsection .fixup,"ax" 116 .pushsection .text.fixup,"ax"
117 .align 2 117 .align 2
118.Lfix: ret r9 @ let the user eat segfaults 118.Lfix: ret r9 @ let the user eat segfaults
119 .popsection 119 .popsection
diff --git a/arch/arm/plat-omap/counter_32k.c b/arch/arm/plat-omap/counter_32k.c
index 61b4d705c267..2438b96004c1 100644
--- a/arch/arm/plat-omap/counter_32k.c
+++ b/arch/arm/plat-omap/counter_32k.c
@@ -44,24 +44,20 @@ static u64 notrace omap_32k_read_sched_clock(void)
44} 44}
45 45
46/** 46/**
47 * omap_read_persistent_clock - Return time from a persistent clock. 47 * omap_read_persistent_clock64 - Return time from a persistent clock.
48 * 48 *
49 * Reads the time from a source which isn't disabled during PM, the 49 * Reads the time from a source which isn't disabled during PM, the
50 * 32k sync timer. Convert the cycles elapsed since last read into 50 * 32k sync timer. Convert the cycles elapsed since last read into
51 * nsecs and adds to a monotonically increasing timespec. 51 * nsecs and adds to a monotonically increasing timespec64.
52 */ 52 */
53static struct timespec persistent_ts; 53static struct timespec64 persistent_ts;
54static cycles_t cycles; 54static cycles_t cycles;
55static unsigned int persistent_mult, persistent_shift; 55static unsigned int persistent_mult, persistent_shift;
56static DEFINE_SPINLOCK(read_persistent_clock_lock);
57 56
58static void omap_read_persistent_clock(struct timespec *ts) 57static void omap_read_persistent_clock64(struct timespec64 *ts)
59{ 58{
60 unsigned long long nsecs; 59 unsigned long long nsecs;
61 cycles_t last_cycles; 60 cycles_t last_cycles;
62 unsigned long flags;
63
64 spin_lock_irqsave(&read_persistent_clock_lock, flags);
65 61
66 last_cycles = cycles; 62 last_cycles = cycles;
67 cycles = sync32k_cnt_reg ? readl_relaxed(sync32k_cnt_reg) : 0; 63 cycles = sync32k_cnt_reg ? readl_relaxed(sync32k_cnt_reg) : 0;
@@ -69,11 +65,9 @@ static void omap_read_persistent_clock(struct timespec *ts)
69 nsecs = clocksource_cyc2ns(cycles - last_cycles, 65 nsecs = clocksource_cyc2ns(cycles - last_cycles,
70 persistent_mult, persistent_shift); 66 persistent_mult, persistent_shift);
71 67
72 timespec_add_ns(&persistent_ts, nsecs); 68 timespec64_add_ns(&persistent_ts, nsecs);
73 69
74 *ts = persistent_ts; 70 *ts = persistent_ts;
75
76 spin_unlock_irqrestore(&read_persistent_clock_lock, flags);
77} 71}
78 72
79/** 73/**
@@ -103,7 +97,7 @@ int __init omap_init_clocksource_32k(void __iomem *vbase)
103 97
104 /* 98 /*
105 * 120000 rough estimate from the calculations in 99 * 120000 rough estimate from the calculations in
106 * __clocksource_updatefreq_scale. 100 * __clocksource_update_freq_scale.
107 */ 101 */
108 clocks_calc_mult_shift(&persistent_mult, &persistent_shift, 102 clocks_calc_mult_shift(&persistent_mult, &persistent_shift,
109 32768, NSEC_PER_SEC, 120000); 103 32768, NSEC_PER_SEC, 120000);
@@ -116,7 +110,7 @@ int __init omap_init_clocksource_32k(void __iomem *vbase)
116 } 110 }
117 111
118 sched_clock_register(omap_32k_read_sched_clock, 32, 32768); 112 sched_clock_register(omap_32k_read_sched_clock, 32, 32768);
119 register_persistent_clock(NULL, omap_read_persistent_clock); 113 register_persistent_clock(NULL, omap_read_persistent_clock64);
120 pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); 114 pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n");
121 115
122 return 0; 116 return 0;
diff --git a/arch/arm/vdso/.gitignore b/arch/arm/vdso/.gitignore
new file mode 100644
index 000000000000..f8b69d84238e
--- /dev/null
+++ b/arch/arm/vdso/.gitignore
@@ -0,0 +1 @@
vdso.lds
diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile
new file mode 100644
index 000000000000..bab0a8be7924
--- /dev/null
+++ b/arch/arm/vdso/Makefile
@@ -0,0 +1,74 @@
1hostprogs-y := vdsomunge
2
3obj-vdso := vgettimeofday.o datapage.o
4
5# Build rules
6targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.so.raw vdso.lds
7obj-vdso := $(addprefix $(obj)/, $(obj-vdso))
8
9ccflags-y := -shared -fPIC -fno-common -fno-builtin -fno-stack-protector
10ccflags-y += -nostdlib -Wl,-soname=linux-vdso.so.1 -DDISABLE_BRANCH_PROFILING
11ccflags-y += -Wl,--no-undefined $(call cc-ldoption, -Wl$(comma)--hash-style=sysv)
12
13obj-y += vdso.o
14extra-y += vdso.lds
15CPPFLAGS_vdso.lds += -P -C -U$(ARCH)
16
17CFLAGS_REMOVE_vdso.o = -pg
18
19# Force -O2 to avoid libgcc dependencies
20CFLAGS_REMOVE_vgettimeofday.o = -pg -Os
21CFLAGS_vgettimeofday.o = -O2
22
23# Disable gcov profiling for VDSO code
24GCOV_PROFILE := n
25
26# Force dependency
27$(obj)/vdso.o : $(obj)/vdso.so
28
29# Link rule for the .so file
30$(obj)/vdso.so.raw: $(src)/vdso.lds $(obj-vdso) FORCE
31 $(call if_changed,vdsold)
32
33$(obj)/vdso.so.dbg: $(obj)/vdso.so.raw $(obj)/vdsomunge FORCE
34 $(call if_changed,vdsomunge)
35
36# Strip rule for the .so file
37$(obj)/%.so: OBJCOPYFLAGS := -S
38$(obj)/%.so: $(obj)/%.so.dbg FORCE
39 $(call if_changed,objcopy)
40
41# Actual build commands
42quiet_cmd_vdsold = VDSO $@
43 cmd_vdsold = $(CC) $(c_flags) -Wl,-T $(filter %.lds,$^) $(filter %.o,$^) \
44 $(call cc-ldoption, -Wl$(comma)--build-id) \
45 -Wl,-Bsymbolic -Wl,-z,max-page-size=4096 \
46 -Wl,-z,common-page-size=4096 -o $@
47
48quiet_cmd_vdsomunge = MUNGE $@
49 cmd_vdsomunge = $(objtree)/$(obj)/vdsomunge $< $@
50
51#
52# Install the unstripped copy of vdso.so.dbg. If our toolchain
53# supports build-id, install .build-id links as well.
54#
55# Cribbed from arch/x86/vdso/Makefile.
56#
57quiet_cmd_vdso_install = INSTALL $<
58define cmd_vdso_install
59 cp $< "$(MODLIB)/vdso/vdso.so"; \
60 if readelf -n $< | grep -q 'Build ID'; then \
61 buildid=`readelf -n $< |grep 'Build ID' |sed -e 's/^.*Build ID: \(.*\)$$/\1/'`; \
62 first=`echo $$buildid | cut -b-2`; \
63 last=`echo $$buildid | cut -b3-`; \
64 mkdir -p "$(MODLIB)/vdso/.build-id/$$first"; \
65 ln -sf "../../vdso.so" "$(MODLIB)/vdso/.build-id/$$first/$$last.debug"; \
66 fi
67endef
68
69$(MODLIB)/vdso: FORCE
70 @mkdir -p $(MODLIB)/vdso
71
72PHONY += vdso_install
73vdso_install: $(obj)/vdso.so.dbg $(MODLIB)/vdso FORCE
74 $(call cmd,vdso_install)
diff --git a/arch/arm/vdso/datapage.S b/arch/arm/vdso/datapage.S
new file mode 100644
index 000000000000..a2e60367931b
--- /dev/null
+++ b/arch/arm/vdso/datapage.S
@@ -0,0 +1,15 @@
1#include <linux/linkage.h>
2#include <asm/asm-offsets.h>
3
4 .align 2
5.L_vdso_data_ptr:
6 .long _start - . - VDSO_DATA_SIZE
7
8ENTRY(__get_datapage)
9 .fnstart
10 adr r0, .L_vdso_data_ptr
11 ldr r1, [r0]
12 add r0, r0, r1
13 bx lr
14 .fnend
15ENDPROC(__get_datapage)
diff --git a/arch/arm/vdso/vdso.S b/arch/arm/vdso/vdso.S
new file mode 100644
index 000000000000..b2b97e3e7bab
--- /dev/null
+++ b/arch/arm/vdso/vdso.S
@@ -0,0 +1,35 @@
1/*
2 * Adapted from arm64 version.
3 *
4 * Copyright (C) 2012 ARM Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 *
18 * Author: Will Deacon <will.deacon@arm.com>
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <linux/const.h>
24#include <asm/page.h>
25
26 __PAGE_ALIGNED_DATA
27
28 .globl vdso_start, vdso_end
29 .balign PAGE_SIZE
30vdso_start:
31 .incbin "arch/arm/vdso/vdso.so"
32 .balign PAGE_SIZE
33vdso_end:
34
35 .previous
diff --git a/arch/arm/vdso/vdso.lds.S b/arch/arm/vdso/vdso.lds.S
new file mode 100644
index 000000000000..89ca89f12d23
--- /dev/null
+++ b/arch/arm/vdso/vdso.lds.S
@@ -0,0 +1,87 @@
1/*
2 * Adapted from arm64 version.
3 *
4 * GNU linker script for the VDSO library.
5 *
6 * Copyright (C) 2012 ARM Limited
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 *
20 * Author: Will Deacon <will.deacon@arm.com>
21 * Heavily based on the vDSO linker scripts for other archs.
22 */
23
24#include <linux/const.h>
25#include <asm/page.h>
26#include <asm/vdso.h>
27
28OUTPUT_FORMAT("elf32-littlearm", "elf32-bigarm", "elf32-littlearm")
29OUTPUT_ARCH(arm)
30
31SECTIONS
32{
33 PROVIDE(_start = .);
34
35 . = SIZEOF_HEADERS;
36
37 .hash : { *(.hash) } :text
38 .gnu.hash : { *(.gnu.hash) }
39 .dynsym : { *(.dynsym) }
40 .dynstr : { *(.dynstr) }
41 .gnu.version : { *(.gnu.version) }
42 .gnu.version_d : { *(.gnu.version_d) }
43 .gnu.version_r : { *(.gnu.version_r) }
44
45 .note : { *(.note.*) } :text :note
46
47
48 .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr
49 .eh_frame : { KEEP (*(.eh_frame)) } :text
50
51 .dynamic : { *(.dynamic) } :text :dynamic
52
53 .rodata : { *(.rodata*) } :text
54
55 .text : { *(.text*) } :text =0xe7f001f2
56
57 .got : { *(.got) }
58 .rel.plt : { *(.rel.plt) }
59
60 /DISCARD/ : {
61 *(.note.GNU-stack)
62 *(.data .data.* .gnu.linkonce.d.* .sdata*)
63 *(.bss .sbss .dynbss .dynsbss)
64 }
65}
66
67/*
68 * We must supply the ELF program headers explicitly to get just one
69 * PT_LOAD segment, and set the flags explicitly to make segments read-only.
70 */
71PHDRS
72{
73 text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */
74 dynamic PT_DYNAMIC FLAGS(4); /* PF_R */
75 note PT_NOTE FLAGS(4); /* PF_R */
76 eh_frame_hdr PT_GNU_EH_FRAME;
77}
78
79VERSION
80{
81 LINUX_2.6 {
82 global:
83 __vdso_clock_gettime;
84 __vdso_gettimeofday;
85 local: *;
86 };
87}
diff --git a/arch/arm/vdso/vdsomunge.c b/arch/arm/vdso/vdsomunge.c
new file mode 100644
index 000000000000..9005b07296c8
--- /dev/null
+++ b/arch/arm/vdso/vdsomunge.c
@@ -0,0 +1,201 @@
1/*
2 * Copyright 2015 Mentor Graphics Corporation.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; version 2 of the
7 * License.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 *
17 *
18 * vdsomunge - Host program which produces a shared object
19 * architecturally specified to be usable by both soft- and hard-float
20 * programs.
21 *
22 * The Procedure Call Standard for the ARM Architecture (ARM IHI
23 * 0042E) says:
24 *
25 * 6.4.1 VFP and Base Standard Compatibility
26 *
27 * Code compiled for the VFP calling standard is compatible with
28 * the base standard (and vice-versa) if no floating-point or
29 * containerized vector arguments or results are used.
30 *
31 * And ELF for the ARM Architecture (ARM IHI 0044E) (Table 4-2) says:
32 *
33 * If both EF_ARM_ABI_FLOAT_XXXX bits are clear, conformance to the
34 * base procedure-call standard is implied.
35 *
36 * The VDSO is built with -msoft-float, as with the rest of the ARM
37 * kernel, and uses no floating point arguments or results. The build
38 * process will produce a shared object that may or may not have the
39 * EF_ARM_ABI_FLOAT_SOFT flag set (it seems to depend on the binutils
40 * version; binutils starting with 2.24 appears to set it). The
41 * EF_ARM_ABI_FLOAT_HARD flag should definitely not be set, and this
42 * program will error out if it is.
43 *
44 * If the soft-float flag is set, this program clears it. That's all
45 * it does.
46 */
47
48#define _GNU_SOURCE
49
50#include <byteswap.h>
51#include <elf.h>
52#include <errno.h>
53#include <error.h>
54#include <fcntl.h>
55#include <stdbool.h>
56#include <stdio.h>
57#include <stdlib.h>
58#include <string.h>
59#include <sys/mman.h>
60#include <sys/stat.h>
61#include <sys/types.h>
62#include <unistd.h>
63
64#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
65#define HOST_ORDER ELFDATA2LSB
66#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
67#define HOST_ORDER ELFDATA2MSB
68#endif
69
70/* Some of the ELF constants we'd like to use were added to <elf.h>
71 * relatively recently.
72 */
73#ifndef EF_ARM_EABI_VER5
74#define EF_ARM_EABI_VER5 0x05000000
75#endif
76
77#ifndef EF_ARM_ABI_FLOAT_SOFT
78#define EF_ARM_ABI_FLOAT_SOFT 0x200
79#endif
80
81#ifndef EF_ARM_ABI_FLOAT_HARD
82#define EF_ARM_ABI_FLOAT_HARD 0x400
83#endif
84
85static const char *outfile;
86
87static void cleanup(void)
88{
89 if (error_message_count > 0 && outfile != NULL)
90 unlink(outfile);
91}
92
93static Elf32_Word read_elf_word(Elf32_Word word, bool swap)
94{
95 return swap ? bswap_32(word) : word;
96}
97
98static Elf32_Half read_elf_half(Elf32_Half half, bool swap)
99{
100 return swap ? bswap_16(half) : half;
101}
102
103static void write_elf_word(Elf32_Word val, Elf32_Word *dst, bool swap)
104{
105 *dst = swap ? bswap_32(val) : val;
106}
107
108int main(int argc, char **argv)
109{
110 const Elf32_Ehdr *inhdr;
111 bool clear_soft_float;
112 const char *infile;
113 Elf32_Word e_flags;
114 const void *inbuf;
115 struct stat stat;
116 void *outbuf;
117 bool swap;
118 int outfd;
119 int infd;
120
121 atexit(cleanup);
122
123 if (argc != 3)
124 error(EXIT_FAILURE, 0, "Usage: %s [infile] [outfile]", argv[0]);
125
126 infile = argv[1];
127 outfile = argv[2];
128
129 infd = open(infile, O_RDONLY);
130 if (infd < 0)
131 error(EXIT_FAILURE, errno, "Cannot open %s", infile);
132
133 if (fstat(infd, &stat) != 0)
134 error(EXIT_FAILURE, errno, "Failed stat for %s", infile);
135
136 inbuf = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, infd, 0);
137 if (inbuf == MAP_FAILED)
138 error(EXIT_FAILURE, errno, "Failed to map %s", infile);
139
140 close(infd);
141
142 inhdr = inbuf;
143
144 if (memcmp(&inhdr->e_ident, ELFMAG, SELFMAG) != 0)
145 error(EXIT_FAILURE, 0, "Not an ELF file");
146
147 if (inhdr->e_ident[EI_CLASS] != ELFCLASS32)
148 error(EXIT_FAILURE, 0, "Unsupported ELF class");
149
150 swap = inhdr->e_ident[EI_DATA] != HOST_ORDER;
151
152 if (read_elf_half(inhdr->e_type, swap) != ET_DYN)
153 error(EXIT_FAILURE, 0, "Not a shared object");
154
155 if (read_elf_half(inhdr->e_machine, swap) != EM_ARM) {
156 error(EXIT_FAILURE, 0, "Unsupported architecture %#x",
157 inhdr->e_machine);
158 }
159
160 e_flags = read_elf_word(inhdr->e_flags, swap);
161
162 if (EF_ARM_EABI_VERSION(e_flags) != EF_ARM_EABI_VER5) {
163 error(EXIT_FAILURE, 0, "Unsupported EABI version %#x",
164 EF_ARM_EABI_VERSION(e_flags));
165 }
166
167 if (e_flags & EF_ARM_ABI_FLOAT_HARD)
168 error(EXIT_FAILURE, 0,
169 "Unexpected hard-float flag set in e_flags");
170
171 clear_soft_float = !!(e_flags & EF_ARM_ABI_FLOAT_SOFT);
172
173 outfd = open(outfile, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
174 if (outfd < 0)
175 error(EXIT_FAILURE, errno, "Cannot open %s", outfile);
176
177 if (ftruncate(outfd, stat.st_size) != 0)
178 error(EXIT_FAILURE, errno, "Cannot truncate %s", outfile);
179
180 outbuf = mmap(NULL, stat.st_size, PROT_READ | PROT_WRITE, MAP_SHARED,
181 outfd, 0);
182 if (outbuf == MAP_FAILED)
183 error(EXIT_FAILURE, errno, "Failed to map %s", outfile);
184
185 close(outfd);
186
187 memcpy(outbuf, inbuf, stat.st_size);
188
189 if (clear_soft_float) {
190 Elf32_Ehdr *outhdr;
191
192 outhdr = outbuf;
193 e_flags &= ~EF_ARM_ABI_FLOAT_SOFT;
194 write_elf_word(e_flags, &outhdr->e_flags, swap);
195 }
196
197 if (msync(outbuf, stat.st_size, MS_SYNC) != 0)
198 error(EXIT_FAILURE, errno, "Failed to sync %s", outfile);
199
200 return EXIT_SUCCESS;
201}
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
new file mode 100644
index 000000000000..79214d5ff097
--- /dev/null
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -0,0 +1,282 @@
1/*
2 * Copyright 2015 Mentor Graphics Corporation.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; version 2 of the
7 * License.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/compiler.h>
19#include <linux/hrtimer.h>
20#include <linux/time.h>
21#include <asm/arch_timer.h>
22#include <asm/barrier.h>
23#include <asm/bug.h>
24#include <asm/page.h>
25#include <asm/unistd.h>
26#include <asm/vdso_datapage.h>
27
28#ifndef CONFIG_AEABI
29#error This code depends on AEABI system call conventions
30#endif
31
32extern struct vdso_data *__get_datapage(void);
33
34static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)
35{
36 u32 seq;
37repeat:
38 seq = ACCESS_ONCE(vdata->seq_count);
39 if (seq & 1) {
40 cpu_relax();
41 goto repeat;
42 }
43 return seq;
44}
45
46static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
47{
48 u32 seq;
49
50 seq = __vdso_read_begin(vdata);
51
52 smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */
53 return seq;
54}
55
56static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
57{
58 smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */
59 return vdata->seq_count != start;
60}
61
62static notrace long clock_gettime_fallback(clockid_t _clkid,
63 struct timespec *_ts)
64{
65 register struct timespec *ts asm("r1") = _ts;
66 register clockid_t clkid asm("r0") = _clkid;
67 register long ret asm ("r0");
68 register long nr asm("r7") = __NR_clock_gettime;
69
70 asm volatile(
71 " swi #0\n"
72 : "=r" (ret)
73 : "r" (clkid), "r" (ts), "r" (nr)
74 : "memory");
75
76 return ret;
77}
78
79static notrace int do_realtime_coarse(struct timespec *ts,
80 struct vdso_data *vdata)
81{
82 u32 seq;
83
84 do {
85 seq = vdso_read_begin(vdata);
86
87 ts->tv_sec = vdata->xtime_coarse_sec;
88 ts->tv_nsec = vdata->xtime_coarse_nsec;
89
90 } while (vdso_read_retry(vdata, seq));
91
92 return 0;
93}
94
95static notrace int do_monotonic_coarse(struct timespec *ts,
96 struct vdso_data *vdata)
97{
98 struct timespec tomono;
99 u32 seq;
100
101 do {
102 seq = vdso_read_begin(vdata);
103
104 ts->tv_sec = vdata->xtime_coarse_sec;
105 ts->tv_nsec = vdata->xtime_coarse_nsec;
106
107 tomono.tv_sec = vdata->wtm_clock_sec;
108 tomono.tv_nsec = vdata->wtm_clock_nsec;
109
110 } while (vdso_read_retry(vdata, seq));
111
112 ts->tv_sec += tomono.tv_sec;
113 timespec_add_ns(ts, tomono.tv_nsec);
114
115 return 0;
116}
117
118#ifdef CONFIG_ARM_ARCH_TIMER
119
120static notrace u64 get_ns(struct vdso_data *vdata)
121{
122 u64 cycle_delta;
123 u64 cycle_now;
124 u64 nsec;
125
126 cycle_now = arch_counter_get_cntvct();
127
128 cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
129
130 nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;
131 nsec >>= vdata->cs_shift;
132
133 return nsec;
134}
135
136static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
137{
138 u64 nsecs;
139 u32 seq;
140
141 do {
142 seq = vdso_read_begin(vdata);
143
144 if (!vdata->tk_is_cntvct)
145 return -1;
146
147 ts->tv_sec = vdata->xtime_clock_sec;
148 nsecs = get_ns(vdata);
149
150 } while (vdso_read_retry(vdata, seq));
151
152 ts->tv_nsec = 0;
153 timespec_add_ns(ts, nsecs);
154
155 return 0;
156}
157
158static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
159{
160 struct timespec tomono;
161 u64 nsecs;
162 u32 seq;
163
164 do {
165 seq = vdso_read_begin(vdata);
166
167 if (!vdata->tk_is_cntvct)
168 return -1;
169
170 ts->tv_sec = vdata->xtime_clock_sec;
171 nsecs = get_ns(vdata);
172
173 tomono.tv_sec = vdata->wtm_clock_sec;
174 tomono.tv_nsec = vdata->wtm_clock_nsec;
175
176 } while (vdso_read_retry(vdata, seq));
177
178 ts->tv_sec += tomono.tv_sec;
179 ts->tv_nsec = 0;
180 timespec_add_ns(ts, nsecs + tomono.tv_nsec);
181
182 return 0;
183}
184
185#else /* CONFIG_ARM_ARCH_TIMER */
186
187static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
188{
189 return -1;
190}
191
192static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
193{
194 return -1;
195}
196
197#endif /* CONFIG_ARM_ARCH_TIMER */
198
199notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts)
200{
201 struct vdso_data *vdata;
202 int ret = -1;
203
204 vdata = __get_datapage();
205
206 switch (clkid) {
207 case CLOCK_REALTIME_COARSE:
208 ret = do_realtime_coarse(ts, vdata);
209 break;
210 case CLOCK_MONOTONIC_COARSE:
211 ret = do_monotonic_coarse(ts, vdata);
212 break;
213 case CLOCK_REALTIME:
214 ret = do_realtime(ts, vdata);
215 break;
216 case CLOCK_MONOTONIC:
217 ret = do_monotonic(ts, vdata);
218 break;
219 default:
220 break;
221 }
222
223 if (ret)
224 ret = clock_gettime_fallback(clkid, ts);
225
226 return ret;
227}
228
229static notrace long gettimeofday_fallback(struct timeval *_tv,
230 struct timezone *_tz)
231{
232 register struct timezone *tz asm("r1") = _tz;
233 register struct timeval *tv asm("r0") = _tv;
234 register long ret asm ("r0");
235 register long nr asm("r7") = __NR_gettimeofday;
236
237 asm volatile(
238 " swi #0\n"
239 : "=r" (ret)
240 : "r" (tv), "r" (tz), "r" (nr)
241 : "memory");
242
243 return ret;
244}
245
246notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
247{
248 struct timespec ts;
249 struct vdso_data *vdata;
250 int ret;
251
252 vdata = __get_datapage();
253
254 ret = do_realtime(&ts, vdata);
255 if (ret)
256 return gettimeofday_fallback(tv, tz);
257
258 if (tv) {
259 tv->tv_sec = ts.tv_sec;
260 tv->tv_usec = ts.tv_nsec / 1000;
261 }
262 if (tz) {
263 tz->tz_minuteswest = vdata->tz_minuteswest;
264 tz->tz_dsttime = vdata->tz_dsttime;
265 }
266
267 return ret;
268}
269
270/* Avoid unresolved references emitted by GCC */
271
272void __aeabi_unwind_cpp_pr0(void)
273{
274}
275
276void __aeabi_unwind_cpp_pr1(void)
277{
278}
279
280void __aeabi_unwind_cpp_pr2(void)
281{
282}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-16 16:42:30 -0500