aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorRussell King <rmk+kernel@arm.linux.org.uk>2013-10-07 10:43:04 -0400
committerRussell King <rmk+kernel@arm.linux.org.uk>2013-10-07 10:43:04 -0400
commita56e74f546b64be93731e42d83baf5b538cc1b11 (patch)
tree18f6dee45d801e57ac9db2a31664b0d5c0762c50 /arch
parentd08e2e09042bd3f7ef66a35cb4bb92794ab26bb2 (diff)
parente4e7f10bfc4069925e99cc4b428c3434e30b6c3f (diff)
Merge branch 'arm-aesbs' of git://git.linaro.org/people/ardbiesheuvel/linux-arm into devel-stable
Diffstat (limited to 'arch')
-rw-r--r--arch/Kconfig3
-rw-r--r--arch/arm/Kconfig3
-rw-r--r--arch/arm/boot/dts/Makefile1
-rw-r--r--arch/arm/boot/dts/am335x-bone-common.dtsi262
-rw-r--r--arch/arm/boot/dts/am335x-bone.dts256
-rw-r--r--arch/arm/boot/dts/am335x-boneblack.dts17
-rw-r--r--arch/arm/boot/dts/imx27.dtsi6
-rw-r--r--arch/arm/boot/dts/imx51.dtsi2
-rw-r--r--arch/arm/boot/dts/imx6q-pinfunc.h4
-rw-r--r--arch/arm/boot/dts/omap3-beagle-xm.dts2
-rw-r--r--arch/arm/boot/dts/omap3-igep.dtsi14
-rw-r--r--arch/arm/boot/dts/omap4-panda-common.dtsi46
-rw-r--r--arch/arm/boot/dts/omap4-sdp.dts39
-rw-r--r--arch/arm/boot/dts/omap5.dtsi7
-rw-r--r--arch/arm/configs/multi_v7_defconfig2
-rw-r--r--arch/arm/crypto/Makefile14
-rw-r--r--arch/arm/crypto/aes-armv4.S6
-rw-r--r--arch/arm/crypto/aes_glue.c22
-rw-r--r--arch/arm/crypto/aes_glue.h19
-rw-r--r--arch/arm/crypto/aesbs-core.S_shipped2544
-rw-r--r--arch/arm/crypto/aesbs-glue.c434
-rw-r--r--arch/arm/crypto/bsaes-armv7.pl2467
-rw-r--r--arch/arm/include/asm/Kbuild1
-rw-r--r--arch/arm/include/asm/uaccess.h7
-rw-r--r--arch/arm/kernel/entry-common.S4
-rw-r--r--arch/arm/kernel/entry-header.S8
-rw-r--r--arch/arm/mach-imx/clk-fixup-mux.c1
-rw-r--r--arch/arm/mach-imx/clk-imx27.c2
-rw-r--r--arch/arm/mach-imx/clk-imx51-imx53.c4
-rw-r--r--arch/arm/mach-imx/mach-imx6q.c9
-rw-r--r--arch/arm/mach-imx/system.c11
-rw-r--r--arch/arm/mach-omap2/cclock44xx_data.c2
-rw-r--r--arch/arm/mach-omap2/cpuidle44xx.c2
-rw-r--r--arch/arm/mach-omap2/gpmc.c4
-rw-r--r--arch/arm/mach-omap2/mux34xx.c2
-rw-r--r--arch/arm/mach-omap2/omap-smp.c2
-rw-r--r--arch/arm/mach-omap2/omap_device.c2
-rw-r--r--arch/arm/mach-sa1100/collie.c2
-rw-r--r--arch/arm/mach-shmobile/clock-r8a73a4.c2
-rw-r--r--arch/arm/mach-shmobile/clock-sh73a0.c2
-rw-r--r--arch/arm/mach-u300/Kconfig10
-rw-r--r--arch/arm/mach-ux500/cache-l2x0.c1
-rw-r--r--arch/arm64/include/asm/hwcap.h2
-rw-r--r--arch/arm64/kernel/process.c21
-rw-r--r--arch/arm64/kernel/setup.c2
-rw-r--r--arch/arm64/mm/fault.c2
-rw-r--r--arch/mips/Makefile3
-rw-r--r--arch/mips/alchemy/common/usb.c3
-rw-r--r--arch/mips/bcm63xx/cpu.c4
l---------arch/mips/boot/dts/include/dt-bindings2
-rw-r--r--arch/mips/cavium-octeon/csrc-octeon.c1
-rw-r--r--arch/mips/dec/prom/init.c1
-rw-r--r--arch/mips/include/asm/cpu-features.h8
-rw-r--r--arch/mips/include/asm/cpu-info.h1
-rw-r--r--arch/mips/include/asm/cpu-type.h203
-rw-r--r--arch/mips/include/asm/cpu.h38
-rw-r--r--arch/mips/include/asm/mach-au1x00/au1000.h4
-rw-r--r--arch/mips/include/asm/mach-ip22/cpu-feature-overrides.h2
-rw-r--r--arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h2
-rw-r--r--arch/mips/include/asm/mach-ip28/cpu-feature-overrides.h2
-rw-r--r--arch/mips/include/asm/mipsregs.h7
-rw-r--r--arch/mips/include/asm/pci.h12
-rw-r--r--arch/mips/include/asm/timex.h33
-rw-r--r--arch/mips/include/asm/vga.h3
-rw-r--r--arch/mips/kernel/cpu-probe.c58
-rw-r--r--arch/mips/kernel/idle.c3
-rw-r--r--arch/mips/kernel/time.c1
-rw-r--r--arch/mips/kernel/traps.c3
-rw-r--r--arch/mips/mm/c-octeon.c6
-rw-r--r--arch/mips/mm/c-r4k.c48
-rw-r--r--arch/mips/mm/dma-default.c13
-rw-r--r--arch/mips/mm/page.c1
-rw-r--r--arch/mips/mm/sc-mips.c3
-rw-r--r--arch/mips/mm/tlb-r4k.c1
-rw-r--r--arch/mips/mm/tlbex.c1
-rw-r--r--arch/mips/mti-malta/malta-time.c5
-rw-r--r--arch/mips/mti-sead3/sead3-time.c3
-rw-r--r--arch/mips/netlogic/xlr/fmn-config.c3
-rw-r--r--arch/mips/oprofile/common.c1
-rw-r--r--arch/mips/pci/pci-bcm1480.c1
-rw-r--r--arch/mips/sibyte/bcm1480/setup.c3
-rw-r--r--arch/mips/sibyte/sb1250/setup.c3
-rw-r--r--arch/mips/sni/setup.c3
-rw-r--r--arch/openrisc/include/asm/prom.h44
-rw-r--r--arch/powerpc/boot/Makefile4
-rw-r--r--arch/powerpc/boot/epapr-wrapper.c9
-rw-r--r--arch/powerpc/boot/epapr.c4
-rw-r--r--arch/powerpc/boot/of.c16
-rwxr-xr-xarch/powerpc/boot/wrapper9
-rw-r--r--arch/powerpc/include/asm/irq.h4
-rw-r--r--arch/powerpc/include/asm/processor.h4
-rw-r--r--arch/powerpc/kernel/asm-offsets.c3
-rw-r--r--arch/powerpc/kernel/irq.c100
-rw-r--r--arch/powerpc/kernel/misc_32.S25
-rw-r--r--arch/powerpc/kernel/misc_64.S10
-rw-r--r--arch/powerpc/kernel/process.c3
-rw-r--r--arch/powerpc/kernel/prom_init.c21
-rw-r--r--arch/powerpc/lib/sstep.c3
-rw-r--r--arch/powerpc/platforms/pseries/smp.c26
-rw-r--r--arch/s390/Kconfig2
-rw-r--r--arch/s390/include/asm/mutex.h2
-rw-r--r--arch/s390/include/asm/processor.h2
-rw-r--r--arch/s390/include/asm/spinlock.h5
-rw-r--r--arch/tile/Kconfig2
-rw-r--r--arch/tile/gxio/iorpc_mpipe.c90
-rw-r--r--arch/tile/gxio/iorpc_mpipe_info.c15
-rw-r--r--arch/tile/gxio/iorpc_trio.c28
-rw-r--r--arch/tile/gxio/iorpc_usb_host.c8
-rw-r--r--arch/tile/gxio/usb_host.c8
-rw-r--r--arch/tile/include/arch/mpipe.h24
-rw-r--r--arch/tile/include/arch/mpipe_constants.h6
-rw-r--r--arch/tile/include/arch/mpipe_shm.h54
-rw-r--r--arch/tile/include/arch/trio_constants.h10
-rw-r--r--arch/tile/include/asm/page.h5
-rw-r--r--arch/tile/include/asm/pgtable_32.h12
-rw-r--r--arch/tile/include/asm/pgtable_64.h4
-rw-r--r--arch/tile/include/gxio/iorpc_mpipe.h52
-rw-r--r--arch/tile/include/gxio/iorpc_mpipe_info.h12
-rw-r--r--arch/tile/include/gxio/iorpc_trio.h28
-rw-r--r--arch/tile/include/gxio/iorpc_usb_host.h8
-rw-r--r--arch/tile/include/gxio/usb_host.h8
-rw-r--r--arch/tile/kernel/compat.c2
-rw-r--r--arch/tile/kernel/futex_64.S55
-rw-r--r--arch/tile/kernel/setup.c3
-rw-r--r--arch/tile/kernel/unaligned.c4
-rw-r--r--arch/tile/mm/fault.c2
-rw-r--r--arch/tile/mm/init.c4
-rw-r--r--arch/tile/mm/pgtable.c3
-rw-r--r--arch/x86/Kconfig5
-rw-r--r--arch/x86/include/asm/xen/page.h31
-rw-r--r--arch/x86/kernel/cpu/perf_event.c12
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c5
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_ds.c1
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.c10
-rw-r--r--arch/x86/kernel/entry_64.S15
-rw-r--r--arch/x86/kernel/microcode_amd.c1
-rw-r--r--arch/x86/kernel/reboot.c18
-rw-r--r--arch/x86/kernel/smpboot.c3
-rw-r--r--arch/x86/kvm/emulate.c14
-rw-r--r--arch/x86/kvm/paging_tmpl.h20
-rw-r--r--arch/x86/kvm/vmx.c13
-rw-r--r--arch/x86/platform/efi/efi.c11
-rw-r--r--arch/x86/xen/p2m.c10
-rw-r--r--arch/x86/xen/spinlock.c26
144 files changed, 6844 insertions, 858 deletions
diff --git a/arch/Kconfig b/arch/Kconfig
index 1feb169274fe..af2cc6eabcc7 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -286,9 +286,6 @@ config HAVE_PERF_USER_STACK_DUMP
286config HAVE_ARCH_JUMP_LABEL 286config HAVE_ARCH_JUMP_LABEL
287 bool 287 bool
288 288
289config HAVE_ARCH_MUTEX_CPU_RELAX
290 bool
291
292config HAVE_RCU_TABLE_FREE 289config HAVE_RCU_TABLE_FREE
293 bool 290 bool
294 291
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index b3135378ac39..22efc5d9c952 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -2243,8 +2243,7 @@ config NEON
2243 2243
2244config KERNEL_MODE_NEON 2244config KERNEL_MODE_NEON
2245 bool "Support for NEON in kernel mode" 2245 bool "Support for NEON in kernel mode"
2246 default n 2246 depends on NEON && AEABI
2247 depends on NEON
2248 help 2247 help
2249 Say Y to include support for NEON in kernel mode. 2248 Say Y to include support for NEON in kernel mode.
2250 2249
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index cc0f1fb61753..e95af3f5433b 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -183,6 +183,7 @@ dtb-$(CONFIG_ARCH_OMAP2PLUS) += omap2420-h4.dtb \
183 am335x-evm.dtb \ 183 am335x-evm.dtb \
184 am335x-evmsk.dtb \ 184 am335x-evmsk.dtb \
185 am335x-bone.dtb \ 185 am335x-bone.dtb \
186 am335x-boneblack.dtb \
186 am3517-evm.dtb \ 187 am3517-evm.dtb \
187 am3517_mt_ventoux.dtb \ 188 am3517_mt_ventoux.dtb \
188 am43x-epos-evm.dtb 189 am43x-epos-evm.dtb
diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi
new file mode 100644
index 000000000000..2f66deda9f5c
--- /dev/null
+++ b/arch/arm/boot/dts/am335x-bone-common.dtsi
@@ -0,0 +1,262 @@
1/*
2 * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9/ {
10 model = "TI AM335x BeagleBone";
11 compatible = "ti,am335x-bone", "ti,am33xx";
12
13 cpus {
14 cpu@0 {
15 cpu0-supply = <&dcdc2_reg>;
16 };
17 };
18
19 memory {
20 device_type = "memory";
21 reg = <0x80000000 0x10000000>; /* 256 MB */
22 };
23
24 am33xx_pinmux: pinmux@44e10800 {
25 pinctrl-names = "default";
26 pinctrl-0 = <&clkout2_pin>;
27
28 user_leds_s0: user_leds_s0 {
29 pinctrl-single,pins = <
30 0x54 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */
31 0x58 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */
32 0x5c (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */
33 0x60 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */
34 >;
35 };
36
37 i2c0_pins: pinmux_i2c0_pins {
38 pinctrl-single,pins = <
39 0x188 (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */
40 0x18c (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */
41 >;
42 };
43
44 uart0_pins: pinmux_uart0_pins {
45 pinctrl-single,pins = <
46 0x170 (PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */
47 0x174 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */
48 >;
49 };
50
51 clkout2_pin: pinmux_clkout2_pin {
52 pinctrl-single,pins = <
53 0x1b4 (PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr1.clkout2 */
54 >;
55 };
56
57 cpsw_default: cpsw_default {
58 pinctrl-single,pins = <
59 /* Slave 1 */
60 0x110 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxerr.mii1_rxerr */
61 0x114 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txen.mii1_txen */
62 0x118 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxdv.mii1_rxdv */
63 0x11c (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd3.mii1_txd3 */
64 0x120 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd2.mii1_txd2 */
65 0x124 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd1.mii1_txd1 */
66 0x128 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd0.mii1_txd0 */
67 0x12c (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_txclk.mii1_txclk */
68 0x130 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxclk.mii1_rxclk */
69 0x134 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd3.mii1_rxd3 */
70 0x138 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd2.mii1_rxd2 */
71 0x13c (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd1.mii1_rxd1 */
72 0x140 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd0.mii1_rxd0 */
73 >;
74 };
75
76 cpsw_sleep: cpsw_sleep {
77 pinctrl-single,pins = <
78 /* Slave 1 reset value */
79 0x110 (PIN_INPUT_PULLDOWN | MUX_MODE7)
80 0x114 (PIN_INPUT_PULLDOWN | MUX_MODE7)
81 0x118 (PIN_INPUT_PULLDOWN | MUX_MODE7)
82 0x11c (PIN_INPUT_PULLDOWN | MUX_MODE7)
83 0x120 (PIN_INPUT_PULLDOWN | MUX_MODE7)
84 0x124 (PIN_INPUT_PULLDOWN | MUX_MODE7)
85 0x128 (PIN_INPUT_PULLDOWN | MUX_MODE7)
86 0x12c (PIN_INPUT_PULLDOWN | MUX_MODE7)
87 0x130 (PIN_INPUT_PULLDOWN | MUX_MODE7)
88 0x134 (PIN_INPUT_PULLDOWN | MUX_MODE7)
89 0x138 (PIN_INPUT_PULLDOWN | MUX_MODE7)
90 0x13c (PIN_INPUT_PULLDOWN | MUX_MODE7)
91 0x140 (PIN_INPUT_PULLDOWN | MUX_MODE7)
92 >;
93 };
94
95 davinci_mdio_default: davinci_mdio_default {
96 pinctrl-single,pins = <
97 /* MDIO */
98 0x148 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */
99 0x14c (PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */
100 >;
101 };
102
103 davinci_mdio_sleep: davinci_mdio_sleep {
104 pinctrl-single,pins = <
105 /* MDIO reset value */
106 0x148 (PIN_INPUT_PULLDOWN | MUX_MODE7)
107 0x14c (PIN_INPUT_PULLDOWN | MUX_MODE7)
108 >;
109 };
110 };
111
112 ocp {
113 uart0: serial@44e09000 {
114 pinctrl-names = "default";
115 pinctrl-0 = <&uart0_pins>;
116
117 status = "okay";
118 };
119
120 musb: usb@47400000 {
121 status = "okay";
122
123 control@44e10000 {
124 status = "okay";
125 };
126
127 usb-phy@47401300 {
128 status = "okay";
129 };
130
131 usb-phy@47401b00 {
132 status = "okay";
133 };
134
135 usb@47401000 {
136 status = "okay";
137 };
138
139 usb@47401800 {
140 status = "okay";
141 dr_mode = "host";
142 };
143
144 dma-controller@07402000 {
145 status = "okay";
146 };
147 };
148
149 i2c0: i2c@44e0b000 {
150 pinctrl-names = "default";
151 pinctrl-0 = <&i2c0_pins>;
152
153 status = "okay";
154 clock-frequency = <400000>;
155
156 tps: tps@24 {
157 reg = <0x24>;
158 };
159
160 };
161 };
162
163 leds {
164 pinctrl-names = "default";
165 pinctrl-0 = <&user_leds_s0>;
166
167 compatible = "gpio-leds";
168
169 led@2 {
170 label = "beaglebone:green:heartbeat";
171 gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
172 linux,default-trigger = "heartbeat";
173 default-state = "off";
174 };
175
176 led@3 {
177 label = "beaglebone:green:mmc0";
178 gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
179 linux,default-trigger = "mmc0";
180 default-state = "off";
181 };
182
183 led@4 {
184 label = "beaglebone:green:usr2";
185 gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>;
186 default-state = "off";
187 };
188
189 led@5 {
190 label = "beaglebone:green:usr3";
191 gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
192 default-state = "off";
193 };
194 };
195};
196
197/include/ "tps65217.dtsi"
198
199&tps {
200 regulators {
201 dcdc1_reg: regulator@0 {
202 regulator-always-on;
203 };
204
205 dcdc2_reg: regulator@1 {
206 /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */
207 regulator-name = "vdd_mpu";
208 regulator-min-microvolt = <925000>;
209 regulator-max-microvolt = <1325000>;
210 regulator-boot-on;
211 regulator-always-on;
212 };
213
214 dcdc3_reg: regulator@2 {
215 /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */
216 regulator-name = "vdd_core";
217 regulator-min-microvolt = <925000>;
218 regulator-max-microvolt = <1150000>;
219 regulator-boot-on;
220 regulator-always-on;
221 };
222
223 ldo1_reg: regulator@3 {
224 regulator-always-on;
225 };
226
227 ldo2_reg: regulator@4 {
228 regulator-always-on;
229 };
230
231 ldo3_reg: regulator@5 {
232 regulator-always-on;
233 };
234
235 ldo4_reg: regulator@6 {
236 regulator-always-on;
237 };
238 };
239};
240
241&cpsw_emac0 {
242 phy_id = <&davinci_mdio>, <0>;
243 phy-mode = "mii";
244};
245
246&cpsw_emac1 {
247 phy_id = <&davinci_mdio>, <1>;
248 phy-mode = "mii";
249};
250
251&mac {
252 pinctrl-names = "default", "sleep";
253 pinctrl-0 = <&cpsw_default>;
254 pinctrl-1 = <&cpsw_sleep>;
255
256};
257
258&davinci_mdio {
259 pinctrl-names = "default", "sleep";
260 pinctrl-0 = <&davinci_mdio_default>;
261 pinctrl-1 = <&davinci_mdio_sleep>;
262};
diff --git a/arch/arm/boot/dts/am335x-bone.dts b/arch/arm/boot/dts/am335x-bone.dts
index d318987d44a1..7993c489982c 100644
--- a/arch/arm/boot/dts/am335x-bone.dts
+++ b/arch/arm/boot/dts/am335x-bone.dts
@@ -8,258 +8,4 @@
8/dts-v1/; 8/dts-v1/;
9 9
10#include "am33xx.dtsi" 10#include "am33xx.dtsi"
11 11#include "am335x-bone-common.dtsi"
12/ {
13 model = "TI AM335x BeagleBone";
14 compatible = "ti,am335x-bone", "ti,am33xx";
15
16 cpus {
17 cpu@0 {
18 cpu0-supply = <&dcdc2_reg>;
19 };
20 };
21
22 memory {
23 device_type = "memory";
24 reg = <0x80000000 0x10000000>; /* 256 MB */
25 };
26
27 am33xx_pinmux: pinmux@44e10800 {
28 pinctrl-names = "default";
29 pinctrl-0 = <&clkout2_pin>;
30
31 user_leds_s0: user_leds_s0 {
32 pinctrl-single,pins = <
33 0x54 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */
34 0x58 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */
35 0x5c (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */
36 0x60 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */
37 >;
38 };
39
40 i2c0_pins: pinmux_i2c0_pins {
41 pinctrl-single,pins = <
42 0x188 (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */
43 0x18c (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */
44 >;
45 };
46
47 uart0_pins: pinmux_uart0_pins {
48 pinctrl-single,pins = <
49 0x170 (PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */
50 0x174 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */
51 >;
52 };
53
54 clkout2_pin: pinmux_clkout2_pin {
55 pinctrl-single,pins = <
56 0x1b4 (PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr1.clkout2 */
57 >;
58 };
59
60 cpsw_default: cpsw_default {
61 pinctrl-single,pins = <
62 /* Slave 1 */
63 0x110 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxerr.mii1_rxerr */
64 0x114 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txen.mii1_txen */
65 0x118 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxdv.mii1_rxdv */
66 0x11c (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd3.mii1_txd3 */
67 0x120 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd2.mii1_txd2 */
68 0x124 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd1.mii1_txd1 */
69 0x128 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd0.mii1_txd0 */
70 0x12c (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_txclk.mii1_txclk */
71 0x130 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxclk.mii1_rxclk */
72 0x134 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd3.mii1_rxd3 */
73 0x138 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd2.mii1_rxd2 */
74 0x13c (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd1.mii1_rxd1 */
75 0x140 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd0.mii1_rxd0 */
76 >;
77 };
78
79 cpsw_sleep: cpsw_sleep {
80 pinctrl-single,pins = <
81 /* Slave 1 reset value */
82 0x110 (PIN_INPUT_PULLDOWN | MUX_MODE7)
83 0x114 (PIN_INPUT_PULLDOWN | MUX_MODE7)
84 0x118 (PIN_INPUT_PULLDOWN | MUX_MODE7)
85 0x11c (PIN_INPUT_PULLDOWN | MUX_MODE7)
86 0x120 (PIN_INPUT_PULLDOWN | MUX_MODE7)
87 0x124 (PIN_INPUT_PULLDOWN | MUX_MODE7)
88 0x128 (PIN_INPUT_PULLDOWN | MUX_MODE7)
89 0x12c (PIN_INPUT_PULLDOWN | MUX_MODE7)
90 0x130 (PIN_INPUT_PULLDOWN | MUX_MODE7)
91 0x134 (PIN_INPUT_PULLDOWN | MUX_MODE7)
92 0x138 (PIN_INPUT_PULLDOWN | MUX_MODE7)
93 0x13c (PIN_INPUT_PULLDOWN | MUX_MODE7)
94 0x140 (PIN_INPUT_PULLDOWN | MUX_MODE7)
95 >;
96 };
97
98 davinci_mdio_default: davinci_mdio_default {
99 pinctrl-single,pins = <
100 /* MDIO */
101 0x148 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */
102 0x14c (PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */
103 >;
104 };
105
106 davinci_mdio_sleep: davinci_mdio_sleep {
107 pinctrl-single,pins = <
108 /* MDIO reset value */
109 0x148 (PIN_INPUT_PULLDOWN | MUX_MODE7)
110 0x14c (PIN_INPUT_PULLDOWN | MUX_MODE7)
111 >;
112 };
113 };
114
115 ocp {
116 uart0: serial@44e09000 {
117 pinctrl-names = "default";
118 pinctrl-0 = <&uart0_pins>;
119
120 status = "okay";
121 };
122
123 musb: usb@47400000 {
124 status = "okay";
125
126 control@44e10000 {
127 status = "okay";
128 };
129
130 usb-phy@47401300 {
131 status = "okay";
132 };
133
134 usb-phy@47401b00 {
135 status = "okay";
136 };
137
138 usb@47401000 {
139 status = "okay";
140 };
141
142 usb@47401800 {
143 status = "okay";
144 dr_mode = "host";
145 };
146
147 dma-controller@07402000 {
148 status = "okay";
149 };
150 };
151
152 i2c0: i2c@44e0b000 {
153 pinctrl-names = "default";
154 pinctrl-0 = <&i2c0_pins>;
155
156 status = "okay";
157 clock-frequency = <400000>;
158
159 tps: tps@24 {
160 reg = <0x24>;
161 };
162
163 };
164 };
165
166 leds {
167 pinctrl-names = "default";
168 pinctrl-0 = <&user_leds_s0>;
169
170 compatible = "gpio-leds";
171
172 led@2 {
173 label = "beaglebone:green:heartbeat";
174 gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
175 linux,default-trigger = "heartbeat";
176 default-state = "off";
177 };
178
179 led@3 {
180 label = "beaglebone:green:mmc0";
181 gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
182 linux,default-trigger = "mmc0";
183 default-state = "off";
184 };
185
186 led@4 {
187 label = "beaglebone:green:usr2";
188 gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>;
189 default-state = "off";
190 };
191
192 led@5 {
193 label = "beaglebone:green:usr3";
194 gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
195 default-state = "off";
196 };
197 };
198};
199
200/include/ "tps65217.dtsi"
201
202&tps {
203 regulators {
204 dcdc1_reg: regulator@0 {
205 regulator-always-on;
206 };
207
208 dcdc2_reg: regulator@1 {
209 /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */
210 regulator-name = "vdd_mpu";
211 regulator-min-microvolt = <925000>;
212 regulator-max-microvolt = <1325000>;
213 regulator-boot-on;
214 regulator-always-on;
215 };
216
217 dcdc3_reg: regulator@2 {
218 /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */
219 regulator-name = "vdd_core";
220 regulator-min-microvolt = <925000>;
221 regulator-max-microvolt = <1150000>;
222 regulator-boot-on;
223 regulator-always-on;
224 };
225
226 ldo1_reg: regulator@3 {
227 regulator-always-on;
228 };
229
230 ldo2_reg: regulator@4 {
231 regulator-always-on;
232 };
233
234 ldo3_reg: regulator@5 {
235 regulator-always-on;
236 };
237
238 ldo4_reg: regulator@6 {
239 regulator-always-on;
240 };
241 };
242};
243
244&cpsw_emac0 {
245 phy_id = <&davinci_mdio>, <0>;
246 phy-mode = "mii";
247};
248
249&cpsw_emac1 {
250 phy_id = <&davinci_mdio>, <1>;
251 phy-mode = "mii";
252};
253
254&mac {
255 pinctrl-names = "default", "sleep";
256 pinctrl-0 = <&cpsw_default>;
257 pinctrl-1 = <&cpsw_sleep>;
258
259};
260
261&davinci_mdio {
262 pinctrl-names = "default", "sleep";
263 pinctrl-0 = <&davinci_mdio_default>;
264 pinctrl-1 = <&davinci_mdio_sleep>;
265};
diff --git a/arch/arm/boot/dts/am335x-boneblack.dts b/arch/arm/boot/dts/am335x-boneblack.dts
new file mode 100644
index 000000000000..197cadf72d2c
--- /dev/null
+++ b/arch/arm/boot/dts/am335x-boneblack.dts
@@ -0,0 +1,17 @@
1/*
2 * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8/dts-v1/;
9
10#include "am33xx.dtsi"
11#include "am335x-bone-common.dtsi"
12
13&ldo3_reg {
14 regulator-min-microvolt = <1800000>;
15 regulator-max-microvolt = <1800000>;
16 regulator-always-on;
17};
diff --git a/arch/arm/boot/dts/imx27.dtsi b/arch/arm/boot/dts/imx27.dtsi
index c037c223619a..b7a1c6d950b9 100644
--- a/arch/arm/boot/dts/imx27.dtsi
+++ b/arch/arm/boot/dts/imx27.dtsi
@@ -187,7 +187,7 @@
187 compatible = "fsl,imx27-cspi"; 187 compatible = "fsl,imx27-cspi";
188 reg = <0x1000e000 0x1000>; 188 reg = <0x1000e000 0x1000>;
189 interrupts = <16>; 189 interrupts = <16>;
190 clocks = <&clks 53>, <&clks 53>; 190 clocks = <&clks 53>, <&clks 60>;
191 clock-names = "ipg", "per"; 191 clock-names = "ipg", "per";
192 status = "disabled"; 192 status = "disabled";
193 }; 193 };
@@ -198,7 +198,7 @@
198 compatible = "fsl,imx27-cspi"; 198 compatible = "fsl,imx27-cspi";
199 reg = <0x1000f000 0x1000>; 199 reg = <0x1000f000 0x1000>;
200 interrupts = <15>; 200 interrupts = <15>;
201 clocks = <&clks 52>, <&clks 52>; 201 clocks = <&clks 52>, <&clks 60>;
202 clock-names = "ipg", "per"; 202 clock-names = "ipg", "per";
203 status = "disabled"; 203 status = "disabled";
204 }; 204 };
@@ -309,7 +309,7 @@
309 compatible = "fsl,imx27-cspi"; 309 compatible = "fsl,imx27-cspi";
310 reg = <0x10017000 0x1000>; 310 reg = <0x10017000 0x1000>;
311 interrupts = <6>; 311 interrupts = <6>;
312 clocks = <&clks 51>, <&clks 51>; 312 clocks = <&clks 51>, <&clks 60>;
313 clock-names = "ipg", "per"; 313 clock-names = "ipg", "per";
314 status = "disabled"; 314 status = "disabled";
315 }; 315 };
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index a85abb424c34..54cee6517902 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -474,7 +474,7 @@
474 compatible = "fsl,imx51-pata", "fsl,imx27-pata"; 474 compatible = "fsl,imx51-pata", "fsl,imx27-pata";
475 reg = <0x83fe0000 0x4000>; 475 reg = <0x83fe0000 0x4000>;
476 interrupts = <70>; 476 interrupts = <70>;
477 clocks = <&clks 161>; 477 clocks = <&clks 172>;
478 status = "disabled"; 478 status = "disabled";
479 }; 479 };
480 480
diff --git a/arch/arm/boot/dts/imx6q-pinfunc.h b/arch/arm/boot/dts/imx6q-pinfunc.h
index c0e38a45e4bb..9bbe82bdee41 100644
--- a/arch/arm/boot/dts/imx6q-pinfunc.h
+++ b/arch/arm/boot/dts/imx6q-pinfunc.h
@@ -207,8 +207,8 @@
207#define MX6QDL_PAD_EIM_D29__ECSPI4_SS0 0x0c8 0x3dc 0x824 0x2 0x1 207#define MX6QDL_PAD_EIM_D29__ECSPI4_SS0 0x0c8 0x3dc 0x824 0x2 0x1
208#define MX6QDL_PAD_EIM_D29__UART2_RTS_B 0x0c8 0x3dc 0x924 0x4 0x1 208#define MX6QDL_PAD_EIM_D29__UART2_RTS_B 0x0c8 0x3dc 0x924 0x4 0x1
209#define MX6QDL_PAD_EIM_D29__UART2_CTS_B 0x0c8 0x3dc 0x000 0x4 0x0 209#define MX6QDL_PAD_EIM_D29__UART2_CTS_B 0x0c8 0x3dc 0x000 0x4 0x0
210#define MX6QDL_PAD_EIM_D29__UART2_DTE_RTS_B 0x0c4 0x3dc 0x000 0x4 0x0 210#define MX6QDL_PAD_EIM_D29__UART2_DTE_RTS_B 0x0c8 0x3dc 0x000 0x4 0x0
211#define MX6QDL_PAD_EIM_D29__UART2_DTE_CTS_B 0x0c4 0x3dc 0x924 0x4 0x1 211#define MX6QDL_PAD_EIM_D29__UART2_DTE_CTS_B 0x0c8 0x3dc 0x924 0x4 0x1
212#define MX6QDL_PAD_EIM_D29__GPIO3_IO29 0x0c8 0x3dc 0x000 0x5 0x0 212#define MX6QDL_PAD_EIM_D29__GPIO3_IO29 0x0c8 0x3dc 0x000 0x5 0x0
213#define MX6QDL_PAD_EIM_D29__IPU2_CSI1_VSYNC 0x0c8 0x3dc 0x8e4 0x6 0x0 213#define MX6QDL_PAD_EIM_D29__IPU2_CSI1_VSYNC 0x0c8 0x3dc 0x8e4 0x6 0x0
214#define MX6QDL_PAD_EIM_D29__IPU1_DI0_PIN14 0x0c8 0x3dc 0x000 0x7 0x0 214#define MX6QDL_PAD_EIM_D29__IPU1_DI0_PIN14 0x0c8 0x3dc 0x000 0x7 0x0
diff --git a/arch/arm/boot/dts/omap3-beagle-xm.dts b/arch/arm/boot/dts/omap3-beagle-xm.dts
index afdb16417d4e..0c514dc8460c 100644
--- a/arch/arm/boot/dts/omap3-beagle-xm.dts
+++ b/arch/arm/boot/dts/omap3-beagle-xm.dts
@@ -11,7 +11,7 @@
11 11
12/ { 12/ {
13 model = "TI OMAP3 BeagleBoard xM"; 13 model = "TI OMAP3 BeagleBoard xM";
14 compatible = "ti,omap3-beagle-xm, ti,omap3-beagle", "ti,omap3"; 14 compatible = "ti,omap3-beagle-xm", "ti,omap3-beagle", "ti,omap3";
15 15
16 cpus { 16 cpus {
17 cpu@0 { 17 cpu@0 {
diff --git a/arch/arm/boot/dts/omap3-igep.dtsi b/arch/arm/boot/dts/omap3-igep.dtsi
index bc48b114eae6..2326d11462a5 100644
--- a/arch/arm/boot/dts/omap3-igep.dtsi
+++ b/arch/arm/boot/dts/omap3-igep.dtsi
@@ -48,6 +48,15 @@
48 >; 48 >;
49 }; 49 };
50 50
51 mcbsp2_pins: pinmux_mcbsp2_pins {
52 pinctrl-single,pins = <
53 0x10c (PIN_INPUT | MUX_MODE0) /* mcbsp2_fsx.mcbsp2_fsx */
54 0x10e (PIN_INPUT | MUX_MODE0) /* mcbsp2_clkx.mcbsp2_clkx */
55 0x110 (PIN_INPUT | MUX_MODE0) /* mcbsp2_dr.mcbsp2.dr */
56 0x112 (PIN_OUTPUT | MUX_MODE0) /* mcbsp2_dx.mcbsp2_dx */
57 >;
58 };
59
51 mmc1_pins: pinmux_mmc1_pins { 60 mmc1_pins: pinmux_mmc1_pins {
52 pinctrl-single,pins = < 61 pinctrl-single,pins = <
53 0x114 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_clk.sdmmc1_clk */ 62 0x114 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_clk.sdmmc1_clk */
@@ -93,6 +102,11 @@
93 clock-frequency = <400000>; 102 clock-frequency = <400000>;
94}; 103};
95 104
105&mcbsp2 {
106 pinctrl-names = "default";
107 pinctrl-0 = <&mcbsp2_pins>;
108};
109
96&mmc1 { 110&mmc1 {
97 pinctrl-names = "default"; 111 pinctrl-names = "default";
98 pinctrl-0 = <&mmc1_pins>; 112 pinctrl-0 = <&mmc1_pins>;
diff --git a/arch/arm/boot/dts/omap4-panda-common.dtsi b/arch/arm/boot/dts/omap4-panda-common.dtsi
index faa95b5b242e..814ab67c8c29 100644
--- a/arch/arm/boot/dts/omap4-panda-common.dtsi
+++ b/arch/arm/boot/dts/omap4-panda-common.dtsi
@@ -107,6 +107,19 @@
107 */ 107 */
108 clock-frequency = <19200000>; 108 clock-frequency = <19200000>;
109 }; 109 };
110
111 /* regulator for wl12xx on sdio5 */
112 wl12xx_vmmc: wl12xx_vmmc {
113 pinctrl-names = "default";
114 pinctrl-0 = <&wl12xx_gpio>;
115 compatible = "regulator-fixed";
116 regulator-name = "vwl1271";
117 regulator-min-microvolt = <1800000>;
118 regulator-max-microvolt = <1800000>;
119 gpio = <&gpio2 11 0>;
120 startup-delay-us = <70000>;
121 enable-active-high;
122 };
110}; 123};
111 124
112&omap4_pmx_wkup { 125&omap4_pmx_wkup {
@@ -235,6 +248,33 @@
235 0x1c (PIN_OUTPUT | MUX_MODE3) /* gpio_wk8 */ 248 0x1c (PIN_OUTPUT | MUX_MODE3) /* gpio_wk8 */
236 >; 249 >;
237 }; 250 };
251
252 /*
253 * wl12xx GPIO outputs for WLAN_EN, BT_EN, FM_EN, BT_WAKEUP
254 * REVISIT: Are the pull-ups needed for GPIO 48 and 49?
255 */
256 wl12xx_gpio: pinmux_wl12xx_gpio {
257 pinctrl-single,pins = <
258 0x26 (PIN_OUTPUT | MUX_MODE3) /* gpmc_a19.gpio_43 */
259 0x2c (PIN_OUTPUT | MUX_MODE3) /* gpmc_a22.gpio_46 */
260 0x30 (PIN_OUTPUT_PULLUP | MUX_MODE3) /* gpmc_a24.gpio_48 */
261 0x32 (PIN_OUTPUT_PULLUP | MUX_MODE3) /* gpmc_a25.gpio_49 */
262 >;
263 };
264
265 /* wl12xx GPIO inputs and SDIO pins */
266 wl12xx_pins: pinmux_wl12xx_pins {
267 pinctrl-single,pins = <
268 0x38 (PIN_INPUT | MUX_MODE3) /* gpmc_ncs2.gpio_52 */
269 0x3a (PIN_INPUT | MUX_MODE3) /* gpmc_ncs3.gpio_53 */
270 0x108 (PIN_OUTPUT | MUX_MODE0) /* sdmmc5_clk.sdmmc5_clk */
271 0x10a (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc5_cmd.sdmmc5_cmd */
272 0x10c (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc5_dat0.sdmmc5_dat0 */
273 0x10e (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc5_dat1.sdmmc5_dat1 */
274 0x110 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc5_dat2.sdmmc5_dat2 */
275 0x112 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc5_dat3.sdmmc5_dat3 */
276 >;
277 };
238}; 278};
239 279
240&i2c1 { 280&i2c1 {
@@ -314,8 +354,12 @@
314}; 354};
315 355
316&mmc5 { 356&mmc5 {
317 ti,non-removable; 357 pinctrl-names = "default";
358 pinctrl-0 = <&wl12xx_pins>;
359 vmmc-supply = <&wl12xx_vmmc>;
360 non-removable;
318 bus-width = <4>; 361 bus-width = <4>;
362 cap-power-off-card;
319}; 363};
320 364
321&emif1 { 365&emif1 {
diff --git a/arch/arm/boot/dts/omap4-sdp.dts b/arch/arm/boot/dts/omap4-sdp.dts
index 7951b4ea500a..4f78380ecdb8 100644
--- a/arch/arm/boot/dts/omap4-sdp.dts
+++ b/arch/arm/boot/dts/omap4-sdp.dts
@@ -140,6 +140,19 @@
140 "DMic", "Digital Mic", 140 "DMic", "Digital Mic",
141 "Digital Mic", "Digital Mic1 Bias"; 141 "Digital Mic", "Digital Mic1 Bias";
142 }; 142 };
143
144 /* regulator for wl12xx on sdio5 */
145 wl12xx_vmmc: wl12xx_vmmc {
146 pinctrl-names = "default";
147 pinctrl-0 = <&wl12xx_gpio>;
148 compatible = "regulator-fixed";
149 regulator-name = "vwl1271";
150 regulator-min-microvolt = <1800000>;
151 regulator-max-microvolt = <1800000>;
152 gpio = <&gpio2 22 0>;
153 startup-delay-us = <70000>;
154 enable-active-high;
155 };
143}; 156};
144 157
145&omap4_pmx_wkup { 158&omap4_pmx_wkup {
@@ -295,6 +308,26 @@
295 0xf0 (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c4_sda */ 308 0xf0 (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c4_sda */
296 >; 309 >;
297 }; 310 };
311
312 /* wl12xx GPIO output for WLAN_EN */
313 wl12xx_gpio: pinmux_wl12xx_gpio {
314 pinctrl-single,pins = <
315 0x3c (PIN_OUTPUT | MUX_MODE3) /* gpmc_nwp.gpio_54 */
316 >;
317 };
318
319 /* wl12xx GPIO inputs and SDIO pins */
320 wl12xx_pins: pinmux_wl12xx_pins {
321 pinctrl-single,pins = <
322 0x3a (PIN_INPUT | MUX_MODE3) /* gpmc_ncs3.gpio_53 */
323 0x108 (PIN_OUTPUT | MUX_MODE3) /* sdmmc5_clk.sdmmc5_clk */
324 0x10a (PIN_INPUT_PULLUP | MUX_MODE3) /* sdmmc5_cmd.sdmmc5_cmd */
325 0x10c (PIN_INPUT_PULLUP | MUX_MODE3) /* sdmmc5_dat0.sdmmc5_dat0 */
326 0x10e (PIN_INPUT_PULLUP | MUX_MODE3) /* sdmmc5_dat1.sdmmc5_dat1 */
327 0x110 (PIN_INPUT_PULLUP | MUX_MODE3) /* sdmmc5_dat2.sdmmc5_dat2 */
328 0x112 (PIN_INPUT_PULLUP | MUX_MODE3) /* sdmmc5_dat3.sdmmc5_dat3 */
329 >;
330 };
298}; 331};
299 332
300&i2c1 { 333&i2c1 {
@@ -420,8 +453,12 @@
420}; 453};
421 454
422&mmc5 { 455&mmc5 {
456 pinctrl-names = "default";
457 pinctrl-0 = <&wl12xx_pins>;
458 vmmc-supply = <&wl12xx_vmmc>;
459 non-removable;
423 bus-width = <4>; 460 bus-width = <4>;
424 ti,non-removable; 461 cap-power-off-card;
425}; 462};
426 463
427&emif1 { 464&emif1 {
diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi
index 07be2cd7b318..7cdea1bfea09 100644
--- a/arch/arm/boot/dts/omap5.dtsi
+++ b/arch/arm/boot/dts/omap5.dtsi
@@ -637,7 +637,7 @@
637 omap_dwc3@4a020000 { 637 omap_dwc3@4a020000 {
638 compatible = "ti,dwc3"; 638 compatible = "ti,dwc3";
639 ti,hwmods = "usb_otg_ss"; 639 ti,hwmods = "usb_otg_ss";
640 reg = <0x4a020000 0x1000>; 640 reg = <0x4a020000 0x10000>;
641 interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>; 641 interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
642 #address-cells = <1>; 642 #address-cells = <1>;
643 #size-cells = <1>; 643 #size-cells = <1>;
@@ -645,17 +645,18 @@
645 ranges; 645 ranges;
646 dwc3@4a030000 { 646 dwc3@4a030000 {
647 compatible = "snps,dwc3"; 647 compatible = "snps,dwc3";
648 reg = <0x4a030000 0x1000>; 648 reg = <0x4a030000 0x10000>;
649 interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>; 649 interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
650 usb-phy = <&usb2_phy>, <&usb3_phy>; 650 usb-phy = <&usb2_phy>, <&usb3_phy>;
651 tx-fifo-resize; 651 tx-fifo-resize;
652 }; 652 };
653 }; 653 };
654 654
655 ocp2scp { 655 ocp2scp@4a080000 {
656 compatible = "ti,omap-ocp2scp"; 656 compatible = "ti,omap-ocp2scp";
657 #address-cells = <1>; 657 #address-cells = <1>;
658 #size-cells = <1>; 658 #size-cells = <1>;
659 reg = <0x4a080000 0x20>;
659 ranges; 660 ranges;
660 ti,hwmods = "ocp2scp1"; 661 ti,hwmods = "ocp2scp1";
661 usb2_phy: usb2phy@4a084000 { 662 usb2_phy: usb2phy@4a084000 {
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index 6e572c64cf5a..f3935b46df29 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -36,6 +36,7 @@ CONFIG_ARCH_TEGRA_114_SOC=y
36CONFIG_TEGRA_PCI=y 36CONFIG_TEGRA_PCI=y
37CONFIG_TEGRA_EMC_SCALING_ENABLE=y 37CONFIG_TEGRA_EMC_SCALING_ENABLE=y
38CONFIG_ARCH_U8500=y 38CONFIG_ARCH_U8500=y
39CONFIG_MACH_HREFV60=y
39CONFIG_MACH_SNOWBALL=y 40CONFIG_MACH_SNOWBALL=y
40CONFIG_MACH_UX500_DT=y 41CONFIG_MACH_UX500_DT=y
41CONFIG_ARCH_VEXPRESS=y 42CONFIG_ARCH_VEXPRESS=y
@@ -46,6 +47,7 @@ CONFIG_ARCH_ZYNQ=y
46CONFIG_SMP=y 47CONFIG_SMP=y
47CONFIG_HIGHPTE=y 48CONFIG_HIGHPTE=y
48CONFIG_ARM_APPENDED_DTB=y 49CONFIG_ARM_APPENDED_DTB=y
50CONFIG_ARM_ATAG_DTB_COMPAT=y
49CONFIG_NET=y 51CONFIG_NET=y
50CONFIG_UNIX=y 52CONFIG_UNIX=y
51CONFIG_INET=y 53CONFIG_INET=y
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index a2c83851bc90..81cda39860c5 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -3,7 +3,17 @@
3# 3#
4 4
5obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o 5obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o
6obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
6obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o 7obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
7 8
8aes-arm-y := aes-armv4.o aes_glue.o 9aes-arm-y := aes-armv4.o aes_glue.o
9sha1-arm-y := sha1-armv4-large.o sha1_glue.o 10aes-arm-bs-y := aesbs-core.o aesbs-glue.o
11sha1-arm-y := sha1-armv4-large.o sha1_glue.o
12
13quiet_cmd_perl = PERL $@
14 cmd_perl = $(PERL) $(<) > $(@)
15
16$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl
17 $(call cmd,perl)
18
19.PRECIOUS: $(obj)/aesbs-core.S
diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S
index 19d6cd6f29f9..3a14ea8fe97e 100644
--- a/arch/arm/crypto/aes-armv4.S
+++ b/arch/arm/crypto/aes-armv4.S
@@ -148,7 +148,7 @@ AES_Te:
148@ const AES_KEY *key) { 148@ const AES_KEY *key) {
149.align 5 149.align 5
150ENTRY(AES_encrypt) 150ENTRY(AES_encrypt)
151 sub r3,pc,#8 @ AES_encrypt 151 adr r3,AES_encrypt
152 stmdb sp!,{r1,r4-r12,lr} 152 stmdb sp!,{r1,r4-r12,lr}
153 mov r12,r0 @ inp 153 mov r12,r0 @ inp
154 mov r11,r2 154 mov r11,r2
@@ -381,7 +381,7 @@ _armv4_AES_encrypt:
381.align 5 381.align 5
382ENTRY(private_AES_set_encrypt_key) 382ENTRY(private_AES_set_encrypt_key)
383_armv4_AES_set_encrypt_key: 383_armv4_AES_set_encrypt_key:
384 sub r3,pc,#8 @ AES_set_encrypt_key 384 adr r3,_armv4_AES_set_encrypt_key
385 teq r0,#0 385 teq r0,#0
386 moveq r0,#-1 386 moveq r0,#-1
387 beq .Labrt 387 beq .Labrt
@@ -843,7 +843,7 @@ AES_Td:
843@ const AES_KEY *key) { 843@ const AES_KEY *key) {
844.align 5 844.align 5
845ENTRY(AES_decrypt) 845ENTRY(AES_decrypt)
846 sub r3,pc,#8 @ AES_decrypt 846 adr r3,AES_decrypt
847 stmdb sp!,{r1,r4-r12,lr} 847 stmdb sp!,{r1,r4-r12,lr}
848 mov r12,r0 @ inp 848 mov r12,r0 @ inp
849 mov r11,r2 849 mov r11,r2
diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c
index 59f7877ead6a..3003fa1f6fb4 100644
--- a/arch/arm/crypto/aes_glue.c
+++ b/arch/arm/crypto/aes_glue.c
@@ -6,22 +6,12 @@
6#include <linux/crypto.h> 6#include <linux/crypto.h>
7#include <crypto/aes.h> 7#include <crypto/aes.h>
8 8
9#define AES_MAXNR 14 9#include "aes_glue.h"
10 10
11typedef struct { 11EXPORT_SYMBOL(AES_encrypt);
12 unsigned int rd_key[4 *(AES_MAXNR + 1)]; 12EXPORT_SYMBOL(AES_decrypt);
13 int rounds; 13EXPORT_SYMBOL(private_AES_set_encrypt_key);
14} AES_KEY; 14EXPORT_SYMBOL(private_AES_set_decrypt_key);
15
16struct AES_CTX {
17 AES_KEY enc_key;
18 AES_KEY dec_key;
19};
20
21asmlinkage void AES_encrypt(const u8 *in, u8 *out, AES_KEY *ctx);
22asmlinkage void AES_decrypt(const u8 *in, u8 *out, AES_KEY *ctx);
23asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key);
24asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key);
25 15
26static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 16static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
27{ 17{
@@ -81,7 +71,7 @@ static struct crypto_alg aes_alg = {
81 .cipher = { 71 .cipher = {
82 .cia_min_keysize = AES_MIN_KEY_SIZE, 72 .cia_min_keysize = AES_MIN_KEY_SIZE,
83 .cia_max_keysize = AES_MAX_KEY_SIZE, 73 .cia_max_keysize = AES_MAX_KEY_SIZE,
84 .cia_setkey = aes_set_key, 74 .cia_setkey = aes_set_key,
85 .cia_encrypt = aes_encrypt, 75 .cia_encrypt = aes_encrypt,
86 .cia_decrypt = aes_decrypt 76 .cia_decrypt = aes_decrypt
87 } 77 }
diff --git a/arch/arm/crypto/aes_glue.h b/arch/arm/crypto/aes_glue.h
new file mode 100644
index 000000000000..cca3e51eb606
--- /dev/null
+++ b/arch/arm/crypto/aes_glue.h
@@ -0,0 +1,19 @@
1
2#define AES_MAXNR 14
3
4struct AES_KEY {
5 unsigned int rd_key[4 * (AES_MAXNR + 1)];
6 int rounds;
7};
8
9struct AES_CTX {
10 struct AES_KEY enc_key;
11 struct AES_KEY dec_key;
12};
13
14asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct AES_KEY *ctx);
15asmlinkage void AES_decrypt(const u8 *in, u8 *out, struct AES_KEY *ctx);
16asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey,
17 const int bits, struct AES_KEY *key);
18asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey,
19 const int bits, struct AES_KEY *key);
diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped
new file mode 100644
index 000000000000..64205d453260
--- /dev/null
+++ b/arch/arm/crypto/aesbs-core.S_shipped
@@ -0,0 +1,2544 @@
1
2@ ====================================================================
3@ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
4@ project. The module is, however, dual licensed under OpenSSL and
5@ CRYPTOGAMS licenses depending on where you obtain it. For further
6@ details see http://www.openssl.org/~appro/cryptogams/.
7@
8@ Specific modes and adaptation for Linux kernel by Ard Biesheuvel
9@ <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
10@ granted.
11@ ====================================================================
12
13@ Bit-sliced AES for ARM NEON
14@
15@ February 2012.
16@
17@ This implementation is direct adaptation of bsaes-x86_64 module for
18@ ARM NEON. Except that this module is endian-neutral [in sense that
19@ it can be compiled for either endianness] by courtesy of vld1.8's
20@ neutrality. Initial version doesn't implement interface to OpenSSL,
21@ only low-level primitives and unsupported entry points, just enough
22@ to collect performance results, which for Cortex-A8 core are:
23@
24@ encrypt 19.5 cycles per byte processed with 128-bit key
25@ decrypt 22.1 cycles per byte processed with 128-bit key
26@ key conv. 440 cycles per 128-bit key/0.18 of 8x block
27@
28@ Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7,
29@ which is [much] worse than anticipated (for further details see
30@ http://www.openssl.org/~appro/Snapdragon-S4.html).
31@
32@ Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code
33@ manages in 20.0 cycles].
34@
35@ When comparing to x86_64 results keep in mind that NEON unit is
36@ [mostly] single-issue and thus can't [fully] benefit from
37@ instruction-level parallelism. And when comparing to aes-armv4
38@ results keep in mind key schedule conversion overhead (see
39@ bsaes-x86_64.pl for further details)...
40@
41@ <appro@openssl.org>
42
43@ April-August 2013
44@
45@ Add CBC, CTR and XTS subroutines, adapt for kernel use.
46@
47@ <ard.biesheuvel@linaro.org>
48
49#ifndef __KERNEL__
50# include "arm_arch.h"
51
52# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
53# define VFP_ABI_POP vldmia sp!,{d8-d15}
54# define VFP_ABI_FRAME 0x40
55#else
56# define VFP_ABI_PUSH
57# define VFP_ABI_POP
58# define VFP_ABI_FRAME 0
59# define BSAES_ASM_EXTENDED_KEY
60# define XTS_CHAIN_TWEAK
61# define __ARM_ARCH__ __LINUX_ARM_ARCH__
62#endif
63
64#ifdef __thumb__
65# define adrl adr
66#endif
67
68#if __ARM_ARCH__>=7
69.text
70.syntax unified @ ARMv7-capable assembler is expected to handle this
71#ifdef __thumb2__
72.thumb
73#else
74.code 32
75#endif
76
77.fpu neon
78
79.type _bsaes_decrypt8,%function
80.align 4
81_bsaes_decrypt8:
82 adr r6,_bsaes_decrypt8
83 vldmia r4!, {q9} @ round 0 key
84 add r6,r6,#.LM0ISR-_bsaes_decrypt8
85
86 vldmia r6!, {q8} @ .LM0ISR
87 veor q10, q0, q9 @ xor with round0 key
88 veor q11, q1, q9
89 vtbl.8 d0, {q10}, d16
90 vtbl.8 d1, {q10}, d17
91 veor q12, q2, q9
92 vtbl.8 d2, {q11}, d16
93 vtbl.8 d3, {q11}, d17
94 veor q13, q3, q9
95 vtbl.8 d4, {q12}, d16
96 vtbl.8 d5, {q12}, d17
97 veor q14, q4, q9
98 vtbl.8 d6, {q13}, d16
99 vtbl.8 d7, {q13}, d17
100 veor q15, q5, q9
101 vtbl.8 d8, {q14}, d16
102 vtbl.8 d9, {q14}, d17
103 veor q10, q6, q9
104 vtbl.8 d10, {q15}, d16
105 vtbl.8 d11, {q15}, d17
106 veor q11, q7, q9
107 vtbl.8 d12, {q10}, d16
108 vtbl.8 d13, {q10}, d17
109 vtbl.8 d14, {q11}, d16
110 vtbl.8 d15, {q11}, d17
111 vmov.i8 q8,#0x55 @ compose .LBS0
112 vmov.i8 q9,#0x33 @ compose .LBS1
113 vshr.u64 q10, q6, #1
114 vshr.u64 q11, q4, #1
115 veor q10, q10, q7
116 veor q11, q11, q5
117 vand q10, q10, q8
118 vand q11, q11, q8
119 veor q7, q7, q10
120 vshl.u64 q10, q10, #1
121 veor q5, q5, q11
122 vshl.u64 q11, q11, #1
123 veor q6, q6, q10
124 veor q4, q4, q11
125 vshr.u64 q10, q2, #1
126 vshr.u64 q11, q0, #1
127 veor q10, q10, q3
128 veor q11, q11, q1
129 vand q10, q10, q8
130 vand q11, q11, q8
131 veor q3, q3, q10
132 vshl.u64 q10, q10, #1
133 veor q1, q1, q11
134 vshl.u64 q11, q11, #1
135 veor q2, q2, q10
136 veor q0, q0, q11
137 vmov.i8 q8,#0x0f @ compose .LBS2
138 vshr.u64 q10, q5, #2
139 vshr.u64 q11, q4, #2
140 veor q10, q10, q7
141 veor q11, q11, q6
142 vand q10, q10, q9
143 vand q11, q11, q9
144 veor q7, q7, q10
145 vshl.u64 q10, q10, #2
146 veor q6, q6, q11
147 vshl.u64 q11, q11, #2
148 veor q5, q5, q10
149 veor q4, q4, q11
150 vshr.u64 q10, q1, #2
151 vshr.u64 q11, q0, #2
152 veor q10, q10, q3
153 veor q11, q11, q2
154 vand q10, q10, q9
155 vand q11, q11, q9
156 veor q3, q3, q10
157 vshl.u64 q10, q10, #2
158 veor q2, q2, q11
159 vshl.u64 q11, q11, #2
160 veor q1, q1, q10
161 veor q0, q0, q11
162 vshr.u64 q10, q3, #4
163 vshr.u64 q11, q2, #4
164 veor q10, q10, q7
165 veor q11, q11, q6
166 vand q10, q10, q8
167 vand q11, q11, q8
168 veor q7, q7, q10
169 vshl.u64 q10, q10, #4
170 veor q6, q6, q11
171 vshl.u64 q11, q11, #4
172 veor q3, q3, q10
173 veor q2, q2, q11
174 vshr.u64 q10, q1, #4
175 vshr.u64 q11, q0, #4
176 veor q10, q10, q5
177 veor q11, q11, q4
178 vand q10, q10, q8
179 vand q11, q11, q8
180 veor q5, q5, q10
181 vshl.u64 q10, q10, #4
182 veor q4, q4, q11
183 vshl.u64 q11, q11, #4
184 veor q1, q1, q10
185 veor q0, q0, q11
186 sub r5,r5,#1
187 b .Ldec_sbox
188.align 4
189.Ldec_loop:
190 vldmia r4!, {q8-q11}
191 veor q8, q8, q0
192 veor q9, q9, q1
193 vtbl.8 d0, {q8}, d24
194 vtbl.8 d1, {q8}, d25
195 vldmia r4!, {q8}
196 veor q10, q10, q2
197 vtbl.8 d2, {q9}, d24
198 vtbl.8 d3, {q9}, d25
199 vldmia r4!, {q9}
200 veor q11, q11, q3
201 vtbl.8 d4, {q10}, d24
202 vtbl.8 d5, {q10}, d25
203 vldmia r4!, {q10}
204 vtbl.8 d6, {q11}, d24
205 vtbl.8 d7, {q11}, d25
206 vldmia r4!, {q11}
207 veor q8, q8, q4
208 veor q9, q9, q5
209 vtbl.8 d8, {q8}, d24
210 vtbl.8 d9, {q8}, d25
211 veor q10, q10, q6
212 vtbl.8 d10, {q9}, d24
213 vtbl.8 d11, {q9}, d25
214 veor q11, q11, q7
215 vtbl.8 d12, {q10}, d24
216 vtbl.8 d13, {q10}, d25
217 vtbl.8 d14, {q11}, d24
218 vtbl.8 d15, {q11}, d25
219.Ldec_sbox:
220 veor q1, q1, q4
221 veor q3, q3, q4
222
223 veor q4, q4, q7
224 veor q1, q1, q6
225 veor q2, q2, q7
226 veor q6, q6, q4
227
228 veor q0, q0, q1
229 veor q2, q2, q5
230 veor q7, q7, q6
231 veor q3, q3, q0
232 veor q5, q5, q0
233 veor q1, q1, q3
234 veor q11, q3, q0
235 veor q10, q7, q4
236 veor q9, q1, q6
237 veor q13, q4, q0
238 vmov q8, q10
239 veor q12, q5, q2
240
241 vorr q10, q10, q9
242 veor q15, q11, q8
243 vand q14, q11, q12
244 vorr q11, q11, q12
245 veor q12, q12, q9
246 vand q8, q8, q9
247 veor q9, q6, q2
248 vand q15, q15, q12
249 vand q13, q13, q9
250 veor q9, q3, q7
251 veor q12, q1, q5
252 veor q11, q11, q13
253 veor q10, q10, q13
254 vand q13, q9, q12
255 vorr q9, q9, q12
256 veor q11, q11, q15
257 veor q8, q8, q13
258 veor q10, q10, q14
259 veor q9, q9, q15
260 veor q8, q8, q14
261 vand q12, q4, q6
262 veor q9, q9, q14
263 vand q13, q0, q2
264 vand q14, q7, q1
265 vorr q15, q3, q5
266 veor q11, q11, q12
267 veor q9, q9, q14
268 veor q8, q8, q15
269 veor q10, q10, q13
270
271 @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3
272
273 @ new smaller inversion
274
275 vand q14, q11, q9
276 vmov q12, q8
277
278 veor q13, q10, q14
279 veor q15, q8, q14
280 veor q14, q8, q14 @ q14=q15
281
282 vbsl q13, q9, q8
283 vbsl q15, q11, q10
284 veor q11, q11, q10
285
286 vbsl q12, q13, q14
287 vbsl q8, q14, q13
288
289 vand q14, q12, q15
290 veor q9, q9, q8
291
292 veor q14, q14, q11
293 veor q12, q5, q2
294 veor q8, q1, q6
295 veor q10, q15, q14
296 vand q10, q10, q5
297 veor q5, q5, q1
298 vand q11, q1, q15
299 vand q5, q5, q14
300 veor q1, q11, q10
301 veor q5, q5, q11
302 veor q15, q15, q13
303 veor q14, q14, q9
304 veor q11, q15, q14
305 veor q10, q13, q9
306 vand q11, q11, q12
307 vand q10, q10, q2
308 veor q12, q12, q8
309 veor q2, q2, q6
310 vand q8, q8, q15
311 vand q6, q6, q13
312 vand q12, q12, q14
313 vand q2, q2, q9
314 veor q8, q8, q12
315 veor q2, q2, q6
316 veor q12, q12, q11
317 veor q6, q6, q10
318 veor q5, q5, q12
319 veor q2, q2, q12
320 veor q1, q1, q8
321 veor q6, q6, q8
322
323 veor q12, q3, q0
324 veor q8, q7, q4
325 veor q11, q15, q14
326 veor q10, q13, q9
327 vand q11, q11, q12
328 vand q10, q10, q0
329 veor q12, q12, q8
330 veor q0, q0, q4
331 vand q8, q8, q15
332 vand q4, q4, q13
333 vand q12, q12, q14
334 vand q0, q0, q9
335 veor q8, q8, q12
336 veor q0, q0, q4
337 veor q12, q12, q11
338 veor q4, q4, q10
339 veor q15, q15, q13
340 veor q14, q14, q9
341 veor q10, q15, q14
342 vand q10, q10, q3
343 veor q3, q3, q7
344 vand q11, q7, q15
345 vand q3, q3, q14
346 veor q7, q11, q10
347 veor q3, q3, q11
348 veor q3, q3, q12
349 veor q0, q0, q12
350 veor q7, q7, q8
351 veor q4, q4, q8
352 veor q1, q1, q7
353 veor q6, q6, q5
354
355 veor q4, q4, q1
356 veor q2, q2, q7
357 veor q5, q5, q7
358 veor q4, q4, q2
359 veor q7, q7, q0
360 veor q4, q4, q5
361 veor q3, q3, q6
362 veor q6, q6, q1
363 veor q3, q3, q4
364
365 veor q4, q4, q0
366 veor q7, q7, q3
367 subs r5,r5,#1
368 bcc .Ldec_done
369 @ multiplication by 0x05-0x00-0x04-0x00
370 vext.8 q8, q0, q0, #8
371 vext.8 q14, q3, q3, #8
372 vext.8 q15, q5, q5, #8
373 veor q8, q8, q0
374 vext.8 q9, q1, q1, #8
375 veor q14, q14, q3
376 vext.8 q10, q6, q6, #8
377 veor q15, q15, q5
378 vext.8 q11, q4, q4, #8
379 veor q9, q9, q1
380 vext.8 q12, q2, q2, #8
381 veor q10, q10, q6
382 vext.8 q13, q7, q7, #8
383 veor q11, q11, q4
384 veor q12, q12, q2
385 veor q13, q13, q7
386
387 veor q0, q0, q14
388 veor q1, q1, q14
389 veor q6, q6, q8
390 veor q2, q2, q10
391 veor q4, q4, q9
392 veor q1, q1, q15
393 veor q6, q6, q15
394 veor q2, q2, q14
395 veor q7, q7, q11
396 veor q4, q4, q14
397 veor q3, q3, q12
398 veor q2, q2, q15
399 veor q7, q7, q15
400 veor q5, q5, q13
401 vext.8 q8, q0, q0, #12 @ x0 <<< 32
402 vext.8 q9, q1, q1, #12
403 veor q0, q0, q8 @ x0 ^ (x0 <<< 32)
404 vext.8 q10, q6, q6, #12
405 veor q1, q1, q9
406 vext.8 q11, q4, q4, #12
407 veor q6, q6, q10
408 vext.8 q12, q2, q2, #12
409 veor q4, q4, q11
410 vext.8 q13, q7, q7, #12
411 veor q2, q2, q12
412 vext.8 q14, q3, q3, #12
413 veor q7, q7, q13
414 vext.8 q15, q5, q5, #12
415 veor q3, q3, q14
416
417 veor q9, q9, q0
418 veor q5, q5, q15
419 vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
420 veor q10, q10, q1
421 veor q8, q8, q5
422 veor q9, q9, q5
423 vext.8 q1, q1, q1, #8
424 veor q13, q13, q2
425 veor q0, q0, q8
426 veor q14, q14, q7
427 veor q1, q1, q9
428 vext.8 q8, q2, q2, #8
429 veor q12, q12, q4
430 vext.8 q9, q7, q7, #8
431 veor q15, q15, q3
432 vext.8 q2, q4, q4, #8
433 veor q11, q11, q6
434 vext.8 q7, q5, q5, #8
435 veor q12, q12, q5
436 vext.8 q4, q3, q3, #8
437 veor q11, q11, q5
438 vext.8 q3, q6, q6, #8
439 veor q5, q9, q13
440 veor q11, q11, q2
441 veor q7, q7, q15
442 veor q6, q4, q14
443 veor q4, q8, q12
444 veor q2, q3, q10
445 vmov q3, q11
446 @ vmov q5, q9
447 vldmia r6, {q12} @ .LISR
448 ite eq @ Thumb2 thing, sanity check in ARM
449 addeq r6,r6,#0x10
450 bne .Ldec_loop
451 vldmia r6, {q12} @ .LISRM0
452 b .Ldec_loop
453.align 4
454.Ldec_done:
455 vmov.i8 q8,#0x55 @ compose .LBS0
456 vmov.i8 q9,#0x33 @ compose .LBS1
457 vshr.u64 q10, q3, #1
458 vshr.u64 q11, q2, #1
459 veor q10, q10, q5
460 veor q11, q11, q7
461 vand q10, q10, q8
462 vand q11, q11, q8
463 veor q5, q5, q10
464 vshl.u64 q10, q10, #1
465 veor q7, q7, q11
466 vshl.u64 q11, q11, #1
467 veor q3, q3, q10
468 veor q2, q2, q11
469 vshr.u64 q10, q6, #1
470 vshr.u64 q11, q0, #1
471 veor q10, q10, q4
472 veor q11, q11, q1
473 vand q10, q10, q8
474 vand q11, q11, q8
475 veor q4, q4, q10
476 vshl.u64 q10, q10, #1
477 veor q1, q1, q11
478 vshl.u64 q11, q11, #1
479 veor q6, q6, q10
480 veor q0, q0, q11
481 vmov.i8 q8,#0x0f @ compose .LBS2
482 vshr.u64 q10, q7, #2
483 vshr.u64 q11, q2, #2
484 veor q10, q10, q5
485 veor q11, q11, q3
486 vand q10, q10, q9
487 vand q11, q11, q9
488 veor q5, q5, q10
489 vshl.u64 q10, q10, #2
490 veor q3, q3, q11
491 vshl.u64 q11, q11, #2
492 veor q7, q7, q10
493 veor q2, q2, q11
494 vshr.u64 q10, q1, #2
495 vshr.u64 q11, q0, #2
496 veor q10, q10, q4
497 veor q11, q11, q6
498 vand q10, q10, q9
499 vand q11, q11, q9
500 veor q4, q4, q10
501 vshl.u64 q10, q10, #2
502 veor q6, q6, q11
503 vshl.u64 q11, q11, #2
504 veor q1, q1, q10
505 veor q0, q0, q11
506 vshr.u64 q10, q4, #4
507 vshr.u64 q11, q6, #4
508 veor q10, q10, q5
509 veor q11, q11, q3
510 vand q10, q10, q8
511 vand q11, q11, q8
512 veor q5, q5, q10
513 vshl.u64 q10, q10, #4
514 veor q3, q3, q11
515 vshl.u64 q11, q11, #4
516 veor q4, q4, q10
517 veor q6, q6, q11
518 vshr.u64 q10, q1, #4
519 vshr.u64 q11, q0, #4
520 veor q10, q10, q7
521 veor q11, q11, q2
522 vand q10, q10, q8
523 vand q11, q11, q8
524 veor q7, q7, q10
525 vshl.u64 q10, q10, #4
526 veor q2, q2, q11
527 vshl.u64 q11, q11, #4
528 veor q1, q1, q10
529 veor q0, q0, q11
530 vldmia r4, {q8} @ last round key
531 veor q6, q6, q8
532 veor q4, q4, q8
533 veor q2, q2, q8
534 veor q7, q7, q8
535 veor q3, q3, q8
536 veor q5, q5, q8
537 veor q0, q0, q8
538 veor q1, q1, q8
539 bx lr
540.size _bsaes_decrypt8,.-_bsaes_decrypt8
541
542.type _bsaes_const,%object
543.align 6
544_bsaes_const:
545.LM0ISR: @ InvShiftRows constants
546 .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
547.LISR:
548 .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
549.LISRM0:
550 .quad 0x01040b0e0205080f, 0x0306090c00070a0d
551.LM0SR: @ ShiftRows constants
552 .quad 0x0a0e02060f03070b, 0x0004080c05090d01
553.LSR:
554 .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
555.LSRM0:
556 .quad 0x0304090e00050a0f, 0x01060b0c0207080d
557.LM0:
558 .quad 0x02060a0e03070b0f, 0x0004080c0105090d
559.LREVM0SR:
560 .quad 0x090d01050c000408, 0x03070b0f060a0e02
561.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro@openssl.org>"
562.align 6
563.size _bsaes_const,.-_bsaes_const
564
565.type _bsaes_encrypt8,%function
566.align 4
567_bsaes_encrypt8:
568 adr r6,_bsaes_encrypt8
569 vldmia r4!, {q9} @ round 0 key
570 sub r6,r6,#_bsaes_encrypt8-.LM0SR
571
572 vldmia r6!, {q8} @ .LM0SR
573_bsaes_encrypt8_alt:
574 veor q10, q0, q9 @ xor with round0 key
575 veor q11, q1, q9
576 vtbl.8 d0, {q10}, d16
577 vtbl.8 d1, {q10}, d17
578 veor q12, q2, q9
579 vtbl.8 d2, {q11}, d16
580 vtbl.8 d3, {q11}, d17
581 veor q13, q3, q9
582 vtbl.8 d4, {q12}, d16
583 vtbl.8 d5, {q12}, d17
584 veor q14, q4, q9
585 vtbl.8 d6, {q13}, d16
586 vtbl.8 d7, {q13}, d17
587 veor q15, q5, q9
588 vtbl.8 d8, {q14}, d16
589 vtbl.8 d9, {q14}, d17
590 veor q10, q6, q9
591 vtbl.8 d10, {q15}, d16
592 vtbl.8 d11, {q15}, d17
593 veor q11, q7, q9
594 vtbl.8 d12, {q10}, d16
595 vtbl.8 d13, {q10}, d17
596 vtbl.8 d14, {q11}, d16
597 vtbl.8 d15, {q11}, d17
598_bsaes_encrypt8_bitslice:
599 vmov.i8 q8,#0x55 @ compose .LBS0
600 vmov.i8 q9,#0x33 @ compose .LBS1
601 vshr.u64 q10, q6, #1
602 vshr.u64 q11, q4, #1
603 veor q10, q10, q7
604 veor q11, q11, q5
605 vand q10, q10, q8
606 vand q11, q11, q8
607 veor q7, q7, q10
608 vshl.u64 q10, q10, #1
609 veor q5, q5, q11
610 vshl.u64 q11, q11, #1
611 veor q6, q6, q10
612 veor q4, q4, q11
613 vshr.u64 q10, q2, #1
614 vshr.u64 q11, q0, #1
615 veor q10, q10, q3
616 veor q11, q11, q1
617 vand q10, q10, q8
618 vand q11, q11, q8
619 veor q3, q3, q10
620 vshl.u64 q10, q10, #1
621 veor q1, q1, q11
622 vshl.u64 q11, q11, #1
623 veor q2, q2, q10
624 veor q0, q0, q11
625 vmov.i8 q8,#0x0f @ compose .LBS2
626 vshr.u64 q10, q5, #2
627 vshr.u64 q11, q4, #2
628 veor q10, q10, q7
629 veor q11, q11, q6
630 vand q10, q10, q9
631 vand q11, q11, q9
632 veor q7, q7, q10
633 vshl.u64 q10, q10, #2
634 veor q6, q6, q11
635 vshl.u64 q11, q11, #2
636 veor q5, q5, q10
637 veor q4, q4, q11
638 vshr.u64 q10, q1, #2
639 vshr.u64 q11, q0, #2
640 veor q10, q10, q3
641 veor q11, q11, q2
642 vand q10, q10, q9
643 vand q11, q11, q9
644 veor q3, q3, q10
645 vshl.u64 q10, q10, #2
646 veor q2, q2, q11
647 vshl.u64 q11, q11, #2
648 veor q1, q1, q10
649 veor q0, q0, q11
650 vshr.u64 q10, q3, #4
651 vshr.u64 q11, q2, #4
652 veor q10, q10, q7
653 veor q11, q11, q6
654 vand q10, q10, q8
655 vand q11, q11, q8
656 veor q7, q7, q10
657 vshl.u64 q10, q10, #4
658 veor q6, q6, q11
659 vshl.u64 q11, q11, #4
660 veor q3, q3, q10
661 veor q2, q2, q11
662 vshr.u64 q10, q1, #4
663 vshr.u64 q11, q0, #4
664 veor q10, q10, q5
665 veor q11, q11, q4
666 vand q10, q10, q8
667 vand q11, q11, q8
668 veor q5, q5, q10
669 vshl.u64 q10, q10, #4
670 veor q4, q4, q11
671 vshl.u64 q11, q11, #4
672 veor q1, q1, q10
673 veor q0, q0, q11
674 sub r5,r5,#1
675 b .Lenc_sbox
676.align 4
677.Lenc_loop:
678 vldmia r4!, {q8-q11}
679 veor q8, q8, q0
680 veor q9, q9, q1
681 vtbl.8 d0, {q8}, d24
682 vtbl.8 d1, {q8}, d25
683 vldmia r4!, {q8}
684 veor q10, q10, q2
685 vtbl.8 d2, {q9}, d24
686 vtbl.8 d3, {q9}, d25
687 vldmia r4!, {q9}
688 veor q11, q11, q3
689 vtbl.8 d4, {q10}, d24
690 vtbl.8 d5, {q10}, d25
691 vldmia r4!, {q10}
692 vtbl.8 d6, {q11}, d24
693 vtbl.8 d7, {q11}, d25
694 vldmia r4!, {q11}
695 veor q8, q8, q4
696 veor q9, q9, q5
697 vtbl.8 d8, {q8}, d24
698 vtbl.8 d9, {q8}, d25
699 veor q10, q10, q6
700 vtbl.8 d10, {q9}, d24
701 vtbl.8 d11, {q9}, d25
702 veor q11, q11, q7
703 vtbl.8 d12, {q10}, d24
704 vtbl.8 d13, {q10}, d25
705 vtbl.8 d14, {q11}, d24
706 vtbl.8 d15, {q11}, d25
707.Lenc_sbox:
708 veor q2, q2, q1
709 veor q5, q5, q6
710 veor q3, q3, q0
711 veor q6, q6, q2
712 veor q5, q5, q0
713
714 veor q6, q6, q3
715 veor q3, q3, q7
716 veor q7, q7, q5
717 veor q3, q3, q4
718 veor q4, q4, q5
719
720 veor q2, q2, q7
721 veor q3, q3, q1
722 veor q1, q1, q5
723 veor q11, q7, q4
724 veor q10, q1, q2
725 veor q9, q5, q3
726 veor q13, q2, q4
727 vmov q8, q10
728 veor q12, q6, q0
729
730 vorr q10, q10, q9
731 veor q15, q11, q8
732 vand q14, q11, q12
733 vorr q11, q11, q12
734 veor q12, q12, q9
735 vand q8, q8, q9
736 veor q9, q3, q0
737 vand q15, q15, q12
738 vand q13, q13, q9
739 veor q9, q7, q1
740 veor q12, q5, q6
741 veor q11, q11, q13
742 veor q10, q10, q13
743 vand q13, q9, q12
744 vorr q9, q9, q12
745 veor q11, q11, q15
746 veor q8, q8, q13
747 veor q10, q10, q14
748 veor q9, q9, q15
749 veor q8, q8, q14
750 vand q12, q2, q3
751 veor q9, q9, q14
752 vand q13, q4, q0
753 vand q14, q1, q5
754 vorr q15, q7, q6
755 veor q11, q11, q12
756 veor q9, q9, q14
757 veor q8, q8, q15
758 veor q10, q10, q13
759
760 @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3
761
762 @ new smaller inversion
763
764 vand q14, q11, q9
765 vmov q12, q8
766
767 veor q13, q10, q14
768 veor q15, q8, q14
769 veor q14, q8, q14 @ q14=q15
770
771 vbsl q13, q9, q8
772 vbsl q15, q11, q10
773 veor q11, q11, q10
774
775 vbsl q12, q13, q14
776 vbsl q8, q14, q13
777
778 vand q14, q12, q15
779 veor q9, q9, q8
780
781 veor q14, q14, q11
782 veor q12, q6, q0
783 veor q8, q5, q3
784 veor q10, q15, q14
785 vand q10, q10, q6
786 veor q6, q6, q5
787 vand q11, q5, q15
788 vand q6, q6, q14
789 veor q5, q11, q10
790 veor q6, q6, q11
791 veor q15, q15, q13
792 veor q14, q14, q9
793 veor q11, q15, q14
794 veor q10, q13, q9
795 vand q11, q11, q12
796 vand q10, q10, q0
797 veor q12, q12, q8
798 veor q0, q0, q3
799 vand q8, q8, q15
800 vand q3, q3, q13
801 vand q12, q12, q14
802 vand q0, q0, q9
803 veor q8, q8, q12
804 veor q0, q0, q3
805 veor q12, q12, q11
806 veor q3, q3, q10
807 veor q6, q6, q12
808 veor q0, q0, q12
809 veor q5, q5, q8
810 veor q3, q3, q8
811
812 veor q12, q7, q4
813 veor q8, q1, q2
814 veor q11, q15, q14
815 veor q10, q13, q9
816 vand q11, q11, q12
817 vand q10, q10, q4
818 veor q12, q12, q8
819 veor q4, q4, q2
820 vand q8, q8, q15
821 vand q2, q2, q13
822 vand q12, q12, q14
823 vand q4, q4, q9
824 veor q8, q8, q12
825 veor q4, q4, q2
826 veor q12, q12, q11
827 veor q2, q2, q10
828 veor q15, q15, q13
829 veor q14, q14, q9
830 veor q10, q15, q14
831 vand q10, q10, q7
832 veor q7, q7, q1
833 vand q11, q1, q15
834 vand q7, q7, q14
835 veor q1, q11, q10
836 veor q7, q7, q11
837 veor q7, q7, q12
838 veor q4, q4, q12
839 veor q1, q1, q8
840 veor q2, q2, q8
841 veor q7, q7, q0
842 veor q1, q1, q6
843 veor q6, q6, q0
844 veor q4, q4, q7
845 veor q0, q0, q1
846
847 veor q1, q1, q5
848 veor q5, q5, q2
849 veor q2, q2, q3
850 veor q3, q3, q5
851 veor q4, q4, q5
852
853 veor q6, q6, q3
854 subs r5,r5,#1
855 bcc .Lenc_done
856 vext.8 q8, q0, q0, #12 @ x0 <<< 32
857 vext.8 q9, q1, q1, #12
858 veor q0, q0, q8 @ x0 ^ (x0 <<< 32)
859 vext.8 q10, q4, q4, #12
860 veor q1, q1, q9
861 vext.8 q11, q6, q6, #12
862 veor q4, q4, q10
863 vext.8 q12, q3, q3, #12
864 veor q6, q6, q11
865 vext.8 q13, q7, q7, #12
866 veor q3, q3, q12
867 vext.8 q14, q2, q2, #12
868 veor q7, q7, q13
869 vext.8 q15, q5, q5, #12
870 veor q2, q2, q14
871
872 veor q9, q9, q0
873 veor q5, q5, q15
874 vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
875 veor q10, q10, q1
876 veor q8, q8, q5
877 veor q9, q9, q5
878 vext.8 q1, q1, q1, #8
879 veor q13, q13, q3
880 veor q0, q0, q8
881 veor q14, q14, q7
882 veor q1, q1, q9
883 vext.8 q8, q3, q3, #8
884 veor q12, q12, q6
885 vext.8 q9, q7, q7, #8
886 veor q15, q15, q2
887 vext.8 q3, q6, q6, #8
888 veor q11, q11, q4
889 vext.8 q7, q5, q5, #8
890 veor q12, q12, q5
891 vext.8 q6, q2, q2, #8
892 veor q11, q11, q5
893 vext.8 q2, q4, q4, #8
894 veor q5, q9, q13
895 veor q4, q8, q12
896 veor q3, q3, q11
897 veor q7, q7, q15
898 veor q6, q6, q14
899 @ vmov q4, q8
900 veor q2, q2, q10
901 @ vmov q5, q9
902 vldmia r6, {q12} @ .LSR
903 ite eq @ Thumb2 thing, samity check in ARM
904 addeq r6,r6,#0x10
905 bne .Lenc_loop
906 vldmia r6, {q12} @ .LSRM0
907 b .Lenc_loop
908.align 4
909.Lenc_done:
910 vmov.i8 q8,#0x55 @ compose .LBS0
911 vmov.i8 q9,#0x33 @ compose .LBS1
912 vshr.u64 q10, q2, #1
913 vshr.u64 q11, q3, #1
914 veor q10, q10, q5
915 veor q11, q11, q7
916 vand q10, q10, q8
917 vand q11, q11, q8
918 veor q5, q5, q10
919 vshl.u64 q10, q10, #1
920 veor q7, q7, q11
921 vshl.u64 q11, q11, #1
922 veor q2, q2, q10
923 veor q3, q3, q11
924 vshr.u64 q10, q4, #1
925 vshr.u64 q11, q0, #1
926 veor q10, q10, q6
927 veor q11, q11, q1
928 vand q10, q10, q8
929 vand q11, q11, q8
930 veor q6, q6, q10
931 vshl.u64 q10, q10, #1
932 veor q1, q1, q11
933 vshl.u64 q11, q11, #1
934 veor q4, q4, q10
935 veor q0, q0, q11
936 vmov.i8 q8,#0x0f @ compose .LBS2
937 vshr.u64 q10, q7, #2
938 vshr.u64 q11, q3, #2
939 veor q10, q10, q5
940 veor q11, q11, q2
941 vand q10, q10, q9
942 vand q11, q11, q9
943 veor q5, q5, q10
944 vshl.u64 q10, q10, #2
945 veor q2, q2, q11
946 vshl.u64 q11, q11, #2
947 veor q7, q7, q10
948 veor q3, q3, q11
949 vshr.u64 q10, q1, #2
950 vshr.u64 q11, q0, #2
951 veor q10, q10, q6
952 veor q11, q11, q4
953 vand q10, q10, q9
954 vand q11, q11, q9
955 veor q6, q6, q10
956 vshl.u64 q10, q10, #2
957 veor q4, q4, q11
958 vshl.u64 q11, q11, #2
959 veor q1, q1, q10
960 veor q0, q0, q11
961 vshr.u64 q10, q6, #4
962 vshr.u64 q11, q4, #4
963 veor q10, q10, q5
964 veor q11, q11, q2
965 vand q10, q10, q8
966 vand q11, q11, q8
967 veor q5, q5, q10
968 vshl.u64 q10, q10, #4
969 veor q2, q2, q11
970 vshl.u64 q11, q11, #4
971 veor q6, q6, q10
972 veor q4, q4, q11
973 vshr.u64 q10, q1, #4
974 vshr.u64 q11, q0, #4
975 veor q10, q10, q7
976 veor q11, q11, q3
977 vand q10, q10, q8
978 vand q11, q11, q8
979 veor q7, q7, q10
980 vshl.u64 q10, q10, #4
981 veor q3, q3, q11
982 vshl.u64 q11, q11, #4
983 veor q1, q1, q10
984 veor q0, q0, q11
985 vldmia r4, {q8} @ last round key
986 veor q4, q4, q8
987 veor q6, q6, q8
988 veor q3, q3, q8
989 veor q7, q7, q8
990 veor q2, q2, q8
991 veor q5, q5, q8
992 veor q0, q0, q8
993 veor q1, q1, q8
994 bx lr
995.size _bsaes_encrypt8,.-_bsaes_encrypt8
996.type _bsaes_key_convert,%function
997.align 4
998_bsaes_key_convert:
999 adr r6,_bsaes_key_convert
1000 vld1.8 {q7}, [r4]! @ load round 0 key
1001 sub r6,r6,#_bsaes_key_convert-.LM0
1002 vld1.8 {q15}, [r4]! @ load round 1 key
1003
1004 vmov.i8 q8, #0x01 @ bit masks
1005 vmov.i8 q9, #0x02
1006 vmov.i8 q10, #0x04
1007 vmov.i8 q11, #0x08
1008 vmov.i8 q12, #0x10
1009 vmov.i8 q13, #0x20
1010 vldmia r6, {q14} @ .LM0
1011
1012#ifdef __ARMEL__
1013 vrev32.8 q7, q7
1014 vrev32.8 q15, q15
1015#endif
1016 sub r5,r5,#1
1017 vstmia r12!, {q7} @ save round 0 key
1018 b .Lkey_loop
1019
1020.align 4
1021.Lkey_loop:
1022 vtbl.8 d14,{q15},d28
1023 vtbl.8 d15,{q15},d29
1024 vmov.i8 q6, #0x40
1025 vmov.i8 q15, #0x80
1026
1027 vtst.8 q0, q7, q8
1028 vtst.8 q1, q7, q9
1029 vtst.8 q2, q7, q10
1030 vtst.8 q3, q7, q11
1031 vtst.8 q4, q7, q12
1032 vtst.8 q5, q7, q13
1033 vtst.8 q6, q7, q6
1034 vtst.8 q7, q7, q15
1035 vld1.8 {q15}, [r4]! @ load next round key
1036 vmvn q0, q0 @ "pnot"
1037 vmvn q1, q1
1038 vmvn q5, q5
1039 vmvn q6, q6
1040#ifdef __ARMEL__
1041 vrev32.8 q15, q15
1042#endif
1043 subs r5,r5,#1
1044 vstmia r12!,{q0-q7} @ write bit-sliced round key
1045 bne .Lkey_loop
1046
1047 vmov.i8 q7,#0x63 @ compose .L63
1048 @ don't save last round key
1049 bx lr
1050.size _bsaes_key_convert,.-_bsaes_key_convert
1051.extern AES_cbc_encrypt
1052.extern AES_decrypt
1053
1054.global bsaes_cbc_encrypt
1055.type bsaes_cbc_encrypt,%function
1056.align 5
1057bsaes_cbc_encrypt:
1058#ifndef __KERNEL__
1059 cmp r2, #128
1060#ifndef __thumb__
1061 blo AES_cbc_encrypt
1062#else
1063 bhs 1f
1064 b AES_cbc_encrypt
10651:
1066#endif
1067#endif
1068
1069 @ it is up to the caller to make sure we are called with enc == 0
1070
1071 mov ip, sp
1072 stmdb sp!, {r4-r10, lr}
1073 VFP_ABI_PUSH
1074 ldr r8, [ip] @ IV is 1st arg on the stack
1075 mov r2, r2, lsr#4 @ len in 16 byte blocks
1076 sub sp, #0x10 @ scratch space to carry over the IV
1077 mov r9, sp @ save sp
1078
1079 ldr r10, [r3, #240] @ get # of rounds
1080#ifndef BSAES_ASM_EXTENDED_KEY
1081 @ allocate the key schedule on the stack
1082 sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key
1083 add r12, #96 @ sifze of bit-slices key schedule
1084
1085 @ populate the key schedule
1086 mov r4, r3 @ pass key
1087 mov r5, r10 @ pass # of rounds
1088 mov sp, r12 @ sp is sp
1089 bl _bsaes_key_convert
1090 vldmia sp, {q6}
1091 vstmia r12, {q15} @ save last round key
1092 veor q7, q7, q6 @ fix up round 0 key
1093 vstmia sp, {q7}
1094#else
1095 ldr r12, [r3, #244]
1096 eors r12, #1
1097 beq 0f
1098
1099 @ populate the key schedule
1100 str r12, [r3, #244]
1101 mov r4, r3 @ pass key
1102 mov r5, r10 @ pass # of rounds
1103 add r12, r3, #248 @ pass key schedule
1104 bl _bsaes_key_convert
1105 add r4, r3, #248
1106 vldmia r4, {q6}
1107 vstmia r12, {q15} @ save last round key
1108 veor q7, q7, q6 @ fix up round 0 key
1109 vstmia r4, {q7}
1110
1111.align 2
11120:
1113#endif
1114
1115 vld1.8 {q15}, [r8] @ load IV
1116 b .Lcbc_dec_loop
1117
1118.align 4
1119.Lcbc_dec_loop:
1120 subs r2, r2, #0x8
1121 bmi .Lcbc_dec_loop_finish
1122
1123 vld1.8 {q0-q1}, [r0]! @ load input
1124 vld1.8 {q2-q3}, [r0]!
1125#ifndef BSAES_ASM_EXTENDED_KEY
1126 mov r4, sp @ pass the key
1127#else
1128 add r4, r3, #248
1129#endif
1130 vld1.8 {q4-q5}, [r0]!
1131 mov r5, r10
1132 vld1.8 {q6-q7}, [r0]
1133 sub r0, r0, #0x60
1134 vstmia r9, {q15} @ put aside IV
1135
1136 bl _bsaes_decrypt8
1137
1138 vldmia r9, {q14} @ reload IV
1139 vld1.8 {q8-q9}, [r0]! @ reload input
1140 veor q0, q0, q14 @ ^= IV
1141 vld1.8 {q10-q11}, [r0]!
1142 veor q1, q1, q8
1143 veor q6, q6, q9
1144 vld1.8 {q12-q13}, [r0]!
1145 veor q4, q4, q10
1146 veor q2, q2, q11
1147 vld1.8 {q14-q15}, [r0]!
1148 veor q7, q7, q12
1149 vst1.8 {q0-q1}, [r1]! @ write output
1150 veor q3, q3, q13
1151 vst1.8 {q6}, [r1]!
1152 veor q5, q5, q14
1153 vst1.8 {q4}, [r1]!
1154 vst1.8 {q2}, [r1]!
1155 vst1.8 {q7}, [r1]!
1156 vst1.8 {q3}, [r1]!
1157 vst1.8 {q5}, [r1]!
1158
1159 b .Lcbc_dec_loop
1160
1161.Lcbc_dec_loop_finish:
1162 adds r2, r2, #8
1163 beq .Lcbc_dec_done
1164
1165 vld1.8 {q0}, [r0]! @ load input
1166 cmp r2, #2
1167 blo .Lcbc_dec_one
1168 vld1.8 {q1}, [r0]!
1169#ifndef BSAES_ASM_EXTENDED_KEY
1170 mov r4, sp @ pass the key
1171#else
1172 add r4, r3, #248
1173#endif
1174 mov r5, r10
1175 vstmia r9, {q15} @ put aside IV
1176 beq .Lcbc_dec_two
1177 vld1.8 {q2}, [r0]!
1178 cmp r2, #4
1179 blo .Lcbc_dec_three
1180 vld1.8 {q3}, [r0]!
1181 beq .Lcbc_dec_four
1182 vld1.8 {q4}, [r0]!
1183 cmp r2, #6
1184 blo .Lcbc_dec_five
1185 vld1.8 {q5}, [r0]!
1186 beq .Lcbc_dec_six
1187 vld1.8 {q6}, [r0]!
1188 sub r0, r0, #0x70
1189
1190 bl _bsaes_decrypt8
1191
1192 vldmia r9, {q14} @ reload IV
1193 vld1.8 {q8-q9}, [r0]! @ reload input
1194 veor q0, q0, q14 @ ^= IV
1195 vld1.8 {q10-q11}, [r0]!
1196 veor q1, q1, q8
1197 veor q6, q6, q9
1198 vld1.8 {q12-q13}, [r0]!
1199 veor q4, q4, q10
1200 veor q2, q2, q11
1201 vld1.8 {q15}, [r0]!
1202 veor q7, q7, q12
1203 vst1.8 {q0-q1}, [r1]! @ write output
1204 veor q3, q3, q13
1205 vst1.8 {q6}, [r1]!
1206 vst1.8 {q4}, [r1]!
1207 vst1.8 {q2}, [r1]!
1208 vst1.8 {q7}, [r1]!
1209 vst1.8 {q3}, [r1]!
1210 b .Lcbc_dec_done
1211.align 4
1212.Lcbc_dec_six:
1213 sub r0, r0, #0x60
1214 bl _bsaes_decrypt8
1215 vldmia r9,{q14} @ reload IV
1216 vld1.8 {q8-q9}, [r0]! @ reload input
1217 veor q0, q0, q14 @ ^= IV
1218 vld1.8 {q10-q11}, [r0]!
1219 veor q1, q1, q8
1220 veor q6, q6, q9
1221 vld1.8 {q12}, [r0]!
1222 veor q4, q4, q10
1223 veor q2, q2, q11
1224 vld1.8 {q15}, [r0]!
1225 veor q7, q7, q12
1226 vst1.8 {q0-q1}, [r1]! @ write output
1227 vst1.8 {q6}, [r1]!
1228 vst1.8 {q4}, [r1]!
1229 vst1.8 {q2}, [r1]!
1230 vst1.8 {q7}, [r1]!
1231 b .Lcbc_dec_done
1232.align 4
1233.Lcbc_dec_five:
1234 sub r0, r0, #0x50
1235 bl _bsaes_decrypt8
1236 vldmia r9, {q14} @ reload IV
1237 vld1.8 {q8-q9}, [r0]! @ reload input
1238 veor q0, q0, q14 @ ^= IV
1239 vld1.8 {q10-q11}, [r0]!
1240 veor q1, q1, q8
1241 veor q6, q6, q9
1242 vld1.8 {q15}, [r0]!
1243 veor q4, q4, q10
1244 vst1.8 {q0-q1}, [r1]! @ write output
1245 veor q2, q2, q11
1246 vst1.8 {q6}, [r1]!
1247 vst1.8 {q4}, [r1]!
1248 vst1.8 {q2}, [r1]!
1249 b .Lcbc_dec_done
1250.align 4
1251.Lcbc_dec_four:
1252 sub r0, r0, #0x40
1253 bl _bsaes_decrypt8
1254 vldmia r9, {q14} @ reload IV
1255 vld1.8 {q8-q9}, [r0]! @ reload input
1256 veor q0, q0, q14 @ ^= IV
1257 vld1.8 {q10}, [r0]!
1258 veor q1, q1, q8
1259 veor q6, q6, q9
1260 vld1.8 {q15}, [r0]!
1261 veor q4, q4, q10
1262 vst1.8 {q0-q1}, [r1]! @ write output
1263 vst1.8 {q6}, [r1]!
1264 vst1.8 {q4}, [r1]!
1265 b .Lcbc_dec_done
1266.align 4
1267.Lcbc_dec_three:
1268 sub r0, r0, #0x30
1269 bl _bsaes_decrypt8
1270 vldmia r9, {q14} @ reload IV
1271 vld1.8 {q8-q9}, [r0]! @ reload input
1272 veor q0, q0, q14 @ ^= IV
1273 vld1.8 {q15}, [r0]!
1274 veor q1, q1, q8
1275 veor q6, q6, q9
1276 vst1.8 {q0-q1}, [r1]! @ write output
1277 vst1.8 {q6}, [r1]!
1278 b .Lcbc_dec_done
1279.align 4
1280.Lcbc_dec_two:
1281 sub r0, r0, #0x20
1282 bl _bsaes_decrypt8
1283 vldmia r9, {q14} @ reload IV
1284 vld1.8 {q8}, [r0]! @ reload input
1285 veor q0, q0, q14 @ ^= IV
1286 vld1.8 {q15}, [r0]! @ reload input
1287 veor q1, q1, q8
1288 vst1.8 {q0-q1}, [r1]! @ write output
1289 b .Lcbc_dec_done
1290.align 4
1291.Lcbc_dec_one:
1292 sub r0, r0, #0x10
1293 mov r10, r1 @ save original out pointer
1294 mov r1, r9 @ use the iv scratch space as out buffer
1295 mov r2, r3
1296 vmov q4,q15 @ just in case ensure that IV
1297 vmov q5,q0 @ and input are preserved
1298 bl AES_decrypt
1299 vld1.8 {q0}, [r9,:64] @ load result
1300 veor q0, q0, q4 @ ^= IV
1301 vmov q15, q5 @ q5 holds input
1302 vst1.8 {q0}, [r10] @ write output
1303
1304.Lcbc_dec_done:
1305#ifndef BSAES_ASM_EXTENDED_KEY
1306 vmov.i32 q0, #0
1307 vmov.i32 q1, #0
1308.Lcbc_dec_bzero: @ wipe key schedule [if any]
1309 vstmia sp!, {q0-q1}
1310 cmp sp, r9
1311 bne .Lcbc_dec_bzero
1312#endif
1313
1314 mov sp, r9
1315 add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb
1316 vst1.8 {q15}, [r8] @ return IV
1317 VFP_ABI_POP
1318 ldmia sp!, {r4-r10, pc}
1319.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
1320.extern AES_encrypt
1321.global bsaes_ctr32_encrypt_blocks
1322.type bsaes_ctr32_encrypt_blocks,%function
1323.align 5
1324bsaes_ctr32_encrypt_blocks:
1325 cmp r2, #8 @ use plain AES for
1326 blo .Lctr_enc_short @ small sizes
1327
1328 mov ip, sp
1329 stmdb sp!, {r4-r10, lr}
1330 VFP_ABI_PUSH
1331 ldr r8, [ip] @ ctr is 1st arg on the stack
1332 sub sp, sp, #0x10 @ scratch space to carry over the ctr
1333 mov r9, sp @ save sp
1334
1335 ldr r10, [r3, #240] @ get # of rounds
1336#ifndef BSAES_ASM_EXTENDED_KEY
1337 @ allocate the key schedule on the stack
1338 sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key
1339 add r12, #96 @ size of bit-sliced key schedule
1340
1341 @ populate the key schedule
1342 mov r4, r3 @ pass key
1343 mov r5, r10 @ pass # of rounds
1344 mov sp, r12 @ sp is sp
1345 bl _bsaes_key_convert
1346 veor q7,q7,q15 @ fix up last round key
1347 vstmia r12, {q7} @ save last round key
1348
1349 vld1.8 {q0}, [r8] @ load counter
1350 add r8, r6, #.LREVM0SR-.LM0 @ borrow r8
1351 vldmia sp, {q4} @ load round0 key
1352#else
1353 ldr r12, [r3, #244]
1354 eors r12, #1
1355 beq 0f
1356
1357 @ populate the key schedule
1358 str r12, [r3, #244]
1359 mov r4, r3 @ pass key
1360 mov r5, r10 @ pass # of rounds
1361 add r12, r3, #248 @ pass key schedule
1362 bl _bsaes_key_convert
1363 veor q7,q7,q15 @ fix up last round key
1364 vstmia r12, {q7} @ save last round key
1365
1366.align 2
13670: add r12, r3, #248
1368 vld1.8 {q0}, [r8] @ load counter
1369 adrl r8, .LREVM0SR @ borrow r8
1370 vldmia r12, {q4} @ load round0 key
1371 sub sp, #0x10 @ place for adjusted round0 key
1372#endif
1373
1374 vmov.i32 q8,#1 @ compose 1<<96
1375 veor q9,q9,q9
1376 vrev32.8 q0,q0
1377 vext.8 q8,q9,q8,#4
1378 vrev32.8 q4,q4
1379 vadd.u32 q9,q8,q8 @ compose 2<<96
1380 vstmia sp, {q4} @ save adjusted round0 key
1381 b .Lctr_enc_loop
1382
1383.align 4
1384.Lctr_enc_loop:
1385 vadd.u32 q10, q8, q9 @ compose 3<<96
1386 vadd.u32 q1, q0, q8 @ +1
1387 vadd.u32 q2, q0, q9 @ +2
1388 vadd.u32 q3, q0, q10 @ +3
1389 vadd.u32 q4, q1, q10
1390 vadd.u32 q5, q2, q10
1391 vadd.u32 q6, q3, q10
1392 vadd.u32 q7, q4, q10
1393 vadd.u32 q10, q5, q10 @ next counter
1394
1395 @ Borrow prologue from _bsaes_encrypt8 to use the opportunity
1396 @ to flip byte order in 32-bit counter
1397
1398 vldmia sp, {q9} @ load round0 key
1399#ifndef BSAES_ASM_EXTENDED_KEY
1400 add r4, sp, #0x10 @ pass next round key
1401#else
1402 add r4, r3, #264
1403#endif
1404 vldmia r8, {q8} @ .LREVM0SR
1405 mov r5, r10 @ pass rounds
1406 vstmia r9, {q10} @ save next counter
1407 sub r6, r8, #.LREVM0SR-.LSR @ pass constants
1408
1409 bl _bsaes_encrypt8_alt
1410
1411 subs r2, r2, #8
1412 blo .Lctr_enc_loop_done
1413
1414 vld1.8 {q8-q9}, [r0]! @ load input
1415 vld1.8 {q10-q11}, [r0]!
1416 veor q0, q8
1417 veor q1, q9
1418 vld1.8 {q12-q13}, [r0]!
1419 veor q4, q10
1420 veor q6, q11
1421 vld1.8 {q14-q15}, [r0]!
1422 veor q3, q12
1423 vst1.8 {q0-q1}, [r1]! @ write output
1424 veor q7, q13
1425 veor q2, q14
1426 vst1.8 {q4}, [r1]!
1427 veor q5, q15
1428 vst1.8 {q6}, [r1]!
1429 vmov.i32 q8, #1 @ compose 1<<96
1430 vst1.8 {q3}, [r1]!
1431 veor q9, q9, q9
1432 vst1.8 {q7}, [r1]!
1433 vext.8 q8, q9, q8, #4
1434 vst1.8 {q2}, [r1]!
1435 vadd.u32 q9,q8,q8 @ compose 2<<96
1436 vst1.8 {q5}, [r1]!
1437 vldmia r9, {q0} @ load counter
1438
1439 bne .Lctr_enc_loop
1440 b .Lctr_enc_done
1441
1442.align 4
1443.Lctr_enc_loop_done:
1444 add r2, r2, #8
1445 vld1.8 {q8}, [r0]! @ load input
1446 veor q0, q8
1447 vst1.8 {q0}, [r1]! @ write output
1448 cmp r2, #2
1449 blo .Lctr_enc_done
1450 vld1.8 {q9}, [r0]!
1451 veor q1, q9
1452 vst1.8 {q1}, [r1]!
1453 beq .Lctr_enc_done
1454 vld1.8 {q10}, [r0]!
1455 veor q4, q10
1456 vst1.8 {q4}, [r1]!
1457 cmp r2, #4
1458 blo .Lctr_enc_done
1459 vld1.8 {q11}, [r0]!
1460 veor q6, q11
1461 vst1.8 {q6}, [r1]!
1462 beq .Lctr_enc_done
1463 vld1.8 {q12}, [r0]!
1464 veor q3, q12
1465 vst1.8 {q3}, [r1]!
1466 cmp r2, #6
1467 blo .Lctr_enc_done
1468 vld1.8 {q13}, [r0]!
1469 veor q7, q13
1470 vst1.8 {q7}, [r1]!
1471 beq .Lctr_enc_done
1472 vld1.8 {q14}, [r0]
1473 veor q2, q14
1474 vst1.8 {q2}, [r1]!
1475
1476.Lctr_enc_done:
1477 vmov.i32 q0, #0
1478 vmov.i32 q1, #0
1479#ifndef BSAES_ASM_EXTENDED_KEY
1480.Lctr_enc_bzero: @ wipe key schedule [if any]
1481 vstmia sp!, {q0-q1}
1482 cmp sp, r9
1483 bne .Lctr_enc_bzero
1484#else
1485 vstmia sp, {q0-q1}
1486#endif
1487
1488 mov sp, r9
1489 add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb
1490 VFP_ABI_POP
1491 ldmia sp!, {r4-r10, pc} @ return
1492
1493.align 4
1494.Lctr_enc_short:
1495 ldr ip, [sp] @ ctr pointer is passed on stack
1496 stmdb sp!, {r4-r8, lr}
1497
1498 mov r4, r0 @ copy arguments
1499 mov r5, r1
1500 mov r6, r2
1501 mov r7, r3
1502 ldr r8, [ip, #12] @ load counter LSW
1503 vld1.8 {q1}, [ip] @ load whole counter value
1504#ifdef __ARMEL__
1505 rev r8, r8
1506#endif
1507 sub sp, sp, #0x10
1508 vst1.8 {q1}, [sp,:64] @ copy counter value
1509 sub sp, sp, #0x10
1510
1511.Lctr_enc_short_loop:
1512 add r0, sp, #0x10 @ input counter value
1513 mov r1, sp @ output on the stack
1514 mov r2, r7 @ key
1515
1516 bl AES_encrypt
1517
1518 vld1.8 {q0}, [r4]! @ load input
1519 vld1.8 {q1}, [sp,:64] @ load encrypted counter
1520 add r8, r8, #1
1521#ifdef __ARMEL__
1522 rev r0, r8
1523 str r0, [sp, #0x1c] @ next counter value
1524#else
1525 str r8, [sp, #0x1c] @ next counter value
1526#endif
1527 veor q0,q0,q1
1528 vst1.8 {q0}, [r5]! @ store output
1529 subs r6, r6, #1
1530 bne .Lctr_enc_short_loop
1531
1532 vmov.i32 q0, #0
1533 vmov.i32 q1, #0
1534 vstmia sp!, {q0-q1}
1535
1536 ldmia sp!, {r4-r8, pc}
1537.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
1538.globl bsaes_xts_encrypt
1539.type bsaes_xts_encrypt,%function
1540.align 4
1541bsaes_xts_encrypt:
1542 mov ip, sp
1543 stmdb sp!, {r4-r10, lr} @ 0x20
1544 VFP_ABI_PUSH
1545 mov r6, sp @ future r3
1546
1547 mov r7, r0
1548 mov r8, r1
1549 mov r9, r2
1550 mov r10, r3
1551
1552 sub r0, sp, #0x10 @ 0x10
1553 bic r0, #0xf @ align at 16 bytes
1554 mov sp, r0
1555
1556#ifdef XTS_CHAIN_TWEAK
1557 ldr r0, [ip] @ pointer to input tweak
1558#else
1559 @ generate initial tweak
1560 ldr r0, [ip, #4] @ iv[]
1561 mov r1, sp
1562 ldr r2, [ip, #0] @ key2
1563 bl AES_encrypt
1564 mov r0,sp @ pointer to initial tweak
1565#endif
1566
1567 ldr r1, [r10, #240] @ get # of rounds
1568 mov r3, r6
1569#ifndef BSAES_ASM_EXTENDED_KEY
1570 @ allocate the key schedule on the stack
1571 sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key
1572 @ add r12, #96 @ size of bit-sliced key schedule
1573 sub r12, #48 @ place for tweak[9]
1574
1575 @ populate the key schedule
1576 mov r4, r10 @ pass key
1577 mov r5, r1 @ pass # of rounds
1578 mov sp, r12
1579 add r12, #0x90 @ pass key schedule
1580 bl _bsaes_key_convert
1581 veor q7, q7, q15 @ fix up last round key
1582 vstmia r12, {q7} @ save last round key
1583#else
1584 ldr r12, [r10, #244]
1585 eors r12, #1
1586 beq 0f
1587
1588 str r12, [r10, #244]
1589 mov r4, r10 @ pass key
1590 mov r5, r1 @ pass # of rounds
1591 add r12, r10, #248 @ pass key schedule
1592 bl _bsaes_key_convert
1593 veor q7, q7, q15 @ fix up last round key
1594 vstmia r12, {q7}
1595
1596.align 2
15970: sub sp, #0x90 @ place for tweak[9]
1598#endif
1599
1600 vld1.8 {q8}, [r0] @ initial tweak
1601 adr r2, .Lxts_magic
1602
1603 subs r9, #0x80
1604 blo .Lxts_enc_short
1605 b .Lxts_enc_loop
1606
1607.align 4
1608.Lxts_enc_loop:
1609 vldmia r2, {q5} @ load XTS magic
1610 vshr.s64 q6, q8, #63
1611 mov r0, sp
1612 vand q6, q6, q5
1613 vadd.u64 q9, q8, q8
1614 vst1.64 {q8}, [r0,:128]!
1615 vswp d13,d12
1616 vshr.s64 q7, q9, #63
1617 veor q9, q9, q6
1618 vand q7, q7, q5
1619 vadd.u64 q10, q9, q9
1620 vst1.64 {q9}, [r0,:128]!
1621 vswp d15,d14
1622 vshr.s64 q6, q10, #63
1623 veor q10, q10, q7
1624 vand q6, q6, q5
1625 vld1.8 {q0}, [r7]!
1626 vadd.u64 q11, q10, q10
1627 vst1.64 {q10}, [r0,:128]!
1628 vswp d13,d12
1629 vshr.s64 q7, q11, #63
1630 veor q11, q11, q6
1631 vand q7, q7, q5
1632 vld1.8 {q1}, [r7]!
1633 veor q0, q0, q8
1634 vadd.u64 q12, q11, q11
1635 vst1.64 {q11}, [r0,:128]!
1636 vswp d15,d14
1637 vshr.s64 q6, q12, #63
1638 veor q12, q12, q7
1639 vand q6, q6, q5
1640 vld1.8 {q2}, [r7]!
1641 veor q1, q1, q9
1642 vadd.u64 q13, q12, q12
1643 vst1.64 {q12}, [r0,:128]!
1644 vswp d13,d12
1645 vshr.s64 q7, q13, #63
1646 veor q13, q13, q6
1647 vand q7, q7, q5
1648 vld1.8 {q3}, [r7]!
1649 veor q2, q2, q10
1650 vadd.u64 q14, q13, q13
1651 vst1.64 {q13}, [r0,:128]!
1652 vswp d15,d14
1653 vshr.s64 q6, q14, #63
1654 veor q14, q14, q7
1655 vand q6, q6, q5
1656 vld1.8 {q4}, [r7]!
1657 veor q3, q3, q11
1658 vadd.u64 q15, q14, q14
1659 vst1.64 {q14}, [r0,:128]!
1660 vswp d13,d12
1661 vshr.s64 q7, q15, #63
1662 veor q15, q15, q6
1663 vand q7, q7, q5
1664 vld1.8 {q5}, [r7]!
1665 veor q4, q4, q12
1666 vadd.u64 q8, q15, q15
1667 vst1.64 {q15}, [r0,:128]!
1668 vswp d15,d14
1669 veor q8, q8, q7
1670 vst1.64 {q8}, [r0,:128] @ next round tweak
1671
1672 vld1.8 {q6-q7}, [r7]!
1673 veor q5, q5, q13
1674#ifndef BSAES_ASM_EXTENDED_KEY
1675 add r4, sp, #0x90 @ pass key schedule
1676#else
1677 add r4, r10, #248 @ pass key schedule
1678#endif
1679 veor q6, q6, q14
1680 mov r5, r1 @ pass rounds
1681 veor q7, q7, q15
1682 mov r0, sp
1683
1684 bl _bsaes_encrypt8
1685
1686 vld1.64 {q8-q9}, [r0,:128]!
1687 vld1.64 {q10-q11}, [r0,:128]!
1688 veor q0, q0, q8
1689 vld1.64 {q12-q13}, [r0,:128]!
1690 veor q1, q1, q9
1691 veor q8, q4, q10
1692 vst1.8 {q0-q1}, [r8]!
1693 veor q9, q6, q11
1694 vld1.64 {q14-q15}, [r0,:128]!
1695 veor q10, q3, q12
1696 vst1.8 {q8-q9}, [r8]!
1697 veor q11, q7, q13
1698 veor q12, q2, q14
1699 vst1.8 {q10-q11}, [r8]!
1700 veor q13, q5, q15
1701 vst1.8 {q12-q13}, [r8]!
1702
1703 vld1.64 {q8}, [r0,:128] @ next round tweak
1704
1705 subs r9, #0x80
1706 bpl .Lxts_enc_loop
1707
1708.Lxts_enc_short:
1709 adds r9, #0x70
1710 bmi .Lxts_enc_done
1711
1712 vldmia r2, {q5} @ load XTS magic
1713 vshr.s64 q7, q8, #63
1714 mov r0, sp
1715 vand q7, q7, q5
1716 vadd.u64 q9, q8, q8
1717 vst1.64 {q8}, [r0,:128]!
1718 vswp d15,d14
1719 vshr.s64 q6, q9, #63
1720 veor q9, q9, q7
1721 vand q6, q6, q5
1722 vadd.u64 q10, q9, q9
1723 vst1.64 {q9}, [r0,:128]!
1724 vswp d13,d12
1725 vshr.s64 q7, q10, #63
1726 veor q10, q10, q6
1727 vand q7, q7, q5
1728 vld1.8 {q0}, [r7]!
1729 subs r9, #0x10
1730 bmi .Lxts_enc_1
1731 vadd.u64 q11, q10, q10
1732 vst1.64 {q10}, [r0,:128]!
1733 vswp d15,d14
1734 vshr.s64 q6, q11, #63
1735 veor q11, q11, q7
1736 vand q6, q6, q5
1737 vld1.8 {q1}, [r7]!
1738 subs r9, #0x10
1739 bmi .Lxts_enc_2
1740 veor q0, q0, q8
1741 vadd.u64 q12, q11, q11
1742 vst1.64 {q11}, [r0,:128]!
1743 vswp d13,d12
1744 vshr.s64 q7, q12, #63
1745 veor q12, q12, q6
1746 vand q7, q7, q5
1747 vld1.8 {q2}, [r7]!
1748 subs r9, #0x10
1749 bmi .Lxts_enc_3
1750 veor q1, q1, q9
1751 vadd.u64 q13, q12, q12
1752 vst1.64 {q12}, [r0,:128]!
1753 vswp d15,d14
1754 vshr.s64 q6, q13, #63
1755 veor q13, q13, q7
1756 vand q6, q6, q5
1757 vld1.8 {q3}, [r7]!
1758 subs r9, #0x10
1759 bmi .Lxts_enc_4
1760 veor q2, q2, q10
1761 vadd.u64 q14, q13, q13
1762 vst1.64 {q13}, [r0,:128]!
1763 vswp d13,d12
1764 vshr.s64 q7, q14, #63
1765 veor q14, q14, q6
1766 vand q7, q7, q5
1767 vld1.8 {q4}, [r7]!
1768 subs r9, #0x10
1769 bmi .Lxts_enc_5
1770 veor q3, q3, q11
1771 vadd.u64 q15, q14, q14
1772 vst1.64 {q14}, [r0,:128]!
1773 vswp d15,d14
1774 vshr.s64 q6, q15, #63
1775 veor q15, q15, q7
1776 vand q6, q6, q5
1777 vld1.8 {q5}, [r7]!
1778 subs r9, #0x10
1779 bmi .Lxts_enc_6
1780 veor q4, q4, q12
1781 sub r9, #0x10
1782 vst1.64 {q15}, [r0,:128] @ next round tweak
1783
1784 vld1.8 {q6}, [r7]!
1785 veor q5, q5, q13
1786#ifndef BSAES_ASM_EXTENDED_KEY
1787 add r4, sp, #0x90 @ pass key schedule
1788#else
1789 add r4, r10, #248 @ pass key schedule
1790#endif
1791 veor q6, q6, q14
1792 mov r5, r1 @ pass rounds
1793 mov r0, sp
1794
1795 bl _bsaes_encrypt8
1796
1797 vld1.64 {q8-q9}, [r0,:128]!
1798 vld1.64 {q10-q11}, [r0,:128]!
1799 veor q0, q0, q8
1800 vld1.64 {q12-q13}, [r0,:128]!
1801 veor q1, q1, q9
1802 veor q8, q4, q10
1803 vst1.8 {q0-q1}, [r8]!
1804 veor q9, q6, q11
1805 vld1.64 {q14}, [r0,:128]!
1806 veor q10, q3, q12
1807 vst1.8 {q8-q9}, [r8]!
1808 veor q11, q7, q13
1809 veor q12, q2, q14
1810 vst1.8 {q10-q11}, [r8]!
1811 vst1.8 {q12}, [r8]!
1812
1813 vld1.64 {q8}, [r0,:128] @ next round tweak
1814 b .Lxts_enc_done
1815.align 4
1816.Lxts_enc_6:
1817 vst1.64 {q14}, [r0,:128] @ next round tweak
1818
1819 veor q4, q4, q12
1820#ifndef BSAES_ASM_EXTENDED_KEY
1821 add r4, sp, #0x90 @ pass key schedule
1822#else
1823 add r4, r10, #248 @ pass key schedule
1824#endif
1825 veor q5, q5, q13
1826 mov r5, r1 @ pass rounds
1827 mov r0, sp
1828
1829 bl _bsaes_encrypt8
1830
1831 vld1.64 {q8-q9}, [r0,:128]!
1832 vld1.64 {q10-q11}, [r0,:128]!
1833 veor q0, q0, q8
1834 vld1.64 {q12-q13}, [r0,:128]!
1835 veor q1, q1, q9
1836 veor q8, q4, q10
1837 vst1.8 {q0-q1}, [r8]!
1838 veor q9, q6, q11
1839 veor q10, q3, q12
1840 vst1.8 {q8-q9}, [r8]!
1841 veor q11, q7, q13
1842 vst1.8 {q10-q11}, [r8]!
1843
1844 vld1.64 {q8}, [r0,:128] @ next round tweak
1845 b .Lxts_enc_done
1846
1847@ put this in range for both ARM and Thumb mode adr instructions
1848.align 5
1849.Lxts_magic:
1850 .quad 1, 0x87
1851
1852.align 5
1853.Lxts_enc_5:
1854 vst1.64 {q13}, [r0,:128] @ next round tweak
1855
1856 veor q3, q3, q11
1857#ifndef BSAES_ASM_EXTENDED_KEY
1858 add r4, sp, #0x90 @ pass key schedule
1859#else
1860 add r4, r10, #248 @ pass key schedule
1861#endif
1862 veor q4, q4, q12
1863 mov r5, r1 @ pass rounds
1864 mov r0, sp
1865
1866 bl _bsaes_encrypt8
1867
1868 vld1.64 {q8-q9}, [r0,:128]!
1869 vld1.64 {q10-q11}, [r0,:128]!
1870 veor q0, q0, q8
1871 vld1.64 {q12}, [r0,:128]!
1872 veor q1, q1, q9
1873 veor q8, q4, q10
1874 vst1.8 {q0-q1}, [r8]!
1875 veor q9, q6, q11
1876 veor q10, q3, q12
1877 vst1.8 {q8-q9}, [r8]!
1878 vst1.8 {q10}, [r8]!
1879
1880 vld1.64 {q8}, [r0,:128] @ next round tweak
1881 b .Lxts_enc_done
1882.align 4
1883.Lxts_enc_4:
1884 vst1.64 {q12}, [r0,:128] @ next round tweak
1885
1886 veor q2, q2, q10
1887#ifndef BSAES_ASM_EXTENDED_KEY
1888 add r4, sp, #0x90 @ pass key schedule
1889#else
1890 add r4, r10, #248 @ pass key schedule
1891#endif
1892 veor q3, q3, q11
1893 mov r5, r1 @ pass rounds
1894 mov r0, sp
1895
1896 bl _bsaes_encrypt8
1897
1898 vld1.64 {q8-q9}, [r0,:128]!
1899 vld1.64 {q10-q11}, [r0,:128]!
1900 veor q0, q0, q8
1901 veor q1, q1, q9
1902 veor q8, q4, q10
1903 vst1.8 {q0-q1}, [r8]!
1904 veor q9, q6, q11
1905 vst1.8 {q8-q9}, [r8]!
1906
1907 vld1.64 {q8}, [r0,:128] @ next round tweak
1908 b .Lxts_enc_done
1909.align 4
1910.Lxts_enc_3:
1911 vst1.64 {q11}, [r0,:128] @ next round tweak
1912
1913 veor q1, q1, q9
1914#ifndef BSAES_ASM_EXTENDED_KEY
1915 add r4, sp, #0x90 @ pass key schedule
1916#else
1917 add r4, r10, #248 @ pass key schedule
1918#endif
1919 veor q2, q2, q10
1920 mov r5, r1 @ pass rounds
1921 mov r0, sp
1922
1923 bl _bsaes_encrypt8
1924
1925 vld1.64 {q8-q9}, [r0,:128]!
1926 vld1.64 {q10}, [r0,:128]!
1927 veor q0, q0, q8
1928 veor q1, q1, q9
1929 veor q8, q4, q10
1930 vst1.8 {q0-q1}, [r8]!
1931 vst1.8 {q8}, [r8]!
1932
1933 vld1.64 {q8}, [r0,:128] @ next round tweak
1934 b .Lxts_enc_done
1935.align 4
1936.Lxts_enc_2:
1937 vst1.64 {q10}, [r0,:128] @ next round tweak
1938
1939 veor q0, q0, q8
1940#ifndef BSAES_ASM_EXTENDED_KEY
1941 add r4, sp, #0x90 @ pass key schedule
1942#else
1943 add r4, r10, #248 @ pass key schedule
1944#endif
1945 veor q1, q1, q9
1946 mov r5, r1 @ pass rounds
1947 mov r0, sp
1948
1949 bl _bsaes_encrypt8
1950
1951 vld1.64 {q8-q9}, [r0,:128]!
1952 veor q0, q0, q8
1953 veor q1, q1, q9
1954 vst1.8 {q0-q1}, [r8]!
1955
1956 vld1.64 {q8}, [r0,:128] @ next round tweak
1957 b .Lxts_enc_done
1958.align 4
1959.Lxts_enc_1:
1960 mov r0, sp
1961 veor q0, q8
1962 mov r1, sp
1963 vst1.8 {q0}, [sp,:128]
1964 mov r2, r10
1965 mov r4, r3 @ preserve fp
1966
1967 bl AES_encrypt
1968
1969 vld1.8 {q0}, [sp,:128]
1970 veor q0, q0, q8
1971 vst1.8 {q0}, [r8]!
1972 mov r3, r4
1973
1974 vmov q8, q9 @ next round tweak
1975
1976.Lxts_enc_done:
1977#ifndef XTS_CHAIN_TWEAK
1978 adds r9, #0x10
1979 beq .Lxts_enc_ret
1980 sub r6, r8, #0x10
1981
1982.Lxts_enc_steal:
1983 ldrb r0, [r7], #1
1984 ldrb r1, [r8, #-0x10]
1985 strb r0, [r8, #-0x10]
1986 strb r1, [r8], #1
1987
1988 subs r9, #1
1989 bhi .Lxts_enc_steal
1990
1991 vld1.8 {q0}, [r6]
1992 mov r0, sp
1993 veor q0, q0, q8
1994 mov r1, sp
1995 vst1.8 {q0}, [sp,:128]
1996 mov r2, r10
1997 mov r4, r3 @ preserve fp
1998
1999 bl AES_encrypt
2000
2001 vld1.8 {q0}, [sp,:128]
2002 veor q0, q0, q8
2003 vst1.8 {q0}, [r6]
2004 mov r3, r4
2005#endif
2006
2007.Lxts_enc_ret:
2008 bic r0, r3, #0xf
2009 vmov.i32 q0, #0
2010 vmov.i32 q1, #0
2011#ifdef XTS_CHAIN_TWEAK
2012 ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak
2013#endif
2014.Lxts_enc_bzero: @ wipe key schedule [if any]
2015 vstmia sp!, {q0-q1}
2016 cmp sp, r0
2017 bne .Lxts_enc_bzero
2018
2019 mov sp, r3
2020#ifdef XTS_CHAIN_TWEAK
2021 vst1.8 {q8}, [r1]
2022#endif
2023 VFP_ABI_POP
2024 ldmia sp!, {r4-r10, pc} @ return
2025
2026.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
2027
2028.globl bsaes_xts_decrypt
2029.type bsaes_xts_decrypt,%function
2030.align 4
2031bsaes_xts_decrypt:
2032 mov ip, sp
2033 stmdb sp!, {r4-r10, lr} @ 0x20
2034 VFP_ABI_PUSH
2035 mov r6, sp @ future r3
2036
2037 mov r7, r0
2038 mov r8, r1
2039 mov r9, r2
2040 mov r10, r3
2041
2042 sub r0, sp, #0x10 @ 0x10
2043 bic r0, #0xf @ align at 16 bytes
2044 mov sp, r0
2045
2046#ifdef XTS_CHAIN_TWEAK
2047 ldr r0, [ip] @ pointer to input tweak
2048#else
2049 @ generate initial tweak
2050 ldr r0, [ip, #4] @ iv[]
2051 mov r1, sp
2052 ldr r2, [ip, #0] @ key2
2053 bl AES_encrypt
2054 mov r0, sp @ pointer to initial tweak
2055#endif
2056
2057 ldr r1, [r10, #240] @ get # of rounds
2058 mov r3, r6
2059#ifndef BSAES_ASM_EXTENDED_KEY
2060 @ allocate the key schedule on the stack
2061 sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key
2062 @ add r12, #96 @ size of bit-sliced key schedule
2063 sub r12, #48 @ place for tweak[9]
2064
2065 @ populate the key schedule
2066 mov r4, r10 @ pass key
2067 mov r5, r1 @ pass # of rounds
2068 mov sp, r12
2069 add r12, #0x90 @ pass key schedule
2070 bl _bsaes_key_convert
2071 add r4, sp, #0x90
2072 vldmia r4, {q6}
2073 vstmia r12, {q15} @ save last round key
2074 veor q7, q7, q6 @ fix up round 0 key
2075 vstmia r4, {q7}
2076#else
2077 ldr r12, [r10, #244]
2078 eors r12, #1
2079 beq 0f
2080
2081 str r12, [r10, #244]
2082 mov r4, r10 @ pass key
2083 mov r5, r1 @ pass # of rounds
2084 add r12, r10, #248 @ pass key schedule
2085 bl _bsaes_key_convert
2086 add r4, r10, #248
2087 vldmia r4, {q6}
2088 vstmia r12, {q15} @ save last round key
2089 veor q7, q7, q6 @ fix up round 0 key
2090 vstmia r4, {q7}
2091
2092.align 2
20930: sub sp, #0x90 @ place for tweak[9]
2094#endif
2095 vld1.8 {q8}, [r0] @ initial tweak
2096 adr r2, .Lxts_magic
2097
2098 tst r9, #0xf @ if not multiple of 16
2099 it ne @ Thumb2 thing, sanity check in ARM
2100 subne r9, #0x10 @ subtract another 16 bytes
2101 subs r9, #0x80
2102
2103 blo .Lxts_dec_short
2104 b .Lxts_dec_loop
2105
2106.align 4
2107.Lxts_dec_loop:
2108 vldmia r2, {q5} @ load XTS magic
2109 vshr.s64 q6, q8, #63
2110 mov r0, sp
2111 vand q6, q6, q5
2112 vadd.u64 q9, q8, q8
2113 vst1.64 {q8}, [r0,:128]!
2114 vswp d13,d12
2115 vshr.s64 q7, q9, #63
2116 veor q9, q9, q6
2117 vand q7, q7, q5
2118 vadd.u64 q10, q9, q9
2119 vst1.64 {q9}, [r0,:128]!
2120 vswp d15,d14
2121 vshr.s64 q6, q10, #63
2122 veor q10, q10, q7
2123 vand q6, q6, q5
2124 vld1.8 {q0}, [r7]!
2125 vadd.u64 q11, q10, q10
2126 vst1.64 {q10}, [r0,:128]!
2127 vswp d13,d12
2128 vshr.s64 q7, q11, #63
2129 veor q11, q11, q6
2130 vand q7, q7, q5
2131 vld1.8 {q1}, [r7]!
2132 veor q0, q0, q8
2133 vadd.u64 q12, q11, q11
2134 vst1.64 {q11}, [r0,:128]!
2135 vswp d15,d14
2136 vshr.s64 q6, q12, #63
2137 veor q12, q12, q7
2138 vand q6, q6, q5
2139 vld1.8 {q2}, [r7]!
2140 veor q1, q1, q9
2141 vadd.u64 q13, q12, q12
2142 vst1.64 {q12}, [r0,:128]!
2143 vswp d13,d12
2144 vshr.s64 q7, q13, #63
2145 veor q13, q13, q6
2146 vand q7, q7, q5
2147 vld1.8 {q3}, [r7]!
2148 veor q2, q2, q10
2149 vadd.u64 q14, q13, q13
2150 vst1.64 {q13}, [r0,:128]!
2151 vswp d15,d14
2152 vshr.s64 q6, q14, #63
2153 veor q14, q14, q7
2154 vand q6, q6, q5
2155 vld1.8 {q4}, [r7]!
2156 veor q3, q3, q11
2157 vadd.u64 q15, q14, q14
2158 vst1.64 {q14}, [r0,:128]!
2159 vswp d13,d12
2160 vshr.s64 q7, q15, #63
2161 veor q15, q15, q6
2162 vand q7, q7, q5
2163 vld1.8 {q5}, [r7]!
2164 veor q4, q4, q12
2165 vadd.u64 q8, q15, q15
2166 vst1.64 {q15}, [r0,:128]!
2167 vswp d15,d14
2168 veor q8, q8, q7
2169 vst1.64 {q8}, [r0,:128] @ next round tweak
2170
2171 vld1.8 {q6-q7}, [r7]!
2172 veor q5, q5, q13
2173#ifndef BSAES_ASM_EXTENDED_KEY
2174 add r4, sp, #0x90 @ pass key schedule
2175#else
2176 add r4, r10, #248 @ pass key schedule
2177#endif
2178 veor q6, q6, q14
2179 mov r5, r1 @ pass rounds
2180 veor q7, q7, q15
2181 mov r0, sp
2182
2183 bl _bsaes_decrypt8
2184
2185 vld1.64 {q8-q9}, [r0,:128]!
2186 vld1.64 {q10-q11}, [r0,:128]!
2187 veor q0, q0, q8
2188 vld1.64 {q12-q13}, [r0,:128]!
2189 veor q1, q1, q9
2190 veor q8, q6, q10
2191 vst1.8 {q0-q1}, [r8]!
2192 veor q9, q4, q11
2193 vld1.64 {q14-q15}, [r0,:128]!
2194 veor q10, q2, q12
2195 vst1.8 {q8-q9}, [r8]!
2196 veor q11, q7, q13
2197 veor q12, q3, q14
2198 vst1.8 {q10-q11}, [r8]!
2199 veor q13, q5, q15
2200 vst1.8 {q12-q13}, [r8]!
2201
2202 vld1.64 {q8}, [r0,:128] @ next round tweak
2203
2204 subs r9, #0x80
2205 bpl .Lxts_dec_loop
2206
2207.Lxts_dec_short:
2208 adds r9, #0x70
2209 bmi .Lxts_dec_done
2210
2211 vldmia r2, {q5} @ load XTS magic
2212 vshr.s64 q7, q8, #63
2213 mov r0, sp
2214 vand q7, q7, q5
2215 vadd.u64 q9, q8, q8
2216 vst1.64 {q8}, [r0,:128]!
2217 vswp d15,d14
2218 vshr.s64 q6, q9, #63
2219 veor q9, q9, q7
2220 vand q6, q6, q5
2221 vadd.u64 q10, q9, q9
2222 vst1.64 {q9}, [r0,:128]!
2223 vswp d13,d12
2224 vshr.s64 q7, q10, #63
2225 veor q10, q10, q6
2226 vand q7, q7, q5
2227 vld1.8 {q0}, [r7]!
2228 subs r9, #0x10
2229 bmi .Lxts_dec_1
2230 vadd.u64 q11, q10, q10
2231 vst1.64 {q10}, [r0,:128]!
2232 vswp d15,d14
2233 vshr.s64 q6, q11, #63
2234 veor q11, q11, q7
2235 vand q6, q6, q5
2236 vld1.8 {q1}, [r7]!
2237 subs r9, #0x10
2238 bmi .Lxts_dec_2
2239 veor q0, q0, q8
2240 vadd.u64 q12, q11, q11
2241 vst1.64 {q11}, [r0,:128]!
2242 vswp d13,d12
2243 vshr.s64 q7, q12, #63
2244 veor q12, q12, q6
2245 vand q7, q7, q5
2246 vld1.8 {q2}, [r7]!
2247 subs r9, #0x10
2248 bmi .Lxts_dec_3
2249 veor q1, q1, q9
2250 vadd.u64 q13, q12, q12
2251 vst1.64 {q12}, [r0,:128]!
2252 vswp d15,d14
2253 vshr.s64 q6, q13, #63
2254 veor q13, q13, q7
2255 vand q6, q6, q5
2256 vld1.8 {q3}, [r7]!
2257 subs r9, #0x10
2258 bmi .Lxts_dec_4
2259 veor q2, q2, q10
2260 vadd.u64 q14, q13, q13
2261 vst1.64 {q13}, [r0,:128]!
2262 vswp d13,d12
2263 vshr.s64 q7, q14, #63
2264 veor q14, q14, q6
2265 vand q7, q7, q5
2266 vld1.8 {q4}, [r7]!
2267 subs r9, #0x10
2268 bmi .Lxts_dec_5
2269 veor q3, q3, q11
2270 vadd.u64 q15, q14, q14
2271 vst1.64 {q14}, [r0,:128]!
2272 vswp d15,d14
2273 vshr.s64 q6, q15, #63
2274 veor q15, q15, q7
2275 vand q6, q6, q5
2276 vld1.8 {q5}, [r7]!
2277 subs r9, #0x10
2278 bmi .Lxts_dec_6
2279 veor q4, q4, q12
2280 sub r9, #0x10
2281 vst1.64 {q15}, [r0,:128] @ next round tweak
2282
2283 vld1.8 {q6}, [r7]!
2284 veor q5, q5, q13
2285#ifndef BSAES_ASM_EXTENDED_KEY
2286 add r4, sp, #0x90 @ pass key schedule
2287#else
2288 add r4, r10, #248 @ pass key schedule
2289#endif
2290 veor q6, q6, q14
2291 mov r5, r1 @ pass rounds
2292 mov r0, sp
2293
2294 bl _bsaes_decrypt8
2295
2296 vld1.64 {q8-q9}, [r0,:128]!
2297 vld1.64 {q10-q11}, [r0,:128]!
2298 veor q0, q0, q8
2299 vld1.64 {q12-q13}, [r0,:128]!
2300 veor q1, q1, q9
2301 veor q8, q6, q10
2302 vst1.8 {q0-q1}, [r8]!
2303 veor q9, q4, q11
2304 vld1.64 {q14}, [r0,:128]!
2305 veor q10, q2, q12
2306 vst1.8 {q8-q9}, [r8]!
2307 veor q11, q7, q13
2308 veor q12, q3, q14
2309 vst1.8 {q10-q11}, [r8]!
2310 vst1.8 {q12}, [r8]!
2311
2312 vld1.64 {q8}, [r0,:128] @ next round tweak
2313 b .Lxts_dec_done
2314.align 4
2315.Lxts_dec_6:
2316 vst1.64 {q14}, [r0,:128] @ next round tweak
2317
2318 veor q4, q4, q12
2319#ifndef BSAES_ASM_EXTENDED_KEY
2320 add r4, sp, #0x90 @ pass key schedule
2321#else
2322 add r4, r10, #248 @ pass key schedule
2323#endif
2324 veor q5, q5, q13
2325 mov r5, r1 @ pass rounds
2326 mov r0, sp
2327
2328 bl _bsaes_decrypt8
2329
2330 vld1.64 {q8-q9}, [r0,:128]!
2331 vld1.64 {q10-q11}, [r0,:128]!
2332 veor q0, q0, q8
2333 vld1.64 {q12-q13}, [r0,:128]!
2334 veor q1, q1, q9
2335 veor q8, q6, q10
2336 vst1.8 {q0-q1}, [r8]!
2337 veor q9, q4, q11
2338 veor q10, q2, q12
2339 vst1.8 {q8-q9}, [r8]!
2340 veor q11, q7, q13
2341 vst1.8 {q10-q11}, [r8]!
2342
2343 vld1.64 {q8}, [r0,:128] @ next round tweak
2344 b .Lxts_dec_done
2345.align 4
2346.Lxts_dec_5:
2347 vst1.64 {q13}, [r0,:128] @ next round tweak
2348
2349 veor q3, q3, q11
2350#ifndef BSAES_ASM_EXTENDED_KEY
2351 add r4, sp, #0x90 @ pass key schedule
2352#else
2353 add r4, r10, #248 @ pass key schedule
2354#endif
2355 veor q4, q4, q12
2356 mov r5, r1 @ pass rounds
2357 mov r0, sp
2358
2359 bl _bsaes_decrypt8
2360
2361 vld1.64 {q8-q9}, [r0,:128]!
2362 vld1.64 {q10-q11}, [r0,:128]!
2363 veor q0, q0, q8
2364 vld1.64 {q12}, [r0,:128]!
2365 veor q1, q1, q9
2366 veor q8, q6, q10
2367 vst1.8 {q0-q1}, [r8]!
2368 veor q9, q4, q11
2369 veor q10, q2, q12
2370 vst1.8 {q8-q9}, [r8]!
2371 vst1.8 {q10}, [r8]!
2372
2373 vld1.64 {q8}, [r0,:128] @ next round tweak
2374 b .Lxts_dec_done
2375.align 4
2376.Lxts_dec_4:
2377 vst1.64 {q12}, [r0,:128] @ next round tweak
2378
2379 veor q2, q2, q10
2380#ifndef BSAES_ASM_EXTENDED_KEY
2381 add r4, sp, #0x90 @ pass key schedule
2382#else
2383 add r4, r10, #248 @ pass key schedule
2384#endif
2385 veor q3, q3, q11
2386 mov r5, r1 @ pass rounds
2387 mov r0, sp
2388
2389 bl _bsaes_decrypt8
2390
2391 vld1.64 {q8-q9}, [r0,:128]!
2392 vld1.64 {q10-q11}, [r0,:128]!
2393 veor q0, q0, q8
2394 veor q1, q1, q9
2395 veor q8, q6, q10
2396 vst1.8 {q0-q1}, [r8]!
2397 veor q9, q4, q11
2398 vst1.8 {q8-q9}, [r8]!
2399
2400 vld1.64 {q8}, [r0,:128] @ next round tweak
2401 b .Lxts_dec_done
2402.align 4
2403.Lxts_dec_3:
2404 vst1.64 {q11}, [r0,:128] @ next round tweak
2405
2406 veor q1, q1, q9
2407#ifndef BSAES_ASM_EXTENDED_KEY
2408 add r4, sp, #0x90 @ pass key schedule
2409#else
2410 add r4, r10, #248 @ pass key schedule
2411#endif
2412 veor q2, q2, q10
2413 mov r5, r1 @ pass rounds
2414 mov r0, sp
2415
2416 bl _bsaes_decrypt8
2417
2418 vld1.64 {q8-q9}, [r0,:128]!
2419 vld1.64 {q10}, [r0,:128]!
2420 veor q0, q0, q8
2421 veor q1, q1, q9
2422 veor q8, q6, q10
2423 vst1.8 {q0-q1}, [r8]!
2424 vst1.8 {q8}, [r8]!
2425
2426 vld1.64 {q8}, [r0,:128] @ next round tweak
2427 b .Lxts_dec_done
2428.align 4
2429.Lxts_dec_2:
2430 vst1.64 {q10}, [r0,:128] @ next round tweak
2431
2432 veor q0, q0, q8
2433#ifndef BSAES_ASM_EXTENDED_KEY
2434 add r4, sp, #0x90 @ pass key schedule
2435#else
2436 add r4, r10, #248 @ pass key schedule
2437#endif
2438 veor q1, q1, q9
2439 mov r5, r1 @ pass rounds
2440 mov r0, sp
2441
2442 bl _bsaes_decrypt8
2443
2444 vld1.64 {q8-q9}, [r0,:128]!
2445 veor q0, q0, q8
2446 veor q1, q1, q9
2447 vst1.8 {q0-q1}, [r8]!
2448
2449 vld1.64 {q8}, [r0,:128] @ next round tweak
2450 b .Lxts_dec_done
2451.align 4
2452.Lxts_dec_1:
2453 mov r0, sp
2454 veor q0, q8
2455 mov r1, sp
2456 vst1.8 {q0}, [sp,:128]
2457 mov r2, r10
2458 mov r4, r3 @ preserve fp
2459 mov r5, r2 @ preserve magic
2460
2461 bl AES_decrypt
2462
2463 vld1.8 {q0}, [sp,:128]
2464 veor q0, q0, q8
2465 vst1.8 {q0}, [r8]!
2466 mov r3, r4
2467 mov r2, r5
2468
2469 vmov q8, q9 @ next round tweak
2470
2471.Lxts_dec_done:
2472#ifndef XTS_CHAIN_TWEAK
2473 adds r9, #0x10
2474 beq .Lxts_dec_ret
2475
2476 @ calculate one round of extra tweak for the stolen ciphertext
2477 vldmia r2, {q5}
2478 vshr.s64 q6, q8, #63
2479 vand q6, q6, q5
2480 vadd.u64 q9, q8, q8
2481 vswp d13,d12
2482 veor q9, q9, q6
2483
2484 @ perform the final decryption with the last tweak value
2485 vld1.8 {q0}, [r7]!
2486 mov r0, sp
2487 veor q0, q0, q9
2488 mov r1, sp
2489 vst1.8 {q0}, [sp,:128]
2490 mov r2, r10
2491 mov r4, r3 @ preserve fp
2492
2493 bl AES_decrypt
2494
2495 vld1.8 {q0}, [sp,:128]
2496 veor q0, q0, q9
2497 vst1.8 {q0}, [r8]
2498
2499 mov r6, r8
2500.Lxts_dec_steal:
2501 ldrb r1, [r8]
2502 ldrb r0, [r7], #1
2503 strb r1, [r8, #0x10]
2504 strb r0, [r8], #1
2505
2506 subs r9, #1
2507 bhi .Lxts_dec_steal
2508
2509 vld1.8 {q0}, [r6]
2510 mov r0, sp
2511 veor q0, q8
2512 mov r1, sp
2513 vst1.8 {q0}, [sp,:128]
2514 mov r2, r10
2515
2516 bl AES_decrypt
2517
2518 vld1.8 {q0}, [sp,:128]
2519 veor q0, q0, q8
2520 vst1.8 {q0}, [r6]
2521 mov r3, r4
2522#endif
2523
2524.Lxts_dec_ret:
2525 bic r0, r3, #0xf
2526 vmov.i32 q0, #0
2527 vmov.i32 q1, #0
2528#ifdef XTS_CHAIN_TWEAK
2529 ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak
2530#endif
2531.Lxts_dec_bzero: @ wipe key schedule [if any]
2532 vstmia sp!, {q0-q1}
2533 cmp sp, r0
2534 bne .Lxts_dec_bzero
2535
2536 mov sp, r3
2537#ifdef XTS_CHAIN_TWEAK
2538 vst1.8 {q8}, [r1]
2539#endif
2540 VFP_ABI_POP
2541 ldmia sp!, {r4-r10, pc} @ return
2542
2543.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
2544#endif
diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c
new file mode 100644
index 000000000000..4522366da759
--- /dev/null
+++ b/arch/arm/crypto/aesbs-glue.c
@@ -0,0 +1,434 @@
1/*
2 * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES
3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <crypto/aes.h>
13#include <crypto/ablk_helper.h>
14#include <crypto/algapi.h>
15#include <linux/module.h>
16
17#include "aes_glue.h"
18
19#define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE)
20
21struct BS_KEY {
22 struct AES_KEY rk;
23 int converted;
24 u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE];
25} __aligned(8);
26
27asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in);
28asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in);
29
30asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes,
31 struct BS_KEY *key, u8 iv[]);
32
33asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks,
34 struct BS_KEY *key, u8 const iv[]);
35
36asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes,
37 struct BS_KEY *key, u8 tweak[]);
38
39asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes,
40 struct BS_KEY *key, u8 tweak[]);
41
42struct aesbs_cbc_ctx {
43 struct AES_KEY enc;
44 struct BS_KEY dec;
45};
46
47struct aesbs_ctr_ctx {
48 struct BS_KEY enc;
49};
50
51struct aesbs_xts_ctx {
52 struct BS_KEY enc;
53 struct BS_KEY dec;
54 struct AES_KEY twkey;
55};
56
57static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key,
58 unsigned int key_len)
59{
60 struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
61 int bits = key_len * 8;
62
63 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) {
64 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
65 return -EINVAL;
66 }
67 ctx->dec.rk = ctx->enc;
68 private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
69 ctx->dec.converted = 0;
70 return 0;
71}
72
73static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key,
74 unsigned int key_len)
75{
76 struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
77 int bits = key_len * 8;
78
79 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
80 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
81 return -EINVAL;
82 }
83 ctx->enc.converted = 0;
84 return 0;
85}
86
87static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
88 unsigned int key_len)
89{
90 struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
91 int bits = key_len * 4;
92
93 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
94 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
95 return -EINVAL;
96 }
97 ctx->dec.rk = ctx->enc.rk;
98 private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
99 private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey);
100 ctx->enc.converted = ctx->dec.converted = 0;
101 return 0;
102}
103
104static int aesbs_cbc_encrypt(struct blkcipher_desc *desc,
105 struct scatterlist *dst,
106 struct scatterlist *src, unsigned int nbytes)
107{
108 struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
109 struct blkcipher_walk walk;
110 int err;
111
112 blkcipher_walk_init(&walk, dst, src, nbytes);
113 err = blkcipher_walk_virt(desc, &walk);
114
115 while (walk.nbytes) {
116 u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
117 u8 *src = walk.src.virt.addr;
118
119 if (walk.dst.virt.addr == walk.src.virt.addr) {
120 u8 *iv = walk.iv;
121
122 do {
123 crypto_xor(src, iv, AES_BLOCK_SIZE);
124 AES_encrypt(src, src, &ctx->enc);
125 iv = src;
126 src += AES_BLOCK_SIZE;
127 } while (--blocks);
128 memcpy(walk.iv, iv, AES_BLOCK_SIZE);
129 } else {
130 u8 *dst = walk.dst.virt.addr;
131
132 do {
133 crypto_xor(walk.iv, src, AES_BLOCK_SIZE);
134 AES_encrypt(walk.iv, dst, &ctx->enc);
135 memcpy(walk.iv, dst, AES_BLOCK_SIZE);
136 src += AES_BLOCK_SIZE;
137 dst += AES_BLOCK_SIZE;
138 } while (--blocks);
139 }
140 err = blkcipher_walk_done(desc, &walk, 0);
141 }
142 return err;
143}
144
145static int aesbs_cbc_decrypt(struct blkcipher_desc *desc,
146 struct scatterlist *dst,
147 struct scatterlist *src, unsigned int nbytes)
148{
149 struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
150 struct blkcipher_walk walk;
151 int err;
152
153 blkcipher_walk_init(&walk, dst, src, nbytes);
154 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
155
156 while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) {
157 kernel_neon_begin();
158 bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
159 walk.nbytes, &ctx->dec, walk.iv);
160 kernel_neon_end();
161 err = blkcipher_walk_done(desc, &walk, 0);
162 }
163 while (walk.nbytes) {
164 u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
165 u8 *dst = walk.dst.virt.addr;
166 u8 *src = walk.src.virt.addr;
167 u8 bk[2][AES_BLOCK_SIZE];
168 u8 *iv = walk.iv;
169
170 do {
171 if (walk.dst.virt.addr == walk.src.virt.addr)
172 memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE);
173
174 AES_decrypt(src, dst, &ctx->dec.rk);
175 crypto_xor(dst, iv, AES_BLOCK_SIZE);
176
177 if (walk.dst.virt.addr == walk.src.virt.addr)
178 iv = bk[blocks & 1];
179 else
180 iv = src;
181
182 dst += AES_BLOCK_SIZE;
183 src += AES_BLOCK_SIZE;
184 } while (--blocks);
185 err = blkcipher_walk_done(desc, &walk, 0);
186 }
187 return err;
188}
189
190static void inc_be128_ctr(__be32 ctr[], u32 addend)
191{
192 int i;
193
194 for (i = 3; i >= 0; i--, addend = 1) {
195 u32 n = be32_to_cpu(ctr[i]) + addend;
196
197 ctr[i] = cpu_to_be32(n);
198 if (n >= addend)
199 break;
200 }
201}
202
203static int aesbs_ctr_encrypt(struct blkcipher_desc *desc,
204 struct scatterlist *dst, struct scatterlist *src,
205 unsigned int nbytes)
206{
207 struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
208 struct blkcipher_walk walk;
209 u32 blocks;
210 int err;
211
212 blkcipher_walk_init(&walk, dst, src, nbytes);
213 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
214
215 while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) {
216 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
217 __be32 *ctr = (__be32 *)walk.iv;
218 u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]);
219
220 /* avoid 32 bit counter overflow in the NEON code */
221 if (unlikely(headroom < blocks)) {
222 blocks = headroom + 1;
223 tail = walk.nbytes - blocks * AES_BLOCK_SIZE;
224 }
225 kernel_neon_begin();
226 bsaes_ctr32_encrypt_blocks(walk.src.virt.addr,
227 walk.dst.virt.addr, blocks,
228 &ctx->enc, walk.iv);
229 kernel_neon_end();
230 inc_be128_ctr(ctr, blocks);
231
232 nbytes -= blocks * AES_BLOCK_SIZE;
233 if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE)
234 break;
235
236 err = blkcipher_walk_done(desc, &walk, tail);
237 }
238 if (walk.nbytes) {
239 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
240 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
241 u8 ks[AES_BLOCK_SIZE];
242
243 AES_encrypt(walk.iv, ks, &ctx->enc.rk);
244 if (tdst != tsrc)
245 memcpy(tdst, tsrc, nbytes);
246 crypto_xor(tdst, ks, nbytes);
247 err = blkcipher_walk_done(desc, &walk, 0);
248 }
249 return err;
250}
251
252static int aesbs_xts_encrypt(struct blkcipher_desc *desc,
253 struct scatterlist *dst,
254 struct scatterlist *src, unsigned int nbytes)
255{
256 struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
257 struct blkcipher_walk walk;
258 int err;
259
260 blkcipher_walk_init(&walk, dst, src, nbytes);
261 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
262
263 /* generate the initial tweak */
264 AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
265
266 while (walk.nbytes) {
267 kernel_neon_begin();
268 bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
269 walk.nbytes, &ctx->enc, walk.iv);
270 kernel_neon_end();
271 err = blkcipher_walk_done(desc, &walk, 0);
272 }
273 return err;
274}
275
276static int aesbs_xts_decrypt(struct blkcipher_desc *desc,
277 struct scatterlist *dst,
278 struct scatterlist *src, unsigned int nbytes)
279{
280 struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
281 struct blkcipher_walk walk;
282 int err;
283
284 blkcipher_walk_init(&walk, dst, src, nbytes);
285 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
286
287 /* generate the initial tweak */
288 AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
289
290 while (walk.nbytes) {
291 kernel_neon_begin();
292 bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
293 walk.nbytes, &ctx->dec, walk.iv);
294 kernel_neon_end();
295 err = blkcipher_walk_done(desc, &walk, 0);
296 }
297 return err;
298}
299
300static struct crypto_alg aesbs_algs[] = { {
301 .cra_name = "__cbc-aes-neonbs",
302 .cra_driver_name = "__driver-cbc-aes-neonbs",
303 .cra_priority = 0,
304 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
305 .cra_blocksize = AES_BLOCK_SIZE,
306 .cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
307 .cra_alignmask = 7,
308 .cra_type = &crypto_blkcipher_type,
309 .cra_module = THIS_MODULE,
310 .cra_blkcipher = {
311 .min_keysize = AES_MIN_KEY_SIZE,
312 .max_keysize = AES_MAX_KEY_SIZE,
313 .ivsize = AES_BLOCK_SIZE,
314 .setkey = aesbs_cbc_set_key,
315 .encrypt = aesbs_cbc_encrypt,
316 .decrypt = aesbs_cbc_decrypt,
317 },
318}, {
319 .cra_name = "__ctr-aes-neonbs",
320 .cra_driver_name = "__driver-ctr-aes-neonbs",
321 .cra_priority = 0,
322 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
323 .cra_blocksize = 1,
324 .cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
325 .cra_alignmask = 7,
326 .cra_type = &crypto_blkcipher_type,
327 .cra_module = THIS_MODULE,
328 .cra_blkcipher = {
329 .min_keysize = AES_MIN_KEY_SIZE,
330 .max_keysize = AES_MAX_KEY_SIZE,
331 .ivsize = AES_BLOCK_SIZE,
332 .setkey = aesbs_ctr_set_key,
333 .encrypt = aesbs_ctr_encrypt,
334 .decrypt = aesbs_ctr_encrypt,
335 },
336}, {
337 .cra_name = "__xts-aes-neonbs",
338 .cra_driver_name = "__driver-xts-aes-neonbs",
339 .cra_priority = 0,
340 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
341 .cra_blocksize = AES_BLOCK_SIZE,
342 .cra_ctxsize = sizeof(struct aesbs_xts_ctx),
343 .cra_alignmask = 7,
344 .cra_type = &crypto_blkcipher_type,
345 .cra_module = THIS_MODULE,
346 .cra_blkcipher = {
347 .min_keysize = 2 * AES_MIN_KEY_SIZE,
348 .max_keysize = 2 * AES_MAX_KEY_SIZE,
349 .ivsize = AES_BLOCK_SIZE,
350 .setkey = aesbs_xts_set_key,
351 .encrypt = aesbs_xts_encrypt,
352 .decrypt = aesbs_xts_decrypt,
353 },
354}, {
355 .cra_name = "cbc(aes)",
356 .cra_driver_name = "cbc-aes-neonbs",
357 .cra_priority = 300,
358 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
359 .cra_blocksize = AES_BLOCK_SIZE,
360 .cra_ctxsize = sizeof(struct async_helper_ctx),
361 .cra_alignmask = 7,
362 .cra_type = &crypto_ablkcipher_type,
363 .cra_module = THIS_MODULE,
364 .cra_init = ablk_init,
365 .cra_exit = ablk_exit,
366 .cra_ablkcipher = {
367 .min_keysize = AES_MIN_KEY_SIZE,
368 .max_keysize = AES_MAX_KEY_SIZE,
369 .ivsize = AES_BLOCK_SIZE,
370 .setkey = ablk_set_key,
371 .encrypt = __ablk_encrypt,
372 .decrypt = ablk_decrypt,
373 }
374}, {
375 .cra_name = "ctr(aes)",
376 .cra_driver_name = "ctr-aes-neonbs",
377 .cra_priority = 300,
378 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
379 .cra_blocksize = 1,
380 .cra_ctxsize = sizeof(struct async_helper_ctx),
381 .cra_alignmask = 7,
382 .cra_type = &crypto_ablkcipher_type,
383 .cra_module = THIS_MODULE,
384 .cra_init = ablk_init,
385 .cra_exit = ablk_exit,
386 .cra_ablkcipher = {
387 .min_keysize = AES_MIN_KEY_SIZE,
388 .max_keysize = AES_MAX_KEY_SIZE,
389 .ivsize = AES_BLOCK_SIZE,
390 .setkey = ablk_set_key,
391 .encrypt = ablk_encrypt,
392 .decrypt = ablk_decrypt,
393 }
394}, {
395 .cra_name = "xts(aes)",
396 .cra_driver_name = "xts-aes-neonbs",
397 .cra_priority = 300,
398 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
399 .cra_blocksize = AES_BLOCK_SIZE,
400 .cra_ctxsize = sizeof(struct async_helper_ctx),
401 .cra_alignmask = 7,
402 .cra_type = &crypto_ablkcipher_type,
403 .cra_module = THIS_MODULE,
404 .cra_init = ablk_init,
405 .cra_exit = ablk_exit,
406 .cra_ablkcipher = {
407 .min_keysize = 2 * AES_MIN_KEY_SIZE,
408 .max_keysize = 2 * AES_MAX_KEY_SIZE,
409 .ivsize = AES_BLOCK_SIZE,
410 .setkey = ablk_set_key,
411 .encrypt = ablk_encrypt,
412 .decrypt = ablk_decrypt,
413 }
414} };
415
416static int __init aesbs_mod_init(void)
417{
418 if (!cpu_has_neon())
419 return -ENODEV;
420
421 return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
422}
423
424static void __exit aesbs_mod_exit(void)
425{
426 crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
427}
428
429module_init(aesbs_mod_init);
430module_exit(aesbs_mod_exit);
431
432MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON");
433MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
434MODULE_LICENSE("GPL");
diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl
new file mode 100644
index 000000000000..f3d96d932573
--- /dev/null
+++ b/arch/arm/crypto/bsaes-armv7.pl
@@ -0,0 +1,2467 @@
1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8#
9# Specific modes and adaptation for Linux kernel by Ard Biesheuvel
10# <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
11# granted.
12# ====================================================================
13
14# Bit-sliced AES for ARM NEON
15#
16# February 2012.
17#
18# This implementation is direct adaptation of bsaes-x86_64 module for
19# ARM NEON. Except that this module is endian-neutral [in sense that
20# it can be compiled for either endianness] by courtesy of vld1.8's
21# neutrality. Initial version doesn't implement interface to OpenSSL,
22# only low-level primitives and unsupported entry points, just enough
23# to collect performance results, which for Cortex-A8 core are:
24#
25# encrypt 19.5 cycles per byte processed with 128-bit key
26# decrypt 22.1 cycles per byte processed with 128-bit key
27# key conv. 440 cycles per 128-bit key/0.18 of 8x block
28#
29# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7,
30# which is [much] worse than anticipated (for further details see
31# http://www.openssl.org/~appro/Snapdragon-S4.html).
32#
33# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code
34# manages in 20.0 cycles].
35#
36# When comparing to x86_64 results keep in mind that NEON unit is
37# [mostly] single-issue and thus can't [fully] benefit from
38# instruction-level parallelism. And when comparing to aes-armv4
39# results keep in mind key schedule conversion overhead (see
40# bsaes-x86_64.pl for further details)...
41#
42# <appro@openssl.org>
43
44# April-August 2013
45#
46# Add CBC, CTR and XTS subroutines, adapt for kernel use.
47#
48# <ard.biesheuvel@linaro.org>
49
50while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
51open STDOUT,">$output";
52
53my ($inp,$out,$len,$key)=("r0","r1","r2","r3");
54my @XMM=map("q$_",(0..15));
55
56{
57my ($key,$rounds,$const)=("r4","r5","r6");
58
59sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
60sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
61
62sub Sbox {
63# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
64# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb
65my @b=@_[0..7];
66my @t=@_[8..11];
67my @s=@_[12..15];
68 &InBasisChange (@b);
69 &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s);
70 &OutBasisChange (@b[7,1,4,2,6,5,0,3]);
71}
72
73sub InBasisChange {
74# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
75# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
76my @b=@_[0..7];
77$code.=<<___;
78 veor @b[2], @b[2], @b[1]
79 veor @b[5], @b[5], @b[6]
80 veor @b[3], @b[3], @b[0]
81 veor @b[6], @b[6], @b[2]
82 veor @b[5], @b[5], @b[0]
83
84 veor @b[6], @b[6], @b[3]
85 veor @b[3], @b[3], @b[7]
86 veor @b[7], @b[7], @b[5]
87 veor @b[3], @b[3], @b[4]
88 veor @b[4], @b[4], @b[5]
89
90 veor @b[2], @b[2], @b[7]
91 veor @b[3], @b[3], @b[1]
92 veor @b[1], @b[1], @b[5]
93___
94}
95
96sub OutBasisChange {
97# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
98# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb
99my @b=@_[0..7];
100$code.=<<___;
101 veor @b[0], @b[0], @b[6]
102 veor @b[1], @b[1], @b[4]
103 veor @b[4], @b[4], @b[6]
104 veor @b[2], @b[2], @b[0]
105 veor @b[6], @b[6], @b[1]
106
107 veor @b[1], @b[1], @b[5]
108 veor @b[5], @b[5], @b[3]
109 veor @b[3], @b[3], @b[7]
110 veor @b[7], @b[7], @b[5]
111 veor @b[2], @b[2], @b[5]
112
113 veor @b[4], @b[4], @b[7]
114___
115}
116
117sub InvSbox {
118# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
119# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb
120my @b=@_[0..7];
121my @t=@_[8..11];
122my @s=@_[12..15];
123 &InvInBasisChange (@b);
124 &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s);
125 &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]);
126}
127
128sub InvInBasisChange { # OutBasisChange in reverse (with twist)
129my @b=@_[5,1,2,6,3,7,0,4];
130$code.=<<___
131 veor @b[1], @b[1], @b[7]
132 veor @b[4], @b[4], @b[7]
133
134 veor @b[7], @b[7], @b[5]
135 veor @b[1], @b[1], @b[3]
136 veor @b[2], @b[2], @b[5]
137 veor @b[3], @b[3], @b[7]
138
139 veor @b[6], @b[6], @b[1]
140 veor @b[2], @b[2], @b[0]
141 veor @b[5], @b[5], @b[3]
142 veor @b[4], @b[4], @b[6]
143 veor @b[0], @b[0], @b[6]
144 veor @b[1], @b[1], @b[4]
145___
146}
147
148sub InvOutBasisChange { # InBasisChange in reverse
149my @b=@_[2,5,7,3,6,1,0,4];
150$code.=<<___;
151 veor @b[1], @b[1], @b[5]
152 veor @b[2], @b[2], @b[7]
153
154 veor @b[3], @b[3], @b[1]
155 veor @b[4], @b[4], @b[5]
156 veor @b[7], @b[7], @b[5]
157 veor @b[3], @b[3], @b[4]
158 veor @b[5], @b[5], @b[0]
159 veor @b[3], @b[3], @b[7]
160 veor @b[6], @b[6], @b[2]
161 veor @b[2], @b[2], @b[1]
162 veor @b[6], @b[6], @b[3]
163
164 veor @b[3], @b[3], @b[0]
165 veor @b[5], @b[5], @b[6]
166___
167}
168
169sub Mul_GF4 {
170#;*************************************************************
171#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) *
172#;*************************************************************
173my ($x0,$x1,$y0,$y1,$t0,$t1)=@_;
174$code.=<<___;
175 veor $t0, $y0, $y1
176 vand $t0, $t0, $x0
177 veor $x0, $x0, $x1
178 vand $t1, $x1, $y0
179 vand $x0, $x0, $y1
180 veor $x1, $t1, $t0
181 veor $x0, $x0, $t1
182___
183}
184
185sub Mul_GF4_N { # not used, see next subroutine
186# multiply and scale by N
187my ($x0,$x1,$y0,$y1,$t0)=@_;
188$code.=<<___;
189 veor $t0, $y0, $y1
190 vand $t0, $t0, $x0
191 veor $x0, $x0, $x1
192 vand $x1, $x1, $y0
193 vand $x0, $x0, $y1
194 veor $x1, $x1, $x0
195 veor $x0, $x0, $t0
196___
197}
198
199sub Mul_GF4_N_GF4 {
200# interleaved Mul_GF4_N and Mul_GF4
201my ($x0,$x1,$y0,$y1,$t0,
202 $x2,$x3,$y2,$y3,$t1)=@_;
203$code.=<<___;
204 veor $t0, $y0, $y1
205 veor $t1, $y2, $y3
206 vand $t0, $t0, $x0
207 vand $t1, $t1, $x2
208 veor $x0, $x0, $x1
209 veor $x2, $x2, $x3
210 vand $x1, $x1, $y0
211 vand $x3, $x3, $y2
212 vand $x0, $x0, $y1
213 vand $x2, $x2, $y3
214 veor $x1, $x1, $x0
215 veor $x2, $x2, $x3
216 veor $x0, $x0, $t0
217 veor $x3, $x3, $t1
218___
219}
220sub Mul_GF16_2 {
221my @x=@_[0..7];
222my @y=@_[8..11];
223my @t=@_[12..15];
224$code.=<<___;
225 veor @t[0], @x[0], @x[2]
226 veor @t[1], @x[1], @x[3]
227___
228 &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]);
229$code.=<<___;
230 veor @y[0], @y[0], @y[2]
231 veor @y[1], @y[1], @y[3]
232___
233 Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3],
234 @x[2], @x[3], @y[2], @y[3], @t[2]);
235$code.=<<___;
236 veor @x[0], @x[0], @t[0]
237 veor @x[2], @x[2], @t[0]
238 veor @x[1], @x[1], @t[1]
239 veor @x[3], @x[3], @t[1]
240
241 veor @t[0], @x[4], @x[6]
242 veor @t[1], @x[5], @x[7]
243___
244 &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3],
245 @x[6], @x[7], @y[2], @y[3], @t[2]);
246$code.=<<___;
247 veor @y[0], @y[0], @y[2]
248 veor @y[1], @y[1], @y[3]
249___
250 &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]);
251$code.=<<___;
252 veor @x[4], @x[4], @t[0]
253 veor @x[6], @x[6], @t[0]
254 veor @x[5], @x[5], @t[1]
255 veor @x[7], @x[7], @t[1]
256___
257}
258sub Inv_GF256 {
259#;********************************************************************
260#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) *
261#;********************************************************************
262my @x=@_[0..7];
263my @t=@_[8..11];
264my @s=@_[12..15];
265# direct optimizations from hardware
266$code.=<<___;
267 veor @t[3], @x[4], @x[6]
268 veor @t[2], @x[5], @x[7]
269 veor @t[1], @x[1], @x[3]
270 veor @s[1], @x[7], @x[6]
271 vmov @t[0], @t[2]
272 veor @s[0], @x[0], @x[2]
273
274 vorr @t[2], @t[2], @t[1]
275 veor @s[3], @t[3], @t[0]
276 vand @s[2], @t[3], @s[0]
277 vorr @t[3], @t[3], @s[0]
278 veor @s[0], @s[0], @t[1]
279 vand @t[0], @t[0], @t[1]
280 veor @t[1], @x[3], @x[2]
281 vand @s[3], @s[3], @s[0]
282 vand @s[1], @s[1], @t[1]
283 veor @t[1], @x[4], @x[5]
284 veor @s[0], @x[1], @x[0]
285 veor @t[3], @t[3], @s[1]
286 veor @t[2], @t[2], @s[1]
287 vand @s[1], @t[1], @s[0]
288 vorr @t[1], @t[1], @s[0]
289 veor @t[3], @t[3], @s[3]
290 veor @t[0], @t[0], @s[1]
291 veor @t[2], @t[2], @s[2]
292 veor @t[1], @t[1], @s[3]
293 veor @t[0], @t[0], @s[2]
294 vand @s[0], @x[7], @x[3]
295 veor @t[1], @t[1], @s[2]
296 vand @s[1], @x[6], @x[2]
297 vand @s[2], @x[5], @x[1]
298 vorr @s[3], @x[4], @x[0]
299 veor @t[3], @t[3], @s[0]
300 veor @t[1], @t[1], @s[2]
301 veor @t[0], @t[0], @s[3]
302 veor @t[2], @t[2], @s[1]
303
304 @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
305
306 @ new smaller inversion
307
308 vand @s[2], @t[3], @t[1]
309 vmov @s[0], @t[0]
310
311 veor @s[1], @t[2], @s[2]
312 veor @s[3], @t[0], @s[2]
313 veor @s[2], @t[0], @s[2] @ @s[2]=@s[3]
314
315 vbsl @s[1], @t[1], @t[0]
316 vbsl @s[3], @t[3], @t[2]
317 veor @t[3], @t[3], @t[2]
318
319 vbsl @s[0], @s[1], @s[2]
320 vbsl @t[0], @s[2], @s[1]
321
322 vand @s[2], @s[0], @s[3]
323 veor @t[1], @t[1], @t[0]
324
325 veor @s[2], @s[2], @t[3]
326___
327# output in s3, s2, s1, t1
328
329# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3
330
331# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
332 &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]);
333
334### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb
335}
336
337# AES linear components
338
339sub ShiftRows {
340my @x=@_[0..7];
341my @t=@_[8..11];
342my $mask=pop;
343$code.=<<___;
344 vldmia $key!, {@t[0]-@t[3]}
345 veor @t[0], @t[0], @x[0]
346 veor @t[1], @t[1], @x[1]
347 vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)`
348 vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)`
349 vldmia $key!, {@t[0]}
350 veor @t[2], @t[2], @x[2]
351 vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)`
352 vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)`
353 vldmia $key!, {@t[1]}
354 veor @t[3], @t[3], @x[3]
355 vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)`
356 vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)`
357 vldmia $key!, {@t[2]}
358 vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)`
359 vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)`
360 vldmia $key!, {@t[3]}
361 veor @t[0], @t[0], @x[4]
362 veor @t[1], @t[1], @x[5]
363 vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)`
364 vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)`
365 veor @t[2], @t[2], @x[6]
366 vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)`
367 vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)`
368 veor @t[3], @t[3], @x[7]
369 vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)`
370 vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)`
371 vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)`
372 vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)`
373___
374}
375
376sub MixColumns {
377# modified to emit output in order suitable for feeding back to aesenc[last]
378my @x=@_[0..7];
379my @t=@_[8..15];
380my $inv=@_[16]; # optional
381$code.=<<___;
382 vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32
383 vext.8 @t[1], @x[1], @x[1], #12
384 veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32)
385 vext.8 @t[2], @x[2], @x[2], #12
386 veor @x[1], @x[1], @t[1]
387 vext.8 @t[3], @x[3], @x[3], #12
388 veor @x[2], @x[2], @t[2]
389 vext.8 @t[4], @x[4], @x[4], #12
390 veor @x[3], @x[3], @t[3]
391 vext.8 @t[5], @x[5], @x[5], #12
392 veor @x[4], @x[4], @t[4]
393 vext.8 @t[6], @x[6], @x[6], #12
394 veor @x[5], @x[5], @t[5]
395 vext.8 @t[7], @x[7], @x[7], #12
396 veor @x[6], @x[6], @t[6]
397
398 veor @t[1], @t[1], @x[0]
399 veor @x[7], @x[7], @t[7]
400 vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
401 veor @t[2], @t[2], @x[1]
402 veor @t[0], @t[0], @x[7]
403 veor @t[1], @t[1], @x[7]
404 vext.8 @x[1], @x[1], @x[1], #8
405 veor @t[5], @t[5], @x[4]
406 veor @x[0], @x[0], @t[0]
407 veor @t[6], @t[6], @x[5]
408 veor @x[1], @x[1], @t[1]
409 vext.8 @t[0], @x[4], @x[4], #8
410 veor @t[4], @t[4], @x[3]
411 vext.8 @t[1], @x[5], @x[5], #8
412 veor @t[7], @t[7], @x[6]
413 vext.8 @x[4], @x[3], @x[3], #8
414 veor @t[3], @t[3], @x[2]
415 vext.8 @x[5], @x[7], @x[7], #8
416 veor @t[4], @t[4], @x[7]
417 vext.8 @x[3], @x[6], @x[6], #8
418 veor @t[3], @t[3], @x[7]
419 vext.8 @x[6], @x[2], @x[2], #8
420 veor @x[7], @t[1], @t[5]
421___
422$code.=<<___ if (!$inv);
423 veor @x[2], @t[0], @t[4]
424 veor @x[4], @x[4], @t[3]
425 veor @x[5], @x[5], @t[7]
426 veor @x[3], @x[3], @t[6]
427 @ vmov @x[2], @t[0]
428 veor @x[6], @x[6], @t[2]
429 @ vmov @x[7], @t[1]
430___
431$code.=<<___ if ($inv);
432 veor @t[3], @t[3], @x[4]
433 veor @x[5], @x[5], @t[7]
434 veor @x[2], @x[3], @t[6]
435 veor @x[3], @t[0], @t[4]
436 veor @x[4], @x[6], @t[2]
437 vmov @x[6], @t[3]
438 @ vmov @x[7], @t[1]
439___
440}
441
442sub InvMixColumns_orig {
443my @x=@_[0..7];
444my @t=@_[8..15];
445
446$code.=<<___;
447 @ multiplication by 0x0e
448 vext.8 @t[7], @x[7], @x[7], #12
449 vmov @t[2], @x[2]
450 veor @x[2], @x[2], @x[5] @ 2 5
451 veor @x[7], @x[7], @x[5] @ 7 5
452 vext.8 @t[0], @x[0], @x[0], #12
453 vmov @t[5], @x[5]
454 veor @x[5], @x[5], @x[0] @ 5 0 [1]
455 veor @x[0], @x[0], @x[1] @ 0 1
456 vext.8 @t[1], @x[1], @x[1], #12
457 veor @x[1], @x[1], @x[2] @ 1 25
458 veor @x[0], @x[0], @x[6] @ 01 6 [2]
459 vext.8 @t[3], @x[3], @x[3], #12
460 veor @x[1], @x[1], @x[3] @ 125 3 [4]
461 veor @x[2], @x[2], @x[0] @ 25 016 [3]
462 veor @x[3], @x[3], @x[7] @ 3 75
463 veor @x[7], @x[7], @x[6] @ 75 6 [0]
464 vext.8 @t[6], @x[6], @x[6], #12
465 vmov @t[4], @x[4]
466 veor @x[6], @x[6], @x[4] @ 6 4
467 veor @x[4], @x[4], @x[3] @ 4 375 [6]
468 veor @x[3], @x[3], @x[7] @ 375 756=36
469 veor @x[6], @x[6], @t[5] @ 64 5 [7]
470 veor @x[3], @x[3], @t[2] @ 36 2
471 vext.8 @t[5], @t[5], @t[5], #12
472 veor @x[3], @x[3], @t[4] @ 362 4 [5]
473___
474 my @y = @x[7,5,0,2,1,3,4,6];
475$code.=<<___;
476 @ multiplication by 0x0b
477 veor @y[1], @y[1], @y[0]
478 veor @y[0], @y[0], @t[0]
479 vext.8 @t[2], @t[2], @t[2], #12
480 veor @y[1], @y[1], @t[1]
481 veor @y[0], @y[0], @t[5]
482 vext.8 @t[4], @t[4], @t[4], #12
483 veor @y[1], @y[1], @t[6]
484 veor @y[0], @y[0], @t[7]
485 veor @t[7], @t[7], @t[6] @ clobber t[7]
486
487 veor @y[3], @y[3], @t[0]
488 veor @y[1], @y[1], @y[0]
489 vext.8 @t[0], @t[0], @t[0], #12
490 veor @y[2], @y[2], @t[1]
491 veor @y[4], @y[4], @t[1]
492 vext.8 @t[1], @t[1], @t[1], #12
493 veor @y[2], @y[2], @t[2]
494 veor @y[3], @y[3], @t[2]
495 veor @y[5], @y[5], @t[2]
496 veor @y[2], @y[2], @t[7]
497 vext.8 @t[2], @t[2], @t[2], #12
498 veor @y[3], @y[3], @t[3]
499 veor @y[6], @y[6], @t[3]
500 veor @y[4], @y[4], @t[3]
501 veor @y[7], @y[7], @t[4]
502 vext.8 @t[3], @t[3], @t[3], #12
503 veor @y[5], @y[5], @t[4]
504 veor @y[7], @y[7], @t[7]
505 veor @t[7], @t[7], @t[5] @ clobber t[7] even more
506 veor @y[3], @y[3], @t[5]
507 veor @y[4], @y[4], @t[4]
508
509 veor @y[5], @y[5], @t[7]
510 vext.8 @t[4], @t[4], @t[4], #12
511 veor @y[6], @y[6], @t[7]
512 veor @y[4], @y[4], @t[7]
513
514 veor @t[7], @t[7], @t[5]
515 vext.8 @t[5], @t[5], @t[5], #12
516
517 @ multiplication by 0x0d
518 veor @y[4], @y[4], @y[7]
519 veor @t[7], @t[7], @t[6] @ restore t[7]
520 veor @y[7], @y[7], @t[4]
521 vext.8 @t[6], @t[6], @t[6], #12
522 veor @y[2], @y[2], @t[0]
523 veor @y[7], @y[7], @t[5]
524 vext.8 @t[7], @t[7], @t[7], #12
525 veor @y[2], @y[2], @t[2]
526
527 veor @y[3], @y[3], @y[1]
528 veor @y[1], @y[1], @t[1]
529 veor @y[0], @y[0], @t[0]
530 veor @y[3], @y[3], @t[0]
531 veor @y[1], @y[1], @t[5]
532 veor @y[0], @y[0], @t[5]
533 vext.8 @t[0], @t[0], @t[0], #12
534 veor @y[1], @y[1], @t[7]
535 veor @y[0], @y[0], @t[6]
536 veor @y[3], @y[3], @y[1]
537 veor @y[4], @y[4], @t[1]
538 vext.8 @t[1], @t[1], @t[1], #12
539
540 veor @y[7], @y[7], @t[7]
541 veor @y[4], @y[4], @t[2]
542 veor @y[5], @y[5], @t[2]
543 veor @y[2], @y[2], @t[6]
544 veor @t[6], @t[6], @t[3] @ clobber t[6]
545 vext.8 @t[2], @t[2], @t[2], #12
546 veor @y[4], @y[4], @y[7]
547 veor @y[3], @y[3], @t[6]
548
549 veor @y[6], @y[6], @t[6]
550 veor @y[5], @y[5], @t[5]
551 vext.8 @t[5], @t[5], @t[5], #12
552 veor @y[6], @y[6], @t[4]
553 vext.8 @t[4], @t[4], @t[4], #12
554 veor @y[5], @y[5], @t[6]
555 veor @y[6], @y[6], @t[7]
556 vext.8 @t[7], @t[7], @t[7], #12
557 veor @t[6], @t[6], @t[3] @ restore t[6]
558 vext.8 @t[3], @t[3], @t[3], #12
559
560 @ multiplication by 0x09
561 veor @y[4], @y[4], @y[1]
562 veor @t[1], @t[1], @y[1] @ t[1]=y[1]
563 veor @t[0], @t[0], @t[5] @ clobber t[0]
564 vext.8 @t[6], @t[6], @t[6], #12
565 veor @t[1], @t[1], @t[5]
566 veor @y[3], @y[3], @t[0]
567 veor @t[0], @t[0], @y[0] @ t[0]=y[0]
568 veor @t[1], @t[1], @t[6]
569 veor @t[6], @t[6], @t[7] @ clobber t[6]
570 veor @y[4], @y[4], @t[1]
571 veor @y[7], @y[7], @t[4]
572 veor @y[6], @y[6], @t[3]
573 veor @y[5], @y[5], @t[2]
574 veor @t[4], @t[4], @y[4] @ t[4]=y[4]
575 veor @t[3], @t[3], @y[3] @ t[3]=y[3]
576 veor @t[5], @t[5], @y[5] @ t[5]=y[5]
577 veor @t[2], @t[2], @y[2] @ t[2]=y[2]
578 veor @t[3], @t[3], @t[7]
579 veor @XMM[5], @t[5], @t[6]
580 veor @XMM[6], @t[6], @y[6] @ t[6]=y[6]
581 veor @XMM[2], @t[2], @t[6]
582 veor @XMM[7], @t[7], @y[7] @ t[7]=y[7]
583
584 vmov @XMM[0], @t[0]
585 vmov @XMM[1], @t[1]
586 @ vmov @XMM[2], @t[2]
587 vmov @XMM[3], @t[3]
588 vmov @XMM[4], @t[4]
589 @ vmov @XMM[5], @t[5]
590 @ vmov @XMM[6], @t[6]
591 @ vmov @XMM[7], @t[7]
592___
593}
594
595sub InvMixColumns {
596my @x=@_[0..7];
597my @t=@_[8..15];
598
599# Thanks to Jussi Kivilinna for providing pointer to
600#
601# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 |
602# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 |
603# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 |
604# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 |
605
606$code.=<<___;
607 @ multiplication by 0x05-0x00-0x04-0x00
608 vext.8 @t[0], @x[0], @x[0], #8
609 vext.8 @t[6], @x[6], @x[6], #8
610 vext.8 @t[7], @x[7], @x[7], #8
611 veor @t[0], @t[0], @x[0]
612 vext.8 @t[1], @x[1], @x[1], #8
613 veor @t[6], @t[6], @x[6]
614 vext.8 @t[2], @x[2], @x[2], #8
615 veor @t[7], @t[7], @x[7]
616 vext.8 @t[3], @x[3], @x[3], #8
617 veor @t[1], @t[1], @x[1]
618 vext.8 @t[4], @x[4], @x[4], #8
619 veor @t[2], @t[2], @x[2]
620 vext.8 @t[5], @x[5], @x[5], #8
621 veor @t[3], @t[3], @x[3]
622 veor @t[4], @t[4], @x[4]
623 veor @t[5], @t[5], @x[5]
624
625 veor @x[0], @x[0], @t[6]
626 veor @x[1], @x[1], @t[6]
627 veor @x[2], @x[2], @t[0]
628 veor @x[4], @x[4], @t[2]
629 veor @x[3], @x[3], @t[1]
630 veor @x[1], @x[1], @t[7]
631 veor @x[2], @x[2], @t[7]
632 veor @x[4], @x[4], @t[6]
633 veor @x[5], @x[5], @t[3]
634 veor @x[3], @x[3], @t[6]
635 veor @x[6], @x[6], @t[4]
636 veor @x[4], @x[4], @t[7]
637 veor @x[5], @x[5], @t[7]
638 veor @x[7], @x[7], @t[5]
639___
640 &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6
641}
642
643sub swapmove {
644my ($a,$b,$n,$mask,$t)=@_;
645$code.=<<___;
646 vshr.u64 $t, $b, #$n
647 veor $t, $t, $a
648 vand $t, $t, $mask
649 veor $a, $a, $t
650 vshl.u64 $t, $t, #$n
651 veor $b, $b, $t
652___
653}
654sub swapmove2x {
655my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_;
656$code.=<<___;
657 vshr.u64 $t0, $b0, #$n
658 vshr.u64 $t1, $b1, #$n
659 veor $t0, $t0, $a0
660 veor $t1, $t1, $a1
661 vand $t0, $t0, $mask
662 vand $t1, $t1, $mask
663 veor $a0, $a0, $t0
664 vshl.u64 $t0, $t0, #$n
665 veor $a1, $a1, $t1
666 vshl.u64 $t1, $t1, #$n
667 veor $b0, $b0, $t0
668 veor $b1, $b1, $t1
669___
670}
671
672sub bitslice {
673my @x=reverse(@_[0..7]);
674my ($t0,$t1,$t2,$t3)=@_[8..11];
675$code.=<<___;
676 vmov.i8 $t0,#0x55 @ compose .LBS0
677 vmov.i8 $t1,#0x33 @ compose .LBS1
678___
679 &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3);
680 &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3);
681$code.=<<___;
682 vmov.i8 $t0,#0x0f @ compose .LBS2
683___
684 &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3);
685 &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3);
686
687 &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3);
688 &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3);
689}
690
691$code.=<<___;
692#ifndef __KERNEL__
693# include "arm_arch.h"
694
695# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
696# define VFP_ABI_POP vldmia sp!,{d8-d15}
697# define VFP_ABI_FRAME 0x40
698#else
699# define VFP_ABI_PUSH
700# define VFP_ABI_POP
701# define VFP_ABI_FRAME 0
702# define BSAES_ASM_EXTENDED_KEY
703# define XTS_CHAIN_TWEAK
704# define __ARM_ARCH__ __LINUX_ARM_ARCH__
705#endif
706
707#ifdef __thumb__
708# define adrl adr
709#endif
710
711#if __ARM_ARCH__>=7
712.text
713.syntax unified @ ARMv7-capable assembler is expected to handle this
714#ifdef __thumb2__
715.thumb
716#else
717.code 32
718#endif
719
720.fpu neon
721
722.type _bsaes_decrypt8,%function
723.align 4
724_bsaes_decrypt8:
725 adr $const,_bsaes_decrypt8
726 vldmia $key!, {@XMM[9]} @ round 0 key
727 add $const,$const,#.LM0ISR-_bsaes_decrypt8
728
729 vldmia $const!, {@XMM[8]} @ .LM0ISR
730 veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key
731 veor @XMM[11], @XMM[1], @XMM[9]
732 vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])`
733 vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])`
734 veor @XMM[12], @XMM[2], @XMM[9]
735 vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])`
736 vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])`
737 veor @XMM[13], @XMM[3], @XMM[9]
738 vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])`
739 vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])`
740 veor @XMM[14], @XMM[4], @XMM[9]
741 vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])`
742 vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])`
743 veor @XMM[15], @XMM[5], @XMM[9]
744 vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])`
745 vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])`
746 veor @XMM[10], @XMM[6], @XMM[9]
747 vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])`
748 vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])`
749 veor @XMM[11], @XMM[7], @XMM[9]
750 vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])`
751 vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])`
752 vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])`
753 vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])`
754___
755 &bitslice (@XMM[0..7, 8..11]);
756$code.=<<___;
757 sub $rounds,$rounds,#1
758 b .Ldec_sbox
759.align 4
760.Ldec_loop:
761___
762 &ShiftRows (@XMM[0..7, 8..12]);
763$code.=".Ldec_sbox:\n";
764 &InvSbox (@XMM[0..7, 8..15]);
765$code.=<<___;
766 subs $rounds,$rounds,#1
767 bcc .Ldec_done
768___
769 &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]);
770$code.=<<___;
771 vldmia $const, {@XMM[12]} @ .LISR
772 ite eq @ Thumb2 thing, sanity check in ARM
773 addeq $const,$const,#0x10
774 bne .Ldec_loop
775 vldmia $const, {@XMM[12]} @ .LISRM0
776 b .Ldec_loop
777.align 4
778.Ldec_done:
779___
780 &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]);
781$code.=<<___;
782 vldmia $key, {@XMM[8]} @ last round key
783 veor @XMM[6], @XMM[6], @XMM[8]
784 veor @XMM[4], @XMM[4], @XMM[8]
785 veor @XMM[2], @XMM[2], @XMM[8]
786 veor @XMM[7], @XMM[7], @XMM[8]
787 veor @XMM[3], @XMM[3], @XMM[8]
788 veor @XMM[5], @XMM[5], @XMM[8]
789 veor @XMM[0], @XMM[0], @XMM[8]
790 veor @XMM[1], @XMM[1], @XMM[8]
791 bx lr
792.size _bsaes_decrypt8,.-_bsaes_decrypt8
793
794.type _bsaes_const,%object
795.align 6
796_bsaes_const:
797.LM0ISR: @ InvShiftRows constants
798 .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
799.LISR:
800 .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
801.LISRM0:
802 .quad 0x01040b0e0205080f, 0x0306090c00070a0d
803.LM0SR: @ ShiftRows constants
804 .quad 0x0a0e02060f03070b, 0x0004080c05090d01
805.LSR:
806 .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
807.LSRM0:
808 .quad 0x0304090e00050a0f, 0x01060b0c0207080d
809.LM0:
810 .quad 0x02060a0e03070b0f, 0x0004080c0105090d
811.LREVM0SR:
812 .quad 0x090d01050c000408, 0x03070b0f060a0e02
813.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro\@openssl.org>"
814.align 6
815.size _bsaes_const,.-_bsaes_const
816
817.type _bsaes_encrypt8,%function
818.align 4
819_bsaes_encrypt8:
820 adr $const,_bsaes_encrypt8
821 vldmia $key!, {@XMM[9]} @ round 0 key
822 sub $const,$const,#_bsaes_encrypt8-.LM0SR
823
824 vldmia $const!, {@XMM[8]} @ .LM0SR
825_bsaes_encrypt8_alt:
826 veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key
827 veor @XMM[11], @XMM[1], @XMM[9]
828 vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])`
829 vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])`
830 veor @XMM[12], @XMM[2], @XMM[9]
831 vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])`
832 vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])`
833 veor @XMM[13], @XMM[3], @XMM[9]
834 vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])`
835 vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])`
836 veor @XMM[14], @XMM[4], @XMM[9]
837 vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])`
838 vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])`
839 veor @XMM[15], @XMM[5], @XMM[9]
840 vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])`
841 vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])`
842 veor @XMM[10], @XMM[6], @XMM[9]
843 vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])`
844 vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])`
845 veor @XMM[11], @XMM[7], @XMM[9]
846 vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])`
847 vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])`
848 vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])`
849 vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])`
850_bsaes_encrypt8_bitslice:
851___
852 &bitslice (@XMM[0..7, 8..11]);
853$code.=<<___;
854 sub $rounds,$rounds,#1
855 b .Lenc_sbox
856.align 4
857.Lenc_loop:
858___
859 &ShiftRows (@XMM[0..7, 8..12]);
860$code.=".Lenc_sbox:\n";
861 &Sbox (@XMM[0..7, 8..15]);
862$code.=<<___;
863 subs $rounds,$rounds,#1
864 bcc .Lenc_done
865___
866 &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]);
867$code.=<<___;
868 vldmia $const, {@XMM[12]} @ .LSR
869 ite eq @ Thumb2 thing, samity check in ARM
870 addeq $const,$const,#0x10
871 bne .Lenc_loop
872 vldmia $const, {@XMM[12]} @ .LSRM0
873 b .Lenc_loop
874.align 4
875.Lenc_done:
876___
877 # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb
878 &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]);
879$code.=<<___;
880 vldmia $key, {@XMM[8]} @ last round key
881 veor @XMM[4], @XMM[4], @XMM[8]
882 veor @XMM[6], @XMM[6], @XMM[8]
883 veor @XMM[3], @XMM[3], @XMM[8]
884 veor @XMM[7], @XMM[7], @XMM[8]
885 veor @XMM[2], @XMM[2], @XMM[8]
886 veor @XMM[5], @XMM[5], @XMM[8]
887 veor @XMM[0], @XMM[0], @XMM[8]
888 veor @XMM[1], @XMM[1], @XMM[8]
889 bx lr
890.size _bsaes_encrypt8,.-_bsaes_encrypt8
891___
892}
893{
894my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6");
895
896sub bitslice_key {
897my @x=reverse(@_[0..7]);
898my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12];
899
900 &swapmove (@x[0,1],1,$bs0,$t2,$t3);
901$code.=<<___;
902 @ &swapmove(@x[2,3],1,$t0,$t2,$t3);
903 vmov @x[2], @x[0]
904 vmov @x[3], @x[1]
905___
906 #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3);
907
908 &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3);
909$code.=<<___;
910 @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3);
911 vmov @x[4], @x[0]
912 vmov @x[6], @x[2]
913 vmov @x[5], @x[1]
914 vmov @x[7], @x[3]
915___
916 &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3);
917 &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3);
918}
919
920$code.=<<___;
921.type _bsaes_key_convert,%function
922.align 4
923_bsaes_key_convert:
924 adr $const,_bsaes_key_convert
925 vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key
926 sub $const,$const,#_bsaes_key_convert-.LM0
927 vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key
928
929 vmov.i8 @XMM[8], #0x01 @ bit masks
930 vmov.i8 @XMM[9], #0x02
931 vmov.i8 @XMM[10], #0x04
932 vmov.i8 @XMM[11], #0x08
933 vmov.i8 @XMM[12], #0x10
934 vmov.i8 @XMM[13], #0x20
935 vldmia $const, {@XMM[14]} @ .LM0
936
937#ifdef __ARMEL__
938 vrev32.8 @XMM[7], @XMM[7]
939 vrev32.8 @XMM[15], @XMM[15]
940#endif
941 sub $rounds,$rounds,#1
942 vstmia $out!, {@XMM[7]} @ save round 0 key
943 b .Lkey_loop
944
945.align 4
946.Lkey_loop:
947 vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])`
948 vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])`
949 vmov.i8 @XMM[6], #0x40
950 vmov.i8 @XMM[15], #0x80
951
952 vtst.8 @XMM[0], @XMM[7], @XMM[8]
953 vtst.8 @XMM[1], @XMM[7], @XMM[9]
954 vtst.8 @XMM[2], @XMM[7], @XMM[10]
955 vtst.8 @XMM[3], @XMM[7], @XMM[11]
956 vtst.8 @XMM[4], @XMM[7], @XMM[12]
957 vtst.8 @XMM[5], @XMM[7], @XMM[13]
958 vtst.8 @XMM[6], @XMM[7], @XMM[6]
959 vtst.8 @XMM[7], @XMM[7], @XMM[15]
960 vld1.8 {@XMM[15]}, [$inp]! @ load next round key
961 vmvn @XMM[0], @XMM[0] @ "pnot"
962 vmvn @XMM[1], @XMM[1]
963 vmvn @XMM[5], @XMM[5]
964 vmvn @XMM[6], @XMM[6]
965#ifdef __ARMEL__
966 vrev32.8 @XMM[15], @XMM[15]
967#endif
968 subs $rounds,$rounds,#1
969 vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key
970 bne .Lkey_loop
971
972 vmov.i8 @XMM[7],#0x63 @ compose .L63
973 @ don't save last round key
974 bx lr
975.size _bsaes_key_convert,.-_bsaes_key_convert
976___
977}
978
979if (0) { # following four functions are unsupported interface
980 # used for benchmarking...
981$code.=<<___;
982.globl bsaes_enc_key_convert
983.type bsaes_enc_key_convert,%function
984.align 4
985bsaes_enc_key_convert:
986 stmdb sp!,{r4-r6,lr}
987 vstmdb sp!,{d8-d15} @ ABI specification says so
988
989 ldr r5,[$inp,#240] @ pass rounds
990 mov r4,$inp @ pass key
991 mov r12,$out @ pass key schedule
992 bl _bsaes_key_convert
993 veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
994 vstmia r12, {@XMM[7]} @ save last round key
995
996 vldmia sp!,{d8-d15}
997 ldmia sp!,{r4-r6,pc}
998.size bsaes_enc_key_convert,.-bsaes_enc_key_convert
999
1000.globl bsaes_encrypt_128
1001.type bsaes_encrypt_128,%function
1002.align 4
1003bsaes_encrypt_128:
1004 stmdb sp!,{r4-r6,lr}
1005 vstmdb sp!,{d8-d15} @ ABI specification says so
1006.Lenc128_loop:
1007 vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
1008 vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
1009 mov r4,$key @ pass the key
1010 vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
1011 mov r5,#10 @ pass rounds
1012 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
1013
1014 bl _bsaes_encrypt8
1015
1016 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1017 vst1.8 {@XMM[4]}, [$out]!
1018 vst1.8 {@XMM[6]}, [$out]!
1019 vst1.8 {@XMM[3]}, [$out]!
1020 vst1.8 {@XMM[7]}, [$out]!
1021 vst1.8 {@XMM[2]}, [$out]!
1022 subs $len,$len,#0x80
1023 vst1.8 {@XMM[5]}, [$out]!
1024 bhi .Lenc128_loop
1025
1026 vldmia sp!,{d8-d15}
1027 ldmia sp!,{r4-r6,pc}
1028.size bsaes_encrypt_128,.-bsaes_encrypt_128
1029
1030.globl bsaes_dec_key_convert
1031.type bsaes_dec_key_convert,%function
1032.align 4
1033bsaes_dec_key_convert:
1034 stmdb sp!,{r4-r6,lr}
1035 vstmdb sp!,{d8-d15} @ ABI specification says so
1036
1037 ldr r5,[$inp,#240] @ pass rounds
1038 mov r4,$inp @ pass key
1039 mov r12,$out @ pass key schedule
1040 bl _bsaes_key_convert
1041 vldmia $out, {@XMM[6]}
1042 vstmia r12, {@XMM[15]} @ save last round key
1043 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
1044 vstmia $out, {@XMM[7]}
1045
1046 vldmia sp!,{d8-d15}
1047 ldmia sp!,{r4-r6,pc}
1048.size bsaes_dec_key_convert,.-bsaes_dec_key_convert
1049
1050.globl bsaes_decrypt_128
1051.type bsaes_decrypt_128,%function
1052.align 4
1053bsaes_decrypt_128:
1054 stmdb sp!,{r4-r6,lr}
1055 vstmdb sp!,{d8-d15} @ ABI specification says so
1056.Ldec128_loop:
1057 vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
1058 vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
1059 mov r4,$key @ pass the key
1060 vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
1061 mov r5,#10 @ pass rounds
1062 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
1063
1064 bl _bsaes_decrypt8
1065
1066 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1067 vst1.8 {@XMM[6]}, [$out]!
1068 vst1.8 {@XMM[4]}, [$out]!
1069 vst1.8 {@XMM[2]}, [$out]!
1070 vst1.8 {@XMM[7]}, [$out]!
1071 vst1.8 {@XMM[3]}, [$out]!
1072 subs $len,$len,#0x80
1073 vst1.8 {@XMM[5]}, [$out]!
1074 bhi .Ldec128_loop
1075
1076 vldmia sp!,{d8-d15}
1077 ldmia sp!,{r4-r6,pc}
1078.size bsaes_decrypt_128,.-bsaes_decrypt_128
1079___
1080}
1081{
1082my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10));
1083my ($keysched)=("sp");
1084
1085$code.=<<___;
1086.extern AES_cbc_encrypt
1087.extern AES_decrypt
1088
1089.global bsaes_cbc_encrypt
1090.type bsaes_cbc_encrypt,%function
1091.align 5
1092bsaes_cbc_encrypt:
1093#ifndef __KERNEL__
1094 cmp $len, #128
1095#ifndef __thumb__
1096 blo AES_cbc_encrypt
1097#else
1098 bhs 1f
1099 b AES_cbc_encrypt
11001:
1101#endif
1102#endif
1103
1104 @ it is up to the caller to make sure we are called with enc == 0
1105
1106 mov ip, sp
1107 stmdb sp!, {r4-r10, lr}
1108 VFP_ABI_PUSH
1109 ldr $ivp, [ip] @ IV is 1st arg on the stack
1110 mov $len, $len, lsr#4 @ len in 16 byte blocks
1111 sub sp, #0x10 @ scratch space to carry over the IV
1112 mov $fp, sp @ save sp
1113
1114 ldr $rounds, [$key, #240] @ get # of rounds
1115#ifndef BSAES_ASM_EXTENDED_KEY
1116 @ allocate the key schedule on the stack
1117 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
1118 add r12, #`128-32` @ sifze of bit-slices key schedule
1119
1120 @ populate the key schedule
1121 mov r4, $key @ pass key
1122 mov r5, $rounds @ pass # of rounds
1123 mov sp, r12 @ sp is $keysched
1124 bl _bsaes_key_convert
1125 vldmia $keysched, {@XMM[6]}
1126 vstmia r12, {@XMM[15]} @ save last round key
1127 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
1128 vstmia $keysched, {@XMM[7]}
1129#else
1130 ldr r12, [$key, #244]
1131 eors r12, #1
1132 beq 0f
1133
1134 @ populate the key schedule
1135 str r12, [$key, #244]
1136 mov r4, $key @ pass key
1137 mov r5, $rounds @ pass # of rounds
1138 add r12, $key, #248 @ pass key schedule
1139 bl _bsaes_key_convert
1140 add r4, $key, #248
1141 vldmia r4, {@XMM[6]}
1142 vstmia r12, {@XMM[15]} @ save last round key
1143 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
1144 vstmia r4, {@XMM[7]}
1145
1146.align 2
11470:
1148#endif
1149
1150 vld1.8 {@XMM[15]}, [$ivp] @ load IV
1151 b .Lcbc_dec_loop
1152
1153.align 4
1154.Lcbc_dec_loop:
1155 subs $len, $len, #0x8
1156 bmi .Lcbc_dec_loop_finish
1157
1158 vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
1159 vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
1160#ifndef BSAES_ASM_EXTENDED_KEY
1161 mov r4, $keysched @ pass the key
1162#else
1163 add r4, $key, #248
1164#endif
1165 vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
1166 mov r5, $rounds
1167 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]
1168 sub $inp, $inp, #0x60
1169 vstmia $fp, {@XMM[15]} @ put aside IV
1170
1171 bl _bsaes_decrypt8
1172
1173 vldmia $fp, {@XMM[14]} @ reload IV
1174 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1175 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1176 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1177 veor @XMM[1], @XMM[1], @XMM[8]
1178 veor @XMM[6], @XMM[6], @XMM[9]
1179 vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
1180 veor @XMM[4], @XMM[4], @XMM[10]
1181 veor @XMM[2], @XMM[2], @XMM[11]
1182 vld1.8 {@XMM[14]-@XMM[15]}, [$inp]!
1183 veor @XMM[7], @XMM[7], @XMM[12]
1184 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1185 veor @XMM[3], @XMM[3], @XMM[13]
1186 vst1.8 {@XMM[6]}, [$out]!
1187 veor @XMM[5], @XMM[5], @XMM[14]
1188 vst1.8 {@XMM[4]}, [$out]!
1189 vst1.8 {@XMM[2]}, [$out]!
1190 vst1.8 {@XMM[7]}, [$out]!
1191 vst1.8 {@XMM[3]}, [$out]!
1192 vst1.8 {@XMM[5]}, [$out]!
1193
1194 b .Lcbc_dec_loop
1195
1196.Lcbc_dec_loop_finish:
1197 adds $len, $len, #8
1198 beq .Lcbc_dec_done
1199
1200 vld1.8 {@XMM[0]}, [$inp]! @ load input
1201 cmp $len, #2
1202 blo .Lcbc_dec_one
1203 vld1.8 {@XMM[1]}, [$inp]!
1204#ifndef BSAES_ASM_EXTENDED_KEY
1205 mov r4, $keysched @ pass the key
1206#else
1207 add r4, $key, #248
1208#endif
1209 mov r5, $rounds
1210 vstmia $fp, {@XMM[15]} @ put aside IV
1211 beq .Lcbc_dec_two
1212 vld1.8 {@XMM[2]}, [$inp]!
1213 cmp $len, #4
1214 blo .Lcbc_dec_three
1215 vld1.8 {@XMM[3]}, [$inp]!
1216 beq .Lcbc_dec_four
1217 vld1.8 {@XMM[4]}, [$inp]!
1218 cmp $len, #6
1219 blo .Lcbc_dec_five
1220 vld1.8 {@XMM[5]}, [$inp]!
1221 beq .Lcbc_dec_six
1222 vld1.8 {@XMM[6]}, [$inp]!
1223 sub $inp, $inp, #0x70
1224
1225 bl _bsaes_decrypt8
1226
1227 vldmia $fp, {@XMM[14]} @ reload IV
1228 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1229 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1230 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1231 veor @XMM[1], @XMM[1], @XMM[8]
1232 veor @XMM[6], @XMM[6], @XMM[9]
1233 vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
1234 veor @XMM[4], @XMM[4], @XMM[10]
1235 veor @XMM[2], @XMM[2], @XMM[11]
1236 vld1.8 {@XMM[15]}, [$inp]!
1237 veor @XMM[7], @XMM[7], @XMM[12]
1238 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1239 veor @XMM[3], @XMM[3], @XMM[13]
1240 vst1.8 {@XMM[6]}, [$out]!
1241 vst1.8 {@XMM[4]}, [$out]!
1242 vst1.8 {@XMM[2]}, [$out]!
1243 vst1.8 {@XMM[7]}, [$out]!
1244 vst1.8 {@XMM[3]}, [$out]!
1245 b .Lcbc_dec_done
1246.align 4
1247.Lcbc_dec_six:
1248 sub $inp, $inp, #0x60
1249 bl _bsaes_decrypt8
1250 vldmia $fp,{@XMM[14]} @ reload IV
1251 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1252 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1253 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1254 veor @XMM[1], @XMM[1], @XMM[8]
1255 veor @XMM[6], @XMM[6], @XMM[9]
1256 vld1.8 {@XMM[12]}, [$inp]!
1257 veor @XMM[4], @XMM[4], @XMM[10]
1258 veor @XMM[2], @XMM[2], @XMM[11]
1259 vld1.8 {@XMM[15]}, [$inp]!
1260 veor @XMM[7], @XMM[7], @XMM[12]
1261 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1262 vst1.8 {@XMM[6]}, [$out]!
1263 vst1.8 {@XMM[4]}, [$out]!
1264 vst1.8 {@XMM[2]}, [$out]!
1265 vst1.8 {@XMM[7]}, [$out]!
1266 b .Lcbc_dec_done
1267.align 4
1268.Lcbc_dec_five:
1269 sub $inp, $inp, #0x50
1270 bl _bsaes_decrypt8
1271 vldmia $fp, {@XMM[14]} @ reload IV
1272 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1273 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1274 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1275 veor @XMM[1], @XMM[1], @XMM[8]
1276 veor @XMM[6], @XMM[6], @XMM[9]
1277 vld1.8 {@XMM[15]}, [$inp]!
1278 veor @XMM[4], @XMM[4], @XMM[10]
1279 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1280 veor @XMM[2], @XMM[2], @XMM[11]
1281 vst1.8 {@XMM[6]}, [$out]!
1282 vst1.8 {@XMM[4]}, [$out]!
1283 vst1.8 {@XMM[2]}, [$out]!
1284 b .Lcbc_dec_done
1285.align 4
1286.Lcbc_dec_four:
1287 sub $inp, $inp, #0x40
1288 bl _bsaes_decrypt8
1289 vldmia $fp, {@XMM[14]} @ reload IV
1290 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1291 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1292 vld1.8 {@XMM[10]}, [$inp]!
1293 veor @XMM[1], @XMM[1], @XMM[8]
1294 veor @XMM[6], @XMM[6], @XMM[9]
1295 vld1.8 {@XMM[15]}, [$inp]!
1296 veor @XMM[4], @XMM[4], @XMM[10]
1297 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1298 vst1.8 {@XMM[6]}, [$out]!
1299 vst1.8 {@XMM[4]}, [$out]!
1300 b .Lcbc_dec_done
1301.align 4
1302.Lcbc_dec_three:
1303 sub $inp, $inp, #0x30
1304 bl _bsaes_decrypt8
1305 vldmia $fp, {@XMM[14]} @ reload IV
1306 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1307 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1308 vld1.8 {@XMM[15]}, [$inp]!
1309 veor @XMM[1], @XMM[1], @XMM[8]
1310 veor @XMM[6], @XMM[6], @XMM[9]
1311 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1312 vst1.8 {@XMM[6]}, [$out]!
1313 b .Lcbc_dec_done
1314.align 4
1315.Lcbc_dec_two:
1316 sub $inp, $inp, #0x20
1317 bl _bsaes_decrypt8
1318 vldmia $fp, {@XMM[14]} @ reload IV
1319 vld1.8 {@XMM[8]}, [$inp]! @ reload input
1320 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1321 vld1.8 {@XMM[15]}, [$inp]! @ reload input
1322 veor @XMM[1], @XMM[1], @XMM[8]
1323 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1324 b .Lcbc_dec_done
1325.align 4
1326.Lcbc_dec_one:
1327 sub $inp, $inp, #0x10
1328 mov $rounds, $out @ save original out pointer
1329 mov $out, $fp @ use the iv scratch space as out buffer
1330 mov r2, $key
1331 vmov @XMM[4],@XMM[15] @ just in case ensure that IV
1332 vmov @XMM[5],@XMM[0] @ and input are preserved
1333 bl AES_decrypt
1334 vld1.8 {@XMM[0]}, [$fp,:64] @ load result
1335 veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV
1336 vmov @XMM[15], @XMM[5] @ @XMM[5] holds input
1337 vst1.8 {@XMM[0]}, [$rounds] @ write output
1338
1339.Lcbc_dec_done:
1340#ifndef BSAES_ASM_EXTENDED_KEY
1341 vmov.i32 q0, #0
1342 vmov.i32 q1, #0
1343.Lcbc_dec_bzero: @ wipe key schedule [if any]
1344 vstmia $keysched!, {q0-q1}
1345 cmp $keysched, $fp
1346 bne .Lcbc_dec_bzero
1347#endif
1348
1349 mov sp, $fp
1350 add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb
1351 vst1.8 {@XMM[15]}, [$ivp] @ return IV
1352 VFP_ABI_POP
1353 ldmia sp!, {r4-r10, pc}
1354.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
1355___
1356}
1357{
1358my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10)));
1359my $const = "r6"; # shared with _bsaes_encrypt8_alt
1360my $keysched = "sp";
1361
1362$code.=<<___;
1363.extern AES_encrypt
1364.global bsaes_ctr32_encrypt_blocks
1365.type bsaes_ctr32_encrypt_blocks,%function
1366.align 5
1367bsaes_ctr32_encrypt_blocks:
1368 cmp $len, #8 @ use plain AES for
1369 blo .Lctr_enc_short @ small sizes
1370
1371 mov ip, sp
1372 stmdb sp!, {r4-r10, lr}
1373 VFP_ABI_PUSH
1374 ldr $ctr, [ip] @ ctr is 1st arg on the stack
1375 sub sp, sp, #0x10 @ scratch space to carry over the ctr
1376 mov $fp, sp @ save sp
1377
1378 ldr $rounds, [$key, #240] @ get # of rounds
1379#ifndef BSAES_ASM_EXTENDED_KEY
1380 @ allocate the key schedule on the stack
1381 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
1382 add r12, #`128-32` @ size of bit-sliced key schedule
1383
1384 @ populate the key schedule
1385 mov r4, $key @ pass key
1386 mov r5, $rounds @ pass # of rounds
1387 mov sp, r12 @ sp is $keysched
1388 bl _bsaes_key_convert
1389 veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
1390 vstmia r12, {@XMM[7]} @ save last round key
1391
1392 vld1.8 {@XMM[0]}, [$ctr] @ load counter
1393 add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr
1394 vldmia $keysched, {@XMM[4]} @ load round0 key
1395#else
1396 ldr r12, [$key, #244]
1397 eors r12, #1
1398 beq 0f
1399
1400 @ populate the key schedule
1401 str r12, [$key, #244]
1402 mov r4, $key @ pass key
1403 mov r5, $rounds @ pass # of rounds
1404 add r12, $key, #248 @ pass key schedule
1405 bl _bsaes_key_convert
1406 veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
1407 vstmia r12, {@XMM[7]} @ save last round key
1408
1409.align 2
14100: add r12, $key, #248
1411 vld1.8 {@XMM[0]}, [$ctr] @ load counter
1412 adrl $ctr, .LREVM0SR @ borrow $ctr
1413 vldmia r12, {@XMM[4]} @ load round0 key
1414 sub sp, #0x10 @ place for adjusted round0 key
1415#endif
1416
1417 vmov.i32 @XMM[8],#1 @ compose 1<<96
1418 veor @XMM[9],@XMM[9],@XMM[9]
1419 vrev32.8 @XMM[0],@XMM[0]
1420 vext.8 @XMM[8],@XMM[9],@XMM[8],#4
1421 vrev32.8 @XMM[4],@XMM[4]
1422 vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96
1423 vstmia $keysched, {@XMM[4]} @ save adjusted round0 key
1424 b .Lctr_enc_loop
1425
1426.align 4
1427.Lctr_enc_loop:
1428 vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96
1429 vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1
1430 vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2
1431 vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3
1432 vadd.u32 @XMM[4], @XMM[1], @XMM[10]
1433 vadd.u32 @XMM[5], @XMM[2], @XMM[10]
1434 vadd.u32 @XMM[6], @XMM[3], @XMM[10]
1435 vadd.u32 @XMM[7], @XMM[4], @XMM[10]
1436 vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter
1437
1438 @ Borrow prologue from _bsaes_encrypt8 to use the opportunity
1439 @ to flip byte order in 32-bit counter
1440
1441 vldmia $keysched, {@XMM[9]} @ load round0 key
1442#ifndef BSAES_ASM_EXTENDED_KEY
1443 add r4, $keysched, #0x10 @ pass next round key
1444#else
1445 add r4, $key, #`248+16`
1446#endif
1447 vldmia $ctr, {@XMM[8]} @ .LREVM0SR
1448 mov r5, $rounds @ pass rounds
1449 vstmia $fp, {@XMM[10]} @ save next counter
1450 sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants
1451
1452 bl _bsaes_encrypt8_alt
1453
1454 subs $len, $len, #8
1455 blo .Lctr_enc_loop_done
1456
1457 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input
1458 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1459 veor @XMM[0], @XMM[8]
1460 veor @XMM[1], @XMM[9]
1461 vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
1462 veor @XMM[4], @XMM[10]
1463 veor @XMM[6], @XMM[11]
1464 vld1.8 {@XMM[14]-@XMM[15]}, [$inp]!
1465 veor @XMM[3], @XMM[12]
1466 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1467 veor @XMM[7], @XMM[13]
1468 veor @XMM[2], @XMM[14]
1469 vst1.8 {@XMM[4]}, [$out]!
1470 veor @XMM[5], @XMM[15]
1471 vst1.8 {@XMM[6]}, [$out]!
1472 vmov.i32 @XMM[8], #1 @ compose 1<<96
1473 vst1.8 {@XMM[3]}, [$out]!
1474 veor @XMM[9], @XMM[9], @XMM[9]
1475 vst1.8 {@XMM[7]}, [$out]!
1476 vext.8 @XMM[8], @XMM[9], @XMM[8], #4
1477 vst1.8 {@XMM[2]}, [$out]!
1478 vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96
1479 vst1.8 {@XMM[5]}, [$out]!
1480 vldmia $fp, {@XMM[0]} @ load counter
1481
1482 bne .Lctr_enc_loop
1483 b .Lctr_enc_done
1484
1485.align 4
1486.Lctr_enc_loop_done:
1487 add $len, $len, #8
1488 vld1.8 {@XMM[8]}, [$inp]! @ load input
1489 veor @XMM[0], @XMM[8]
1490 vst1.8 {@XMM[0]}, [$out]! @ write output
1491 cmp $len, #2
1492 blo .Lctr_enc_done
1493 vld1.8 {@XMM[9]}, [$inp]!
1494 veor @XMM[1], @XMM[9]
1495 vst1.8 {@XMM[1]}, [$out]!
1496 beq .Lctr_enc_done
1497 vld1.8 {@XMM[10]}, [$inp]!
1498 veor @XMM[4], @XMM[10]
1499 vst1.8 {@XMM[4]}, [$out]!
1500 cmp $len, #4
1501 blo .Lctr_enc_done
1502 vld1.8 {@XMM[11]}, [$inp]!
1503 veor @XMM[6], @XMM[11]
1504 vst1.8 {@XMM[6]}, [$out]!
1505 beq .Lctr_enc_done
1506 vld1.8 {@XMM[12]}, [$inp]!
1507 veor @XMM[3], @XMM[12]
1508 vst1.8 {@XMM[3]}, [$out]!
1509 cmp $len, #6
1510 blo .Lctr_enc_done
1511 vld1.8 {@XMM[13]}, [$inp]!
1512 veor @XMM[7], @XMM[13]
1513 vst1.8 {@XMM[7]}, [$out]!
1514 beq .Lctr_enc_done
1515 vld1.8 {@XMM[14]}, [$inp]
1516 veor @XMM[2], @XMM[14]
1517 vst1.8 {@XMM[2]}, [$out]!
1518
1519.Lctr_enc_done:
1520 vmov.i32 q0, #0
1521 vmov.i32 q1, #0
1522#ifndef BSAES_ASM_EXTENDED_KEY
1523.Lctr_enc_bzero: @ wipe key schedule [if any]
1524 vstmia $keysched!, {q0-q1}
1525 cmp $keysched, $fp
1526 bne .Lctr_enc_bzero
1527#else
1528 vstmia $keysched, {q0-q1}
1529#endif
1530
1531 mov sp, $fp
1532 add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb
1533 VFP_ABI_POP
1534 ldmia sp!, {r4-r10, pc} @ return
1535
1536.align 4
1537.Lctr_enc_short:
1538 ldr ip, [sp] @ ctr pointer is passed on stack
1539 stmdb sp!, {r4-r8, lr}
1540
1541 mov r4, $inp @ copy arguments
1542 mov r5, $out
1543 mov r6, $len
1544 mov r7, $key
1545 ldr r8, [ip, #12] @ load counter LSW
1546 vld1.8 {@XMM[1]}, [ip] @ load whole counter value
1547#ifdef __ARMEL__
1548 rev r8, r8
1549#endif
1550 sub sp, sp, #0x10
1551 vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value
1552 sub sp, sp, #0x10
1553
1554.Lctr_enc_short_loop:
1555 add r0, sp, #0x10 @ input counter value
1556 mov r1, sp @ output on the stack
1557 mov r2, r7 @ key
1558
1559 bl AES_encrypt
1560
1561 vld1.8 {@XMM[0]}, [r4]! @ load input
1562 vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter
1563 add r8, r8, #1
1564#ifdef __ARMEL__
1565 rev r0, r8
1566 str r0, [sp, #0x1c] @ next counter value
1567#else
1568 str r8, [sp, #0x1c] @ next counter value
1569#endif
1570 veor @XMM[0],@XMM[0],@XMM[1]
1571 vst1.8 {@XMM[0]}, [r5]! @ store output
1572 subs r6, r6, #1
1573 bne .Lctr_enc_short_loop
1574
1575 vmov.i32 q0, #0
1576 vmov.i32 q1, #0
1577 vstmia sp!, {q0-q1}
1578
1579 ldmia sp!, {r4-r8, pc}
1580.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
1581___
1582}
1583{
1584######################################################################
1585# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len,
1586# const AES_KEY *key1, const AES_KEY *key2,
1587# const unsigned char iv[16]);
1588#
1589my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3)));
1590my $const="r6"; # returned by _bsaes_key_convert
1591my $twmask=@XMM[5];
1592my @T=@XMM[6..7];
1593
1594$code.=<<___;
1595.globl bsaes_xts_encrypt
1596.type bsaes_xts_encrypt,%function
1597.align 4
1598bsaes_xts_encrypt:
1599 mov ip, sp
1600 stmdb sp!, {r4-r10, lr} @ 0x20
1601 VFP_ABI_PUSH
1602 mov r6, sp @ future $fp
1603
1604 mov $inp, r0
1605 mov $out, r1
1606 mov $len, r2
1607 mov $key, r3
1608
1609 sub r0, sp, #0x10 @ 0x10
1610 bic r0, #0xf @ align at 16 bytes
1611 mov sp, r0
1612
1613#ifdef XTS_CHAIN_TWEAK
1614 ldr r0, [ip] @ pointer to input tweak
1615#else
1616 @ generate initial tweak
1617 ldr r0, [ip, #4] @ iv[]
1618 mov r1, sp
1619 ldr r2, [ip, #0] @ key2
1620 bl AES_encrypt
1621 mov r0,sp @ pointer to initial tweak
1622#endif
1623
1624 ldr $rounds, [$key, #240] @ get # of rounds
1625 mov $fp, r6
1626#ifndef BSAES_ASM_EXTENDED_KEY
1627 @ allocate the key schedule on the stack
1628 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
1629 @ add r12, #`128-32` @ size of bit-sliced key schedule
1630 sub r12, #`32+16` @ place for tweak[9]
1631
1632 @ populate the key schedule
1633 mov r4, $key @ pass key
1634 mov r5, $rounds @ pass # of rounds
1635 mov sp, r12
1636 add r12, #0x90 @ pass key schedule
1637 bl _bsaes_key_convert
1638 veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key
1639 vstmia r12, {@XMM[7]} @ save last round key
1640#else
1641 ldr r12, [$key, #244]
1642 eors r12, #1
1643 beq 0f
1644
1645 str r12, [$key, #244]
1646 mov r4, $key @ pass key
1647 mov r5, $rounds @ pass # of rounds
1648 add r12, $key, #248 @ pass key schedule
1649 bl _bsaes_key_convert
1650 veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key
1651 vstmia r12, {@XMM[7]}
1652
1653.align 2
16540: sub sp, #0x90 @ place for tweak[9]
1655#endif
1656
1657 vld1.8 {@XMM[8]}, [r0] @ initial tweak
1658 adr $magic, .Lxts_magic
1659
1660 subs $len, #0x80
1661 blo .Lxts_enc_short
1662 b .Lxts_enc_loop
1663
1664.align 4
1665.Lxts_enc_loop:
1666 vldmia $magic, {$twmask} @ load XTS magic
1667 vshr.s64 @T[0], @XMM[8], #63
1668 mov r0, sp
1669 vand @T[0], @T[0], $twmask
1670___
1671for($i=9;$i<16;$i++) {
1672$code.=<<___;
1673 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
1674 vst1.64 {@XMM[$i-1]}, [r0,:128]!
1675 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
1676 vshr.s64 @T[1], @XMM[$i], #63
1677 veor @XMM[$i], @XMM[$i], @T[0]
1678 vand @T[1], @T[1], $twmask
1679___
1680 @T=reverse(@T);
1681
1682$code.=<<___ if ($i>=10);
1683 vld1.8 {@XMM[$i-10]}, [$inp]!
1684___
1685$code.=<<___ if ($i>=11);
1686 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
1687___
1688}
1689$code.=<<___;
1690 vadd.u64 @XMM[8], @XMM[15], @XMM[15]
1691 vst1.64 {@XMM[15]}, [r0,:128]!
1692 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
1693 veor @XMM[8], @XMM[8], @T[0]
1694 vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1695
1696 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
1697 veor @XMM[5], @XMM[5], @XMM[13]
1698#ifndef BSAES_ASM_EXTENDED_KEY
1699 add r4, sp, #0x90 @ pass key schedule
1700#else
1701 add r4, $key, #248 @ pass key schedule
1702#endif
1703 veor @XMM[6], @XMM[6], @XMM[14]
1704 mov r5, $rounds @ pass rounds
1705 veor @XMM[7], @XMM[7], @XMM[15]
1706 mov r0, sp
1707
1708 bl _bsaes_encrypt8
1709
1710 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1711 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1712 veor @XMM[0], @XMM[0], @XMM[ 8]
1713 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
1714 veor @XMM[1], @XMM[1], @XMM[ 9]
1715 veor @XMM[8], @XMM[4], @XMM[10]
1716 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1717 veor @XMM[9], @XMM[6], @XMM[11]
1718 vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]!
1719 veor @XMM[10], @XMM[3], @XMM[12]
1720 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1721 veor @XMM[11], @XMM[7], @XMM[13]
1722 veor @XMM[12], @XMM[2], @XMM[14]
1723 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
1724 veor @XMM[13], @XMM[5], @XMM[15]
1725 vst1.8 {@XMM[12]-@XMM[13]}, [$out]!
1726
1727 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1728
1729 subs $len, #0x80
1730 bpl .Lxts_enc_loop
1731
1732.Lxts_enc_short:
1733 adds $len, #0x70
1734 bmi .Lxts_enc_done
1735
1736 vldmia $magic, {$twmask} @ load XTS magic
1737 vshr.s64 @T[0], @XMM[8], #63
1738 mov r0, sp
1739 vand @T[0], @T[0], $twmask
1740___
1741for($i=9;$i<16;$i++) {
1742$code.=<<___;
1743 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
1744 vst1.64 {@XMM[$i-1]}, [r0,:128]!
1745 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
1746 vshr.s64 @T[1], @XMM[$i], #63
1747 veor @XMM[$i], @XMM[$i], @T[0]
1748 vand @T[1], @T[1], $twmask
1749___
1750 @T=reverse(@T);
1751
1752$code.=<<___ if ($i>=10);
1753 vld1.8 {@XMM[$i-10]}, [$inp]!
1754 subs $len, #0x10
1755 bmi .Lxts_enc_`$i-9`
1756___
1757$code.=<<___ if ($i>=11);
1758 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
1759___
1760}
1761$code.=<<___;
1762 sub $len, #0x10
1763 vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak
1764
1765 vld1.8 {@XMM[6]}, [$inp]!
1766 veor @XMM[5], @XMM[5], @XMM[13]
1767#ifndef BSAES_ASM_EXTENDED_KEY
1768 add r4, sp, #0x90 @ pass key schedule
1769#else
1770 add r4, $key, #248 @ pass key schedule
1771#endif
1772 veor @XMM[6], @XMM[6], @XMM[14]
1773 mov r5, $rounds @ pass rounds
1774 mov r0, sp
1775
1776 bl _bsaes_encrypt8
1777
1778 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1779 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1780 veor @XMM[0], @XMM[0], @XMM[ 8]
1781 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
1782 veor @XMM[1], @XMM[1], @XMM[ 9]
1783 veor @XMM[8], @XMM[4], @XMM[10]
1784 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1785 veor @XMM[9], @XMM[6], @XMM[11]
1786 vld1.64 {@XMM[14]}, [r0,:128]!
1787 veor @XMM[10], @XMM[3], @XMM[12]
1788 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1789 veor @XMM[11], @XMM[7], @XMM[13]
1790 veor @XMM[12], @XMM[2], @XMM[14]
1791 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
1792 vst1.8 {@XMM[12]}, [$out]!
1793
1794 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1795 b .Lxts_enc_done
1796.align 4
1797.Lxts_enc_6:
1798 vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak
1799
1800 veor @XMM[4], @XMM[4], @XMM[12]
1801#ifndef BSAES_ASM_EXTENDED_KEY
1802 add r4, sp, #0x90 @ pass key schedule
1803#else
1804 add r4, $key, #248 @ pass key schedule
1805#endif
1806 veor @XMM[5], @XMM[5], @XMM[13]
1807 mov r5, $rounds @ pass rounds
1808 mov r0, sp
1809
1810 bl _bsaes_encrypt8
1811
1812 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1813 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1814 veor @XMM[0], @XMM[0], @XMM[ 8]
1815 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
1816 veor @XMM[1], @XMM[1], @XMM[ 9]
1817 veor @XMM[8], @XMM[4], @XMM[10]
1818 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1819 veor @XMM[9], @XMM[6], @XMM[11]
1820 veor @XMM[10], @XMM[3], @XMM[12]
1821 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1822 veor @XMM[11], @XMM[7], @XMM[13]
1823 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
1824
1825 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1826 b .Lxts_enc_done
1827
1828@ put this in range for both ARM and Thumb mode adr instructions
1829.align 5
1830.Lxts_magic:
1831 .quad 1, 0x87
1832
1833.align 5
1834.Lxts_enc_5:
1835 vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak
1836
1837 veor @XMM[3], @XMM[3], @XMM[11]
1838#ifndef BSAES_ASM_EXTENDED_KEY
1839 add r4, sp, #0x90 @ pass key schedule
1840#else
1841 add r4, $key, #248 @ pass key schedule
1842#endif
1843 veor @XMM[4], @XMM[4], @XMM[12]
1844 mov r5, $rounds @ pass rounds
1845 mov r0, sp
1846
1847 bl _bsaes_encrypt8
1848
1849 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1850 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1851 veor @XMM[0], @XMM[0], @XMM[ 8]
1852 vld1.64 {@XMM[12]}, [r0,:128]!
1853 veor @XMM[1], @XMM[1], @XMM[ 9]
1854 veor @XMM[8], @XMM[4], @XMM[10]
1855 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1856 veor @XMM[9], @XMM[6], @XMM[11]
1857 veor @XMM[10], @XMM[3], @XMM[12]
1858 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1859 vst1.8 {@XMM[10]}, [$out]!
1860
1861 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1862 b .Lxts_enc_done
1863.align 4
1864.Lxts_enc_4:
1865 vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak
1866
1867 veor @XMM[2], @XMM[2], @XMM[10]
1868#ifndef BSAES_ASM_EXTENDED_KEY
1869 add r4, sp, #0x90 @ pass key schedule
1870#else
1871 add r4, $key, #248 @ pass key schedule
1872#endif
1873 veor @XMM[3], @XMM[3], @XMM[11]
1874 mov r5, $rounds @ pass rounds
1875 mov r0, sp
1876
1877 bl _bsaes_encrypt8
1878
1879 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1880 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1881 veor @XMM[0], @XMM[0], @XMM[ 8]
1882 veor @XMM[1], @XMM[1], @XMM[ 9]
1883 veor @XMM[8], @XMM[4], @XMM[10]
1884 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1885 veor @XMM[9], @XMM[6], @XMM[11]
1886 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1887
1888 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1889 b .Lxts_enc_done
1890.align 4
1891.Lxts_enc_3:
1892 vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak
1893
1894 veor @XMM[1], @XMM[1], @XMM[9]
1895#ifndef BSAES_ASM_EXTENDED_KEY
1896 add r4, sp, #0x90 @ pass key schedule
1897#else
1898 add r4, $key, #248 @ pass key schedule
1899#endif
1900 veor @XMM[2], @XMM[2], @XMM[10]
1901 mov r5, $rounds @ pass rounds
1902 mov r0, sp
1903
1904 bl _bsaes_encrypt8
1905
1906 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
1907 vld1.64 {@XMM[10]}, [r0,:128]!
1908 veor @XMM[0], @XMM[0], @XMM[ 8]
1909 veor @XMM[1], @XMM[1], @XMM[ 9]
1910 veor @XMM[8], @XMM[4], @XMM[10]
1911 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1912 vst1.8 {@XMM[8]}, [$out]!
1913
1914 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1915 b .Lxts_enc_done
1916.align 4
1917.Lxts_enc_2:
1918 vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak
1919
1920 veor @XMM[0], @XMM[0], @XMM[8]
1921#ifndef BSAES_ASM_EXTENDED_KEY
1922 add r4, sp, #0x90 @ pass key schedule
1923#else
1924 add r4, $key, #248 @ pass key schedule
1925#endif
1926 veor @XMM[1], @XMM[1], @XMM[9]
1927 mov r5, $rounds @ pass rounds
1928 mov r0, sp
1929
1930 bl _bsaes_encrypt8
1931
1932 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
1933 veor @XMM[0], @XMM[0], @XMM[ 8]
1934 veor @XMM[1], @XMM[1], @XMM[ 9]
1935 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1936
1937 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1938 b .Lxts_enc_done
1939.align 4
1940.Lxts_enc_1:
1941 mov r0, sp
1942 veor @XMM[0], @XMM[8]
1943 mov r1, sp
1944 vst1.8 {@XMM[0]}, [sp,:128]
1945 mov r2, $key
1946 mov r4, $fp @ preserve fp
1947
1948 bl AES_encrypt
1949
1950 vld1.8 {@XMM[0]}, [sp,:128]
1951 veor @XMM[0], @XMM[0], @XMM[8]
1952 vst1.8 {@XMM[0]}, [$out]!
1953 mov $fp, r4
1954
1955 vmov @XMM[8], @XMM[9] @ next round tweak
1956
1957.Lxts_enc_done:
1958#ifndef XTS_CHAIN_TWEAK
1959 adds $len, #0x10
1960 beq .Lxts_enc_ret
1961 sub r6, $out, #0x10
1962
1963.Lxts_enc_steal:
1964 ldrb r0, [$inp], #1
1965 ldrb r1, [$out, #-0x10]
1966 strb r0, [$out, #-0x10]
1967 strb r1, [$out], #1
1968
1969 subs $len, #1
1970 bhi .Lxts_enc_steal
1971
1972 vld1.8 {@XMM[0]}, [r6]
1973 mov r0, sp
1974 veor @XMM[0], @XMM[0], @XMM[8]
1975 mov r1, sp
1976 vst1.8 {@XMM[0]}, [sp,:128]
1977 mov r2, $key
1978 mov r4, $fp @ preserve fp
1979
1980 bl AES_encrypt
1981
1982 vld1.8 {@XMM[0]}, [sp,:128]
1983 veor @XMM[0], @XMM[0], @XMM[8]
1984 vst1.8 {@XMM[0]}, [r6]
1985 mov $fp, r4
1986#endif
1987
1988.Lxts_enc_ret:
1989 bic r0, $fp, #0xf
1990 vmov.i32 q0, #0
1991 vmov.i32 q1, #0
1992#ifdef XTS_CHAIN_TWEAK
1993 ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak
1994#endif
1995.Lxts_enc_bzero: @ wipe key schedule [if any]
1996 vstmia sp!, {q0-q1}
1997 cmp sp, r0
1998 bne .Lxts_enc_bzero
1999
2000 mov sp, $fp
2001#ifdef XTS_CHAIN_TWEAK
2002 vst1.8 {@XMM[8]}, [r1]
2003#endif
2004 VFP_ABI_POP
2005 ldmia sp!, {r4-r10, pc} @ return
2006
2007.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
2008
2009.globl bsaes_xts_decrypt
2010.type bsaes_xts_decrypt,%function
2011.align 4
2012bsaes_xts_decrypt:
2013 mov ip, sp
2014 stmdb sp!, {r4-r10, lr} @ 0x20
2015 VFP_ABI_PUSH
2016 mov r6, sp @ future $fp
2017
2018 mov $inp, r0
2019 mov $out, r1
2020 mov $len, r2
2021 mov $key, r3
2022
2023 sub r0, sp, #0x10 @ 0x10
2024 bic r0, #0xf @ align at 16 bytes
2025 mov sp, r0
2026
2027#ifdef XTS_CHAIN_TWEAK
2028 ldr r0, [ip] @ pointer to input tweak
2029#else
2030 @ generate initial tweak
2031 ldr r0, [ip, #4] @ iv[]
2032 mov r1, sp
2033 ldr r2, [ip, #0] @ key2
2034 bl AES_encrypt
2035 mov r0, sp @ pointer to initial tweak
2036#endif
2037
2038 ldr $rounds, [$key, #240] @ get # of rounds
2039 mov $fp, r6
2040#ifndef BSAES_ASM_EXTENDED_KEY
2041 @ allocate the key schedule on the stack
2042 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
2043 @ add r12, #`128-32` @ size of bit-sliced key schedule
2044 sub r12, #`32+16` @ place for tweak[9]
2045
2046 @ populate the key schedule
2047 mov r4, $key @ pass key
2048 mov r5, $rounds @ pass # of rounds
2049 mov sp, r12
2050 add r12, #0x90 @ pass key schedule
2051 bl _bsaes_key_convert
2052 add r4, sp, #0x90
2053 vldmia r4, {@XMM[6]}
2054 vstmia r12, {@XMM[15]} @ save last round key
2055 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
2056 vstmia r4, {@XMM[7]}
2057#else
2058 ldr r12, [$key, #244]
2059 eors r12, #1
2060 beq 0f
2061
2062 str r12, [$key, #244]
2063 mov r4, $key @ pass key
2064 mov r5, $rounds @ pass # of rounds
2065 add r12, $key, #248 @ pass key schedule
2066 bl _bsaes_key_convert
2067 add r4, $key, #248
2068 vldmia r4, {@XMM[6]}
2069 vstmia r12, {@XMM[15]} @ save last round key
2070 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
2071 vstmia r4, {@XMM[7]}
2072
2073.align 2
20740: sub sp, #0x90 @ place for tweak[9]
2075#endif
2076 vld1.8 {@XMM[8]}, [r0] @ initial tweak
2077 adr $magic, .Lxts_magic
2078
2079 tst $len, #0xf @ if not multiple of 16
2080 it ne @ Thumb2 thing, sanity check in ARM
2081 subne $len, #0x10 @ subtract another 16 bytes
2082 subs $len, #0x80
2083
2084 blo .Lxts_dec_short
2085 b .Lxts_dec_loop
2086
2087.align 4
2088.Lxts_dec_loop:
2089 vldmia $magic, {$twmask} @ load XTS magic
2090 vshr.s64 @T[0], @XMM[8], #63
2091 mov r0, sp
2092 vand @T[0], @T[0], $twmask
2093___
2094for($i=9;$i<16;$i++) {
2095$code.=<<___;
2096 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
2097 vst1.64 {@XMM[$i-1]}, [r0,:128]!
2098 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
2099 vshr.s64 @T[1], @XMM[$i], #63
2100 veor @XMM[$i], @XMM[$i], @T[0]
2101 vand @T[1], @T[1], $twmask
2102___
2103 @T=reverse(@T);
2104
2105$code.=<<___ if ($i>=10);
2106 vld1.8 {@XMM[$i-10]}, [$inp]!
2107___
2108$code.=<<___ if ($i>=11);
2109 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
2110___
2111}
2112$code.=<<___;
2113 vadd.u64 @XMM[8], @XMM[15], @XMM[15]
2114 vst1.64 {@XMM[15]}, [r0,:128]!
2115 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
2116 veor @XMM[8], @XMM[8], @T[0]
2117 vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2118
2119 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
2120 veor @XMM[5], @XMM[5], @XMM[13]
2121#ifndef BSAES_ASM_EXTENDED_KEY
2122 add r4, sp, #0x90 @ pass key schedule
2123#else
2124 add r4, $key, #248 @ pass key schedule
2125#endif
2126 veor @XMM[6], @XMM[6], @XMM[14]
2127 mov r5, $rounds @ pass rounds
2128 veor @XMM[7], @XMM[7], @XMM[15]
2129 mov r0, sp
2130
2131 bl _bsaes_decrypt8
2132
2133 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2134 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2135 veor @XMM[0], @XMM[0], @XMM[ 8]
2136 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
2137 veor @XMM[1], @XMM[1], @XMM[ 9]
2138 veor @XMM[8], @XMM[6], @XMM[10]
2139 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2140 veor @XMM[9], @XMM[4], @XMM[11]
2141 vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]!
2142 veor @XMM[10], @XMM[2], @XMM[12]
2143 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2144 veor @XMM[11], @XMM[7], @XMM[13]
2145 veor @XMM[12], @XMM[3], @XMM[14]
2146 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
2147 veor @XMM[13], @XMM[5], @XMM[15]
2148 vst1.8 {@XMM[12]-@XMM[13]}, [$out]!
2149
2150 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2151
2152 subs $len, #0x80
2153 bpl .Lxts_dec_loop
2154
2155.Lxts_dec_short:
2156 adds $len, #0x70
2157 bmi .Lxts_dec_done
2158
2159 vldmia $magic, {$twmask} @ load XTS magic
2160 vshr.s64 @T[0], @XMM[8], #63
2161 mov r0, sp
2162 vand @T[0], @T[0], $twmask
2163___
2164for($i=9;$i<16;$i++) {
2165$code.=<<___;
2166 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
2167 vst1.64 {@XMM[$i-1]}, [r0,:128]!
2168 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
2169 vshr.s64 @T[1], @XMM[$i], #63
2170 veor @XMM[$i], @XMM[$i], @T[0]
2171 vand @T[1], @T[1], $twmask
2172___
2173 @T=reverse(@T);
2174
2175$code.=<<___ if ($i>=10);
2176 vld1.8 {@XMM[$i-10]}, [$inp]!
2177 subs $len, #0x10
2178 bmi .Lxts_dec_`$i-9`
2179___
2180$code.=<<___ if ($i>=11);
2181 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
2182___
2183}
2184$code.=<<___;
2185 sub $len, #0x10
2186 vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak
2187
2188 vld1.8 {@XMM[6]}, [$inp]!
2189 veor @XMM[5], @XMM[5], @XMM[13]
2190#ifndef BSAES_ASM_EXTENDED_KEY
2191 add r4, sp, #0x90 @ pass key schedule
2192#else
2193 add r4, $key, #248 @ pass key schedule
2194#endif
2195 veor @XMM[6], @XMM[6], @XMM[14]
2196 mov r5, $rounds @ pass rounds
2197 mov r0, sp
2198
2199 bl _bsaes_decrypt8
2200
2201 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2202 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2203 veor @XMM[0], @XMM[0], @XMM[ 8]
2204 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
2205 veor @XMM[1], @XMM[1], @XMM[ 9]
2206 veor @XMM[8], @XMM[6], @XMM[10]
2207 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2208 veor @XMM[9], @XMM[4], @XMM[11]
2209 vld1.64 {@XMM[14]}, [r0,:128]!
2210 veor @XMM[10], @XMM[2], @XMM[12]
2211 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2212 veor @XMM[11], @XMM[7], @XMM[13]
2213 veor @XMM[12], @XMM[3], @XMM[14]
2214 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
2215 vst1.8 {@XMM[12]}, [$out]!
2216
2217 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2218 b .Lxts_dec_done
2219.align 4
2220.Lxts_dec_6:
2221 vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak
2222
2223 veor @XMM[4], @XMM[4], @XMM[12]
2224#ifndef BSAES_ASM_EXTENDED_KEY
2225 add r4, sp, #0x90 @ pass key schedule
2226#else
2227 add r4, $key, #248 @ pass key schedule
2228#endif
2229 veor @XMM[5], @XMM[5], @XMM[13]
2230 mov r5, $rounds @ pass rounds
2231 mov r0, sp
2232
2233 bl _bsaes_decrypt8
2234
2235 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2236 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2237 veor @XMM[0], @XMM[0], @XMM[ 8]
2238 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
2239 veor @XMM[1], @XMM[1], @XMM[ 9]
2240 veor @XMM[8], @XMM[6], @XMM[10]
2241 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2242 veor @XMM[9], @XMM[4], @XMM[11]
2243 veor @XMM[10], @XMM[2], @XMM[12]
2244 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2245 veor @XMM[11], @XMM[7], @XMM[13]
2246 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
2247
2248 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2249 b .Lxts_dec_done
2250.align 4
2251.Lxts_dec_5:
2252 vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak
2253
2254 veor @XMM[3], @XMM[3], @XMM[11]
2255#ifndef BSAES_ASM_EXTENDED_KEY
2256 add r4, sp, #0x90 @ pass key schedule
2257#else
2258 add r4, $key, #248 @ pass key schedule
2259#endif
2260 veor @XMM[4], @XMM[4], @XMM[12]
2261 mov r5, $rounds @ pass rounds
2262 mov r0, sp
2263
2264 bl _bsaes_decrypt8
2265
2266 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2267 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2268 veor @XMM[0], @XMM[0], @XMM[ 8]
2269 vld1.64 {@XMM[12]}, [r0,:128]!
2270 veor @XMM[1], @XMM[1], @XMM[ 9]
2271 veor @XMM[8], @XMM[6], @XMM[10]
2272 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2273 veor @XMM[9], @XMM[4], @XMM[11]
2274 veor @XMM[10], @XMM[2], @XMM[12]
2275 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2276 vst1.8 {@XMM[10]}, [$out]!
2277
2278 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2279 b .Lxts_dec_done
2280.align 4
2281.Lxts_dec_4:
2282 vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak
2283
2284 veor @XMM[2], @XMM[2], @XMM[10]
2285#ifndef BSAES_ASM_EXTENDED_KEY
2286 add r4, sp, #0x90 @ pass key schedule
2287#else
2288 add r4, $key, #248 @ pass key schedule
2289#endif
2290 veor @XMM[3], @XMM[3], @XMM[11]
2291 mov r5, $rounds @ pass rounds
2292 mov r0, sp
2293
2294 bl _bsaes_decrypt8
2295
2296 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2297 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2298 veor @XMM[0], @XMM[0], @XMM[ 8]
2299 veor @XMM[1], @XMM[1], @XMM[ 9]
2300 veor @XMM[8], @XMM[6], @XMM[10]
2301 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2302 veor @XMM[9], @XMM[4], @XMM[11]
2303 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2304
2305 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2306 b .Lxts_dec_done
2307.align 4
2308.Lxts_dec_3:
2309 vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak
2310
2311 veor @XMM[1], @XMM[1], @XMM[9]
2312#ifndef BSAES_ASM_EXTENDED_KEY
2313 add r4, sp, #0x90 @ pass key schedule
2314#else
2315 add r4, $key, #248 @ pass key schedule
2316#endif
2317 veor @XMM[2], @XMM[2], @XMM[10]
2318 mov r5, $rounds @ pass rounds
2319 mov r0, sp
2320
2321 bl _bsaes_decrypt8
2322
2323 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
2324 vld1.64 {@XMM[10]}, [r0,:128]!
2325 veor @XMM[0], @XMM[0], @XMM[ 8]
2326 veor @XMM[1], @XMM[1], @XMM[ 9]
2327 veor @XMM[8], @XMM[6], @XMM[10]
2328 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2329 vst1.8 {@XMM[8]}, [$out]!
2330
2331 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2332 b .Lxts_dec_done
2333.align 4
2334.Lxts_dec_2:
2335 vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak
2336
2337 veor @XMM[0], @XMM[0], @XMM[8]
2338#ifndef BSAES_ASM_EXTENDED_KEY
2339 add r4, sp, #0x90 @ pass key schedule
2340#else
2341 add r4, $key, #248 @ pass key schedule
2342#endif
2343 veor @XMM[1], @XMM[1], @XMM[9]
2344 mov r5, $rounds @ pass rounds
2345 mov r0, sp
2346
2347 bl _bsaes_decrypt8
2348
2349 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
2350 veor @XMM[0], @XMM[0], @XMM[ 8]
2351 veor @XMM[1], @XMM[1], @XMM[ 9]
2352 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2353
2354 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2355 b .Lxts_dec_done
2356.align 4
2357.Lxts_dec_1:
2358 mov r0, sp
2359 veor @XMM[0], @XMM[8]
2360 mov r1, sp
2361 vst1.8 {@XMM[0]}, [sp,:128]
2362 mov r2, $key
2363 mov r4, $fp @ preserve fp
2364 mov r5, $magic @ preserve magic
2365
2366 bl AES_decrypt
2367
2368 vld1.8 {@XMM[0]}, [sp,:128]
2369 veor @XMM[0], @XMM[0], @XMM[8]
2370 vst1.8 {@XMM[0]}, [$out]!
2371 mov $fp, r4
2372 mov $magic, r5
2373
2374 vmov @XMM[8], @XMM[9] @ next round tweak
2375
2376.Lxts_dec_done:
2377#ifndef XTS_CHAIN_TWEAK
2378 adds $len, #0x10
2379 beq .Lxts_dec_ret
2380
2381 @ calculate one round of extra tweak for the stolen ciphertext
2382 vldmia $magic, {$twmask}
2383 vshr.s64 @XMM[6], @XMM[8], #63
2384 vand @XMM[6], @XMM[6], $twmask
2385 vadd.u64 @XMM[9], @XMM[8], @XMM[8]
2386 vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")`
2387 veor @XMM[9], @XMM[9], @XMM[6]
2388
2389 @ perform the final decryption with the last tweak value
2390 vld1.8 {@XMM[0]}, [$inp]!
2391 mov r0, sp
2392 veor @XMM[0], @XMM[0], @XMM[9]
2393 mov r1, sp
2394 vst1.8 {@XMM[0]}, [sp,:128]
2395 mov r2, $key
2396 mov r4, $fp @ preserve fp
2397
2398 bl AES_decrypt
2399
2400 vld1.8 {@XMM[0]}, [sp,:128]
2401 veor @XMM[0], @XMM[0], @XMM[9]
2402 vst1.8 {@XMM[0]}, [$out]
2403
2404 mov r6, $out
2405.Lxts_dec_steal:
2406 ldrb r1, [$out]
2407 ldrb r0, [$inp], #1
2408 strb r1, [$out, #0x10]
2409 strb r0, [$out], #1
2410
2411 subs $len, #1
2412 bhi .Lxts_dec_steal
2413
2414 vld1.8 {@XMM[0]}, [r6]
2415 mov r0, sp
2416 veor @XMM[0], @XMM[8]
2417 mov r1, sp
2418 vst1.8 {@XMM[0]}, [sp,:128]
2419 mov r2, $key
2420
2421 bl AES_decrypt
2422
2423 vld1.8 {@XMM[0]}, [sp,:128]
2424 veor @XMM[0], @XMM[0], @XMM[8]
2425 vst1.8 {@XMM[0]}, [r6]
2426 mov $fp, r4
2427#endif
2428
2429.Lxts_dec_ret:
2430 bic r0, $fp, #0xf
2431 vmov.i32 q0, #0
2432 vmov.i32 q1, #0
2433#ifdef XTS_CHAIN_TWEAK
2434 ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak
2435#endif
2436.Lxts_dec_bzero: @ wipe key schedule [if any]
2437 vstmia sp!, {q0-q1}
2438 cmp sp, r0
2439 bne .Lxts_dec_bzero
2440
2441 mov sp, $fp
2442#ifdef XTS_CHAIN_TWEAK
2443 vst1.8 {@XMM[8]}, [r1]
2444#endif
2445 VFP_ABI_POP
2446 ldmia sp!, {r4-r10, pc} @ return
2447
2448.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
2449___
2450}
2451$code.=<<___;
2452#endif
2453___
2454
2455$code =~ s/\`([^\`]*)\`/eval($1)/gem;
2456
2457open SELF,$0;
2458while(<SELF>) {
2459 next if (/^#!/);
2460 last if (!s/^#/@/ and !/^$/);
2461 print;
2462}
2463close SELF;
2464
2465print $code;
2466
2467close STDOUT;
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index d3db39860b9c..6577b8aeb711 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -24,6 +24,7 @@ generic-y += sembuf.h
24generic-y += serial.h 24generic-y += serial.h
25generic-y += shmbuf.h 25generic-y += shmbuf.h
26generic-y += siginfo.h 26generic-y += siginfo.h
27generic-y += simd.h
27generic-y += sizes.h 28generic-y += sizes.h
28generic-y += socket.h 29generic-y += socket.h
29generic-y += sockios.h 30generic-y += sockios.h
diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index 7e1f76027f66..72abdc541f38 100644
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -19,6 +19,13 @@
19#include <asm/unified.h> 19#include <asm/unified.h>
20#include <asm/compiler.h> 20#include <asm/compiler.h>
21 21
22#if __LINUX_ARM_ARCH__ < 6
23#include <asm-generic/uaccess-unaligned.h>
24#else
25#define __get_user_unaligned __get_user
26#define __put_user_unaligned __put_user
27#endif
28
22#define VERIFY_READ 0 29#define VERIFY_READ 0
23#define VERIFY_WRITE 1 30#define VERIFY_WRITE 1
24 31
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 74ad15d1a065..bc6bd9683ba4 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -442,10 +442,10 @@ local_restart:
442 ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine 442 ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine
443 443
444 add r1, sp, #S_OFF 444 add r1, sp, #S_OFF
445 cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE) 4452: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
446 eor r0, scno, #__NR_SYSCALL_BASE @ put OS number back 446 eor r0, scno, #__NR_SYSCALL_BASE @ put OS number back
447 bcs arm_syscall 447 bcs arm_syscall
4482: mov why, #0 @ no longer a real syscall 448 mov why, #0 @ no longer a real syscall
449 b sys_ni_syscall @ not private func 449 b sys_ni_syscall @ not private func
450 450
451#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI) 451#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
index de23a9beed13..39f89fbd5111 100644
--- a/arch/arm/kernel/entry-header.S
+++ b/arch/arm/kernel/entry-header.S
@@ -329,10 +329,10 @@
329#ifdef CONFIG_CONTEXT_TRACKING 329#ifdef CONFIG_CONTEXT_TRACKING
330 .if \save 330 .if \save
331 stmdb sp!, {r0-r3, ip, lr} 331 stmdb sp!, {r0-r3, ip, lr}
332 bl user_exit 332 bl context_tracking_user_exit
333 ldmia sp!, {r0-r3, ip, lr} 333 ldmia sp!, {r0-r3, ip, lr}
334 .else 334 .else
335 bl user_exit 335 bl context_tracking_user_exit
336 .endif 336 .endif
337#endif 337#endif
338 .endm 338 .endm
@@ -341,10 +341,10 @@
341#ifdef CONFIG_CONTEXT_TRACKING 341#ifdef CONFIG_CONTEXT_TRACKING
342 .if \save 342 .if \save
343 stmdb sp!, {r0-r3, ip, lr} 343 stmdb sp!, {r0-r3, ip, lr}
344 bl user_enter 344 bl context_tracking_user_enter
345 ldmia sp!, {r0-r3, ip, lr} 345 ldmia sp!, {r0-r3, ip, lr}
346 .else 346 .else
347 bl user_enter 347 bl context_tracking_user_enter
348 .endif 348 .endif
349#endif 349#endif
350 .endm 350 .endm
diff --git a/arch/arm/mach-imx/clk-fixup-mux.c b/arch/arm/mach-imx/clk-fixup-mux.c
index deb4b8093b30..0d40b35c557c 100644
--- a/arch/arm/mach-imx/clk-fixup-mux.c
+++ b/arch/arm/mach-imx/clk-fixup-mux.c
@@ -90,6 +90,7 @@ struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg,
90 init.ops = &clk_fixup_mux_ops; 90 init.ops = &clk_fixup_mux_ops;
91 init.parent_names = parents; 91 init.parent_names = parents;
92 init.num_parents = num_parents; 92 init.num_parents = num_parents;
93 init.flags = 0;
93 94
94 fixup_mux->mux.reg = reg; 95 fixup_mux->mux.reg = reg;
95 fixup_mux->mux.shift = shift; 96 fixup_mux->mux.shift = shift;
diff --git a/arch/arm/mach-imx/clk-imx27.c b/arch/arm/mach-imx/clk-imx27.c
index c3cfa4116dc0..c6b40f386786 100644
--- a/arch/arm/mach-imx/clk-imx27.c
+++ b/arch/arm/mach-imx/clk-imx27.c
@@ -285,7 +285,7 @@ int __init mx27_clocks_init(unsigned long fref)
285 clk_register_clkdev(clk[ata_ahb_gate], "ata", NULL); 285 clk_register_clkdev(clk[ata_ahb_gate], "ata", NULL);
286 clk_register_clkdev(clk[rtc_ipg_gate], NULL, "imx21-rtc"); 286 clk_register_clkdev(clk[rtc_ipg_gate], NULL, "imx21-rtc");
287 clk_register_clkdev(clk[scc_ipg_gate], "scc", NULL); 287 clk_register_clkdev(clk[scc_ipg_gate], "scc", NULL);
288 clk_register_clkdev(clk[cpu_div], NULL, "cpufreq-cpu0.0"); 288 clk_register_clkdev(clk[cpu_div], NULL, "cpu0");
289 clk_register_clkdev(clk[emi_ahb_gate], "emi_ahb" , NULL); 289 clk_register_clkdev(clk[emi_ahb_gate], "emi_ahb" , NULL);
290 290
291 mxc_timer_init(MX27_IO_ADDRESS(MX27_GPT1_BASE_ADDR), MX27_INT_GPT1); 291 mxc_timer_init(MX27_IO_ADDRESS(MX27_GPT1_BASE_ADDR), MX27_INT_GPT1);
diff --git a/arch/arm/mach-imx/clk-imx51-imx53.c b/arch/arm/mach-imx/clk-imx51-imx53.c
index 1a56a3319997..7c0dc4540aa4 100644
--- a/arch/arm/mach-imx/clk-imx51-imx53.c
+++ b/arch/arm/mach-imx/clk-imx51-imx53.c
@@ -328,7 +328,7 @@ static void __init mx5_clocks_common_init(unsigned long rate_ckil,
328 clk_register_clkdev(clk[ssi2_ipg_gate], NULL, "imx-ssi.1"); 328 clk_register_clkdev(clk[ssi2_ipg_gate], NULL, "imx-ssi.1");
329 clk_register_clkdev(clk[ssi3_ipg_gate], NULL, "imx-ssi.2"); 329 clk_register_clkdev(clk[ssi3_ipg_gate], NULL, "imx-ssi.2");
330 clk_register_clkdev(clk[sdma_gate], NULL, "imx35-sdma"); 330 clk_register_clkdev(clk[sdma_gate], NULL, "imx35-sdma");
331 clk_register_clkdev(clk[cpu_podf], NULL, "cpufreq-cpu0.0"); 331 clk_register_clkdev(clk[cpu_podf], NULL, "cpu0");
332 clk_register_clkdev(clk[iim_gate], "iim", NULL); 332 clk_register_clkdev(clk[iim_gate], "iim", NULL);
333 clk_register_clkdev(clk[dummy], NULL, "imx2-wdt.0"); 333 clk_register_clkdev(clk[dummy], NULL, "imx2-wdt.0");
334 clk_register_clkdev(clk[dummy], NULL, "imx2-wdt.1"); 334 clk_register_clkdev(clk[dummy], NULL, "imx2-wdt.1");
@@ -397,7 +397,7 @@ int __init mx51_clocks_init(unsigned long rate_ckil, unsigned long rate_osc,
397 mx51_spdif_xtal_sel, ARRAY_SIZE(mx51_spdif_xtal_sel)); 397 mx51_spdif_xtal_sel, ARRAY_SIZE(mx51_spdif_xtal_sel));
398 clk[spdif1_sel] = imx_clk_mux("spdif1_sel", MXC_CCM_CSCMR2, 2, 2, 398 clk[spdif1_sel] = imx_clk_mux("spdif1_sel", MXC_CCM_CSCMR2, 2, 2,
399 spdif_sel, ARRAY_SIZE(spdif_sel)); 399 spdif_sel, ARRAY_SIZE(spdif_sel));
400 clk[spdif1_pred] = imx_clk_divider("spdif1_podf", "spdif1_sel", MXC_CCM_CDCDR, 16, 3); 400 clk[spdif1_pred] = imx_clk_divider("spdif1_pred", "spdif1_sel", MXC_CCM_CDCDR, 16, 3);
401 clk[spdif1_podf] = imx_clk_divider("spdif1_podf", "spdif1_pred", MXC_CCM_CDCDR, 9, 6); 401 clk[spdif1_podf] = imx_clk_divider("spdif1_podf", "spdif1_pred", MXC_CCM_CDCDR, 9, 6);
402 clk[spdif1_com_sel] = imx_clk_mux("spdif1_com_sel", MXC_CCM_CSCMR2, 5, 1, 402 clk[spdif1_com_sel] = imx_clk_mux("spdif1_com_sel", MXC_CCM_CSCMR2, 5, 1,
403 mx51_spdif1_com_sel, ARRAY_SIZE(mx51_spdif1_com_sel)); 403 mx51_spdif1_com_sel, ARRAY_SIZE(mx51_spdif1_com_sel));
diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c
index 85a1b51346c8..90372a21087f 100644
--- a/arch/arm/mach-imx/mach-imx6q.c
+++ b/arch/arm/mach-imx/mach-imx6q.c
@@ -233,10 +233,15 @@ put_node:
233 of_node_put(np); 233 of_node_put(np);
234} 234}
235 235
236static void __init imx6q_opp_init(struct device *cpu_dev) 236static void __init imx6q_opp_init(void)
237{ 237{
238 struct device_node *np; 238 struct device_node *np;
239 struct device *cpu_dev = get_cpu_device(0);
239 240
241 if (!cpu_dev) {
242 pr_warn("failed to get cpu0 device\n");
243 return;
244 }
240 np = of_node_get(cpu_dev->of_node); 245 np = of_node_get(cpu_dev->of_node);
241 if (!np) { 246 if (!np) {
242 pr_warn("failed to find cpu0 node\n"); 247 pr_warn("failed to find cpu0 node\n");
@@ -268,7 +273,7 @@ static void __init imx6q_init_late(void)
268 imx6q_cpuidle_init(); 273 imx6q_cpuidle_init();
269 274
270 if (IS_ENABLED(CONFIG_ARM_IMX6Q_CPUFREQ)) { 275 if (IS_ENABLED(CONFIG_ARM_IMX6Q_CPUFREQ)) {
271 imx6q_opp_init(&imx6q_cpufreq_pdev.dev); 276 imx6q_opp_init();
272 platform_device_register(&imx6q_cpufreq_pdev); 277 platform_device_register(&imx6q_cpufreq_pdev);
273 } 278 }
274} 279}
diff --git a/arch/arm/mach-imx/system.c b/arch/arm/mach-imx/system.c
index 64ff37ea72b1..80c177c36c5f 100644
--- a/arch/arm/mach-imx/system.c
+++ b/arch/arm/mach-imx/system.c
@@ -117,6 +117,17 @@ void __init imx_init_l2cache(void)
117 /* Configure the L2 PREFETCH and POWER registers */ 117 /* Configure the L2 PREFETCH and POWER registers */
118 val = readl_relaxed(l2x0_base + L2X0_PREFETCH_CTRL); 118 val = readl_relaxed(l2x0_base + L2X0_PREFETCH_CTRL);
119 val |= 0x70800000; 119 val |= 0x70800000;
120 /*
121 * The L2 cache controller(PL310) version on the i.MX6D/Q is r3p1-50rel0
122 * The L2 cache controller(PL310) version on the i.MX6DL/SOLO/SL is r3p2
123 * But according to ARM PL310 errata: 752271
124 * ID: 752271: Double linefill feature can cause data corruption
125 * Fault Status: Present in: r3p0, r3p1, r3p1-50rel0. Fixed in r3p2
126 * Workaround: The only workaround to this erratum is to disable the
127 * double linefill feature. This is the default behavior.
128 */
129 if (cpu_is_imx6q())
130 val &= ~(1 << 30 | 1 << 23);
120 writel_relaxed(val, l2x0_base + L2X0_PREFETCH_CTRL); 131 writel_relaxed(val, l2x0_base + L2X0_PREFETCH_CTRL);
121 val = L2X0_DYNAMIC_CLK_GATING_EN | L2X0_STNDBY_MODE_EN; 132 val = L2X0_DYNAMIC_CLK_GATING_EN | L2X0_STNDBY_MODE_EN;
122 writel_relaxed(val, l2x0_base + L2X0_POWER_CTRL); 133 writel_relaxed(val, l2x0_base + L2X0_POWER_CTRL);
diff --git a/arch/arm/mach-omap2/cclock44xx_data.c b/arch/arm/mach-omap2/cclock44xx_data.c
index 1d5b5290d2af..b237950eb8a3 100644
--- a/arch/arm/mach-omap2/cclock44xx_data.c
+++ b/arch/arm/mach-omap2/cclock44xx_data.c
@@ -1632,7 +1632,7 @@ static struct omap_clk omap44xx_clks[] = {
1632 CLK(NULL, "auxclk5_src_ck", &auxclk5_src_ck), 1632 CLK(NULL, "auxclk5_src_ck", &auxclk5_src_ck),
1633 CLK(NULL, "auxclk5_ck", &auxclk5_ck), 1633 CLK(NULL, "auxclk5_ck", &auxclk5_ck),
1634 CLK(NULL, "auxclkreq5_ck", &auxclkreq5_ck), 1634 CLK(NULL, "auxclkreq5_ck", &auxclkreq5_ck),
1635 CLK("omap-gpmc", "fck", &dummy_ck), 1635 CLK("50000000.gpmc", "fck", &dummy_ck),
1636 CLK("omap_i2c.1", "ick", &dummy_ck), 1636 CLK("omap_i2c.1", "ick", &dummy_ck),
1637 CLK("omap_i2c.2", "ick", &dummy_ck), 1637 CLK("omap_i2c.2", "ick", &dummy_ck),
1638 CLK("omap_i2c.3", "ick", &dummy_ck), 1638 CLK("omap_i2c.3", "ick", &dummy_ck),
diff --git a/arch/arm/mach-omap2/cpuidle44xx.c b/arch/arm/mach-omap2/cpuidle44xx.c
index c443f2e97e10..4c8982ae9529 100644
--- a/arch/arm/mach-omap2/cpuidle44xx.c
+++ b/arch/arm/mach-omap2/cpuidle44xx.c
@@ -143,7 +143,7 @@ static int omap_enter_idle_coupled(struct cpuidle_device *dev,
143 * Call idle CPU cluster PM exit notifier chain 143 * Call idle CPU cluster PM exit notifier chain
144 * to restore GIC and wakeupgen context. 144 * to restore GIC and wakeupgen context.
145 */ 145 */
146 if ((cx->mpu_state == PWRDM_POWER_RET) && 146 if (dev->cpu == 0 && (cx->mpu_state == PWRDM_POWER_RET) &&
147 (cx->mpu_logic_state == PWRDM_POWER_OFF)) 147 (cx->mpu_logic_state == PWRDM_POWER_OFF))
148 cpu_cluster_pm_exit(); 148 cpu_cluster_pm_exit();
149 149
diff --git a/arch/arm/mach-omap2/gpmc.c b/arch/arm/mach-omap2/gpmc.c
index 9f4795aff48a..579697adaae7 100644
--- a/arch/arm/mach-omap2/gpmc.c
+++ b/arch/arm/mach-omap2/gpmc.c
@@ -1491,8 +1491,8 @@ static int gpmc_probe_generic_child(struct platform_device *pdev,
1491 */ 1491 */
1492 ret = gpmc_cs_remap(cs, res.start); 1492 ret = gpmc_cs_remap(cs, res.start);
1493 if (ret < 0) { 1493 if (ret < 0) {
1494 dev_err(&pdev->dev, "cannot remap GPMC CS %d to 0x%x\n", 1494 dev_err(&pdev->dev, "cannot remap GPMC CS %d to %pa\n",
1495 cs, res.start); 1495 cs, &res.start);
1496 goto err; 1496 goto err;
1497 } 1497 }
1498 1498
diff --git a/arch/arm/mach-omap2/mux34xx.c b/arch/arm/mach-omap2/mux34xx.c
index c53609f46294..be271f1d585b 100644
--- a/arch/arm/mach-omap2/mux34xx.c
+++ b/arch/arm/mach-omap2/mux34xx.c
@@ -620,7 +620,7 @@ static struct omap_mux __initdata omap3_muxmodes[] = {
620 "uart1_rts", "ssi1_flag_tx", NULL, NULL, 620 "uart1_rts", "ssi1_flag_tx", NULL, NULL,
621 "gpio_149", NULL, NULL, "safe_mode"), 621 "gpio_149", NULL, NULL, "safe_mode"),
622 _OMAP3_MUXENTRY(UART1_RX, 151, 622 _OMAP3_MUXENTRY(UART1_RX, 151,
623 "uart1_rx", "ss1_wake_tx", "mcbsp1_clkr", "mcspi4_clk", 623 "uart1_rx", "ssi1_wake_tx", "mcbsp1_clkr", "mcspi4_clk",
624 "gpio_151", NULL, NULL, "safe_mode"), 624 "gpio_151", NULL, NULL, "safe_mode"),
625 _OMAP3_MUXENTRY(UART1_TX, 148, 625 _OMAP3_MUXENTRY(UART1_TX, 148,
626 "uart1_tx", "ssi1_dat_tx", NULL, NULL, 626 "uart1_tx", "ssi1_dat_tx", NULL, NULL,
diff --git a/arch/arm/mach-omap2/omap-smp.c b/arch/arm/mach-omap2/omap-smp.c
index 8708b2a9da45..891211093295 100644
--- a/arch/arm/mach-omap2/omap-smp.c
+++ b/arch/arm/mach-omap2/omap-smp.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * OMAP4 SMP source file. It contains platform specific fucntions 2 * OMAP4 SMP source file. It contains platform specific functions
3 * needed for the linux smp kernel. 3 * needed for the linux smp kernel.
4 * 4 *
5 * Copyright (C) 2009 Texas Instruments, Inc. 5 * Copyright (C) 2009 Texas Instruments, Inc.
diff --git a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c
index f99f68e1e85b..b69dd9abb50a 100644
--- a/arch/arm/mach-omap2/omap_device.c
+++ b/arch/arm/mach-omap2/omap_device.c
@@ -158,7 +158,7 @@ static int omap_device_build_from_dt(struct platform_device *pdev)
158 } 158 }
159 159
160 od = omap_device_alloc(pdev, hwmods, oh_cnt); 160 od = omap_device_alloc(pdev, hwmods, oh_cnt);
161 if (!od) { 161 if (IS_ERR(od)) {
162 dev_err(&pdev->dev, "Cannot allocate omap_device for :%s\n", 162 dev_err(&pdev->dev, "Cannot allocate omap_device for :%s\n",
163 oh_name); 163 oh_name);
164 ret = PTR_ERR(od); 164 ret = PTR_ERR(od);
diff --git a/arch/arm/mach-sa1100/collie.c b/arch/arm/mach-sa1100/collie.c
index 612a45689770..7fb96ebdc0fb 100644
--- a/arch/arm/mach-sa1100/collie.c
+++ b/arch/arm/mach-sa1100/collie.c
@@ -289,7 +289,7 @@ static void collie_flash_exit(void)
289} 289}
290 290
291static struct flash_platform_data collie_flash_data = { 291static struct flash_platform_data collie_flash_data = {
292 .map_name = "cfi_probe", 292 .map_name = "jedec_probe",
293 .init = collie_flash_init, 293 .init = collie_flash_init,
294 .set_vpp = collie_set_vpp, 294 .set_vpp = collie_set_vpp,
295 .exit = collie_flash_exit, 295 .exit = collie_flash_exit,
diff --git a/arch/arm/mach-shmobile/clock-r8a73a4.c b/arch/arm/mach-shmobile/clock-r8a73a4.c
index 8ea5ef6c79cc..5bd2e851e3c7 100644
--- a/arch/arm/mach-shmobile/clock-r8a73a4.c
+++ b/arch/arm/mach-shmobile/clock-r8a73a4.c
@@ -555,7 +555,7 @@ static struct clk_lookup lookups[] = {
555 CLKDEV_CON_ID("pll2h", &pll2h_clk), 555 CLKDEV_CON_ID("pll2h", &pll2h_clk),
556 556
557 /* CPU clock */ 557 /* CPU clock */
558 CLKDEV_DEV_ID("cpufreq-cpu0", &z_clk), 558 CLKDEV_DEV_ID("cpu0", &z_clk),
559 559
560 /* DIV6 */ 560 /* DIV6 */
561 CLKDEV_CON_ID("zb", &div6_clks[DIV6_ZB]), 561 CLKDEV_CON_ID("zb", &div6_clks[DIV6_ZB]),
diff --git a/arch/arm/mach-shmobile/clock-sh73a0.c b/arch/arm/mach-shmobile/clock-sh73a0.c
index 1942eaef5181..c92c023f0d27 100644
--- a/arch/arm/mach-shmobile/clock-sh73a0.c
+++ b/arch/arm/mach-shmobile/clock-sh73a0.c
@@ -616,7 +616,7 @@ static struct clk_lookup lookups[] = {
616 CLKDEV_DEV_ID("smp_twd", &twd_clk), /* smp_twd */ 616 CLKDEV_DEV_ID("smp_twd", &twd_clk), /* smp_twd */
617 617
618 /* DIV4 clocks */ 618 /* DIV4 clocks */
619 CLKDEV_DEV_ID("cpufreq-cpu0", &div4_clks[DIV4_Z]), 619 CLKDEV_DEV_ID("cpu0", &div4_clks[DIV4_Z]),
620 620
621 /* DIV6 clocks */ 621 /* DIV6 clocks */
622 CLKDEV_CON_ID("vck1_clk", &div6_clks[DIV6_VCK1]), 622 CLKDEV_CON_ID("vck1_clk", &div6_clks[DIV6_VCK1]),
diff --git a/arch/arm/mach-u300/Kconfig b/arch/arm/mach-u300/Kconfig
index a85adcd00882..a1659863bfd5 100644
--- a/arch/arm/mach-u300/Kconfig
+++ b/arch/arm/mach-u300/Kconfig
@@ -1,7 +1,3 @@
1menu "ST-Ericsson AB U300/U335 Platform"
2
3comment "ST-Ericsson Mobile Platform Products"
4
5config ARCH_U300 1config ARCH_U300
6 bool "ST-Ericsson U300 Series" if ARCH_MULTI_V5 2 bool "ST-Ericsson U300 Series" if ARCH_MULTI_V5
7 depends on MMU 3 depends on MMU
@@ -25,7 +21,9 @@ config ARCH_U300
25 help 21 help
26 Support for ST-Ericsson U300 series mobile platforms. 22 Support for ST-Ericsson U300 series mobile platforms.
27 23
28comment "ST-Ericsson U300/U335 Feature Selections" 24if ARCH_U300
25
26menu "ST-Ericsson AB U300/U335 Platform"
29 27
30config MACH_U300 28config MACH_U300
31 depends on ARCH_U300 29 depends on ARCH_U300
@@ -53,3 +51,5 @@ config MACH_U300_SPIDUMMY
53 SPI framework and ARM PL022 support. 51 SPI framework and ARM PL022 support.
54 52
55endmenu 53endmenu
54
55endif
diff --git a/arch/arm/mach-ux500/cache-l2x0.c b/arch/arm/mach-ux500/cache-l2x0.c
index 82ccf1d98735..264f894c0e3d 100644
--- a/arch/arm/mach-ux500/cache-l2x0.c
+++ b/arch/arm/mach-ux500/cache-l2x0.c
@@ -69,6 +69,7 @@ static int __init ux500_l2x0_init(void)
69 * some SMI service available. 69 * some SMI service available.
70 */ 70 */
71 outer_cache.disable = NULL; 71 outer_cache.disable = NULL;
72 outer_cache.set_debug = NULL;
72 73
73 return 0; 74 return 0;
74} 75}
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index 6d4482fa35bc..e2950b098e76 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -43,6 +43,6 @@
43 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\ 43 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
44 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV) 44 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV)
45 45
46extern unsigned int elf_hwcap; 46extern unsigned long elf_hwcap;
47#endif 47#endif
48#endif 48#endif
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 57fb55c44c90..7ae8a1f00c3c 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -143,15 +143,26 @@ void machine_restart(char *cmd)
143 143
144void __show_regs(struct pt_regs *regs) 144void __show_regs(struct pt_regs *regs)
145{ 145{
146 int i; 146 int i, top_reg;
147 u64 lr, sp;
148
149 if (compat_user_mode(regs)) {
150 lr = regs->compat_lr;
151 sp = regs->compat_sp;
152 top_reg = 12;
153 } else {
154 lr = regs->regs[30];
155 sp = regs->sp;
156 top_reg = 29;
157 }
147 158
148 show_regs_print_info(KERN_DEFAULT); 159 show_regs_print_info(KERN_DEFAULT);
149 print_symbol("PC is at %s\n", instruction_pointer(regs)); 160 print_symbol("PC is at %s\n", instruction_pointer(regs));
150 print_symbol("LR is at %s\n", regs->regs[30]); 161 print_symbol("LR is at %s\n", lr);
151 printk("pc : [<%016llx>] lr : [<%016llx>] pstate: %08llx\n", 162 printk("pc : [<%016llx>] lr : [<%016llx>] pstate: %08llx\n",
152 regs->pc, regs->regs[30], regs->pstate); 163 regs->pc, lr, regs->pstate);
153 printk("sp : %016llx\n", regs->sp); 164 printk("sp : %016llx\n", sp);
154 for (i = 29; i >= 0; i--) { 165 for (i = top_reg; i >= 0; i--) {
155 printk("x%-2d: %016llx ", i, regs->regs[i]); 166 printk("x%-2d: %016llx ", i, regs->regs[i]);
156 if (i % 2 == 0) 167 if (i % 2 == 0)
157 printk("\n"); 168 printk("\n");
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 12ad8f3d0cfd..055cfb80e05c 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -57,7 +57,7 @@
57unsigned int processor_id; 57unsigned int processor_id;
58EXPORT_SYMBOL(processor_id); 58EXPORT_SYMBOL(processor_id);
59 59
60unsigned int elf_hwcap __read_mostly; 60unsigned long elf_hwcap __read_mostly;
61EXPORT_SYMBOL_GPL(elf_hwcap); 61EXPORT_SYMBOL_GPL(elf_hwcap);
62 62
63static const char *cpu_name; 63static const char *cpu_name;
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 6d6acf153bff..c23751b06120 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -130,7 +130,7 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
130 force_sig_info(sig, &si, tsk); 130 force_sig_info(sig, &si, tsk);
131} 131}
132 132
133void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) 133static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
134{ 134{
135 struct task_struct *tsk = current; 135 struct task_struct *tsk = current;
136 struct mm_struct *mm = tsk->active_mm; 136 struct mm_struct *mm = tsk->active_mm;
diff --git a/arch/mips/Makefile b/arch/mips/Makefile
index 75a36ad11ff5..ca8f8340d75f 100644
--- a/arch/mips/Makefile
+++ b/arch/mips/Makefile
@@ -288,9 +288,6 @@ endif
288vmlinux.32: vmlinux 288vmlinux.32: vmlinux
289 $(OBJCOPY) -O $(32bit-bfd) $(OBJCOPYFLAGS) $< $@ 289 $(OBJCOPY) -O $(32bit-bfd) $(OBJCOPYFLAGS) $< $@
290 290
291
292#obj-$(CONFIG_KPROBES) += kprobes.o
293
294# 291#
295# The 64-bit ELF tools are pretty broken so at this time we generate 64-bit 292# The 64-bit ELF tools are pretty broken so at this time we generate 64-bit
296# ELF files from 32-bit files by conversion. 293# ELF files from 32-bit files by conversion.
diff --git a/arch/mips/alchemy/common/usb.c b/arch/mips/alchemy/common/usb.c
index fcc695626117..2adc7edda49c 100644
--- a/arch/mips/alchemy/common/usb.c
+++ b/arch/mips/alchemy/common/usb.c
@@ -14,6 +14,7 @@
14#include <linux/module.h> 14#include <linux/module.h>
15#include <linux/spinlock.h> 15#include <linux/spinlock.h>
16#include <linux/syscore_ops.h> 16#include <linux/syscore_ops.h>
17#include <asm/cpu.h>
17#include <asm/mach-au1x00/au1000.h> 18#include <asm/mach-au1x00/au1000.h>
18 19
19/* control register offsets */ 20/* control register offsets */
@@ -358,7 +359,7 @@ static inline int au1200_coherency_bug(void)
358{ 359{
359#if defined(CONFIG_DMA_COHERENT) 360#if defined(CONFIG_DMA_COHERENT)
360 /* Au1200 AB USB does not support coherent memory */ 361 /* Au1200 AB USB does not support coherent memory */
361 if (!(read_c0_prid() & 0xff)) { 362 if (!(read_c0_prid() & PRID_REV_MASK)) {
362 printk(KERN_INFO "Au1200 USB: this is chip revision AB !!\n"); 363 printk(KERN_INFO "Au1200 USB: this is chip revision AB !!\n");
363 printk(KERN_INFO "Au1200 USB: update your board or re-configure" 364 printk(KERN_INFO "Au1200 USB: update your board or re-configure"
364 " the kernel\n"); 365 " the kernel\n");
diff --git a/arch/mips/bcm63xx/cpu.c b/arch/mips/bcm63xx/cpu.c
index 7e17374a9ae8..b713cd64b087 100644
--- a/arch/mips/bcm63xx/cpu.c
+++ b/arch/mips/bcm63xx/cpu.c
@@ -306,14 +306,14 @@ void __init bcm63xx_cpu_init(void)
306 306
307 switch (c->cputype) { 307 switch (c->cputype) {
308 case CPU_BMIPS3300: 308 case CPU_BMIPS3300:
309 if ((read_c0_prid() & 0xff00) != PRID_IMP_BMIPS3300_ALT) 309 if ((read_c0_prid() & PRID_IMP_MASK) != PRID_IMP_BMIPS3300_ALT)
310 __cpu_name[cpu] = "Broadcom BCM6338"; 310 __cpu_name[cpu] = "Broadcom BCM6338";
311 /* fall-through */ 311 /* fall-through */
312 case CPU_BMIPS32: 312 case CPU_BMIPS32:
313 chipid_reg = BCM_6345_PERF_BASE; 313 chipid_reg = BCM_6345_PERF_BASE;
314 break; 314 break;
315 case CPU_BMIPS4350: 315 case CPU_BMIPS4350:
316 switch ((read_c0_prid() & 0xff)) { 316 switch ((read_c0_prid() & PRID_REV_MASK)) {
317 case 0x04: 317 case 0x04:
318 chipid_reg = BCM_3368_PERF_BASE; 318 chipid_reg = BCM_3368_PERF_BASE;
319 break; 319 break;
diff --git a/arch/mips/boot/dts/include/dt-bindings b/arch/mips/boot/dts/include/dt-bindings
index 68ae3887b3e5..08c00e4972fa 120000
--- a/arch/mips/boot/dts/include/dt-bindings
+++ b/arch/mips/boot/dts/include/dt-bindings
@@ -1 +1 @@
../../../../../include/dt-bindings ../../../../../include/dt-bindings \ No newline at end of file
diff --git a/arch/mips/cavium-octeon/csrc-octeon.c b/arch/mips/cavium-octeon/csrc-octeon.c
index 02193953eb9e..b752c4ed0b79 100644
--- a/arch/mips/cavium-octeon/csrc-octeon.c
+++ b/arch/mips/cavium-octeon/csrc-octeon.c
@@ -12,6 +12,7 @@
12#include <linux/smp.h> 12#include <linux/smp.h>
13 13
14#include <asm/cpu-info.h> 14#include <asm/cpu-info.h>
15#include <asm/cpu-type.h>
15#include <asm/time.h> 16#include <asm/time.h>
16 17
17#include <asm/octeon/octeon.h> 18#include <asm/octeon/octeon.h>
diff --git a/arch/mips/dec/prom/init.c b/arch/mips/dec/prom/init.c
index ab169046e442..468f665de7bb 100644
--- a/arch/mips/dec/prom/init.c
+++ b/arch/mips/dec/prom/init.c
@@ -13,6 +13,7 @@
13 13
14#include <asm/bootinfo.h> 14#include <asm/bootinfo.h>
15#include <asm/cpu.h> 15#include <asm/cpu.h>
16#include <asm/cpu-type.h>
16#include <asm/processor.h> 17#include <asm/processor.h>
17 18
18#include <asm/dec/prom.h> 19#include <asm/dec/prom.h>
diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
index fa44f3ec5302..d445d060e346 100644
--- a/arch/mips/include/asm/cpu-features.h
+++ b/arch/mips/include/asm/cpu-features.h
@@ -13,12 +13,6 @@
13#include <asm/cpu-info.h> 13#include <asm/cpu-info.h>
14#include <cpu-feature-overrides.h> 14#include <cpu-feature-overrides.h>
15 15
16#ifndef current_cpu_type
17#define current_cpu_type() current_cpu_data.cputype
18#endif
19
20#define boot_cpu_type() cpu_data[0].cputype
21
22/* 16/*
23 * SMP assumption: Options of CPU 0 are a superset of all processors. 17 * SMP assumption: Options of CPU 0 are a superset of all processors.
24 * This is true for all known MIPS systems. 18 * This is true for all known MIPS systems.
@@ -193,7 +187,7 @@
193 187
194/* 188/*
195 * MIPS32, MIPS64, VR5500, IDT32332, IDT32334 and maybe a few other 189 * MIPS32, MIPS64, VR5500, IDT32332, IDT32334 and maybe a few other
196 * pre-MIPS32/MIPS53 processors have CLO, CLZ. The IDT RC64574 is 64-bit and 190 * pre-MIPS32/MIPS64 processors have CLO, CLZ. The IDT RC64574 is 64-bit and
197 * has CLO and CLZ but not DCLO nor DCLZ. For 64-bit kernels 191 * has CLO and CLZ but not DCLO nor DCLZ. For 64-bit kernels
198 * cpu_has_clo_clz also indicates the availability of DCLO and DCLZ. 192 * cpu_has_clo_clz also indicates the availability of DCLO and DCLZ.
199 */ 193 */
diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h
index 41401d8eb7d1..21c8e29c8f91 100644
--- a/arch/mips/include/asm/cpu-info.h
+++ b/arch/mips/include/asm/cpu-info.h
@@ -84,6 +84,7 @@ struct cpuinfo_mips {
84extern struct cpuinfo_mips cpu_data[]; 84extern struct cpuinfo_mips cpu_data[];
85#define current_cpu_data cpu_data[smp_processor_id()] 85#define current_cpu_data cpu_data[smp_processor_id()]
86#define raw_current_cpu_data cpu_data[raw_smp_processor_id()] 86#define raw_current_cpu_data cpu_data[raw_smp_processor_id()]
87#define boot_cpu_data cpu_data[0]
87 88
88extern void cpu_probe(void); 89extern void cpu_probe(void);
89extern void cpu_report(void); 90extern void cpu_report(void);
diff --git a/arch/mips/include/asm/cpu-type.h b/arch/mips/include/asm/cpu-type.h
new file mode 100644
index 000000000000..4a402cc60c03
--- /dev/null
+++ b/arch/mips/include/asm/cpu-type.h
@@ -0,0 +1,203 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 2004 Ralf Baechle
7 * Copyright (C) 2004 Maciej W. Rozycki
8 */
9#ifndef __ASM_CPU_TYPE_H
10#define __ASM_CPU_TYPE_H
11
12#include <linux/smp.h>
13#include <linux/compiler.h>
14
15static inline int __pure __get_cpu_type(const int cpu_type)
16{
17 switch (cpu_type) {
18#if defined(CONFIG_SYS_HAS_CPU_LOONGSON2E) || \
19 defined(CONFIG_SYS_HAS_CPU_LOONGSON2F)
20 case CPU_LOONGSON2:
21#endif
22
23#ifdef CONFIG_SYS_HAS_CPU_LOONGSON1B
24 case CPU_LOONGSON1:
25#endif
26
27#ifdef CONFIG_SYS_HAS_CPU_MIPS32_R1
28 case CPU_4KC:
29 case CPU_ALCHEMY:
30 case CPU_BMIPS3300:
31 case CPU_BMIPS4350:
32 case CPU_PR4450:
33 case CPU_BMIPS32:
34 case CPU_JZRISC:
35#endif
36
37#if defined(CONFIG_SYS_HAS_CPU_MIPS32_R1) || \
38 defined(CONFIG_SYS_HAS_CPU_MIPS32_R2)
39 case CPU_4KEC:
40#endif
41
42#ifdef CONFIG_SYS_HAS_CPU_MIPS32_R2
43 case CPU_4KSC:
44 case CPU_24K:
45 case CPU_34K:
46 case CPU_1004K:
47 case CPU_74K:
48 case CPU_M14KC:
49 case CPU_M14KEC:
50#endif
51
52#ifdef CONFIG_SYS_HAS_CPU_MIPS64_R1
53 case CPU_5KC:
54 case CPU_5KE:
55 case CPU_20KC:
56 case CPU_25KF:
57 case CPU_SB1:
58 case CPU_SB1A:
59#endif
60
61#ifdef CONFIG_SYS_HAS_CPU_MIPS64_R2
62 /*
63 * All MIPS64 R2 processors have their own special symbols. That is,
64 * there currently is no pure R2 core
65 */
66#endif
67
68#ifdef CONFIG_SYS_HAS_CPU_R3000
69 case CPU_R2000:
70 case CPU_R3000:
71 case CPU_R3000A:
72 case CPU_R3041:
73 case CPU_R3051:
74 case CPU_R3052:
75 case CPU_R3081:
76 case CPU_R3081E:
77#endif
78
79#ifdef CONFIG_SYS_HAS_CPU_TX39XX
80 case CPU_TX3912:
81 case CPU_TX3922:
82 case CPU_TX3927:
83#endif
84
85#ifdef CONFIG_SYS_HAS_CPU_VR41XX
86 case CPU_VR41XX:
87 case CPU_VR4111:
88 case CPU_VR4121:
89 case CPU_VR4122:
90 case CPU_VR4131:
91 case CPU_VR4133:
92 case CPU_VR4181:
93 case CPU_VR4181A:
94#endif
95
96#ifdef CONFIG_SYS_HAS_CPU_R4300
97 case CPU_R4300:
98 case CPU_R4310:
99#endif
100
101#ifdef CONFIG_SYS_HAS_CPU_R4X00
102 case CPU_R4000PC:
103 case CPU_R4000SC:
104 case CPU_R4000MC:
105 case CPU_R4200:
106 case CPU_R4400PC:
107 case CPU_R4400SC:
108 case CPU_R4400MC:
109 case CPU_R4600:
110 case CPU_R4700:
111 case CPU_R4640:
112 case CPU_R4650:
113#endif
114
115#ifdef CONFIG_SYS_HAS_CPU_TX49XX
116 case CPU_TX49XX:
117#endif
118
119#ifdef CONFIG_SYS_HAS_CPU_R5000
120 case CPU_R5000:
121#endif
122
123#ifdef CONFIG_SYS_HAS_CPU_R5432
124 case CPU_R5432:
125#endif
126
127#ifdef CONFIG_SYS_HAS_CPU_R5500
128 case CPU_R5500:
129#endif
130
131#ifdef CONFIG_SYS_HAS_CPU_R6000
132 case CPU_R6000:
133 case CPU_R6000A:
134#endif
135
136#ifdef CONFIG_SYS_HAS_CPU_NEVADA
137 case CPU_NEVADA:
138#endif
139
140#ifdef CONFIG_SYS_HAS_CPU_R8000
141 case CPU_R8000:
142#endif
143
144#ifdef CONFIG_SYS_HAS_CPU_R10000
145 case CPU_R10000:
146 case CPU_R12000:
147 case CPU_R14000:
148#endif
149#ifdef CONFIG_SYS_HAS_CPU_RM7000
150 case CPU_RM7000:
151 case CPU_SR71000:
152#endif
153#ifdef CONFIG_SYS_HAS_CPU_RM9000
154 case CPU_RM9000:
155#endif
156#ifdef CONFIG_SYS_HAS_CPU_SB1
157 case CPU_SB1:
158 case CPU_SB1A:
159#endif
160#ifdef CONFIG_SYS_HAS_CPU_CAVIUM_OCTEON
161 case CPU_CAVIUM_OCTEON:
162 case CPU_CAVIUM_OCTEON_PLUS:
163 case CPU_CAVIUM_OCTEON2:
164#endif
165
166#ifdef CONFIG_SYS_HAS_CPU_BMIPS4380
167 case CPU_BMIPS4380:
168#endif
169
170#ifdef CONFIG_SYS_HAS_CPU_BMIPS5000
171 case CPU_BMIPS5000:
172#endif
173
174#ifdef CONFIG_SYS_HAS_CPU_XLP
175 case CPU_XLP:
176#endif
177
178#ifdef CONFIG_SYS_HAS_CPU_XLR
179 case CPU_XLR:
180#endif
181 break;
182 default:
183 unreachable();
184 }
185
186 return cpu_type;
187}
188
189static inline int __pure current_cpu_type(void)
190{
191 const int cpu_type = current_cpu_data.cputype;
192
193 return __get_cpu_type(cpu_type);
194}
195
196static inline int __pure boot_cpu_type(void)
197{
198 const int cpu_type = cpu_data[0].cputype;
199
200 return __get_cpu_type(cpu_type);
201}
202
203#endif /* __ASM_CPU_TYPE_H */
diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h
index 71b9f1998be7..d2035e16502a 100644
--- a/arch/mips/include/asm/cpu.h
+++ b/arch/mips/include/asm/cpu.h
@@ -3,15 +3,14 @@
3 * various MIPS cpu types. 3 * various MIPS cpu types.
4 * 4 *
5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
6 * Copyright (C) 2004 Maciej W. Rozycki 6 * Copyright (C) 2004, 2013 Maciej W. Rozycki
7 */ 7 */
8#ifndef _ASM_CPU_H 8#ifndef _ASM_CPU_H
9#define _ASM_CPU_H 9#define _ASM_CPU_H
10 10
11/* Assigned Company values for bits 23:16 of the PRId Register 11/*
12 (CP0 register 15, select 0). As of the MIPS32 and MIPS64 specs from 12 As of the MIPS32 and MIPS64 specs from MTI, the PRId register (CP0
13 MTI, the PRId register is defined in this (backwards compatible) 13 register 15, select 0) is defined in this (backwards compatible) way:
14 way:
15 14
16 +----------------+----------------+----------------+----------------+ 15 +----------------+----------------+----------------+----------------+
17 | Company Options| Company ID | Processor ID | Revision | 16 | Company Options| Company ID | Processor ID | Revision |
@@ -23,6 +22,14 @@
23 spec. 22 spec.
24*/ 23*/
25 24
25#define PRID_OPT_MASK 0xff000000
26
27/*
28 * Assigned Company values for bits 23:16 of the PRId register.
29 */
30
31#define PRID_COMP_MASK 0xff0000
32
26#define PRID_COMP_LEGACY 0x000000 33#define PRID_COMP_LEGACY 0x000000
27#define PRID_COMP_MIPS 0x010000 34#define PRID_COMP_MIPS 0x010000
28#define PRID_COMP_BROADCOM 0x020000 35#define PRID_COMP_BROADCOM 0x020000
@@ -38,10 +45,17 @@
38#define PRID_COMP_INGENIC 0xd00000 45#define PRID_COMP_INGENIC 0xd00000
39 46
40/* 47/*
41 * Assigned values for the product ID register. In order to detect a 48 * Assigned Processor ID (implementation) values for bits 15:8 of the PRId
42 * certain CPU type exactly eventually additional registers may need to 49 * register. In order to detect a certain CPU type exactly eventually
43 * be examined. These are valid when 23:16 == PRID_COMP_LEGACY 50 * additional registers may need to be examined.
44 */ 51 */
52
53#define PRID_IMP_MASK 0xff00
54
55/*
56 * These are valid when 23:16 == PRID_COMP_LEGACY
57 */
58
45#define PRID_IMP_R2000 0x0100 59#define PRID_IMP_R2000 0x0100
46#define PRID_IMP_AU1_REV1 0x0100 60#define PRID_IMP_AU1_REV1 0x0100
47#define PRID_IMP_AU1_REV2 0x0200 61#define PRID_IMP_AU1_REV2 0x0200
@@ -182,11 +196,15 @@
182#define PRID_IMP_NETLOGIC_XLP2XX 0x1200 196#define PRID_IMP_NETLOGIC_XLP2XX 0x1200
183 197
184/* 198/*
185 * Definitions for 7:0 on legacy processors 199 * Particular Revision values for bits 7:0 of the PRId register.
186 */ 200 */
187 201
188#define PRID_REV_MASK 0x00ff 202#define PRID_REV_MASK 0x00ff
189 203
204/*
205 * Definitions for 7:0 on legacy processors
206 */
207
190#define PRID_REV_TX4927 0x0022 208#define PRID_REV_TX4927 0x0022
191#define PRID_REV_TX4937 0x0030 209#define PRID_REV_TX4937 0x0030
192#define PRID_REV_R4400 0x0040 210#define PRID_REV_R4400 0x0040
@@ -227,6 +245,8 @@
227 * 31 16 15 8 7 0 245 * 31 16 15 8 7 0
228 */ 246 */
229 247
248#define FPIR_IMP_MASK 0xff00
249
230#define FPIR_IMP_NONE 0x0000 250#define FPIR_IMP_NONE 0x0000
231 251
232enum cpu_type_enum { 252enum cpu_type_enum {
diff --git a/arch/mips/include/asm/mach-au1x00/au1000.h b/arch/mips/include/asm/mach-au1x00/au1000.h
index 3e11a468cdf8..54f9e84db8ac 100644
--- a/arch/mips/include/asm/mach-au1x00/au1000.h
+++ b/arch/mips/include/asm/mach-au1x00/au1000.h
@@ -43,6 +43,8 @@
43#include <linux/io.h> 43#include <linux/io.h>
44#include <linux/irq.h> 44#include <linux/irq.h>
45 45
46#include <asm/cpu.h>
47
46/* cpu pipeline flush */ 48/* cpu pipeline flush */
47void static inline au_sync(void) 49void static inline au_sync(void)
48{ 50{
@@ -140,7 +142,7 @@ static inline int au1xxx_cpu_needs_config_od(void)
140 142
141static inline int alchemy_get_cputype(void) 143static inline int alchemy_get_cputype(void)
142{ 144{
143 switch (read_c0_prid() & 0xffff0000) { 145 switch (read_c0_prid() & (PRID_OPT_MASK | PRID_COMP_MASK)) {
144 case 0x00030000: 146 case 0x00030000:
145 return ALCHEMY_CPU_AU1000; 147 return ALCHEMY_CPU_AU1000;
146 break; 148 break;
diff --git a/arch/mips/include/asm/mach-ip22/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ip22/cpu-feature-overrides.h
index f4caacd25552..1bcb6421205e 100644
--- a/arch/mips/include/asm/mach-ip22/cpu-feature-overrides.h
+++ b/arch/mips/include/asm/mach-ip22/cpu-feature-overrides.h
@@ -8,6 +8,8 @@
8#ifndef __ASM_MACH_IP22_CPU_FEATURE_OVERRIDES_H 8#ifndef __ASM_MACH_IP22_CPU_FEATURE_OVERRIDES_H
9#define __ASM_MACH_IP22_CPU_FEATURE_OVERRIDES_H 9#define __ASM_MACH_IP22_CPU_FEATURE_OVERRIDES_H
10 10
11#include <asm/cpu.h>
12
11/* 13/*
12 * IP22 with a variety of processors so we can't use defaults for everything. 14 * IP22 with a variety of processors so we can't use defaults for everything.
13 */ 15 */
diff --git a/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h
index 1d2b6ff60d33..d6111aa2e886 100644
--- a/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h
+++ b/arch/mips/include/asm/mach-ip27/cpu-feature-overrides.h
@@ -8,6 +8,8 @@
8#ifndef __ASM_MACH_IP27_CPU_FEATURE_OVERRIDES_H 8#ifndef __ASM_MACH_IP27_CPU_FEATURE_OVERRIDES_H
9#define __ASM_MACH_IP27_CPU_FEATURE_OVERRIDES_H 9#define __ASM_MACH_IP27_CPU_FEATURE_OVERRIDES_H
10 10
11#include <asm/cpu.h>
12
11/* 13/*
12 * IP27 only comes with R10000 family processors all using the same config 14 * IP27 only comes with R10000 family processors all using the same config
13 */ 15 */
diff --git a/arch/mips/include/asm/mach-ip28/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ip28/cpu-feature-overrides.h
index 65e9c856390d..4cec06d133db 100644
--- a/arch/mips/include/asm/mach-ip28/cpu-feature-overrides.h
+++ b/arch/mips/include/asm/mach-ip28/cpu-feature-overrides.h
@@ -9,6 +9,8 @@
9#ifndef __ASM_MACH_IP28_CPU_FEATURE_OVERRIDES_H 9#ifndef __ASM_MACH_IP28_CPU_FEATURE_OVERRIDES_H
10#define __ASM_MACH_IP28_CPU_FEATURE_OVERRIDES_H 10#define __ASM_MACH_IP28_CPU_FEATURE_OVERRIDES_H
11 11
12#include <asm/cpu.h>
13
12/* 14/*
13 * IP28 only comes with R10000 family processors all using the same config 15 * IP28 only comes with R10000 family processors all using the same config
14 */ 16 */
diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h
index fed1c3e9b486..e0331414c7d6 100644
--- a/arch/mips/include/asm/mipsregs.h
+++ b/arch/mips/include/asm/mipsregs.h
@@ -603,6 +603,13 @@
603#define MIPS_CONF4_MMUEXTDEF (_ULCAST_(3) << 14) 603#define MIPS_CONF4_MMUEXTDEF (_ULCAST_(3) << 14)
604#define MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT (_ULCAST_(1) << 14) 604#define MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT (_ULCAST_(1) << 14)
605 605
606#define MIPS_CONF5_NF (_ULCAST_(1) << 0)
607#define MIPS_CONF5_UFR (_ULCAST_(1) << 2)
608#define MIPS_CONF5_MSAEN (_ULCAST_(1) << 27)
609#define MIPS_CONF5_EVA (_ULCAST_(1) << 28)
610#define MIPS_CONF5_CV (_ULCAST_(1) << 29)
611#define MIPS_CONF5_K (_ULCAST_(1) << 30)
612
606#define MIPS_CONF6_SYND (_ULCAST_(1) << 13) 613#define MIPS_CONF6_SYND (_ULCAST_(1) << 13)
607 614
608#define MIPS_CONF7_WII (_ULCAST_(1) << 31) 615#define MIPS_CONF7_WII (_ULCAST_(1) << 31)
diff --git a/arch/mips/include/asm/pci.h b/arch/mips/include/asm/pci.h
index f194c08bd057..12d6842962be 100644
--- a/arch/mips/include/asm/pci.h
+++ b/arch/mips/include/asm/pci.h
@@ -83,6 +83,18 @@ static inline void pcibios_penalize_isa_irq(int irq, int active)
83extern int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, 83extern int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma,
84 enum pci_mmap_state mmap_state, int write_combine); 84 enum pci_mmap_state mmap_state, int write_combine);
85 85
86#define HAVE_ARCH_PCI_RESOURCE_TO_USER
87
88static inline void pci_resource_to_user(const struct pci_dev *dev, int bar,
89 const struct resource *rsrc, resource_size_t *start,
90 resource_size_t *end)
91{
92 phys_t size = resource_size(rsrc);
93
94 *start = fixup_bigphys_addr(rsrc->start, size);
95 *end = rsrc->start + size;
96}
97
86/* 98/*
87 * Dynamic DMA mapping stuff. 99 * Dynamic DMA mapping stuff.
88 * MIPS has everything mapped statically. 100 * MIPS has everything mapped statically.
diff --git a/arch/mips/include/asm/timex.h b/arch/mips/include/asm/timex.h
index 6529704aa73a..c5424757da65 100644
--- a/arch/mips/include/asm/timex.h
+++ b/arch/mips/include/asm/timex.h
@@ -10,7 +10,9 @@
10 10
11#ifdef __KERNEL__ 11#ifdef __KERNEL__
12 12
13#include <asm/cpu-features.h>
13#include <asm/mipsregs.h> 14#include <asm/mipsregs.h>
15#include <asm/cpu-type.h>
14 16
15/* 17/*
16 * This is the clock rate of the i8253 PIT. A MIPS system may not have 18 * This is the clock rate of the i8253 PIT. A MIPS system may not have
@@ -33,9 +35,38 @@
33 35
34typedef unsigned int cycles_t; 36typedef unsigned int cycles_t;
35 37
38/*
39 * On R4000/R4400 before version 5.0 an erratum exists such that if the
40 * cycle counter is read in the exact moment that it is matching the
41 * compare register, no interrupt will be generated.
42 *
43 * There is a suggested workaround and also the erratum can't strike if
44 * the compare interrupt isn't being used as the clock source device.
45 * However for now the implementaton of this function doesn't get these
46 * fine details right.
47 */
36static inline cycles_t get_cycles(void) 48static inline cycles_t get_cycles(void)
37{ 49{
38 return 0; 50 switch (boot_cpu_type()) {
51 case CPU_R4400PC:
52 case CPU_R4400SC:
53 case CPU_R4400MC:
54 if ((read_c0_prid() & 0xff) >= 0x0050)
55 return read_c0_count();
56 break;
57
58 case CPU_R4000PC:
59 case CPU_R4000SC:
60 case CPU_R4000MC:
61 break;
62
63 default:
64 if (cpu_has_counter)
65 return read_c0_count();
66 break;
67 }
68
69 return 0; /* no usable counter */
39} 70}
40 71
41#endif /* __KERNEL__ */ 72#endif /* __KERNEL__ */
diff --git a/arch/mips/include/asm/vga.h b/arch/mips/include/asm/vga.h
index f4cff7e4fa8a..f82c83749a08 100644
--- a/arch/mips/include/asm/vga.h
+++ b/arch/mips/include/asm/vga.h
@@ -6,6 +6,7 @@
6#ifndef _ASM_VGA_H 6#ifndef _ASM_VGA_H
7#define _ASM_VGA_H 7#define _ASM_VGA_H
8 8
9#include <asm/addrspace.h>
9#include <asm/byteorder.h> 10#include <asm/byteorder.h>
10 11
11/* 12/*
@@ -13,7 +14,7 @@
13 * access the videoram directly without any black magic. 14 * access the videoram directly without any black magic.
14 */ 15 */
15 16
16#define VGA_MAP_MEM(x, s) (0xb0000000L + (unsigned long)(x)) 17#define VGA_MAP_MEM(x, s) CKSEG1ADDR(0x10000000L + (unsigned long)(x))
17 18
18#define vga_readb(x) (*(x)) 19#define vga_readb(x) (*(x))
19#define vga_writeb(x, y) (*(y) = (x)) 20#define vga_writeb(x, y) (*(y) = (x))
diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c
index 37663c7862a5..5465dc183e5a 100644
--- a/arch/mips/kernel/cpu-probe.c
+++ b/arch/mips/kernel/cpu-probe.c
@@ -20,6 +20,7 @@
20 20
21#include <asm/bugs.h> 21#include <asm/bugs.h>
22#include <asm/cpu.h> 22#include <asm/cpu.h>
23#include <asm/cpu-type.h>
23#include <asm/fpu.h> 24#include <asm/fpu.h>
24#include <asm/mipsregs.h> 25#include <asm/mipsregs.h>
25#include <asm/watch.h> 26#include <asm/watch.h>
@@ -55,7 +56,7 @@ static inline void check_errata(void)
55{ 56{
56 struct cpuinfo_mips *c = &current_cpu_data; 57 struct cpuinfo_mips *c = &current_cpu_data;
57 58
58 switch (c->cputype) { 59 switch (current_cpu_type()) {
59 case CPU_34K: 60 case CPU_34K:
60 /* 61 /*
61 * Erratum "RPS May Cause Incorrect Instruction Execution" 62 * Erratum "RPS May Cause Incorrect Instruction Execution"
@@ -122,7 +123,7 @@ static inline unsigned long cpu_get_fpu_id(void)
122 */ 123 */
123static inline int __cpu_has_fpu(void) 124static inline int __cpu_has_fpu(void)
124{ 125{
125 return ((cpu_get_fpu_id() & 0xff00) != FPIR_IMP_NONE); 126 return ((cpu_get_fpu_id() & FPIR_IMP_MASK) != FPIR_IMP_NONE);
126} 127}
127 128
128static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) 129static inline void cpu_probe_vmbits(struct cpuinfo_mips *c)
@@ -290,6 +291,17 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c)
290 return config4 & MIPS_CONF_M; 291 return config4 & MIPS_CONF_M;
291} 292}
292 293
294static inline unsigned int decode_config5(struct cpuinfo_mips *c)
295{
296 unsigned int config5;
297
298 config5 = read_c0_config5();
299 config5 &= ~MIPS_CONF5_UFR;
300 write_c0_config5(config5);
301
302 return config5 & MIPS_CONF_M;
303}
304
293static void decode_configs(struct cpuinfo_mips *c) 305static void decode_configs(struct cpuinfo_mips *c)
294{ 306{
295 int ok; 307 int ok;
@@ -310,6 +322,8 @@ static void decode_configs(struct cpuinfo_mips *c)
310 ok = decode_config3(c); 322 ok = decode_config3(c);
311 if (ok) 323 if (ok)
312 ok = decode_config4(c); 324 ok = decode_config4(c);
325 if (ok)
326 ok = decode_config5(c);
313 327
314 mips_probe_watch_registers(c); 328 mips_probe_watch_registers(c);
315 329
@@ -322,7 +336,7 @@ static void decode_configs(struct cpuinfo_mips *c)
322 336
323static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) 337static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu)
324{ 338{
325 switch (c->processor_id & 0xff00) { 339 switch (c->processor_id & PRID_IMP_MASK) {
326 case PRID_IMP_R2000: 340 case PRID_IMP_R2000:
327 c->cputype = CPU_R2000; 341 c->cputype = CPU_R2000;
328 __cpu_name[cpu] = "R2000"; 342 __cpu_name[cpu] = "R2000";
@@ -333,7 +347,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu)
333 c->tlbsize = 64; 347 c->tlbsize = 64;
334 break; 348 break;
335 case PRID_IMP_R3000: 349 case PRID_IMP_R3000:
336 if ((c->processor_id & 0xff) == PRID_REV_R3000A) { 350 if ((c->processor_id & PRID_REV_MASK) == PRID_REV_R3000A) {
337 if (cpu_has_confreg()) { 351 if (cpu_has_confreg()) {
338 c->cputype = CPU_R3081E; 352 c->cputype = CPU_R3081E;
339 __cpu_name[cpu] = "R3081"; 353 __cpu_name[cpu] = "R3081";
@@ -353,7 +367,8 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu)
353 break; 367 break;
354 case PRID_IMP_R4000: 368 case PRID_IMP_R4000:
355 if (read_c0_config() & CONF_SC) { 369 if (read_c0_config() & CONF_SC) {
356 if ((c->processor_id & 0xff) >= PRID_REV_R4400) { 370 if ((c->processor_id & PRID_REV_MASK) >=
371 PRID_REV_R4400) {
357 c->cputype = CPU_R4400PC; 372 c->cputype = CPU_R4400PC;
358 __cpu_name[cpu] = "R4400PC"; 373 __cpu_name[cpu] = "R4400PC";
359 } else { 374 } else {
@@ -361,7 +376,8 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu)
361 __cpu_name[cpu] = "R4000PC"; 376 __cpu_name[cpu] = "R4000PC";
362 } 377 }
363 } else { 378 } else {
364 if ((c->processor_id & 0xff) >= PRID_REV_R4400) { 379 if ((c->processor_id & PRID_REV_MASK) >=
380 PRID_REV_R4400) {
365 c->cputype = CPU_R4400SC; 381 c->cputype = CPU_R4400SC;
366 __cpu_name[cpu] = "R4400SC"; 382 __cpu_name[cpu] = "R4400SC";
367 } else { 383 } else {
@@ -454,7 +470,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu)
454 __cpu_name[cpu] = "TX3927"; 470 __cpu_name[cpu] = "TX3927";
455 c->tlbsize = 64; 471 c->tlbsize = 64;
456 } else { 472 } else {
457 switch (c->processor_id & 0xff) { 473 switch (c->processor_id & PRID_REV_MASK) {
458 case PRID_REV_TX3912: 474 case PRID_REV_TX3912:
459 c->cputype = CPU_TX3912; 475 c->cputype = CPU_TX3912;
460 __cpu_name[cpu] = "TX3912"; 476 __cpu_name[cpu] = "TX3912";
@@ -640,7 +656,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu)
640static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) 656static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu)
641{ 657{
642 decode_configs(c); 658 decode_configs(c);
643 switch (c->processor_id & 0xff00) { 659 switch (c->processor_id & PRID_IMP_MASK) {
644 case PRID_IMP_4KC: 660 case PRID_IMP_4KC:
645 c->cputype = CPU_4KC; 661 c->cputype = CPU_4KC;
646 __cpu_name[cpu] = "MIPS 4Kc"; 662 __cpu_name[cpu] = "MIPS 4Kc";
@@ -711,7 +727,7 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu)
711static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu) 727static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu)
712{ 728{
713 decode_configs(c); 729 decode_configs(c);
714 switch (c->processor_id & 0xff00) { 730 switch (c->processor_id & PRID_IMP_MASK) {
715 case PRID_IMP_AU1_REV1: 731 case PRID_IMP_AU1_REV1:
716 case PRID_IMP_AU1_REV2: 732 case PRID_IMP_AU1_REV2:
717 c->cputype = CPU_ALCHEMY; 733 c->cputype = CPU_ALCHEMY;
@@ -730,7 +746,7 @@ static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu)
730 break; 746 break;
731 case 4: 747 case 4:
732 __cpu_name[cpu] = "Au1200"; 748 __cpu_name[cpu] = "Au1200";
733 if ((c->processor_id & 0xff) == 2) 749 if ((c->processor_id & PRID_REV_MASK) == 2)
734 __cpu_name[cpu] = "Au1250"; 750 __cpu_name[cpu] = "Au1250";
735 break; 751 break;
736 case 5: 752 case 5:
@@ -748,12 +764,12 @@ static inline void cpu_probe_sibyte(struct cpuinfo_mips *c, unsigned int cpu)
748{ 764{
749 decode_configs(c); 765 decode_configs(c);
750 766
751 switch (c->processor_id & 0xff00) { 767 switch (c->processor_id & PRID_IMP_MASK) {
752 case PRID_IMP_SB1: 768 case PRID_IMP_SB1:
753 c->cputype = CPU_SB1; 769 c->cputype = CPU_SB1;
754 __cpu_name[cpu] = "SiByte SB1"; 770 __cpu_name[cpu] = "SiByte SB1";
755 /* FPU in pass1 is known to have issues. */ 771 /* FPU in pass1 is known to have issues. */
756 if ((c->processor_id & 0xff) < 0x02) 772 if ((c->processor_id & PRID_REV_MASK) < 0x02)
757 c->options &= ~(MIPS_CPU_FPU | MIPS_CPU_32FPR); 773 c->options &= ~(MIPS_CPU_FPU | MIPS_CPU_32FPR);
758 break; 774 break;
759 case PRID_IMP_SB1A: 775 case PRID_IMP_SB1A:
@@ -766,7 +782,7 @@ static inline void cpu_probe_sibyte(struct cpuinfo_mips *c, unsigned int cpu)
766static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu) 782static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu)
767{ 783{
768 decode_configs(c); 784 decode_configs(c);
769 switch (c->processor_id & 0xff00) { 785 switch (c->processor_id & PRID_IMP_MASK) {
770 case PRID_IMP_SR71000: 786 case PRID_IMP_SR71000:
771 c->cputype = CPU_SR71000; 787 c->cputype = CPU_SR71000;
772 __cpu_name[cpu] = "Sandcraft SR71000"; 788 __cpu_name[cpu] = "Sandcraft SR71000";
@@ -779,7 +795,7 @@ static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu)
779static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu) 795static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu)
780{ 796{
781 decode_configs(c); 797 decode_configs(c);
782 switch (c->processor_id & 0xff00) { 798 switch (c->processor_id & PRID_IMP_MASK) {
783 case PRID_IMP_PR4450: 799 case PRID_IMP_PR4450:
784 c->cputype = CPU_PR4450; 800 c->cputype = CPU_PR4450;
785 __cpu_name[cpu] = "Philips PR4450"; 801 __cpu_name[cpu] = "Philips PR4450";
@@ -791,7 +807,7 @@ static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu)
791static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu) 807static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu)
792{ 808{
793 decode_configs(c); 809 decode_configs(c);
794 switch (c->processor_id & 0xff00) { 810 switch (c->processor_id & PRID_IMP_MASK) {
795 case PRID_IMP_BMIPS32_REV4: 811 case PRID_IMP_BMIPS32_REV4:
796 case PRID_IMP_BMIPS32_REV8: 812 case PRID_IMP_BMIPS32_REV8:
797 c->cputype = CPU_BMIPS32; 813 c->cputype = CPU_BMIPS32;
@@ -806,7 +822,7 @@ static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu)
806 set_elf_platform(cpu, "bmips3300"); 822 set_elf_platform(cpu, "bmips3300");
807 break; 823 break;
808 case PRID_IMP_BMIPS43XX: { 824 case PRID_IMP_BMIPS43XX: {
809 int rev = c->processor_id & 0xff; 825 int rev = c->processor_id & PRID_REV_MASK;
810 826
811 if (rev >= PRID_REV_BMIPS4380_LO && 827 if (rev >= PRID_REV_BMIPS4380_LO &&
812 rev <= PRID_REV_BMIPS4380_HI) { 828 rev <= PRID_REV_BMIPS4380_HI) {
@@ -832,7 +848,7 @@ static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu)
832static inline void cpu_probe_cavium(struct cpuinfo_mips *c, unsigned int cpu) 848static inline void cpu_probe_cavium(struct cpuinfo_mips *c, unsigned int cpu)
833{ 849{
834 decode_configs(c); 850 decode_configs(c);
835 switch (c->processor_id & 0xff00) { 851 switch (c->processor_id & PRID_IMP_MASK) {
836 case PRID_IMP_CAVIUM_CN38XX: 852 case PRID_IMP_CAVIUM_CN38XX:
837 case PRID_IMP_CAVIUM_CN31XX: 853 case PRID_IMP_CAVIUM_CN31XX:
838 case PRID_IMP_CAVIUM_CN30XX: 854 case PRID_IMP_CAVIUM_CN30XX:
@@ -875,7 +891,7 @@ static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu)
875 decode_configs(c); 891 decode_configs(c);
876 /* JZRISC does not implement the CP0 counter. */ 892 /* JZRISC does not implement the CP0 counter. */
877 c->options &= ~MIPS_CPU_COUNTER; 893 c->options &= ~MIPS_CPU_COUNTER;
878 switch (c->processor_id & 0xff00) { 894 switch (c->processor_id & PRID_IMP_MASK) {
879 case PRID_IMP_JZRISC: 895 case PRID_IMP_JZRISC:
880 c->cputype = CPU_JZRISC; 896 c->cputype = CPU_JZRISC;
881 __cpu_name[cpu] = "Ingenic JZRISC"; 897 __cpu_name[cpu] = "Ingenic JZRISC";
@@ -890,7 +906,7 @@ static inline void cpu_probe_netlogic(struct cpuinfo_mips *c, int cpu)
890{ 906{
891 decode_configs(c); 907 decode_configs(c);
892 908
893 if ((c->processor_id & 0xff00) == PRID_IMP_NETLOGIC_AU13XX) { 909 if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_NETLOGIC_AU13XX) {
894 c->cputype = CPU_ALCHEMY; 910 c->cputype = CPU_ALCHEMY;
895 __cpu_name[cpu] = "Au1300"; 911 __cpu_name[cpu] = "Au1300";
896 /* following stuff is not for Alchemy */ 912 /* following stuff is not for Alchemy */
@@ -905,7 +921,7 @@ static inline void cpu_probe_netlogic(struct cpuinfo_mips *c, int cpu)
905 MIPS_CPU_EJTAG | 921 MIPS_CPU_EJTAG |
906 MIPS_CPU_LLSC); 922 MIPS_CPU_LLSC);
907 923
908 switch (c->processor_id & 0xff00) { 924 switch (c->processor_id & PRID_IMP_MASK) {
909 case PRID_IMP_NETLOGIC_XLP2XX: 925 case PRID_IMP_NETLOGIC_XLP2XX:
910 c->cputype = CPU_XLP; 926 c->cputype = CPU_XLP;
911 __cpu_name[cpu] = "Broadcom XLPII"; 927 __cpu_name[cpu] = "Broadcom XLPII";
@@ -984,7 +1000,7 @@ void cpu_probe(void)
984 c->cputype = CPU_UNKNOWN; 1000 c->cputype = CPU_UNKNOWN;
985 1001
986 c->processor_id = read_c0_prid(); 1002 c->processor_id = read_c0_prid();
987 switch (c->processor_id & 0xff0000) { 1003 switch (c->processor_id & PRID_COMP_MASK) {
988 case PRID_COMP_LEGACY: 1004 case PRID_COMP_LEGACY:
989 cpu_probe_legacy(c, cpu); 1005 cpu_probe_legacy(c, cpu);
990 break; 1006 break;
diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
index 42f8875d2444..f7991d95bff9 100644
--- a/arch/mips/kernel/idle.c
+++ b/arch/mips/kernel/idle.c
@@ -18,6 +18,7 @@
18#include <linux/sched.h> 18#include <linux/sched.h>
19#include <asm/cpu.h> 19#include <asm/cpu.h>
20#include <asm/cpu-info.h> 20#include <asm/cpu-info.h>
21#include <asm/cpu-type.h>
21#include <asm/idle.h> 22#include <asm/idle.h>
22#include <asm/mipsregs.h> 23#include <asm/mipsregs.h>
23 24
@@ -136,7 +137,7 @@ void __init check_wait(void)
136 return; 137 return;
137 } 138 }
138 139
139 switch (c->cputype) { 140 switch (current_cpu_type()) {
140 case CPU_R3081: 141 case CPU_R3081:
141 case CPU_R3081E: 142 case CPU_R3081E:
142 cpu_wait = r3081_wait; 143 cpu_wait = r3081_wait;
diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c
index 364d26ae4215..dcb8e5d3bb8a 100644
--- a/arch/mips/kernel/time.c
+++ b/arch/mips/kernel/time.c
@@ -24,6 +24,7 @@
24#include <linux/export.h> 24#include <linux/export.h>
25 25
26#include <asm/cpu-features.h> 26#include <asm/cpu-features.h>
27#include <asm/cpu-type.h>
27#include <asm/div64.h> 28#include <asm/div64.h>
28#include <asm/smtc_ipi.h> 29#include <asm/smtc_ipi.h>
29#include <asm/time.h> 30#include <asm/time.h>
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index aec3408edd4b..524841f02803 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -39,6 +39,7 @@
39#include <asm/break.h> 39#include <asm/break.h>
40#include <asm/cop2.h> 40#include <asm/cop2.h>
41#include <asm/cpu.h> 41#include <asm/cpu.h>
42#include <asm/cpu-type.h>
42#include <asm/dsp.h> 43#include <asm/dsp.h>
43#include <asm/fpu.h> 44#include <asm/fpu.h>
44#include <asm/fpu_emulator.h> 45#include <asm/fpu_emulator.h>
@@ -622,7 +623,7 @@ static int simulate_rdhwr(struct pt_regs *regs, int rd, int rt)
622 regs->regs[rt] = read_c0_count(); 623 regs->regs[rt] = read_c0_count();
623 return 0; 624 return 0;
624 case 3: /* Count register resolution */ 625 case 3: /* Count register resolution */
625 switch (current_cpu_data.cputype) { 626 switch (current_cpu_type()) {
626 case CPU_20KC: 627 case CPU_20KC:
627 case CPU_25KF: 628 case CPU_25KF:
628 regs->regs[rt] = 1; 629 regs->regs[rt] = 1;
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 729e7702b1de..c8efdb5b6ee0 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -19,6 +19,7 @@
19#include <asm/bootinfo.h> 19#include <asm/bootinfo.h>
20#include <asm/cacheops.h> 20#include <asm/cacheops.h>
21#include <asm/cpu-features.h> 21#include <asm/cpu-features.h>
22#include <asm/cpu-type.h>
22#include <asm/page.h> 23#include <asm/page.h>
23#include <asm/pgtable.h> 24#include <asm/pgtable.h>
24#include <asm/r4kcache.h> 25#include <asm/r4kcache.h>
@@ -186,9 +187,10 @@ static void probe_octeon(void)
186 unsigned long dcache_size; 187 unsigned long dcache_size;
187 unsigned int config1; 188 unsigned int config1;
188 struct cpuinfo_mips *c = &current_cpu_data; 189 struct cpuinfo_mips *c = &current_cpu_data;
190 int cputype = current_cpu_type();
189 191
190 config1 = read_c0_config1(); 192 config1 = read_c0_config1();
191 switch (c->cputype) { 193 switch (cputype) {
192 case CPU_CAVIUM_OCTEON: 194 case CPU_CAVIUM_OCTEON:
193 case CPU_CAVIUM_OCTEON_PLUS: 195 case CPU_CAVIUM_OCTEON_PLUS:
194 c->icache.linesz = 2 << ((config1 >> 19) & 7); 196 c->icache.linesz = 2 << ((config1 >> 19) & 7);
@@ -199,7 +201,7 @@ static void probe_octeon(void)
199 c->icache.sets * c->icache.ways * c->icache.linesz; 201 c->icache.sets * c->icache.ways * c->icache.linesz;
200 c->icache.waybit = ffs(icache_size / c->icache.ways) - 1; 202 c->icache.waybit = ffs(icache_size / c->icache.ways) - 1;
201 c->dcache.linesz = 128; 203 c->dcache.linesz = 128;
202 if (c->cputype == CPU_CAVIUM_OCTEON_PLUS) 204 if (cputype == CPU_CAVIUM_OCTEON_PLUS)
203 c->dcache.sets = 2; /* CN5XXX has two Dcache sets */ 205 c->dcache.sets = 2; /* CN5XXX has two Dcache sets */
204 else 206 else
205 c->dcache.sets = 1; /* CN3XXX has one Dcache set */ 207 c->dcache.sets = 1; /* CN3XXX has one Dcache set */
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index f749f687ee87..627883bc6d5f 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -12,6 +12,7 @@
12#include <linux/highmem.h> 12#include <linux/highmem.h>
13#include <linux/kernel.h> 13#include <linux/kernel.h>
14#include <linux/linkage.h> 14#include <linux/linkage.h>
15#include <linux/preempt.h>
15#include <linux/sched.h> 16#include <linux/sched.h>
16#include <linux/smp.h> 17#include <linux/smp.h>
17#include <linux/mm.h> 18#include <linux/mm.h>
@@ -24,6 +25,7 @@
24#include <asm/cacheops.h> 25#include <asm/cacheops.h>
25#include <asm/cpu.h> 26#include <asm/cpu.h>
26#include <asm/cpu-features.h> 27#include <asm/cpu-features.h>
28#include <asm/cpu-type.h>
27#include <asm/io.h> 29#include <asm/io.h>
28#include <asm/page.h> 30#include <asm/page.h>
29#include <asm/pgtable.h> 31#include <asm/pgtable.h>
@@ -601,6 +603,7 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
601 /* Catch bad driver code */ 603 /* Catch bad driver code */
602 BUG_ON(size == 0); 604 BUG_ON(size == 0);
603 605
606 preempt_disable();
604 if (cpu_has_inclusive_pcaches) { 607 if (cpu_has_inclusive_pcaches) {
605 if (size >= scache_size) 608 if (size >= scache_size)
606 r4k_blast_scache(); 609 r4k_blast_scache();
@@ -621,6 +624,7 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
621 R4600_HIT_CACHEOP_WAR_IMPL; 624 R4600_HIT_CACHEOP_WAR_IMPL;
622 blast_dcache_range(addr, addr + size); 625 blast_dcache_range(addr, addr + size);
623 } 626 }
627 preempt_enable();
624 628
625 bc_wback_inv(addr, size); 629 bc_wback_inv(addr, size);
626 __sync(); 630 __sync();
@@ -631,6 +635,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
631 /* Catch bad driver code */ 635 /* Catch bad driver code */
632 BUG_ON(size == 0); 636 BUG_ON(size == 0);
633 637
638 preempt_disable();
634 if (cpu_has_inclusive_pcaches) { 639 if (cpu_has_inclusive_pcaches) {
635 if (size >= scache_size) 640 if (size >= scache_size)
636 r4k_blast_scache(); 641 r4k_blast_scache();
@@ -655,6 +660,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
655 R4600_HIT_CACHEOP_WAR_IMPL; 660 R4600_HIT_CACHEOP_WAR_IMPL;
656 blast_inv_dcache_range(addr, addr + size); 661 blast_inv_dcache_range(addr, addr + size);
657 } 662 }
663 preempt_enable();
658 664
659 bc_inv(addr, size); 665 bc_inv(addr, size);
660 __sync(); 666 __sync();
@@ -780,20 +786,30 @@ static inline void rm7k_erratum31(void)
780 786
781static inline void alias_74k_erratum(struct cpuinfo_mips *c) 787static inline void alias_74k_erratum(struct cpuinfo_mips *c)
782{ 788{
789 unsigned int imp = c->processor_id & PRID_IMP_MASK;
790 unsigned int rev = c->processor_id & PRID_REV_MASK;
791
783 /* 792 /*
784 * Early versions of the 74K do not update the cache tags on a 793 * Early versions of the 74K do not update the cache tags on a
785 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG 794 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
786 * aliases. In this case it is better to treat the cache as always 795 * aliases. In this case it is better to treat the cache as always
787 * having aliases. 796 * having aliases.
788 */ 797 */
789 if ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(2, 4, 0)) 798 switch (imp) {
790 c->dcache.flags |= MIPS_CACHE_VTAG; 799 case PRID_IMP_74K:
791 if ((c->processor_id & 0xff) == PRID_REV_ENCODE_332(2, 4, 0)) 800 if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
792 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 801 c->dcache.flags |= MIPS_CACHE_VTAG;
793 if (((c->processor_id & 0xff00) == PRID_IMP_1074K) && 802 if (rev == PRID_REV_ENCODE_332(2, 4, 0))
794 ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(1, 1, 0))) { 803 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
795 c->dcache.flags |= MIPS_CACHE_VTAG; 804 break;
796 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 805 case PRID_IMP_1074K:
806 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
807 c->dcache.flags |= MIPS_CACHE_VTAG;
808 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
809 }
810 break;
811 default:
812 BUG();
797 } 813 }
798} 814}
799 815
@@ -809,7 +825,7 @@ static void probe_pcache(void)
809 unsigned long config1; 825 unsigned long config1;
810 unsigned int lsize; 826 unsigned int lsize;
811 827
812 switch (c->cputype) { 828 switch (current_cpu_type()) {
813 case CPU_R4600: /* QED style two way caches? */ 829 case CPU_R4600: /* QED style two way caches? */
814 case CPU_R4700: 830 case CPU_R4700:
815 case CPU_R5000: 831 case CPU_R5000:
@@ -1025,7 +1041,8 @@ static void probe_pcache(void)
1025 * presumably no vendor is shipping his hardware in the "bad" 1041 * presumably no vendor is shipping his hardware in the "bad"
1026 * configuration. 1042 * configuration.
1027 */ 1043 */
1028 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 1044 if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1045 (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
1029 !(config & CONF_SC) && c->icache.linesz != 16 && 1046 !(config & CONF_SC) && c->icache.linesz != 16 &&
1030 PAGE_SIZE <= 0x8000) 1047 PAGE_SIZE <= 0x8000)
1031 panic("Improper R4000SC processor configuration detected"); 1048 panic("Improper R4000SC processor configuration detected");
@@ -1045,7 +1062,7 @@ static void probe_pcache(void)
1045 * normally they'd suffer from aliases but magic in the hardware deals 1062 * normally they'd suffer from aliases but magic in the hardware deals
1046 * with that for us so we don't need to take care ourselves. 1063 * with that for us so we don't need to take care ourselves.
1047 */ 1064 */
1048 switch (c->cputype) { 1065 switch (current_cpu_type()) {
1049 case CPU_20KC: 1066 case CPU_20KC:
1050 case CPU_25KF: 1067 case CPU_25KF:
1051 case CPU_SB1: 1068 case CPU_SB1:
@@ -1065,7 +1082,7 @@ static void probe_pcache(void)
1065 case CPU_34K: 1082 case CPU_34K:
1066 case CPU_74K: 1083 case CPU_74K:
1067 case CPU_1004K: 1084 case CPU_1004K:
1068 if (c->cputype == CPU_74K) 1085 if (current_cpu_type() == CPU_74K)
1069 alias_74k_erratum(c); 1086 alias_74k_erratum(c);
1070 if ((read_c0_config7() & (1 << 16))) { 1087 if ((read_c0_config7() & (1 << 16))) {
1071 /* effectively physically indexed dcache, 1088 /* effectively physically indexed dcache,
@@ -1078,7 +1095,7 @@ static void probe_pcache(void)
1078 c->dcache.flags |= MIPS_CACHE_ALIASES; 1095 c->dcache.flags |= MIPS_CACHE_ALIASES;
1079 } 1096 }
1080 1097
1081 switch (c->cputype) { 1098 switch (current_cpu_type()) {
1082 case CPU_20KC: 1099 case CPU_20KC:
1083 /* 1100 /*
1084 * Some older 20Kc chips doesn't have the 'VI' bit in 1101 * Some older 20Kc chips doesn't have the 'VI' bit in
@@ -1207,7 +1224,7 @@ static void setup_scache(void)
1207 * processors don't have a S-cache that would be relevant to the 1224 * processors don't have a S-cache that would be relevant to the
1208 * Linux memory management. 1225 * Linux memory management.
1209 */ 1226 */
1210 switch (c->cputype) { 1227 switch (current_cpu_type()) {
1211 case CPU_R4000SC: 1228 case CPU_R4000SC:
1212 case CPU_R4000MC: 1229 case CPU_R4000MC:
1213 case CPU_R4400SC: 1230 case CPU_R4400SC:
@@ -1384,9 +1401,8 @@ static void r4k_cache_error_setup(void)
1384{ 1401{
1385 extern char __weak except_vec2_generic; 1402 extern char __weak except_vec2_generic;
1386 extern char __weak except_vec2_sb1; 1403 extern char __weak except_vec2_sb1;
1387 struct cpuinfo_mips *c = &current_cpu_data;
1388 1404
1389 switch (c->cputype) { 1405 switch (current_cpu_type()) {
1390 case CPU_SB1: 1406 case CPU_SB1:
1391 case CPU_SB1A: 1407 case CPU_SB1A:
1392 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1408 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index 664e523653d0..5f8b95512580 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -18,6 +18,7 @@
18#include <linux/highmem.h> 18#include <linux/highmem.h>
19 19
20#include <asm/cache.h> 20#include <asm/cache.h>
21#include <asm/cpu-type.h>
21#include <asm/io.h> 22#include <asm/io.h>
22 23
23#include <dma-coherence.h> 24#include <dma-coherence.h>
@@ -307,12 +308,10 @@ static void mips_dma_sync_sg_for_cpu(struct device *dev,
307{ 308{
308 int i; 309 int i;
309 310
310 /* Make sure that gcc doesn't leave the empty loop body. */ 311 if (cpu_needs_post_dma_flush(dev))
311 for (i = 0; i < nelems; i++, sg++) { 312 for (i = 0; i < nelems; i++, sg++)
312 if (cpu_needs_post_dma_flush(dev))
313 __dma_sync(sg_page(sg), sg->offset, sg->length, 313 __dma_sync(sg_page(sg), sg->offset, sg->length,
314 direction); 314 direction);
315 }
316} 315}
317 316
318static void mips_dma_sync_sg_for_device(struct device *dev, 317static void mips_dma_sync_sg_for_device(struct device *dev,
@@ -320,12 +319,10 @@ static void mips_dma_sync_sg_for_device(struct device *dev,
320{ 319{
321 int i; 320 int i;
322 321
323 /* Make sure that gcc doesn't leave the empty loop body. */ 322 if (!plat_device_is_coherent(dev))
324 for (i = 0; i < nelems; i++, sg++) { 323 for (i = 0; i < nelems; i++, sg++)
325 if (!plat_device_is_coherent(dev))
326 __dma_sync(sg_page(sg), sg->offset, sg->length, 324 __dma_sync(sg_page(sg), sg->offset, sg->length,
327 direction); 325 direction);
328 }
329} 326}
330 327
331int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 328int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
index 218c2109a55d..cbd81d17793a 100644
--- a/arch/mips/mm/page.c
+++ b/arch/mips/mm/page.c
@@ -18,6 +18,7 @@
18 18
19#include <asm/bugs.h> 19#include <asm/bugs.h>
20#include <asm/cacheops.h> 20#include <asm/cacheops.h>
21#include <asm/cpu-type.h>
21#include <asm/inst.h> 22#include <asm/inst.h>
22#include <asm/io.h> 23#include <asm/io.h>
23#include <asm/page.h> 24#include <asm/page.h>
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
index 5d01392e3518..08d05aee8788 100644
--- a/arch/mips/mm/sc-mips.c
+++ b/arch/mips/mm/sc-mips.c
@@ -6,6 +6,7 @@
6#include <linux/sched.h> 6#include <linux/sched.h>
7#include <linux/mm.h> 7#include <linux/mm.h>
8 8
9#include <asm/cpu-type.h>
9#include <asm/mipsregs.h> 10#include <asm/mipsregs.h>
10#include <asm/bcache.h> 11#include <asm/bcache.h>
11#include <asm/cacheops.h> 12#include <asm/cacheops.h>
@@ -71,7 +72,7 @@ static inline int mips_sc_is_activated(struct cpuinfo_mips *c)
71 unsigned int tmp; 72 unsigned int tmp;
72 73
73 /* Check the bypass bit (L2B) */ 74 /* Check the bypass bit (L2B) */
74 switch (c->cputype) { 75 switch (current_cpu_type()) {
75 case CPU_34K: 76 case CPU_34K:
76 case CPU_74K: 77 case CPU_74K:
77 case CPU_1004K: 78 case CPU_1004K:
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index 00b26a67a06d..bb3a5f643e97 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -16,6 +16,7 @@
16#include <linux/module.h> 16#include <linux/module.h>
17 17
18#include <asm/cpu.h> 18#include <asm/cpu.h>
19#include <asm/cpu-type.h>
19#include <asm/bootinfo.h> 20#include <asm/bootinfo.h>
20#include <asm/mmu_context.h> 21#include <asm/mmu_context.h>
21#include <asm/pgtable.h> 22#include <asm/pgtable.h>
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 821b45175dc1..9bb3a9363b06 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -30,6 +30,7 @@
30#include <linux/cache.h> 30#include <linux/cache.h>
31 31
32#include <asm/cacheflush.h> 32#include <asm/cacheflush.h>
33#include <asm/cpu-type.h>
33#include <asm/pgtable.h> 34#include <asm/pgtable.h>
34#include <asm/war.h> 35#include <asm/war.h>
35#include <asm/uasm.h> 36#include <asm/uasm.h>
diff --git a/arch/mips/mti-malta/malta-time.c b/arch/mips/mti-malta/malta-time.c
index 53aad4a35375..a18af5fce67e 100644
--- a/arch/mips/mti-malta/malta-time.c
+++ b/arch/mips/mti-malta/malta-time.c
@@ -27,6 +27,7 @@
27#include <linux/timex.h> 27#include <linux/timex.h>
28#include <linux/mc146818rtc.h> 28#include <linux/mc146818rtc.h>
29 29
30#include <asm/cpu.h>
30#include <asm/mipsregs.h> 31#include <asm/mipsregs.h>
31#include <asm/mipsmtregs.h> 32#include <asm/mipsmtregs.h>
32#include <asm/hardirq.h> 33#include <asm/hardirq.h>
@@ -76,7 +77,7 @@ static void __init estimate_frequencies(void)
76#endif 77#endif
77 78
78#if defined (CONFIG_KVM_GUEST) && defined (CONFIG_KVM_HOST_FREQ) 79#if defined (CONFIG_KVM_GUEST) && defined (CONFIG_KVM_HOST_FREQ)
79 unsigned int prid = read_c0_prid() & 0xffff00; 80 unsigned int prid = read_c0_prid() & (PRID_COMP_MASK | PRID_IMP_MASK);
80 81
81 /* 82 /*
82 * XXXKYMA: hardwire the CPU frequency to Host Freq/4 83 * XXXKYMA: hardwire the CPU frequency to Host Freq/4
@@ -169,7 +170,7 @@ unsigned int get_c0_compare_int(void)
169 170
170void __init plat_time_init(void) 171void __init plat_time_init(void)
171{ 172{
172 unsigned int prid = read_c0_prid() & 0xffff00; 173 unsigned int prid = read_c0_prid() & (PRID_COMP_MASK | PRID_IMP_MASK);
173 unsigned int freq; 174 unsigned int freq;
174 175
175 estimate_frequencies(); 176 estimate_frequencies();
diff --git a/arch/mips/mti-sead3/sead3-time.c b/arch/mips/mti-sead3/sead3-time.c
index a43ea3cc0a3b..552d26c34386 100644
--- a/arch/mips/mti-sead3/sead3-time.c
+++ b/arch/mips/mti-sead3/sead3-time.c
@@ -7,6 +7,7 @@
7 */ 7 */
8#include <linux/init.h> 8#include <linux/init.h>
9 9
10#include <asm/cpu.h>
10#include <asm/setup.h> 11#include <asm/setup.h>
11#include <asm/time.h> 12#include <asm/time.h>
12#include <asm/irq.h> 13#include <asm/irq.h>
@@ -34,7 +35,7 @@ static void __iomem *status_reg = (void __iomem *)0xbf000410;
34 */ 35 */
35static unsigned int __init estimate_cpu_frequency(void) 36static unsigned int __init estimate_cpu_frequency(void)
36{ 37{
37 unsigned int prid = read_c0_prid() & 0xffff00; 38 unsigned int prid = read_c0_prid() & (PRID_COMP_MASK | PRID_IMP_MASK);
38 unsigned int tick = 0; 39 unsigned int tick = 0;
39 unsigned int freq; 40 unsigned int freq;
40 unsigned int orig; 41 unsigned int orig;
diff --git a/arch/mips/netlogic/xlr/fmn-config.c b/arch/mips/netlogic/xlr/fmn-config.c
index ed3bf0e3f309..c7622c6e5f67 100644
--- a/arch/mips/netlogic/xlr/fmn-config.c
+++ b/arch/mips/netlogic/xlr/fmn-config.c
@@ -36,6 +36,7 @@
36#include <linux/irq.h> 36#include <linux/irq.h>
37#include <linux/interrupt.h> 37#include <linux/interrupt.h>
38 38
39#include <asm/cpu.h>
39#include <asm/mipsregs.h> 40#include <asm/mipsregs.h>
40#include <asm/netlogic/xlr/fmn.h> 41#include <asm/netlogic/xlr/fmn.h>
41#include <asm/netlogic/xlr/xlr.h> 42#include <asm/netlogic/xlr/xlr.h>
@@ -187,7 +188,7 @@ void xlr_board_info_setup(void)
187 int processor_id, num_core; 188 int processor_id, num_core;
188 189
189 num_core = hweight32(nlm_current_node()->coremask); 190 num_core = hweight32(nlm_current_node()->coremask);
190 processor_id = read_c0_prid() & 0xff00; 191 processor_id = read_c0_prid() & PRID_IMP_MASK;
191 192
192 setup_cpu_fmninfo(cpu, num_core); 193 setup_cpu_fmninfo(cpu, num_core);
193 switch (processor_id) { 194 switch (processor_id) {
diff --git a/arch/mips/oprofile/common.c b/arch/mips/oprofile/common.c
index 5e5424753b56..4d1736fc1955 100644
--- a/arch/mips/oprofile/common.c
+++ b/arch/mips/oprofile/common.c
@@ -12,6 +12,7 @@
12#include <linux/oprofile.h> 12#include <linux/oprofile.h>
13#include <linux/smp.h> 13#include <linux/smp.h>
14#include <asm/cpu-info.h> 14#include <asm/cpu-info.h>
15#include <asm/cpu-type.h>
15 16
16#include "op_impl.h" 17#include "op_impl.h"
17 18
diff --git a/arch/mips/pci/pci-bcm1480.c b/arch/mips/pci/pci-bcm1480.c
index 44dd5aa2e36f..5ec2a7bae02c 100644
--- a/arch/mips/pci/pci-bcm1480.c
+++ b/arch/mips/pci/pci-bcm1480.c
@@ -39,6 +39,7 @@
39#include <linux/mm.h> 39#include <linux/mm.h>
40#include <linux/console.h> 40#include <linux/console.h>
41#include <linux/tty.h> 41#include <linux/tty.h>
42#include <linux/vt.h>
42 43
43#include <asm/sibyte/bcm1480_regs.h> 44#include <asm/sibyte/bcm1480_regs.h>
44#include <asm/sibyte/bcm1480_scd.h> 45#include <asm/sibyte/bcm1480_scd.h>
diff --git a/arch/mips/sibyte/bcm1480/setup.c b/arch/mips/sibyte/bcm1480/setup.c
index 05ed92c92b69..8e2e04f77870 100644
--- a/arch/mips/sibyte/bcm1480/setup.c
+++ b/arch/mips/sibyte/bcm1480/setup.c
@@ -22,6 +22,7 @@
22#include <linux/string.h> 22#include <linux/string.h>
23 23
24#include <asm/bootinfo.h> 24#include <asm/bootinfo.h>
25#include <asm/cpu.h>
25#include <asm/mipsregs.h> 26#include <asm/mipsregs.h>
26#include <asm/io.h> 27#include <asm/io.h>
27#include <asm/sibyte/sb1250.h> 28#include <asm/sibyte/sb1250.h>
@@ -119,7 +120,7 @@ void __init bcm1480_setup(void)
119 uint64_t sys_rev; 120 uint64_t sys_rev;
120 int plldiv; 121 int plldiv;
121 122
122 sb1_pass = read_c0_prid() & 0xff; 123 sb1_pass = read_c0_prid() & PRID_REV_MASK;
123 sys_rev = __raw_readq(IOADDR(A_SCD_SYSTEM_REVISION)); 124 sys_rev = __raw_readq(IOADDR(A_SCD_SYSTEM_REVISION));
124 soc_type = SYS_SOC_TYPE(sys_rev); 125 soc_type = SYS_SOC_TYPE(sys_rev);
125 part_type = G_SYS_PART(sys_rev); 126 part_type = G_SYS_PART(sys_rev);
diff --git a/arch/mips/sibyte/sb1250/setup.c b/arch/mips/sibyte/sb1250/setup.c
index a14bd4cb0bc0..3c02b2a77ae9 100644
--- a/arch/mips/sibyte/sb1250/setup.c
+++ b/arch/mips/sibyte/sb1250/setup.c
@@ -22,6 +22,7 @@
22#include <linux/string.h> 22#include <linux/string.h>
23 23
24#include <asm/bootinfo.h> 24#include <asm/bootinfo.h>
25#include <asm/cpu.h>
25#include <asm/mipsregs.h> 26#include <asm/mipsregs.h>
26#include <asm/io.h> 27#include <asm/io.h>
27#include <asm/sibyte/sb1250.h> 28#include <asm/sibyte/sb1250.h>
@@ -182,7 +183,7 @@ void __init sb1250_setup(void)
182 int plldiv; 183 int plldiv;
183 int bad_config = 0; 184 int bad_config = 0;
184 185
185 sb1_pass = read_c0_prid() & 0xff; 186 sb1_pass = read_c0_prid() & PRID_REV_MASK;
186 sys_rev = __raw_readq(IOADDR(A_SCD_SYSTEM_REVISION)); 187 sys_rev = __raw_readq(IOADDR(A_SCD_SYSTEM_REVISION));
187 soc_type = SYS_SOC_TYPE(sys_rev); 188 soc_type = SYS_SOC_TYPE(sys_rev);
188 soc_pass = G_SYS_REVISION(sys_rev); 189 soc_pass = G_SYS_REVISION(sys_rev);
diff --git a/arch/mips/sni/setup.c b/arch/mips/sni/setup.c
index 5b09b3544edd..efad85c8c823 100644
--- a/arch/mips/sni/setup.c
+++ b/arch/mips/sni/setup.c
@@ -25,6 +25,7 @@
25#endif 25#endif
26 26
27#include <asm/bootinfo.h> 27#include <asm/bootinfo.h>
28#include <asm/cpu.h>
28#include <asm/io.h> 29#include <asm/io.h>
29#include <asm/reboot.h> 30#include <asm/reboot.h>
30#include <asm/sni.h> 31#include <asm/sni.h>
@@ -173,7 +174,7 @@ void __init plat_mem_setup(void)
173 system_type = "RM300-Cxx"; 174 system_type = "RM300-Cxx";
174 break; 175 break;
175 case SNI_BRD_PCI_DESKTOP: 176 case SNI_BRD_PCI_DESKTOP:
176 switch (read_c0_prid() & 0xff00) { 177 switch (read_c0_prid() & PRID_IMP_MASK) {
177 case PRID_IMP_R4600: 178 case PRID_IMP_R4600:
178 case PRID_IMP_R4700: 179 case PRID_IMP_R4700:
179 system_type = "RM200-C20"; 180 system_type = "RM200-C20";
diff --git a/arch/openrisc/include/asm/prom.h b/arch/openrisc/include/asm/prom.h
index eb59bfe23e85..93c9980e1b6b 100644
--- a/arch/openrisc/include/asm/prom.h
+++ b/arch/openrisc/include/asm/prom.h
@@ -14,53 +14,9 @@
14 * the Free Software Foundation; either version 2 of the License, or 14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version. 15 * (at your option) any later version.
16 */ 16 */
17
18#include <linux/of.h> /* linux/of.h gets to determine #include ordering */
19
20#ifndef _ASM_OPENRISC_PROM_H 17#ifndef _ASM_OPENRISC_PROM_H
21#define _ASM_OPENRISC_PROM_H 18#define _ASM_OPENRISC_PROM_H
22#ifdef __KERNEL__
23#ifndef __ASSEMBLY__
24 19
25#include <linux/types.h>
26#include <asm/irq.h>
27#include <linux/irqdomain.h>
28#include <linux/atomic.h>
29#include <linux/of_irq.h>
30#include <linux/of_fdt.h>
31#include <linux/of_address.h>
32#include <linux/proc_fs.h>
33#include <linux/platform_device.h>
34#define HAVE_ARCH_DEVTREE_FIXUPS 20#define HAVE_ARCH_DEVTREE_FIXUPS
35 21
36/* Other Prototypes */
37extern int early_uartlite_console(void);
38
39/* Parse the ibm,dma-window property of an OF node into the busno, phys and
40 * size parameters.
41 */
42void of_parse_dma_window(struct device_node *dn, const void *dma_window_prop,
43 unsigned long *busno, unsigned long *phys, unsigned long *size);
44
45extern void kdump_move_device_tree(void);
46
47/* Get the MAC address */
48extern const void *of_get_mac_address(struct device_node *np);
49
50/**
51 * of_irq_map_pci - Resolve the interrupt for a PCI device
52 * @pdev: the device whose interrupt is to be resolved
53 * @out_irq: structure of_irq filled by this function
54 *
55 * This function resolves the PCI interrupt for a given PCI device. If a
56 * device-node exists for a given pci_dev, it will use normal OF tree
57 * walking. If not, it will implement standard swizzling and walk up the
58 * PCI tree until an device-node is found, at which point it will finish
59 * resolving using the OF tree walking.
60 */
61struct pci_dev;
62extern int of_irq_map_pci(struct pci_dev *pdev, struct of_irq *out_irq);
63
64#endif /* __ASSEMBLY__ */
65#endif /* __KERNEL__ */
66#endif /* _ASM_OPENRISC_PROM_H */ 22#endif /* _ASM_OPENRISC_PROM_H */
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 6a15c968d214..15ca2255f438 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -74,7 +74,7 @@ src-wlib-$(CONFIG_8xx) += mpc8xx.c planetcore.c
74src-wlib-$(CONFIG_PPC_82xx) += pq2.c fsl-soc.c planetcore.c 74src-wlib-$(CONFIG_PPC_82xx) += pq2.c fsl-soc.c planetcore.c
75src-wlib-$(CONFIG_EMBEDDED6xx) += mv64x60.c mv64x60_i2c.c ugecon.c 75src-wlib-$(CONFIG_EMBEDDED6xx) += mv64x60.c mv64x60_i2c.c ugecon.c
76 76
77src-plat-y := of.c 77src-plat-y := of.c epapr.c
78src-plat-$(CONFIG_40x) += fixed-head.S ep405.c cuboot-hotfoot.c \ 78src-plat-$(CONFIG_40x) += fixed-head.S ep405.c cuboot-hotfoot.c \
79 treeboot-walnut.c cuboot-acadia.c \ 79 treeboot-walnut.c cuboot-acadia.c \
80 cuboot-kilauea.c simpleboot.c \ 80 cuboot-kilauea.c simpleboot.c \
@@ -97,7 +97,7 @@ src-plat-$(CONFIG_EMBEDDED6xx) += cuboot-pq2.c cuboot-mpc7448hpc2.c \
97 prpmc2800.c 97 prpmc2800.c
98src-plat-$(CONFIG_AMIGAONE) += cuboot-amigaone.c 98src-plat-$(CONFIG_AMIGAONE) += cuboot-amigaone.c
99src-plat-$(CONFIG_PPC_PS3) += ps3-head.S ps3-hvcall.S ps3.c 99src-plat-$(CONFIG_PPC_PS3) += ps3-head.S ps3-hvcall.S ps3.c
100src-plat-$(CONFIG_EPAPR_BOOT) += epapr.c 100src-plat-$(CONFIG_EPAPR_BOOT) += epapr.c epapr-wrapper.c
101 101
102src-wlib := $(sort $(src-wlib-y)) 102src-wlib := $(sort $(src-wlib-y))
103src-plat := $(sort $(src-plat-y)) 103src-plat := $(sort $(src-plat-y))
diff --git a/arch/powerpc/boot/epapr-wrapper.c b/arch/powerpc/boot/epapr-wrapper.c
new file mode 100644
index 000000000000..c10191006673
--- /dev/null
+++ b/arch/powerpc/boot/epapr-wrapper.c
@@ -0,0 +1,9 @@
1extern void epapr_platform_init(unsigned long r3, unsigned long r4,
2 unsigned long r5, unsigned long r6,
3 unsigned long r7);
4
5void platform_init(unsigned long r3, unsigned long r4, unsigned long r5,
6 unsigned long r6, unsigned long r7)
7{
8 epapr_platform_init(r3, r4, r5, r6, r7);
9}
diff --git a/arch/powerpc/boot/epapr.c b/arch/powerpc/boot/epapr.c
index 06c1961bd124..02e91aa2194a 100644
--- a/arch/powerpc/boot/epapr.c
+++ b/arch/powerpc/boot/epapr.c
@@ -48,8 +48,8 @@ static void platform_fixups(void)
48 fdt_addr, fdt_totalsize((void *)fdt_addr), ima_size); 48 fdt_addr, fdt_totalsize((void *)fdt_addr), ima_size);
49} 49}
50 50
51void platform_init(unsigned long r3, unsigned long r4, unsigned long r5, 51void epapr_platform_init(unsigned long r3, unsigned long r4, unsigned long r5,
52 unsigned long r6, unsigned long r7) 52 unsigned long r6, unsigned long r7)
53{ 53{
54 epapr_magic = r6; 54 epapr_magic = r6;
55 ima_size = r7; 55 ima_size = r7;
diff --git a/arch/powerpc/boot/of.c b/arch/powerpc/boot/of.c
index 61d9899aa0d0..62e2f43ec1df 100644
--- a/arch/powerpc/boot/of.c
+++ b/arch/powerpc/boot/of.c
@@ -26,6 +26,9 @@
26 26
27static unsigned long claim_base; 27static unsigned long claim_base;
28 28
29void epapr_platform_init(unsigned long r3, unsigned long r4, unsigned long r5,
30 unsigned long r6, unsigned long r7);
31
29static void *of_try_claim(unsigned long size) 32static void *of_try_claim(unsigned long size)
30{ 33{
31 unsigned long addr = 0; 34 unsigned long addr = 0;
@@ -61,7 +64,7 @@ static void of_image_hdr(const void *hdr)
61 } 64 }
62} 65}
63 66
64void platform_init(unsigned long a1, unsigned long a2, void *promptr) 67static void of_platform_init(unsigned long a1, unsigned long a2, void *promptr)
65{ 68{
66 platform_ops.image_hdr = of_image_hdr; 69 platform_ops.image_hdr = of_image_hdr;
67 platform_ops.malloc = of_try_claim; 70 platform_ops.malloc = of_try_claim;
@@ -81,3 +84,14 @@ void platform_init(unsigned long a1, unsigned long a2, void *promptr)
81 loader_info.initrd_size = a2; 84 loader_info.initrd_size = a2;
82 } 85 }
83} 86}
87
88void platform_init(unsigned long r3, unsigned long r4, unsigned long r5,
89 unsigned long r6, unsigned long r7)
90{
91 /* Detect OF vs. ePAPR boot */
92 if (r5)
93 of_platform_init(r3, r4, (void *)r5);
94 else
95 epapr_platform_init(r3, r4, r5, r6, r7);
96}
97
diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper
index 6761c746048d..cd7af841ba05 100755
--- a/arch/powerpc/boot/wrapper
+++ b/arch/powerpc/boot/wrapper
@@ -148,18 +148,18 @@ make_space=y
148 148
149case "$platform" in 149case "$platform" in
150pseries) 150pseries)
151 platformo=$object/of.o 151 platformo="$object/of.o $object/epapr.o"
152 link_address='0x4000000' 152 link_address='0x4000000'
153 ;; 153 ;;
154maple) 154maple)
155 platformo=$object/of.o 155 platformo="$object/of.o $object/epapr.o"
156 link_address='0x400000' 156 link_address='0x400000'
157 ;; 157 ;;
158pmac|chrp) 158pmac|chrp)
159 platformo=$object/of.o 159 platformo="$object/of.o $object/epapr.o"
160 ;; 160 ;;
161coff) 161coff)
162 platformo="$object/crt0.o $object/of.o" 162 platformo="$object/crt0.o $object/of.o $object/epapr.o"
163 lds=$object/zImage.coff.lds 163 lds=$object/zImage.coff.lds
164 link_address='0x500000' 164 link_address='0x500000'
165 pie= 165 pie=
@@ -253,6 +253,7 @@ treeboot-iss4xx-mpic)
253 platformo="$object/treeboot-iss4xx.o" 253 platformo="$object/treeboot-iss4xx.o"
254 ;; 254 ;;
255epapr) 255epapr)
256 platformo="$object/epapr.o $object/epapr-wrapper.o"
256 link_address='0x20000000' 257 link_address='0x20000000'
257 pie=-pie 258 pie=-pie
258 ;; 259 ;;
diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h
index 0e40843a1c6e..41f13cec8a8f 100644
--- a/arch/powerpc/include/asm/irq.h
+++ b/arch/powerpc/include/asm/irq.h
@@ -69,9 +69,9 @@ extern struct thread_info *softirq_ctx[NR_CPUS];
69 69
70extern void irq_ctx_init(void); 70extern void irq_ctx_init(void);
71extern void call_do_softirq(struct thread_info *tp); 71extern void call_do_softirq(struct thread_info *tp);
72extern int call_handle_irq(int irq, void *p1, 72extern void call_do_irq(struct pt_regs *regs, struct thread_info *tp);
73 struct thread_info *tp, void *func);
74extern void do_IRQ(struct pt_regs *regs); 73extern void do_IRQ(struct pt_regs *regs);
74extern void __do_irq(struct pt_regs *regs);
75 75
76int irq_choose_cpu(const struct cpumask *mask); 76int irq_choose_cpu(const struct cpumask *mask);
77 77
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index e378cccfca55..ce4de5aed7b5 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -149,8 +149,6 @@ typedef struct {
149 149
150struct thread_struct { 150struct thread_struct {
151 unsigned long ksp; /* Kernel stack pointer */ 151 unsigned long ksp; /* Kernel stack pointer */
152 unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */
153
154#ifdef CONFIG_PPC64 152#ifdef CONFIG_PPC64
155 unsigned long ksp_vsid; 153 unsigned long ksp_vsid;
156#endif 154#endif
@@ -162,6 +160,7 @@ struct thread_struct {
162#endif 160#endif
163#ifdef CONFIG_PPC32 161#ifdef CONFIG_PPC32
164 void *pgdir; /* root of page-table tree */ 162 void *pgdir; /* root of page-table tree */
163 unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */
165#endif 164#endif
166#ifdef CONFIG_PPC_ADV_DEBUG_REGS 165#ifdef CONFIG_PPC_ADV_DEBUG_REGS
167 /* 166 /*
@@ -321,7 +320,6 @@ struct thread_struct {
321#else 320#else
322#define INIT_THREAD { \ 321#define INIT_THREAD { \
323 .ksp = INIT_SP, \ 322 .ksp = INIT_SP, \
324 .ksp_limit = INIT_SP_LIMIT, \
325 .regs = (struct pt_regs *)INIT_SP - 1, /* XXX bogus, I think */ \ 323 .regs = (struct pt_regs *)INIT_SP - 1, /* XXX bogus, I think */ \
326 .fs = KERNEL_DS, \ 324 .fs = KERNEL_DS, \
327 .fpr = {{0}}, \ 325 .fpr = {{0}}, \
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index d8958be5f31a..502c7a4e73f7 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -80,10 +80,11 @@ int main(void)
80 DEFINE(TASKTHREADPPR, offsetof(struct task_struct, thread.ppr)); 80 DEFINE(TASKTHREADPPR, offsetof(struct task_struct, thread.ppr));
81#else 81#else
82 DEFINE(THREAD_INFO, offsetof(struct task_struct, stack)); 82 DEFINE(THREAD_INFO, offsetof(struct task_struct, stack));
83 DEFINE(THREAD_INFO_GAP, _ALIGN_UP(sizeof(struct thread_info), 16));
84 DEFINE(KSP_LIMIT, offsetof(struct thread_struct, ksp_limit));
83#endif /* CONFIG_PPC64 */ 85#endif /* CONFIG_PPC64 */
84 86
85 DEFINE(KSP, offsetof(struct thread_struct, ksp)); 87 DEFINE(KSP, offsetof(struct thread_struct, ksp));
86 DEFINE(KSP_LIMIT, offsetof(struct thread_struct, ksp_limit));
87 DEFINE(PT_REGS, offsetof(struct thread_struct, regs)); 88 DEFINE(PT_REGS, offsetof(struct thread_struct, regs));
88#ifdef CONFIG_BOOKE 89#ifdef CONFIG_BOOKE
89 DEFINE(THREAD_NORMSAVES, offsetof(struct thread_struct, normsave[0])); 90 DEFINE(THREAD_NORMSAVES, offsetof(struct thread_struct, normsave[0]));
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index c69440cef7af..57d286a78f86 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -441,50 +441,6 @@ void migrate_irqs(void)
441} 441}
442#endif 442#endif
443 443
444static inline void handle_one_irq(unsigned int irq)
445{
446 struct thread_info *curtp, *irqtp;
447 unsigned long saved_sp_limit;
448 struct irq_desc *desc;
449
450 desc = irq_to_desc(irq);
451 if (!desc)
452 return;
453
454 /* Switch to the irq stack to handle this */
455 curtp = current_thread_info();
456 irqtp = hardirq_ctx[smp_processor_id()];
457
458 if (curtp == irqtp) {
459 /* We're already on the irq stack, just handle it */
460 desc->handle_irq(irq, desc);
461 return;
462 }
463
464 saved_sp_limit = current->thread.ksp_limit;
465
466 irqtp->task = curtp->task;
467 irqtp->flags = 0;
468
469 /* Copy the softirq bits in preempt_count so that the
470 * softirq checks work in the hardirq context. */
471 irqtp->preempt_count = (irqtp->preempt_count & ~SOFTIRQ_MASK) |
472 (curtp->preempt_count & SOFTIRQ_MASK);
473
474 current->thread.ksp_limit = (unsigned long)irqtp +
475 _ALIGN_UP(sizeof(struct thread_info), 16);
476
477 call_handle_irq(irq, desc, irqtp, desc->handle_irq);
478 current->thread.ksp_limit = saved_sp_limit;
479 irqtp->task = NULL;
480
481 /* Set any flag that may have been set on the
482 * alternate stack
483 */
484 if (irqtp->flags)
485 set_bits(irqtp->flags, &curtp->flags);
486}
487
488static inline void check_stack_overflow(void) 444static inline void check_stack_overflow(void)
489{ 445{
490#ifdef CONFIG_DEBUG_STACKOVERFLOW 446#ifdef CONFIG_DEBUG_STACKOVERFLOW
@@ -501,9 +457,9 @@ static inline void check_stack_overflow(void)
501#endif 457#endif
502} 458}
503 459
504void do_IRQ(struct pt_regs *regs) 460void __do_irq(struct pt_regs *regs)
505{ 461{
506 struct pt_regs *old_regs = set_irq_regs(regs); 462 struct irq_desc *desc;
507 unsigned int irq; 463 unsigned int irq;
508 464
509 irq_enter(); 465 irq_enter();
@@ -519,18 +475,56 @@ void do_IRQ(struct pt_regs *regs)
519 */ 475 */
520 irq = ppc_md.get_irq(); 476 irq = ppc_md.get_irq();
521 477
522 /* We can hard enable interrupts now */ 478 /* We can hard enable interrupts now to allow perf interrupts */
523 may_hard_irq_enable(); 479 may_hard_irq_enable();
524 480
525 /* And finally process it */ 481 /* And finally process it */
526 if (irq != NO_IRQ) 482 if (unlikely(irq == NO_IRQ))
527 handle_one_irq(irq);
528 else
529 __get_cpu_var(irq_stat).spurious_irqs++; 483 __get_cpu_var(irq_stat).spurious_irqs++;
484 else {
485 desc = irq_to_desc(irq);
486 if (likely(desc))
487 desc->handle_irq(irq, desc);
488 }
530 489
531 trace_irq_exit(regs); 490 trace_irq_exit(regs);
532 491
533 irq_exit(); 492 irq_exit();
493}
494
495void do_IRQ(struct pt_regs *regs)
496{
497 struct pt_regs *old_regs = set_irq_regs(regs);
498 struct thread_info *curtp, *irqtp;
499
500 /* Switch to the irq stack to handle this */
501 curtp = current_thread_info();
502 irqtp = hardirq_ctx[raw_smp_processor_id()];
503
504 /* Already there ? */
505 if (unlikely(curtp == irqtp)) {
506 __do_irq(regs);
507 set_irq_regs(old_regs);
508 return;
509 }
510
511 /* Prepare the thread_info in the irq stack */
512 irqtp->task = curtp->task;
513 irqtp->flags = 0;
514
515 /* Copy the preempt_count so that the [soft]irq checks work. */
516 irqtp->preempt_count = curtp->preempt_count;
517
518 /* Switch stack and call */
519 call_do_irq(regs, irqtp);
520
521 /* Restore stack limit */
522 irqtp->task = NULL;
523
524 /* Copy back updates to the thread_info */
525 if (irqtp->flags)
526 set_bits(irqtp->flags, &curtp->flags);
527
534 set_irq_regs(old_regs); 528 set_irq_regs(old_regs);
535} 529}
536 530
@@ -592,28 +586,22 @@ void irq_ctx_init(void)
592 memset((void *)softirq_ctx[i], 0, THREAD_SIZE); 586 memset((void *)softirq_ctx[i], 0, THREAD_SIZE);
593 tp = softirq_ctx[i]; 587 tp = softirq_ctx[i];
594 tp->cpu = i; 588 tp->cpu = i;
595 tp->preempt_count = 0;
596 589
597 memset((void *)hardirq_ctx[i], 0, THREAD_SIZE); 590 memset((void *)hardirq_ctx[i], 0, THREAD_SIZE);
598 tp = hardirq_ctx[i]; 591 tp = hardirq_ctx[i];
599 tp->cpu = i; 592 tp->cpu = i;
600 tp->preempt_count = HARDIRQ_OFFSET;
601 } 593 }
602} 594}
603 595
604static inline void do_softirq_onstack(void) 596static inline void do_softirq_onstack(void)
605{ 597{
606 struct thread_info *curtp, *irqtp; 598 struct thread_info *curtp, *irqtp;
607 unsigned long saved_sp_limit = current->thread.ksp_limit;
608 599
609 curtp = current_thread_info(); 600 curtp = current_thread_info();
610 irqtp = softirq_ctx[smp_processor_id()]; 601 irqtp = softirq_ctx[smp_processor_id()];
611 irqtp->task = curtp->task; 602 irqtp->task = curtp->task;
612 irqtp->flags = 0; 603 irqtp->flags = 0;
613 current->thread.ksp_limit = (unsigned long)irqtp +
614 _ALIGN_UP(sizeof(struct thread_info), 16);
615 call_do_softirq(irqtp); 604 call_do_softirq(irqtp);
616 current->thread.ksp_limit = saved_sp_limit;
617 irqtp->task = NULL; 605 irqtp->task = NULL;
618 606
619 /* Set any flag that may have been set on the 607 /* Set any flag that may have been set on the
diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S
index 777d999f563b..2b0ad9845363 100644
--- a/arch/powerpc/kernel/misc_32.S
+++ b/arch/powerpc/kernel/misc_32.S
@@ -36,26 +36,41 @@
36 36
37 .text 37 .text
38 38
39/*
40 * We store the saved ksp_limit in the unused part
41 * of the STACK_FRAME_OVERHEAD
42 */
39_GLOBAL(call_do_softirq) 43_GLOBAL(call_do_softirq)
40 mflr r0 44 mflr r0
41 stw r0,4(r1) 45 stw r0,4(r1)
46 lwz r10,THREAD+KSP_LIMIT(r2)
47 addi r11,r3,THREAD_INFO_GAP
42 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 48 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
43 mr r1,r3 49 mr r1,r3
50 stw r10,8(r1)
51 stw r11,THREAD+KSP_LIMIT(r2)
44 bl __do_softirq 52 bl __do_softirq
53 lwz r10,8(r1)
45 lwz r1,0(r1) 54 lwz r1,0(r1)
46 lwz r0,4(r1) 55 lwz r0,4(r1)
56 stw r10,THREAD+KSP_LIMIT(r2)
47 mtlr r0 57 mtlr r0
48 blr 58 blr
49 59
50_GLOBAL(call_handle_irq) 60_GLOBAL(call_do_irq)
51 mflr r0 61 mflr r0
52 stw r0,4(r1) 62 stw r0,4(r1)
53 mtctr r6 63 lwz r10,THREAD+KSP_LIMIT(r2)
54 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r5) 64 addi r11,r3,THREAD_INFO_GAP
55 mr r1,r5 65 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
56 bctrl 66 mr r1,r4
67 stw r10,8(r1)
68 stw r11,THREAD+KSP_LIMIT(r2)
69 bl __do_irq
70 lwz r10,8(r1)
57 lwz r1,0(r1) 71 lwz r1,0(r1)
58 lwz r0,4(r1) 72 lwz r0,4(r1)
73 stw r10,THREAD+KSP_LIMIT(r2)
59 mtlr r0 74 mtlr r0
60 blr 75 blr
61 76
diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S
index 971d7e78aff2..e59caf874d05 100644
--- a/arch/powerpc/kernel/misc_64.S
+++ b/arch/powerpc/kernel/misc_64.S
@@ -40,14 +40,12 @@ _GLOBAL(call_do_softirq)
40 mtlr r0 40 mtlr r0
41 blr 41 blr
42 42
43_GLOBAL(call_handle_irq) 43_GLOBAL(call_do_irq)
44 ld r8,0(r6)
45 mflr r0 44 mflr r0
46 std r0,16(r1) 45 std r0,16(r1)
47 mtctr r8 46 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
48 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r5) 47 mr r1,r4
49 mr r1,r5 48 bl .__do_irq
50 bctrl
51 ld r1,0(r1) 49 ld r1,0(r1)
52 ld r0,16(r1) 50 ld r0,16(r1)
53 mtlr r0 51 mtlr r0
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 6f428da53e20..96d2fdf3aa9e 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1000,9 +1000,10 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
1000 kregs = (struct pt_regs *) sp; 1000 kregs = (struct pt_regs *) sp;
1001 sp -= STACK_FRAME_OVERHEAD; 1001 sp -= STACK_FRAME_OVERHEAD;
1002 p->thread.ksp = sp; 1002 p->thread.ksp = sp;
1003#ifdef CONFIG_PPC32
1003 p->thread.ksp_limit = (unsigned long)task_stack_page(p) + 1004 p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
1004 _ALIGN_UP(sizeof(struct thread_info), 16); 1005 _ALIGN_UP(sizeof(struct thread_info), 16);
1005 1006#endif
1006#ifdef CONFIG_HAVE_HW_BREAKPOINT 1007#ifdef CONFIG_HAVE_HW_BREAKPOINT
1007 p->thread.ptrace_bps[0] = NULL; 1008 p->thread.ptrace_bps[0] = NULL;
1008#endif 1009#endif
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index 12e656ffe60e..5fe2842e8bab 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -196,6 +196,8 @@ static int __initdata mem_reserve_cnt;
196 196
197static cell_t __initdata regbuf[1024]; 197static cell_t __initdata regbuf[1024];
198 198
199static bool rtas_has_query_cpu_stopped;
200
199 201
200/* 202/*
201 * Error results ... some OF calls will return "-1" on error, some 203 * Error results ... some OF calls will return "-1" on error, some
@@ -1574,6 +1576,11 @@ static void __init prom_instantiate_rtas(void)
1574 prom_setprop(rtas_node, "/rtas", "linux,rtas-entry", 1576 prom_setprop(rtas_node, "/rtas", "linux,rtas-entry",
1575 &val, sizeof(val)); 1577 &val, sizeof(val));
1576 1578
1579 /* Check if it supports "query-cpu-stopped-state" */
1580 if (prom_getprop(rtas_node, "query-cpu-stopped-state",
1581 &val, sizeof(val)) != PROM_ERROR)
1582 rtas_has_query_cpu_stopped = true;
1583
1577#if defined(CONFIG_PPC_POWERNV) && defined(__BIG_ENDIAN__) 1584#if defined(CONFIG_PPC_POWERNV) && defined(__BIG_ENDIAN__)
1578 /* PowerVN takeover hack */ 1585 /* PowerVN takeover hack */
1579 prom_rtas_data = base; 1586 prom_rtas_data = base;
@@ -1815,6 +1822,18 @@ static void __init prom_hold_cpus(void)
1815 = (void *) LOW_ADDR(__secondary_hold_acknowledge); 1822 = (void *) LOW_ADDR(__secondary_hold_acknowledge);
1816 unsigned long secondary_hold = LOW_ADDR(__secondary_hold); 1823 unsigned long secondary_hold = LOW_ADDR(__secondary_hold);
1817 1824
1825 /*
1826 * On pseries, if RTAS supports "query-cpu-stopped-state",
1827 * we skip this stage, the CPUs will be started by the
1828 * kernel using RTAS.
1829 */
1830 if ((of_platform == PLATFORM_PSERIES ||
1831 of_platform == PLATFORM_PSERIES_LPAR) &&
1832 rtas_has_query_cpu_stopped) {
1833 prom_printf("prom_hold_cpus: skipped\n");
1834 return;
1835 }
1836
1818 prom_debug("prom_hold_cpus: start...\n"); 1837 prom_debug("prom_hold_cpus: start...\n");
1819 prom_debug(" 1) spinloop = 0x%x\n", (unsigned long)spinloop); 1838 prom_debug(" 1) spinloop = 0x%x\n", (unsigned long)spinloop);
1820 prom_debug(" 1) *spinloop = 0x%x\n", *spinloop); 1839 prom_debug(" 1) *spinloop = 0x%x\n", *spinloop);
@@ -3011,6 +3030,8 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4,
3011 * On non-powermacs, put all CPUs in spin-loops. 3030 * On non-powermacs, put all CPUs in spin-loops.
3012 * 3031 *
3013 * PowerMacs use a different mechanism to spin CPUs 3032 * PowerMacs use a different mechanism to spin CPUs
3033 *
3034 * (This must be done after instanciating RTAS)
3014 */ 3035 */
3015 if (of_platform != PLATFORM_POWERMAC && 3036 if (of_platform != PLATFORM_POWERMAC &&
3016 of_platform != PLATFORM_OPAL) 3037 of_platform != PLATFORM_OPAL)
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index a7ee978fb860..b1faa1593c90 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -1505,6 +1505,7 @@ int __kprobes emulate_step(struct pt_regs *regs, unsigned int instr)
1505 */ 1505 */
1506 if ((ra == 1) && !(regs->msr & MSR_PR) \ 1506 if ((ra == 1) && !(regs->msr & MSR_PR) \
1507 && (val3 >= (regs->gpr[1] - STACK_INT_FRAME_SIZE))) { 1507 && (val3 >= (regs->gpr[1] - STACK_INT_FRAME_SIZE))) {
1508#ifdef CONFIG_PPC32
1508 /* 1509 /*
1509 * Check if we will touch kernel sack overflow 1510 * Check if we will touch kernel sack overflow
1510 */ 1511 */
@@ -1513,7 +1514,7 @@ int __kprobes emulate_step(struct pt_regs *regs, unsigned int instr)
1513 err = -EINVAL; 1514 err = -EINVAL;
1514 break; 1515 break;
1515 } 1516 }
1516 1517#endif /* CONFIG_PPC32 */
1517 /* 1518 /*
1518 * Check if we already set since that means we'll 1519 * Check if we already set since that means we'll
1519 * lose the previous value. 1520 * lose the previous value.
diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c
index 1c1771a40250..24f58cb0a543 100644
--- a/arch/powerpc/platforms/pseries/smp.c
+++ b/arch/powerpc/platforms/pseries/smp.c
@@ -233,18 +233,24 @@ static void __init smp_init_pseries(void)
233 233
234 alloc_bootmem_cpumask_var(&of_spin_mask); 234 alloc_bootmem_cpumask_var(&of_spin_mask);
235 235
236 /* Mark threads which are still spinning in hold loops. */ 236 /*
237 if (cpu_has_feature(CPU_FTR_SMT)) { 237 * Mark threads which are still spinning in hold loops
238 for_each_present_cpu(i) { 238 *
239 if (cpu_thread_in_core(i) == 0) 239 * We know prom_init will not have started them if RTAS supports
240 cpumask_set_cpu(i, of_spin_mask); 240 * query-cpu-stopped-state.
241 } 241 */
242 } else { 242 if (rtas_token("query-cpu-stopped-state") == RTAS_UNKNOWN_SERVICE) {
243 cpumask_copy(of_spin_mask, cpu_present_mask); 243 if (cpu_has_feature(CPU_FTR_SMT)) {
244 for_each_present_cpu(i) {
245 if (cpu_thread_in_core(i) == 0)
246 cpumask_set_cpu(i, of_spin_mask);
247 }
248 } else
249 cpumask_copy(of_spin_mask, cpu_present_mask);
250
251 cpumask_clear_cpu(boot_cpuid, of_spin_mask);
244 } 252 }
245 253
246 cpumask_clear_cpu(boot_cpuid, of_spin_mask);
247
248 /* Non-lpar has additional take/give timebase */ 254 /* Non-lpar has additional take/give timebase */
249 if (rtas_token("freeze-time-base") != RTAS_UNKNOWN_SERVICE) { 255 if (rtas_token("freeze-time-base") != RTAS_UNKNOWN_SERVICE) {
250 smp_ops->give_timebase = rtas_give_timebase; 256 smp_ops->give_timebase = rtas_give_timebase;
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index dcc6ac2d8026..7143793859fa 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -93,6 +93,7 @@ config S390
93 select ARCH_INLINE_WRITE_UNLOCK_IRQ 93 select ARCH_INLINE_WRITE_UNLOCK_IRQ
94 select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE 94 select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
95 select ARCH_SAVE_PAGE_KEYS if HIBERNATION 95 select ARCH_SAVE_PAGE_KEYS if HIBERNATION
96 select ARCH_USE_CMPXCHG_LOCKREF
96 select ARCH_WANT_IPC_PARSE_VERSION 97 select ARCH_WANT_IPC_PARSE_VERSION
97 select BUILDTIME_EXTABLE_SORT 98 select BUILDTIME_EXTABLE_SORT
98 select CLONE_BACKWARDS2 99 select CLONE_BACKWARDS2
@@ -102,7 +103,6 @@ config S390
102 select GENERIC_TIME_VSYSCALL_OLD 103 select GENERIC_TIME_VSYSCALL_OLD
103 select HAVE_ALIGNED_STRUCT_PAGE if SLUB 104 select HAVE_ALIGNED_STRUCT_PAGE if SLUB
104 select HAVE_ARCH_JUMP_LABEL if !MARCH_G5 105 select HAVE_ARCH_JUMP_LABEL if !MARCH_G5
105 select HAVE_ARCH_MUTEX_CPU_RELAX
106 select HAVE_ARCH_SECCOMP_FILTER 106 select HAVE_ARCH_SECCOMP_FILTER
107 select HAVE_ARCH_TRACEHOOK 107 select HAVE_ARCH_TRACEHOOK
108 select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT 108 select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT
diff --git a/arch/s390/include/asm/mutex.h b/arch/s390/include/asm/mutex.h
index 688271f5f2e4..458c1f7fbc18 100644
--- a/arch/s390/include/asm/mutex.h
+++ b/arch/s390/include/asm/mutex.h
@@ -7,5 +7,3 @@
7 */ 7 */
8 8
9#include <asm-generic/mutex-dec.h> 9#include <asm-generic/mutex-dec.h>
10
11#define arch_mutex_cpu_relax() barrier()
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 0eb37505cab1..ca7821f07260 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -198,6 +198,8 @@ static inline void cpu_relax(void)
198 barrier(); 198 barrier();
199} 199}
200 200
201#define arch_mutex_cpu_relax() barrier()
202
201static inline void psw_set_key(unsigned int key) 203static inline void psw_set_key(unsigned int key)
202{ 204{
203 asm volatile("spka 0(%0)" : : "d" (key)); 205 asm volatile("spka 0(%0)" : : "d" (key));
diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h
index 701fe8c59e1f..83e5d216105e 100644
--- a/arch/s390/include/asm/spinlock.h
+++ b/arch/s390/include/asm/spinlock.h
@@ -44,6 +44,11 @@ extern void arch_spin_lock_wait_flags(arch_spinlock_t *, unsigned long flags);
44extern int arch_spin_trylock_retry(arch_spinlock_t *); 44extern int arch_spin_trylock_retry(arch_spinlock_t *);
45extern void arch_spin_relax(arch_spinlock_t *lock); 45extern void arch_spin_relax(arch_spinlock_t *lock);
46 46
47static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
48{
49 return lock.owner_cpu == 0;
50}
51
47static inline void arch_spin_lock(arch_spinlock_t *lp) 52static inline void arch_spin_lock(arch_spinlock_t *lp)
48{ 53{
49 int old; 54 int old;
diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index 8a7cc663b3f8..d45a2c48f185 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -361,7 +361,7 @@ config CMDLINE_OVERRIDE
361 361
362config VMALLOC_RESERVE 362config VMALLOC_RESERVE
363 hex 363 hex
364 default 0x1000000 364 default 0x2000000
365 365
366config HARDWALL 366config HARDWALL
367 bool "Hardwall support to allow access to user dynamic network" 367 bool "Hardwall support to allow access to user dynamic network"
diff --git a/arch/tile/gxio/iorpc_mpipe.c b/arch/tile/gxio/iorpc_mpipe.c
index 4f8f3d619c4a..e19325c4c431 100644
--- a/arch/tile/gxio/iorpc_mpipe.c
+++ b/arch/tile/gxio/iorpc_mpipe.c
@@ -21,7 +21,7 @@ struct alloc_buffer_stacks_param {
21 unsigned int flags; 21 unsigned int flags;
22}; 22};
23 23
24int gxio_mpipe_alloc_buffer_stacks(gxio_mpipe_context_t * context, 24int gxio_mpipe_alloc_buffer_stacks(gxio_mpipe_context_t *context,
25 unsigned int count, unsigned int first, 25 unsigned int count, unsigned int first,
26 unsigned int flags) 26 unsigned int flags)
27{ 27{
@@ -45,7 +45,7 @@ struct init_buffer_stack_aux_param {
45 unsigned int buffer_size_enum; 45 unsigned int buffer_size_enum;
46}; 46};
47 47
48int gxio_mpipe_init_buffer_stack_aux(gxio_mpipe_context_t * context, 48int gxio_mpipe_init_buffer_stack_aux(gxio_mpipe_context_t *context,
49 void *mem_va, size_t mem_size, 49 void *mem_va, size_t mem_size,
50 unsigned int mem_flags, unsigned int stack, 50 unsigned int mem_flags, unsigned int stack,
51 unsigned int buffer_size_enum) 51 unsigned int buffer_size_enum)
@@ -80,7 +80,7 @@ struct alloc_notif_rings_param {
80 unsigned int flags; 80 unsigned int flags;
81}; 81};
82 82
83int gxio_mpipe_alloc_notif_rings(gxio_mpipe_context_t * context, 83int gxio_mpipe_alloc_notif_rings(gxio_mpipe_context_t *context,
84 unsigned int count, unsigned int first, 84 unsigned int count, unsigned int first,
85 unsigned int flags) 85 unsigned int flags)
86{ 86{
@@ -102,7 +102,7 @@ struct init_notif_ring_aux_param {
102 unsigned int ring; 102 unsigned int ring;
103}; 103};
104 104
105int gxio_mpipe_init_notif_ring_aux(gxio_mpipe_context_t * context, void *mem_va, 105int gxio_mpipe_init_notif_ring_aux(gxio_mpipe_context_t *context, void *mem_va,
106 size_t mem_size, unsigned int mem_flags, 106 size_t mem_size, unsigned int mem_flags,
107 unsigned int ring) 107 unsigned int ring)
108{ 108{
@@ -133,7 +133,7 @@ struct request_notif_ring_interrupt_param {
133 unsigned int ring; 133 unsigned int ring;
134}; 134};
135 135
136int gxio_mpipe_request_notif_ring_interrupt(gxio_mpipe_context_t * context, 136int gxio_mpipe_request_notif_ring_interrupt(gxio_mpipe_context_t *context,
137 int inter_x, int inter_y, 137 int inter_x, int inter_y,
138 int inter_ipi, int inter_event, 138 int inter_ipi, int inter_event,
139 unsigned int ring) 139 unsigned int ring)
@@ -158,7 +158,7 @@ struct enable_notif_ring_interrupt_param {
158 unsigned int ring; 158 unsigned int ring;
159}; 159};
160 160
161int gxio_mpipe_enable_notif_ring_interrupt(gxio_mpipe_context_t * context, 161int gxio_mpipe_enable_notif_ring_interrupt(gxio_mpipe_context_t *context,
162 unsigned int ring) 162 unsigned int ring)
163{ 163{
164 struct enable_notif_ring_interrupt_param temp; 164 struct enable_notif_ring_interrupt_param temp;
@@ -179,7 +179,7 @@ struct alloc_notif_groups_param {
179 unsigned int flags; 179 unsigned int flags;
180}; 180};
181 181
182int gxio_mpipe_alloc_notif_groups(gxio_mpipe_context_t * context, 182int gxio_mpipe_alloc_notif_groups(gxio_mpipe_context_t *context,
183 unsigned int count, unsigned int first, 183 unsigned int count, unsigned int first,
184 unsigned int flags) 184 unsigned int flags)
185{ 185{
@@ -201,7 +201,7 @@ struct init_notif_group_param {
201 gxio_mpipe_notif_group_bits_t bits; 201 gxio_mpipe_notif_group_bits_t bits;
202}; 202};
203 203
204int gxio_mpipe_init_notif_group(gxio_mpipe_context_t * context, 204int gxio_mpipe_init_notif_group(gxio_mpipe_context_t *context,
205 unsigned int group, 205 unsigned int group,
206 gxio_mpipe_notif_group_bits_t bits) 206 gxio_mpipe_notif_group_bits_t bits)
207{ 207{
@@ -223,7 +223,7 @@ struct alloc_buckets_param {
223 unsigned int flags; 223 unsigned int flags;
224}; 224};
225 225
226int gxio_mpipe_alloc_buckets(gxio_mpipe_context_t * context, unsigned int count, 226int gxio_mpipe_alloc_buckets(gxio_mpipe_context_t *context, unsigned int count,
227 unsigned int first, unsigned int flags) 227 unsigned int first, unsigned int flags)
228{ 228{
229 struct alloc_buckets_param temp; 229 struct alloc_buckets_param temp;
@@ -244,7 +244,7 @@ struct init_bucket_param {
244 MPIPE_LBL_INIT_DAT_BSTS_TBL_t bucket_info; 244 MPIPE_LBL_INIT_DAT_BSTS_TBL_t bucket_info;
245}; 245};
246 246
247int gxio_mpipe_init_bucket(gxio_mpipe_context_t * context, unsigned int bucket, 247int gxio_mpipe_init_bucket(gxio_mpipe_context_t *context, unsigned int bucket,
248 MPIPE_LBL_INIT_DAT_BSTS_TBL_t bucket_info) 248 MPIPE_LBL_INIT_DAT_BSTS_TBL_t bucket_info)
249{ 249{
250 struct init_bucket_param temp; 250 struct init_bucket_param temp;
@@ -265,7 +265,7 @@ struct alloc_edma_rings_param {
265 unsigned int flags; 265 unsigned int flags;
266}; 266};
267 267
268int gxio_mpipe_alloc_edma_rings(gxio_mpipe_context_t * context, 268int gxio_mpipe_alloc_edma_rings(gxio_mpipe_context_t *context,
269 unsigned int count, unsigned int first, 269 unsigned int count, unsigned int first,
270 unsigned int flags) 270 unsigned int flags)
271{ 271{
@@ -288,7 +288,7 @@ struct init_edma_ring_aux_param {
288 unsigned int channel; 288 unsigned int channel;
289}; 289};
290 290
291int gxio_mpipe_init_edma_ring_aux(gxio_mpipe_context_t * context, void *mem_va, 291int gxio_mpipe_init_edma_ring_aux(gxio_mpipe_context_t *context, void *mem_va,
292 size_t mem_size, unsigned int mem_flags, 292 size_t mem_size, unsigned int mem_flags,
293 unsigned int ring, unsigned int channel) 293 unsigned int ring, unsigned int channel)
294{ 294{
@@ -315,7 +315,7 @@ int gxio_mpipe_init_edma_ring_aux(gxio_mpipe_context_t * context, void *mem_va,
315EXPORT_SYMBOL(gxio_mpipe_init_edma_ring_aux); 315EXPORT_SYMBOL(gxio_mpipe_init_edma_ring_aux);
316 316
317 317
318int gxio_mpipe_commit_rules(gxio_mpipe_context_t * context, const void *blob, 318int gxio_mpipe_commit_rules(gxio_mpipe_context_t *context, const void *blob,
319 size_t blob_size) 319 size_t blob_size)
320{ 320{
321 const void *params = blob; 321 const void *params = blob;
@@ -332,7 +332,7 @@ struct register_client_memory_param {
332 unsigned int flags; 332 unsigned int flags;
333}; 333};
334 334
335int gxio_mpipe_register_client_memory(gxio_mpipe_context_t * context, 335int gxio_mpipe_register_client_memory(gxio_mpipe_context_t *context,
336 unsigned int iotlb, HV_PTE pte, 336 unsigned int iotlb, HV_PTE pte,
337 unsigned int flags) 337 unsigned int flags)
338{ 338{
@@ -355,7 +355,7 @@ struct link_open_aux_param {
355 unsigned int flags; 355 unsigned int flags;
356}; 356};
357 357
358int gxio_mpipe_link_open_aux(gxio_mpipe_context_t * context, 358int gxio_mpipe_link_open_aux(gxio_mpipe_context_t *context,
359 _gxio_mpipe_link_name_t name, unsigned int flags) 359 _gxio_mpipe_link_name_t name, unsigned int flags)
360{ 360{
361 struct link_open_aux_param temp; 361 struct link_open_aux_param temp;
@@ -374,7 +374,7 @@ struct link_close_aux_param {
374 int mac; 374 int mac;
375}; 375};
376 376
377int gxio_mpipe_link_close_aux(gxio_mpipe_context_t * context, int mac) 377int gxio_mpipe_link_close_aux(gxio_mpipe_context_t *context, int mac)
378{ 378{
379 struct link_close_aux_param temp; 379 struct link_close_aux_param temp;
380 struct link_close_aux_param *params = &temp; 380 struct link_close_aux_param *params = &temp;
@@ -393,7 +393,7 @@ struct link_set_attr_aux_param {
393 int64_t val; 393 int64_t val;
394}; 394};
395 395
396int gxio_mpipe_link_set_attr_aux(gxio_mpipe_context_t * context, int mac, 396int gxio_mpipe_link_set_attr_aux(gxio_mpipe_context_t *context, int mac,
397 uint32_t attr, int64_t val) 397 uint32_t attr, int64_t val)
398{ 398{
399 struct link_set_attr_aux_param temp; 399 struct link_set_attr_aux_param temp;
@@ -415,8 +415,8 @@ struct get_timestamp_aux_param {
415 uint64_t cycles; 415 uint64_t cycles;
416}; 416};
417 417
418int gxio_mpipe_get_timestamp_aux(gxio_mpipe_context_t * context, uint64_t * sec, 418int gxio_mpipe_get_timestamp_aux(gxio_mpipe_context_t *context, uint64_t *sec,
419 uint64_t * nsec, uint64_t * cycles) 419 uint64_t *nsec, uint64_t *cycles)
420{ 420{
421 int __result; 421 int __result;
422 struct get_timestamp_aux_param temp; 422 struct get_timestamp_aux_param temp;
@@ -440,7 +440,7 @@ struct set_timestamp_aux_param {
440 uint64_t cycles; 440 uint64_t cycles;
441}; 441};
442 442
443int gxio_mpipe_set_timestamp_aux(gxio_mpipe_context_t * context, uint64_t sec, 443int gxio_mpipe_set_timestamp_aux(gxio_mpipe_context_t *context, uint64_t sec,
444 uint64_t nsec, uint64_t cycles) 444 uint64_t nsec, uint64_t cycles)
445{ 445{
446 struct set_timestamp_aux_param temp; 446 struct set_timestamp_aux_param temp;
@@ -460,8 +460,7 @@ struct adjust_timestamp_aux_param {
460 int64_t nsec; 460 int64_t nsec;
461}; 461};
462 462
463int gxio_mpipe_adjust_timestamp_aux(gxio_mpipe_context_t * context, 463int gxio_mpipe_adjust_timestamp_aux(gxio_mpipe_context_t *context, int64_t nsec)
464 int64_t nsec)
465{ 464{
466 struct adjust_timestamp_aux_param temp; 465 struct adjust_timestamp_aux_param temp;
467 struct adjust_timestamp_aux_param *params = &temp; 466 struct adjust_timestamp_aux_param *params = &temp;
@@ -475,25 +474,6 @@ int gxio_mpipe_adjust_timestamp_aux(gxio_mpipe_context_t * context,
475 474
476EXPORT_SYMBOL(gxio_mpipe_adjust_timestamp_aux); 475EXPORT_SYMBOL(gxio_mpipe_adjust_timestamp_aux);
477 476
478struct adjust_timestamp_freq_param {
479 int32_t ppb;
480};
481
482int gxio_mpipe_adjust_timestamp_freq(gxio_mpipe_context_t * context,
483 int32_t ppb)
484{
485 struct adjust_timestamp_freq_param temp;
486 struct adjust_timestamp_freq_param *params = &temp;
487
488 params->ppb = ppb;
489
490 return hv_dev_pwrite(context->fd, 0, (HV_VirtAddr) params,
491 sizeof(*params),
492 GXIO_MPIPE_OP_ADJUST_TIMESTAMP_FREQ);
493}
494
495EXPORT_SYMBOL(gxio_mpipe_adjust_timestamp_freq);
496
497struct config_edma_ring_blks_param { 477struct config_edma_ring_blks_param {
498 unsigned int ering; 478 unsigned int ering;
499 unsigned int max_blks; 479 unsigned int max_blks;
@@ -501,7 +481,7 @@ struct config_edma_ring_blks_param {
501 unsigned int db; 481 unsigned int db;
502}; 482};
503 483
504int gxio_mpipe_config_edma_ring_blks(gxio_mpipe_context_t * context, 484int gxio_mpipe_config_edma_ring_blks(gxio_mpipe_context_t *context,
505 unsigned int ering, unsigned int max_blks, 485 unsigned int ering, unsigned int max_blks,
506 unsigned int min_snf_blks, unsigned int db) 486 unsigned int min_snf_blks, unsigned int db)
507{ 487{
@@ -520,11 +500,29 @@ int gxio_mpipe_config_edma_ring_blks(gxio_mpipe_context_t * context,
520 500
521EXPORT_SYMBOL(gxio_mpipe_config_edma_ring_blks); 501EXPORT_SYMBOL(gxio_mpipe_config_edma_ring_blks);
522 502
503struct adjust_timestamp_freq_param {
504 int32_t ppb;
505};
506
507int gxio_mpipe_adjust_timestamp_freq(gxio_mpipe_context_t *context, int32_t ppb)
508{
509 struct adjust_timestamp_freq_param temp;
510 struct adjust_timestamp_freq_param *params = &temp;
511
512 params->ppb = ppb;
513
514 return hv_dev_pwrite(context->fd, 0, (HV_VirtAddr) params,
515 sizeof(*params),
516 GXIO_MPIPE_OP_ADJUST_TIMESTAMP_FREQ);
517}
518
519EXPORT_SYMBOL(gxio_mpipe_adjust_timestamp_freq);
520
523struct arm_pollfd_param { 521struct arm_pollfd_param {
524 union iorpc_pollfd pollfd; 522 union iorpc_pollfd pollfd;
525}; 523};
526 524
527int gxio_mpipe_arm_pollfd(gxio_mpipe_context_t * context, int pollfd_cookie) 525int gxio_mpipe_arm_pollfd(gxio_mpipe_context_t *context, int pollfd_cookie)
528{ 526{
529 struct arm_pollfd_param temp; 527 struct arm_pollfd_param temp;
530 struct arm_pollfd_param *params = &temp; 528 struct arm_pollfd_param *params = &temp;
@@ -541,7 +539,7 @@ struct close_pollfd_param {
541 union iorpc_pollfd pollfd; 539 union iorpc_pollfd pollfd;
542}; 540};
543 541
544int gxio_mpipe_close_pollfd(gxio_mpipe_context_t * context, int pollfd_cookie) 542int gxio_mpipe_close_pollfd(gxio_mpipe_context_t *context, int pollfd_cookie)
545{ 543{
546 struct close_pollfd_param temp; 544 struct close_pollfd_param temp;
547 struct close_pollfd_param *params = &temp; 545 struct close_pollfd_param *params = &temp;
@@ -558,7 +556,7 @@ struct get_mmio_base_param {
558 HV_PTE base; 556 HV_PTE base;
559}; 557};
560 558
561int gxio_mpipe_get_mmio_base(gxio_mpipe_context_t * context, HV_PTE *base) 559int gxio_mpipe_get_mmio_base(gxio_mpipe_context_t *context, HV_PTE *base)
562{ 560{
563 int __result; 561 int __result;
564 struct get_mmio_base_param temp; 562 struct get_mmio_base_param temp;
@@ -579,7 +577,7 @@ struct check_mmio_offset_param {
579 unsigned long size; 577 unsigned long size;
580}; 578};
581 579
582int gxio_mpipe_check_mmio_offset(gxio_mpipe_context_t * context, 580int gxio_mpipe_check_mmio_offset(gxio_mpipe_context_t *context,
583 unsigned long offset, unsigned long size) 581 unsigned long offset, unsigned long size)
584{ 582{
585 struct check_mmio_offset_param temp; 583 struct check_mmio_offset_param temp;
diff --git a/arch/tile/gxio/iorpc_mpipe_info.c b/arch/tile/gxio/iorpc_mpipe_info.c
index 64883aabeb9c..77019c6e9b4a 100644
--- a/arch/tile/gxio/iorpc_mpipe_info.c
+++ b/arch/tile/gxio/iorpc_mpipe_info.c
@@ -15,12 +15,11 @@
15/* This file is machine-generated; DO NOT EDIT! */ 15/* This file is machine-generated; DO NOT EDIT! */
16#include "gxio/iorpc_mpipe_info.h" 16#include "gxio/iorpc_mpipe_info.h"
17 17
18
19struct instance_aux_param { 18struct instance_aux_param {
20 _gxio_mpipe_link_name_t name; 19 _gxio_mpipe_link_name_t name;
21}; 20};
22 21
23int gxio_mpipe_info_instance_aux(gxio_mpipe_info_context_t * context, 22int gxio_mpipe_info_instance_aux(gxio_mpipe_info_context_t *context,
24 _gxio_mpipe_link_name_t name) 23 _gxio_mpipe_link_name_t name)
25{ 24{
26 struct instance_aux_param temp; 25 struct instance_aux_param temp;
@@ -39,10 +38,10 @@ struct enumerate_aux_param {
39 _gxio_mpipe_link_mac_t mac; 38 _gxio_mpipe_link_mac_t mac;
40}; 39};
41 40
42int gxio_mpipe_info_enumerate_aux(gxio_mpipe_info_context_t * context, 41int gxio_mpipe_info_enumerate_aux(gxio_mpipe_info_context_t *context,
43 unsigned int idx, 42 unsigned int idx,
44 _gxio_mpipe_link_name_t * name, 43 _gxio_mpipe_link_name_t *name,
45 _gxio_mpipe_link_mac_t * mac) 44 _gxio_mpipe_link_mac_t *mac)
46{ 45{
47 int __result; 46 int __result;
48 struct enumerate_aux_param temp; 47 struct enumerate_aux_param temp;
@@ -50,7 +49,7 @@ int gxio_mpipe_info_enumerate_aux(gxio_mpipe_info_context_t * context,
50 49
51 __result = 50 __result =
52 hv_dev_pread(context->fd, 0, (HV_VirtAddr) params, sizeof(*params), 51 hv_dev_pread(context->fd, 0, (HV_VirtAddr) params, sizeof(*params),
53 (((uint64_t) idx << 32) | 52 (((uint64_t)idx << 32) |
54 GXIO_MPIPE_INFO_OP_ENUMERATE_AUX)); 53 GXIO_MPIPE_INFO_OP_ENUMERATE_AUX));
55 *name = params->name; 54 *name = params->name;
56 *mac = params->mac; 55 *mac = params->mac;
@@ -64,7 +63,7 @@ struct get_mmio_base_param {
64 HV_PTE base; 63 HV_PTE base;
65}; 64};
66 65
67int gxio_mpipe_info_get_mmio_base(gxio_mpipe_info_context_t * context, 66int gxio_mpipe_info_get_mmio_base(gxio_mpipe_info_context_t *context,
68 HV_PTE *base) 67 HV_PTE *base)
69{ 68{
70 int __result; 69 int __result;
@@ -86,7 +85,7 @@ struct check_mmio_offset_param {
86 unsigned long size; 85 unsigned long size;
87}; 86};
88 87
89int gxio_mpipe_info_check_mmio_offset(gxio_mpipe_info_context_t * context, 88int gxio_mpipe_info_check_mmio_offset(gxio_mpipe_info_context_t *context,
90 unsigned long offset, unsigned long size) 89 unsigned long offset, unsigned long size)
91{ 90{
92 struct check_mmio_offset_param temp; 91 struct check_mmio_offset_param temp;
diff --git a/arch/tile/gxio/iorpc_trio.c b/arch/tile/gxio/iorpc_trio.c
index da6e18e049c3..1d3cedb9aeb4 100644
--- a/arch/tile/gxio/iorpc_trio.c
+++ b/arch/tile/gxio/iorpc_trio.c
@@ -21,7 +21,7 @@ struct alloc_asids_param {
21 unsigned int flags; 21 unsigned int flags;
22}; 22};
23 23
24int gxio_trio_alloc_asids(gxio_trio_context_t * context, unsigned int count, 24int gxio_trio_alloc_asids(gxio_trio_context_t *context, unsigned int count,
25 unsigned int first, unsigned int flags) 25 unsigned int first, unsigned int flags)
26{ 26{
27 struct alloc_asids_param temp; 27 struct alloc_asids_param temp;
@@ -44,7 +44,7 @@ struct alloc_memory_maps_param {
44 unsigned int flags; 44 unsigned int flags;
45}; 45};
46 46
47int gxio_trio_alloc_memory_maps(gxio_trio_context_t * context, 47int gxio_trio_alloc_memory_maps(gxio_trio_context_t *context,
48 unsigned int count, unsigned int first, 48 unsigned int count, unsigned int first,
49 unsigned int flags) 49 unsigned int flags)
50{ 50{
@@ -67,7 +67,7 @@ struct alloc_scatter_queues_param {
67 unsigned int flags; 67 unsigned int flags;
68}; 68};
69 69
70int gxio_trio_alloc_scatter_queues(gxio_trio_context_t * context, 70int gxio_trio_alloc_scatter_queues(gxio_trio_context_t *context,
71 unsigned int count, unsigned int first, 71 unsigned int count, unsigned int first,
72 unsigned int flags) 72 unsigned int flags)
73{ 73{
@@ -91,7 +91,7 @@ struct alloc_pio_regions_param {
91 unsigned int flags; 91 unsigned int flags;
92}; 92};
93 93
94int gxio_trio_alloc_pio_regions(gxio_trio_context_t * context, 94int gxio_trio_alloc_pio_regions(gxio_trio_context_t *context,
95 unsigned int count, unsigned int first, 95 unsigned int count, unsigned int first,
96 unsigned int flags) 96 unsigned int flags)
97{ 97{
@@ -115,7 +115,7 @@ struct init_pio_region_aux_param {
115 unsigned int flags; 115 unsigned int flags;
116}; 116};
117 117
118int gxio_trio_init_pio_region_aux(gxio_trio_context_t * context, 118int gxio_trio_init_pio_region_aux(gxio_trio_context_t *context,
119 unsigned int pio_region, unsigned int mac, 119 unsigned int pio_region, unsigned int mac,
120 uint32_t bus_address_hi, unsigned int flags) 120 uint32_t bus_address_hi, unsigned int flags)
121{ 121{
@@ -145,7 +145,7 @@ struct init_memory_map_mmu_aux_param {
145 unsigned int order_mode; 145 unsigned int order_mode;
146}; 146};
147 147
148int gxio_trio_init_memory_map_mmu_aux(gxio_trio_context_t * context, 148int gxio_trio_init_memory_map_mmu_aux(gxio_trio_context_t *context,
149 unsigned int map, unsigned long va, 149 unsigned int map, unsigned long va,
150 uint64_t size, unsigned int asid, 150 uint64_t size, unsigned int asid,
151 unsigned int mac, uint64_t bus_address, 151 unsigned int mac, uint64_t bus_address,
@@ -175,7 +175,7 @@ struct get_port_property_param {
175 struct pcie_trio_ports_property trio_ports; 175 struct pcie_trio_ports_property trio_ports;
176}; 176};
177 177
178int gxio_trio_get_port_property(gxio_trio_context_t * context, 178int gxio_trio_get_port_property(gxio_trio_context_t *context,
179 struct pcie_trio_ports_property *trio_ports) 179 struct pcie_trio_ports_property *trio_ports)
180{ 180{
181 int __result; 181 int __result;
@@ -198,7 +198,7 @@ struct config_legacy_intr_param {
198 unsigned int intx; 198 unsigned int intx;
199}; 199};
200 200
201int gxio_trio_config_legacy_intr(gxio_trio_context_t * context, int inter_x, 201int gxio_trio_config_legacy_intr(gxio_trio_context_t *context, int inter_x,
202 int inter_y, int inter_ipi, int inter_event, 202 int inter_y, int inter_ipi, int inter_event,
203 unsigned int mac, unsigned int intx) 203 unsigned int mac, unsigned int intx)
204{ 204{
@@ -227,7 +227,7 @@ struct config_msi_intr_param {
227 unsigned int asid; 227 unsigned int asid;
228}; 228};
229 229
230int gxio_trio_config_msi_intr(gxio_trio_context_t * context, int inter_x, 230int gxio_trio_config_msi_intr(gxio_trio_context_t *context, int inter_x,
231 int inter_y, int inter_ipi, int inter_event, 231 int inter_y, int inter_ipi, int inter_event,
232 unsigned int mac, unsigned int mem_map, 232 unsigned int mac, unsigned int mem_map,
233 uint64_t mem_map_base, uint64_t mem_map_limit, 233 uint64_t mem_map_base, uint64_t mem_map_limit,
@@ -259,7 +259,7 @@ struct set_mps_mrs_param {
259 unsigned int mac; 259 unsigned int mac;
260}; 260};
261 261
262int gxio_trio_set_mps_mrs(gxio_trio_context_t * context, uint16_t mps, 262int gxio_trio_set_mps_mrs(gxio_trio_context_t *context, uint16_t mps,
263 uint16_t mrs, unsigned int mac) 263 uint16_t mrs, unsigned int mac)
264{ 264{
265 struct set_mps_mrs_param temp; 265 struct set_mps_mrs_param temp;
@@ -279,7 +279,7 @@ struct force_rc_link_up_param {
279 unsigned int mac; 279 unsigned int mac;
280}; 280};
281 281
282int gxio_trio_force_rc_link_up(gxio_trio_context_t * context, unsigned int mac) 282int gxio_trio_force_rc_link_up(gxio_trio_context_t *context, unsigned int mac)
283{ 283{
284 struct force_rc_link_up_param temp; 284 struct force_rc_link_up_param temp;
285 struct force_rc_link_up_param *params = &temp; 285 struct force_rc_link_up_param *params = &temp;
@@ -296,7 +296,7 @@ struct force_ep_link_up_param {
296 unsigned int mac; 296 unsigned int mac;
297}; 297};
298 298
299int gxio_trio_force_ep_link_up(gxio_trio_context_t * context, unsigned int mac) 299int gxio_trio_force_ep_link_up(gxio_trio_context_t *context, unsigned int mac)
300{ 300{
301 struct force_ep_link_up_param temp; 301 struct force_ep_link_up_param temp;
302 struct force_ep_link_up_param *params = &temp; 302 struct force_ep_link_up_param *params = &temp;
@@ -313,7 +313,7 @@ struct get_mmio_base_param {
313 HV_PTE base; 313 HV_PTE base;
314}; 314};
315 315
316int gxio_trio_get_mmio_base(gxio_trio_context_t * context, HV_PTE *base) 316int gxio_trio_get_mmio_base(gxio_trio_context_t *context, HV_PTE *base)
317{ 317{
318 int __result; 318 int __result;
319 struct get_mmio_base_param temp; 319 struct get_mmio_base_param temp;
@@ -334,7 +334,7 @@ struct check_mmio_offset_param {
334 unsigned long size; 334 unsigned long size;
335}; 335};
336 336
337int gxio_trio_check_mmio_offset(gxio_trio_context_t * context, 337int gxio_trio_check_mmio_offset(gxio_trio_context_t *context,
338 unsigned long offset, unsigned long size) 338 unsigned long offset, unsigned long size)
339{ 339{
340 struct check_mmio_offset_param temp; 340 struct check_mmio_offset_param temp;
diff --git a/arch/tile/gxio/iorpc_usb_host.c b/arch/tile/gxio/iorpc_usb_host.c
index cf3c3cc12204..9c820073bfc0 100644
--- a/arch/tile/gxio/iorpc_usb_host.c
+++ b/arch/tile/gxio/iorpc_usb_host.c
@@ -19,7 +19,7 @@ struct cfg_interrupt_param {
19 union iorpc_interrupt interrupt; 19 union iorpc_interrupt interrupt;
20}; 20};
21 21
22int gxio_usb_host_cfg_interrupt(gxio_usb_host_context_t * context, int inter_x, 22int gxio_usb_host_cfg_interrupt(gxio_usb_host_context_t *context, int inter_x,
23 int inter_y, int inter_ipi, int inter_event) 23 int inter_y, int inter_ipi, int inter_event)
24{ 24{
25 struct cfg_interrupt_param temp; 25 struct cfg_interrupt_param temp;
@@ -41,7 +41,7 @@ struct register_client_memory_param {
41 unsigned int flags; 41 unsigned int flags;
42}; 42};
43 43
44int gxio_usb_host_register_client_memory(gxio_usb_host_context_t * context, 44int gxio_usb_host_register_client_memory(gxio_usb_host_context_t *context,
45 HV_PTE pte, unsigned int flags) 45 HV_PTE pte, unsigned int flags)
46{ 46{
47 struct register_client_memory_param temp; 47 struct register_client_memory_param temp;
@@ -61,7 +61,7 @@ struct get_mmio_base_param {
61 HV_PTE base; 61 HV_PTE base;
62}; 62};
63 63
64int gxio_usb_host_get_mmio_base(gxio_usb_host_context_t * context, HV_PTE *base) 64int gxio_usb_host_get_mmio_base(gxio_usb_host_context_t *context, HV_PTE *base)
65{ 65{
66 int __result; 66 int __result;
67 struct get_mmio_base_param temp; 67 struct get_mmio_base_param temp;
@@ -82,7 +82,7 @@ struct check_mmio_offset_param {
82 unsigned long size; 82 unsigned long size;
83}; 83};
84 84
85int gxio_usb_host_check_mmio_offset(gxio_usb_host_context_t * context, 85int gxio_usb_host_check_mmio_offset(gxio_usb_host_context_t *context,
86 unsigned long offset, unsigned long size) 86 unsigned long offset, unsigned long size)
87{ 87{
88 struct check_mmio_offset_param temp; 88 struct check_mmio_offset_param temp;
diff --git a/arch/tile/gxio/usb_host.c b/arch/tile/gxio/usb_host.c
index 66b002f54ecc..785afad7922e 100644
--- a/arch/tile/gxio/usb_host.c
+++ b/arch/tile/gxio/usb_host.c
@@ -26,7 +26,7 @@
26#include <gxio/kiorpc.h> 26#include <gxio/kiorpc.h>
27#include <gxio/usb_host.h> 27#include <gxio/usb_host.h>
28 28
29int gxio_usb_host_init(gxio_usb_host_context_t * context, int usb_index, 29int gxio_usb_host_init(gxio_usb_host_context_t *context, int usb_index,
30 int is_ehci) 30 int is_ehci)
31{ 31{
32 char file[32]; 32 char file[32];
@@ -63,7 +63,7 @@ int gxio_usb_host_init(gxio_usb_host_context_t * context, int usb_index,
63 63
64EXPORT_SYMBOL_GPL(gxio_usb_host_init); 64EXPORT_SYMBOL_GPL(gxio_usb_host_init);
65 65
66int gxio_usb_host_destroy(gxio_usb_host_context_t * context) 66int gxio_usb_host_destroy(gxio_usb_host_context_t *context)
67{ 67{
68 iounmap((void __force __iomem *)(context->mmio_base)); 68 iounmap((void __force __iomem *)(context->mmio_base));
69 hv_dev_close(context->fd); 69 hv_dev_close(context->fd);
@@ -76,14 +76,14 @@ int gxio_usb_host_destroy(gxio_usb_host_context_t * context)
76 76
77EXPORT_SYMBOL_GPL(gxio_usb_host_destroy); 77EXPORT_SYMBOL_GPL(gxio_usb_host_destroy);
78 78
79void *gxio_usb_host_get_reg_start(gxio_usb_host_context_t * context) 79void *gxio_usb_host_get_reg_start(gxio_usb_host_context_t *context)
80{ 80{
81 return context->mmio_base; 81 return context->mmio_base;
82} 82}
83 83
84EXPORT_SYMBOL_GPL(gxio_usb_host_get_reg_start); 84EXPORT_SYMBOL_GPL(gxio_usb_host_get_reg_start);
85 85
86size_t gxio_usb_host_get_reg_len(gxio_usb_host_context_t * context) 86size_t gxio_usb_host_get_reg_len(gxio_usb_host_context_t *context)
87{ 87{
88 return HV_USB_HOST_MMIO_SIZE; 88 return HV_USB_HOST_MMIO_SIZE;
89} 89}
diff --git a/arch/tile/include/arch/mpipe.h b/arch/tile/include/arch/mpipe.h
index 8a33912fd6cc..904538e754d8 100644
--- a/arch/tile/include/arch/mpipe.h
+++ b/arch/tile/include/arch/mpipe.h
@@ -176,7 +176,18 @@ typedef union
176 */ 176 */
177 uint_reg_t stack_idx : 5; 177 uint_reg_t stack_idx : 5;
178 /* Reserved. */ 178 /* Reserved. */
179 uint_reg_t __reserved_2 : 5; 179 uint_reg_t __reserved_2 : 3;
180 /*
181 * Instance ID. For devices that support automatic buffer return between
182 * mPIPE instances, this field indicates the buffer owner. If the INST
183 * field does not match the mPIPE's instance number when a packet is
184 * egressed, buffers with HWB set will be returned to the other mPIPE
185 * instance. Note that not all devices support multi-mPIPE buffer
186 * return. The MPIPE_EDMA_INFO.REMOTE_BUFF_RTN_SUPPORT bit indicates
187 * whether the INST field in the buffer descriptor is populated by iDMA
188 * hardware. This field is ignored on writes.
189 */
190 uint_reg_t inst : 2;
180 /* 191 /*
181 * Reads as one to indicate that this is a hardware managed buffer. 192 * Reads as one to indicate that this is a hardware managed buffer.
182 * Ignored on writes since all buffers on a given stack are the same size. 193 * Ignored on writes since all buffers on a given stack are the same size.
@@ -205,7 +216,8 @@ typedef union
205 uint_reg_t c : 2; 216 uint_reg_t c : 2;
206 uint_reg_t size : 3; 217 uint_reg_t size : 3;
207 uint_reg_t hwb : 1; 218 uint_reg_t hwb : 1;
208 uint_reg_t __reserved_2 : 5; 219 uint_reg_t inst : 2;
220 uint_reg_t __reserved_2 : 3;
209 uint_reg_t stack_idx : 5; 221 uint_reg_t stack_idx : 5;
210 uint_reg_t __reserved_1 : 6; 222 uint_reg_t __reserved_1 : 6;
211 int_reg_t va : 35; 223 int_reg_t va : 35;
@@ -231,9 +243,9 @@ typedef union
231 /* Reserved. */ 243 /* Reserved. */
232 uint_reg_t __reserved_0 : 3; 244 uint_reg_t __reserved_0 : 3;
233 /* eDMA ring being accessed */ 245 /* eDMA ring being accessed */
234 uint_reg_t ring : 5; 246 uint_reg_t ring : 6;
235 /* Reserved. */ 247 /* Reserved. */
236 uint_reg_t __reserved_1 : 18; 248 uint_reg_t __reserved_1 : 17;
237 /* 249 /*
238 * This field of the address selects the region (address space) to be 250 * This field of the address selects the region (address space) to be
239 * accessed. For the egress DMA post region, this field must be 5. 251 * accessed. For the egress DMA post region, this field must be 5.
@@ -250,8 +262,8 @@ typedef union
250 uint_reg_t svc_dom : 5; 262 uint_reg_t svc_dom : 5;
251 uint_reg_t __reserved_2 : 6; 263 uint_reg_t __reserved_2 : 6;
252 uint_reg_t region : 3; 264 uint_reg_t region : 3;
253 uint_reg_t __reserved_1 : 18; 265 uint_reg_t __reserved_1 : 17;
254 uint_reg_t ring : 5; 266 uint_reg_t ring : 6;
255 uint_reg_t __reserved_0 : 3; 267 uint_reg_t __reserved_0 : 3;
256#endif 268#endif
257 }; 269 };
diff --git a/arch/tile/include/arch/mpipe_constants.h b/arch/tile/include/arch/mpipe_constants.h
index 410a0400e055..84022ac5fe82 100644
--- a/arch/tile/include/arch/mpipe_constants.h
+++ b/arch/tile/include/arch/mpipe_constants.h
@@ -16,13 +16,13 @@
16#ifndef __ARCH_MPIPE_CONSTANTS_H__ 16#ifndef __ARCH_MPIPE_CONSTANTS_H__
17#define __ARCH_MPIPE_CONSTANTS_H__ 17#define __ARCH_MPIPE_CONSTANTS_H__
18 18
19#define MPIPE_NUM_CLASSIFIERS 10 19#define MPIPE_NUM_CLASSIFIERS 16
20#define MPIPE_CLS_MHZ 1200 20#define MPIPE_CLS_MHZ 1200
21 21
22#define MPIPE_NUM_EDMA_RINGS 32 22#define MPIPE_NUM_EDMA_RINGS 64
23 23
24#define MPIPE_NUM_SGMII_MACS 16 24#define MPIPE_NUM_SGMII_MACS 16
25#define MPIPE_NUM_XAUI_MACS 4 25#define MPIPE_NUM_XAUI_MACS 16
26#define MPIPE_NUM_LOOPBACK_CHANNELS 4 26#define MPIPE_NUM_LOOPBACK_CHANNELS 4
27#define MPIPE_NUM_NON_LB_CHANNELS 28 27#define MPIPE_NUM_NON_LB_CHANNELS 28
28 28
diff --git a/arch/tile/include/arch/mpipe_shm.h b/arch/tile/include/arch/mpipe_shm.h
index f2e9e122818d..13b3c4300e50 100644
--- a/arch/tile/include/arch/mpipe_shm.h
+++ b/arch/tile/include/arch/mpipe_shm.h
@@ -44,8 +44,14 @@ typedef union
44 * descriptors toggles each time the ring tail pointer wraps. 44 * descriptors toggles each time the ring tail pointer wraps.
45 */ 45 */
46 uint_reg_t gen : 1; 46 uint_reg_t gen : 1;
47 /**
48 * For devices with EDMA reorder support, this field allows the
49 * descriptor to select the egress FIFO. The associated DMA ring must
50 * have ALLOW_EFIFO_SEL enabled.
51 */
52 uint_reg_t efifo_sel : 6;
47 /** Reserved. Must be zero. */ 53 /** Reserved. Must be zero. */
48 uint_reg_t r0 : 7; 54 uint_reg_t r0 : 1;
49 /** Checksum generation enabled for this transfer. */ 55 /** Checksum generation enabled for this transfer. */
50 uint_reg_t csum : 1; 56 uint_reg_t csum : 1;
51 /** 57 /**
@@ -110,7 +116,8 @@ typedef union
110 uint_reg_t notif : 1; 116 uint_reg_t notif : 1;
111 uint_reg_t ns : 1; 117 uint_reg_t ns : 1;
112 uint_reg_t csum : 1; 118 uint_reg_t csum : 1;
113 uint_reg_t r0 : 7; 119 uint_reg_t r0 : 1;
120 uint_reg_t efifo_sel : 6;
114 uint_reg_t gen : 1; 121 uint_reg_t gen : 1;
115#endif 122#endif
116 123
@@ -126,14 +133,16 @@ typedef union
126 /** Reserved. */ 133 /** Reserved. */
127 uint_reg_t __reserved_1 : 3; 134 uint_reg_t __reserved_1 : 3;
128 /** 135 /**
129 * Instance ID. For devices that support more than one mPIPE instance, 136 * Instance ID. For devices that support automatic buffer return between
130 * this field indicates the buffer owner. If the INST field does not 137 * mPIPE instances, this field indicates the buffer owner. If the INST
131 * match the mPIPE's instance number when a packet is egressed, buffers 138 * field does not match the mPIPE's instance number when a packet is
132 * with HWB set will be returned to the other mPIPE instance. 139 * egressed, buffers with HWB set will be returned to the other mPIPE
140 * instance. Note that not all devices support multi-mPIPE buffer
141 * return. The MPIPE_EDMA_INFO.REMOTE_BUFF_RTN_SUPPORT bit indicates
142 * whether the INST field in the buffer descriptor is populated by iDMA
143 * hardware.
133 */ 144 */
134 uint_reg_t inst : 1; 145 uint_reg_t inst : 2;
135 /** Reserved. */
136 uint_reg_t __reserved_2 : 1;
137 /** 146 /**
138 * Always set to one by hardware in iDMA packet descriptors. For eDMA, 147 * Always set to one by hardware in iDMA packet descriptors. For eDMA,
139 * indicates whether the buffer will be released to the buffer stack 148 * indicates whether the buffer will be released to the buffer stack
@@ -166,8 +175,7 @@ typedef union
166 uint_reg_t c : 2; 175 uint_reg_t c : 2;
167 uint_reg_t size : 3; 176 uint_reg_t size : 3;
168 uint_reg_t hwb : 1; 177 uint_reg_t hwb : 1;
169 uint_reg_t __reserved_2 : 1; 178 uint_reg_t inst : 2;
170 uint_reg_t inst : 1;
171 uint_reg_t __reserved_1 : 3; 179 uint_reg_t __reserved_1 : 3;
172 uint_reg_t stack_idx : 5; 180 uint_reg_t stack_idx : 5;
173 uint_reg_t __reserved_0 : 6; 181 uint_reg_t __reserved_0 : 6;
@@ -408,7 +416,10 @@ typedef union
408 /** 416 /**
409 * Sequence number applied when packet is distributed. Classifier 417 * Sequence number applied when packet is distributed. Classifier
410 * selects which sequence number is to be applied by writing the 13-bit 418 * selects which sequence number is to be applied by writing the 13-bit
411 * SQN-selector into this field. 419 * SQN-selector into this field. For devices that support EXT_SQN (as
420 * indicated in IDMA_INFO.EXT_SQN_SUPPORT), the GP_SQN can be extended to
421 * 32-bits via the IDMA_CTL.EXT_SQN register. In this case the
422 * PACKET_SQN will be reduced to 32 bits.
412 */ 423 */
413 uint_reg_t gp_sqn : 16; 424 uint_reg_t gp_sqn : 16;
414 /** 425 /**
@@ -451,14 +462,16 @@ typedef union
451 /** Reserved. */ 462 /** Reserved. */
452 uint_reg_t __reserved_5 : 3; 463 uint_reg_t __reserved_5 : 3;
453 /** 464 /**
454 * Instance ID. For devices that support more than one mPIPE instance, 465 * Instance ID. For devices that support automatic buffer return between
455 * this field indicates the buffer owner. If the INST field does not 466 * mPIPE instances, this field indicates the buffer owner. If the INST
456 * match the mPIPE's instance number when a packet is egressed, buffers 467 * field does not match the mPIPE's instance number when a packet is
457 * with HWB set will be returned to the other mPIPE instance. 468 * egressed, buffers with HWB set will be returned to the other mPIPE
469 * instance. Note that not all devices support multi-mPIPE buffer
470 * return. The MPIPE_EDMA_INFO.REMOTE_BUFF_RTN_SUPPORT bit indicates
471 * whether the INST field in the buffer descriptor is populated by iDMA
472 * hardware.
458 */ 473 */
459 uint_reg_t inst : 1; 474 uint_reg_t inst : 2;
460 /** Reserved. */
461 uint_reg_t __reserved_6 : 1;
462 /** 475 /**
463 * Always set to one by hardware in iDMA packet descriptors. For eDMA, 476 * Always set to one by hardware in iDMA packet descriptors. For eDMA,
464 * indicates whether the buffer will be released to the buffer stack 477 * indicates whether the buffer will be released to the buffer stack
@@ -491,8 +504,7 @@ typedef union
491 uint_reg_t c : 2; 504 uint_reg_t c : 2;
492 uint_reg_t size : 3; 505 uint_reg_t size : 3;
493 uint_reg_t hwb : 1; 506 uint_reg_t hwb : 1;
494 uint_reg_t __reserved_6 : 1; 507 uint_reg_t inst : 2;
495 uint_reg_t inst : 1;
496 uint_reg_t __reserved_5 : 3; 508 uint_reg_t __reserved_5 : 3;
497 uint_reg_t stack_idx : 5; 509 uint_reg_t stack_idx : 5;
498 uint_reg_t __reserved_4 : 6; 510 uint_reg_t __reserved_4 : 6;
diff --git a/arch/tile/include/arch/trio_constants.h b/arch/tile/include/arch/trio_constants.h
index 628b045436b8..85647e91a458 100644
--- a/arch/tile/include/arch/trio_constants.h
+++ b/arch/tile/include/arch/trio_constants.h
@@ -16,21 +16,21 @@
16#ifndef __ARCH_TRIO_CONSTANTS_H__ 16#ifndef __ARCH_TRIO_CONSTANTS_H__
17#define __ARCH_TRIO_CONSTANTS_H__ 17#define __ARCH_TRIO_CONSTANTS_H__
18 18
19#define TRIO_NUM_ASIDS 16 19#define TRIO_NUM_ASIDS 32
20#define TRIO_NUM_TLBS_PER_ASID 16 20#define TRIO_NUM_TLBS_PER_ASID 16
21 21
22#define TRIO_NUM_TPIO_REGIONS 8 22#define TRIO_NUM_TPIO_REGIONS 8
23#define TRIO_LOG2_NUM_TPIO_REGIONS 3 23#define TRIO_LOG2_NUM_TPIO_REGIONS 3
24 24
25#define TRIO_NUM_MAP_MEM_REGIONS 16 25#define TRIO_NUM_MAP_MEM_REGIONS 32
26#define TRIO_LOG2_NUM_MAP_MEM_REGIONS 4 26#define TRIO_LOG2_NUM_MAP_MEM_REGIONS 5
27#define TRIO_NUM_MAP_SQ_REGIONS 8 27#define TRIO_NUM_MAP_SQ_REGIONS 8
28#define TRIO_LOG2_NUM_MAP_SQ_REGIONS 3 28#define TRIO_LOG2_NUM_MAP_SQ_REGIONS 3
29 29
30#define TRIO_LOG2_NUM_SQ_FIFO_ENTRIES 6 30#define TRIO_LOG2_NUM_SQ_FIFO_ENTRIES 6
31 31
32#define TRIO_NUM_PUSH_DMA_RINGS 32 32#define TRIO_NUM_PUSH_DMA_RINGS 64
33 33
34#define TRIO_NUM_PULL_DMA_RINGS 32 34#define TRIO_NUM_PULL_DMA_RINGS 64
35 35
36#endif /* __ARCH_TRIO_CONSTANTS_H__ */ 36#endif /* __ARCH_TRIO_CONSTANTS_H__ */
diff --git a/arch/tile/include/asm/page.h b/arch/tile/include/asm/page.h
index 6346888f7bdc..672768008618 100644
--- a/arch/tile/include/asm/page.h
+++ b/arch/tile/include/asm/page.h
@@ -182,10 +182,9 @@ static inline __attribute_const__ int get_order(unsigned long size)
182 182
183#define PAGE_OFFSET (-(_AC(1, UL) << (MAX_VA_WIDTH - 1))) 183#define PAGE_OFFSET (-(_AC(1, UL) << (MAX_VA_WIDTH - 1)))
184#define KERNEL_HIGH_VADDR _AC(0xfffffff800000000, UL) /* high 32GB */ 184#define KERNEL_HIGH_VADDR _AC(0xfffffff800000000, UL) /* high 32GB */
185#define FIXADDR_BASE (KERNEL_HIGH_VADDR - 0x400000000) /* 4 GB */ 185#define FIXADDR_BASE (KERNEL_HIGH_VADDR - 0x300000000) /* 4 GB */
186#define FIXADDR_TOP (KERNEL_HIGH_VADDR - 0x300000000) /* 4 GB */ 186#define FIXADDR_TOP (KERNEL_HIGH_VADDR - 0x200000000) /* 4 GB */
187#define _VMALLOC_START FIXADDR_TOP 187#define _VMALLOC_START FIXADDR_TOP
188#define HUGE_VMAP_BASE (KERNEL_HIGH_VADDR - 0x200000000) /* 4 GB */
189#define MEM_SV_START (KERNEL_HIGH_VADDR - 0x100000000) /* 256 MB */ 188#define MEM_SV_START (KERNEL_HIGH_VADDR - 0x100000000) /* 256 MB */
190#define MEM_MODULE_START (MEM_SV_START + (256*1024*1024)) /* 256 MB */ 189#define MEM_MODULE_START (MEM_SV_START + (256*1024*1024)) /* 256 MB */
191#define MEM_MODULE_END (MEM_MODULE_START + (256*1024*1024)) 190#define MEM_MODULE_END (MEM_MODULE_START + (256*1024*1024))
diff --git a/arch/tile/include/asm/pgtable_32.h b/arch/tile/include/asm/pgtable_32.h
index 63142ab3b3dd..d26a42279036 100644
--- a/arch/tile/include/asm/pgtable_32.h
+++ b/arch/tile/include/asm/pgtable_32.h
@@ -55,17 +55,9 @@
55#define PKMAP_BASE ((FIXADDR_BOOT_START - PAGE_SIZE*LAST_PKMAP) & PGDIR_MASK) 55#define PKMAP_BASE ((FIXADDR_BOOT_START - PAGE_SIZE*LAST_PKMAP) & PGDIR_MASK)
56 56
57#ifdef CONFIG_HIGHMEM 57#ifdef CONFIG_HIGHMEM
58# define __VMAPPING_END (PKMAP_BASE & ~(HPAGE_SIZE-1)) 58# define _VMALLOC_END (PKMAP_BASE & ~(HPAGE_SIZE-1))
59#else 59#else
60# define __VMAPPING_END (FIXADDR_START & ~(HPAGE_SIZE-1)) 60# define _VMALLOC_END (FIXADDR_START & ~(HPAGE_SIZE-1))
61#endif
62
63#ifdef CONFIG_HUGEVMAP
64#define HUGE_VMAP_END __VMAPPING_END
65#define HUGE_VMAP_BASE (HUGE_VMAP_END - CONFIG_NR_HUGE_VMAPS * HPAGE_SIZE)
66#define _VMALLOC_END HUGE_VMAP_BASE
67#else
68#define _VMALLOC_END __VMAPPING_END
69#endif 61#endif
70 62
71/* 63/*
diff --git a/arch/tile/include/asm/pgtable_64.h b/arch/tile/include/asm/pgtable_64.h
index 3421177f7370..2c8a9cd102d3 100644
--- a/arch/tile/include/asm/pgtable_64.h
+++ b/arch/tile/include/asm/pgtable_64.h
@@ -52,12 +52,10 @@
52 * memory allocation code). The vmalloc code puts in an internal 52 * memory allocation code). The vmalloc code puts in an internal
53 * guard page between each allocation. 53 * guard page between each allocation.
54 */ 54 */
55#define _VMALLOC_END HUGE_VMAP_BASE 55#define _VMALLOC_END MEM_SV_START
56#define VMALLOC_END _VMALLOC_END 56#define VMALLOC_END _VMALLOC_END
57#define VMALLOC_START _VMALLOC_START 57#define VMALLOC_START _VMALLOC_START
58 58
59#define HUGE_VMAP_END (HUGE_VMAP_BASE + PGDIR_SIZE)
60
61#ifndef __ASSEMBLY__ 59#ifndef __ASSEMBLY__
62 60
63/* We have no pud since we are a three-level page table. */ 61/* We have no pud since we are a three-level page table. */
diff --git a/arch/tile/include/gxio/iorpc_mpipe.h b/arch/tile/include/gxio/iorpc_mpipe.h
index fdd07f88cfd7..4cda03de734f 100644
--- a/arch/tile/include/gxio/iorpc_mpipe.h
+++ b/arch/tile/include/gxio/iorpc_mpipe.h
@@ -56,89 +56,89 @@
56#define GXIO_MPIPE_OP_GET_MMIO_BASE IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8000) 56#define GXIO_MPIPE_OP_GET_MMIO_BASE IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8000)
57#define GXIO_MPIPE_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001) 57#define GXIO_MPIPE_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001)
58 58
59int gxio_mpipe_alloc_buffer_stacks(gxio_mpipe_context_t * context, 59int gxio_mpipe_alloc_buffer_stacks(gxio_mpipe_context_t *context,
60 unsigned int count, unsigned int first, 60 unsigned int count, unsigned int first,
61 unsigned int flags); 61 unsigned int flags);
62 62
63int gxio_mpipe_init_buffer_stack_aux(gxio_mpipe_context_t * context, 63int gxio_mpipe_init_buffer_stack_aux(gxio_mpipe_context_t *context,
64 void *mem_va, size_t mem_size, 64 void *mem_va, size_t mem_size,
65 unsigned int mem_flags, unsigned int stack, 65 unsigned int mem_flags, unsigned int stack,
66 unsigned int buffer_size_enum); 66 unsigned int buffer_size_enum);
67 67
68 68
69int gxio_mpipe_alloc_notif_rings(gxio_mpipe_context_t * context, 69int gxio_mpipe_alloc_notif_rings(gxio_mpipe_context_t *context,
70 unsigned int count, unsigned int first, 70 unsigned int count, unsigned int first,
71 unsigned int flags); 71 unsigned int flags);
72 72
73int gxio_mpipe_init_notif_ring_aux(gxio_mpipe_context_t * context, void *mem_va, 73int gxio_mpipe_init_notif_ring_aux(gxio_mpipe_context_t *context, void *mem_va,
74 size_t mem_size, unsigned int mem_flags, 74 size_t mem_size, unsigned int mem_flags,
75 unsigned int ring); 75 unsigned int ring);
76 76
77int gxio_mpipe_request_notif_ring_interrupt(gxio_mpipe_context_t * context, 77int gxio_mpipe_request_notif_ring_interrupt(gxio_mpipe_context_t *context,
78 int inter_x, int inter_y, 78 int inter_x, int inter_y,
79 int inter_ipi, int inter_event, 79 int inter_ipi, int inter_event,
80 unsigned int ring); 80 unsigned int ring);
81 81
82int gxio_mpipe_enable_notif_ring_interrupt(gxio_mpipe_context_t * context, 82int gxio_mpipe_enable_notif_ring_interrupt(gxio_mpipe_context_t *context,
83 unsigned int ring); 83 unsigned int ring);
84 84
85int gxio_mpipe_alloc_notif_groups(gxio_mpipe_context_t * context, 85int gxio_mpipe_alloc_notif_groups(gxio_mpipe_context_t *context,
86 unsigned int count, unsigned int first, 86 unsigned int count, unsigned int first,
87 unsigned int flags); 87 unsigned int flags);
88 88
89int gxio_mpipe_init_notif_group(gxio_mpipe_context_t * context, 89int gxio_mpipe_init_notif_group(gxio_mpipe_context_t *context,
90 unsigned int group, 90 unsigned int group,
91 gxio_mpipe_notif_group_bits_t bits); 91 gxio_mpipe_notif_group_bits_t bits);
92 92
93int gxio_mpipe_alloc_buckets(gxio_mpipe_context_t * context, unsigned int count, 93int gxio_mpipe_alloc_buckets(gxio_mpipe_context_t *context, unsigned int count,
94 unsigned int first, unsigned int flags); 94 unsigned int first, unsigned int flags);
95 95
96int gxio_mpipe_init_bucket(gxio_mpipe_context_t * context, unsigned int bucket, 96int gxio_mpipe_init_bucket(gxio_mpipe_context_t *context, unsigned int bucket,
97 MPIPE_LBL_INIT_DAT_BSTS_TBL_t bucket_info); 97 MPIPE_LBL_INIT_DAT_BSTS_TBL_t bucket_info);
98 98
99int gxio_mpipe_alloc_edma_rings(gxio_mpipe_context_t * context, 99int gxio_mpipe_alloc_edma_rings(gxio_mpipe_context_t *context,
100 unsigned int count, unsigned int first, 100 unsigned int count, unsigned int first,
101 unsigned int flags); 101 unsigned int flags);
102 102
103int gxio_mpipe_init_edma_ring_aux(gxio_mpipe_context_t * context, void *mem_va, 103int gxio_mpipe_init_edma_ring_aux(gxio_mpipe_context_t *context, void *mem_va,
104 size_t mem_size, unsigned int mem_flags, 104 size_t mem_size, unsigned int mem_flags,
105 unsigned int ring, unsigned int channel); 105 unsigned int ring, unsigned int channel);
106 106
107 107
108int gxio_mpipe_commit_rules(gxio_mpipe_context_t * context, const void *blob, 108int gxio_mpipe_commit_rules(gxio_mpipe_context_t *context, const void *blob,
109 size_t blob_size); 109 size_t blob_size);
110 110
111int gxio_mpipe_register_client_memory(gxio_mpipe_context_t * context, 111int gxio_mpipe_register_client_memory(gxio_mpipe_context_t *context,
112 unsigned int iotlb, HV_PTE pte, 112 unsigned int iotlb, HV_PTE pte,
113 unsigned int flags); 113 unsigned int flags);
114 114
115int gxio_mpipe_link_open_aux(gxio_mpipe_context_t * context, 115int gxio_mpipe_link_open_aux(gxio_mpipe_context_t *context,
116 _gxio_mpipe_link_name_t name, unsigned int flags); 116 _gxio_mpipe_link_name_t name, unsigned int flags);
117 117
118int gxio_mpipe_link_close_aux(gxio_mpipe_context_t * context, int mac); 118int gxio_mpipe_link_close_aux(gxio_mpipe_context_t *context, int mac);
119 119
120int gxio_mpipe_link_set_attr_aux(gxio_mpipe_context_t * context, int mac, 120int gxio_mpipe_link_set_attr_aux(gxio_mpipe_context_t *context, int mac,
121 uint32_t attr, int64_t val); 121 uint32_t attr, int64_t val);
122 122
123int gxio_mpipe_get_timestamp_aux(gxio_mpipe_context_t * context, uint64_t * sec, 123int gxio_mpipe_get_timestamp_aux(gxio_mpipe_context_t *context, uint64_t *sec,
124 uint64_t * nsec, uint64_t * cycles); 124 uint64_t *nsec, uint64_t *cycles);
125 125
126int gxio_mpipe_set_timestamp_aux(gxio_mpipe_context_t * context, uint64_t sec, 126int gxio_mpipe_set_timestamp_aux(gxio_mpipe_context_t *context, uint64_t sec,
127 uint64_t nsec, uint64_t cycles); 127 uint64_t nsec, uint64_t cycles);
128 128
129int gxio_mpipe_adjust_timestamp_aux(gxio_mpipe_context_t * context, 129int gxio_mpipe_adjust_timestamp_aux(gxio_mpipe_context_t *context,
130 int64_t nsec); 130 int64_t nsec);
131 131
132int gxio_mpipe_adjust_timestamp_freq(gxio_mpipe_context_t * context, 132int gxio_mpipe_adjust_timestamp_freq(gxio_mpipe_context_t *context,
133 int32_t ppb); 133 int32_t ppb);
134 134
135int gxio_mpipe_arm_pollfd(gxio_mpipe_context_t * context, int pollfd_cookie); 135int gxio_mpipe_arm_pollfd(gxio_mpipe_context_t *context, int pollfd_cookie);
136 136
137int gxio_mpipe_close_pollfd(gxio_mpipe_context_t * context, int pollfd_cookie); 137int gxio_mpipe_close_pollfd(gxio_mpipe_context_t *context, int pollfd_cookie);
138 138
139int gxio_mpipe_get_mmio_base(gxio_mpipe_context_t * context, HV_PTE *base); 139int gxio_mpipe_get_mmio_base(gxio_mpipe_context_t *context, HV_PTE *base);
140 140
141int gxio_mpipe_check_mmio_offset(gxio_mpipe_context_t * context, 141int gxio_mpipe_check_mmio_offset(gxio_mpipe_context_t *context,
142 unsigned long offset, unsigned long size); 142 unsigned long offset, unsigned long size);
143 143
144#endif /* !__GXIO_MPIPE_LINUX_RPC_H__ */ 144#endif /* !__GXIO_MPIPE_LINUX_RPC_H__ */
diff --git a/arch/tile/include/gxio/iorpc_mpipe_info.h b/arch/tile/include/gxio/iorpc_mpipe_info.h
index 476c5e5ca22c..f0b04284468b 100644
--- a/arch/tile/include/gxio/iorpc_mpipe_info.h
+++ b/arch/tile/include/gxio/iorpc_mpipe_info.h
@@ -33,18 +33,18 @@
33#define GXIO_MPIPE_INFO_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001) 33#define GXIO_MPIPE_INFO_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001)
34 34
35 35
36int gxio_mpipe_info_instance_aux(gxio_mpipe_info_context_t * context, 36int gxio_mpipe_info_instance_aux(gxio_mpipe_info_context_t *context,
37 _gxio_mpipe_link_name_t name); 37 _gxio_mpipe_link_name_t name);
38 38
39int gxio_mpipe_info_enumerate_aux(gxio_mpipe_info_context_t * context, 39int gxio_mpipe_info_enumerate_aux(gxio_mpipe_info_context_t *context,
40 unsigned int idx, 40 unsigned int idx,
41 _gxio_mpipe_link_name_t * name, 41 _gxio_mpipe_link_name_t *name,
42 _gxio_mpipe_link_mac_t * mac); 42 _gxio_mpipe_link_mac_t *mac);
43 43
44int gxio_mpipe_info_get_mmio_base(gxio_mpipe_info_context_t * context, 44int gxio_mpipe_info_get_mmio_base(gxio_mpipe_info_context_t *context,
45 HV_PTE *base); 45 HV_PTE *base);
46 46
47int gxio_mpipe_info_check_mmio_offset(gxio_mpipe_info_context_t * context, 47int gxio_mpipe_info_check_mmio_offset(gxio_mpipe_info_context_t *context,
48 unsigned long offset, unsigned long size); 48 unsigned long offset, unsigned long size);
49 49
50#endif /* !__GXIO_MPIPE_INFO_LINUX_RPC_H__ */ 50#endif /* !__GXIO_MPIPE_INFO_LINUX_RPC_H__ */
diff --git a/arch/tile/include/gxio/iorpc_trio.h b/arch/tile/include/gxio/iorpc_trio.h
index d95b96fd6c93..376a4f771167 100644
--- a/arch/tile/include/gxio/iorpc_trio.h
+++ b/arch/tile/include/gxio/iorpc_trio.h
@@ -46,59 +46,59 @@
46#define GXIO_TRIO_OP_GET_MMIO_BASE IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8000) 46#define GXIO_TRIO_OP_GET_MMIO_BASE IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8000)
47#define GXIO_TRIO_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001) 47#define GXIO_TRIO_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001)
48 48
49int gxio_trio_alloc_asids(gxio_trio_context_t * context, unsigned int count, 49int gxio_trio_alloc_asids(gxio_trio_context_t *context, unsigned int count,
50 unsigned int first, unsigned int flags); 50 unsigned int first, unsigned int flags);
51 51
52 52
53int gxio_trio_alloc_memory_maps(gxio_trio_context_t * context, 53int gxio_trio_alloc_memory_maps(gxio_trio_context_t *context,
54 unsigned int count, unsigned int first, 54 unsigned int count, unsigned int first,
55 unsigned int flags); 55 unsigned int flags);
56 56
57 57
58int gxio_trio_alloc_scatter_queues(gxio_trio_context_t * context, 58int gxio_trio_alloc_scatter_queues(gxio_trio_context_t *context,
59 unsigned int count, unsigned int first, 59 unsigned int count, unsigned int first,
60 unsigned int flags); 60 unsigned int flags);
61 61
62int gxio_trio_alloc_pio_regions(gxio_trio_context_t * context, 62int gxio_trio_alloc_pio_regions(gxio_trio_context_t *context,
63 unsigned int count, unsigned int first, 63 unsigned int count, unsigned int first,
64 unsigned int flags); 64 unsigned int flags);
65 65
66int gxio_trio_init_pio_region_aux(gxio_trio_context_t * context, 66int gxio_trio_init_pio_region_aux(gxio_trio_context_t *context,
67 unsigned int pio_region, unsigned int mac, 67 unsigned int pio_region, unsigned int mac,
68 uint32_t bus_address_hi, unsigned int flags); 68 uint32_t bus_address_hi, unsigned int flags);
69 69
70 70
71int gxio_trio_init_memory_map_mmu_aux(gxio_trio_context_t * context, 71int gxio_trio_init_memory_map_mmu_aux(gxio_trio_context_t *context,
72 unsigned int map, unsigned long va, 72 unsigned int map, unsigned long va,
73 uint64_t size, unsigned int asid, 73 uint64_t size, unsigned int asid,
74 unsigned int mac, uint64_t bus_address, 74 unsigned int mac, uint64_t bus_address,
75 unsigned int node, 75 unsigned int node,
76 unsigned int order_mode); 76 unsigned int order_mode);
77 77
78int gxio_trio_get_port_property(gxio_trio_context_t * context, 78int gxio_trio_get_port_property(gxio_trio_context_t *context,
79 struct pcie_trio_ports_property *trio_ports); 79 struct pcie_trio_ports_property *trio_ports);
80 80
81int gxio_trio_config_legacy_intr(gxio_trio_context_t * context, int inter_x, 81int gxio_trio_config_legacy_intr(gxio_trio_context_t *context, int inter_x,
82 int inter_y, int inter_ipi, int inter_event, 82 int inter_y, int inter_ipi, int inter_event,
83 unsigned int mac, unsigned int intx); 83 unsigned int mac, unsigned int intx);
84 84
85int gxio_trio_config_msi_intr(gxio_trio_context_t * context, int inter_x, 85int gxio_trio_config_msi_intr(gxio_trio_context_t *context, int inter_x,
86 int inter_y, int inter_ipi, int inter_event, 86 int inter_y, int inter_ipi, int inter_event,
87 unsigned int mac, unsigned int mem_map, 87 unsigned int mac, unsigned int mem_map,
88 uint64_t mem_map_base, uint64_t mem_map_limit, 88 uint64_t mem_map_base, uint64_t mem_map_limit,
89 unsigned int asid); 89 unsigned int asid);
90 90
91 91
92int gxio_trio_set_mps_mrs(gxio_trio_context_t * context, uint16_t mps, 92int gxio_trio_set_mps_mrs(gxio_trio_context_t *context, uint16_t mps,
93 uint16_t mrs, unsigned int mac); 93 uint16_t mrs, unsigned int mac);
94 94
95int gxio_trio_force_rc_link_up(gxio_trio_context_t * context, unsigned int mac); 95int gxio_trio_force_rc_link_up(gxio_trio_context_t *context, unsigned int mac);
96 96
97int gxio_trio_force_ep_link_up(gxio_trio_context_t * context, unsigned int mac); 97int gxio_trio_force_ep_link_up(gxio_trio_context_t *context, unsigned int mac);
98 98
99int gxio_trio_get_mmio_base(gxio_trio_context_t * context, HV_PTE *base); 99int gxio_trio_get_mmio_base(gxio_trio_context_t *context, HV_PTE *base);
100 100
101int gxio_trio_check_mmio_offset(gxio_trio_context_t * context, 101int gxio_trio_check_mmio_offset(gxio_trio_context_t *context,
102 unsigned long offset, unsigned long size); 102 unsigned long offset, unsigned long size);
103 103
104#endif /* !__GXIO_TRIO_LINUX_RPC_H__ */ 104#endif /* !__GXIO_TRIO_LINUX_RPC_H__ */
diff --git a/arch/tile/include/gxio/iorpc_usb_host.h b/arch/tile/include/gxio/iorpc_usb_host.h
index 8622e7d126ad..79962a97de8e 100644
--- a/arch/tile/include/gxio/iorpc_usb_host.h
+++ b/arch/tile/include/gxio/iorpc_usb_host.h
@@ -31,16 +31,16 @@
31#define GXIO_USB_HOST_OP_GET_MMIO_BASE IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8000) 31#define GXIO_USB_HOST_OP_GET_MMIO_BASE IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8000)
32#define GXIO_USB_HOST_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001) 32#define GXIO_USB_HOST_OP_CHECK_MMIO_OFFSET IORPC_OPCODE(IORPC_FORMAT_NONE_NOUSER, 0x8001)
33 33
34int gxio_usb_host_cfg_interrupt(gxio_usb_host_context_t * context, int inter_x, 34int gxio_usb_host_cfg_interrupt(gxio_usb_host_context_t *context, int inter_x,
35 int inter_y, int inter_ipi, int inter_event); 35 int inter_y, int inter_ipi, int inter_event);
36 36
37int gxio_usb_host_register_client_memory(gxio_usb_host_context_t * context, 37int gxio_usb_host_register_client_memory(gxio_usb_host_context_t *context,
38 HV_PTE pte, unsigned int flags); 38 HV_PTE pte, unsigned int flags);
39 39
40int gxio_usb_host_get_mmio_base(gxio_usb_host_context_t * context, 40int gxio_usb_host_get_mmio_base(gxio_usb_host_context_t *context,
41 HV_PTE *base); 41 HV_PTE *base);
42 42
43int gxio_usb_host_check_mmio_offset(gxio_usb_host_context_t * context, 43int gxio_usb_host_check_mmio_offset(gxio_usb_host_context_t *context,
44 unsigned long offset, unsigned long size); 44 unsigned long offset, unsigned long size);
45 45
46#endif /* !__GXIO_USB_HOST_LINUX_RPC_H__ */ 46#endif /* !__GXIO_USB_HOST_LINUX_RPC_H__ */
diff --git a/arch/tile/include/gxio/usb_host.h b/arch/tile/include/gxio/usb_host.h
index 5eedec0e988e..93c9636d2dd7 100644
--- a/arch/tile/include/gxio/usb_host.h
+++ b/arch/tile/include/gxio/usb_host.h
@@ -53,7 +53,7 @@ typedef struct {
53 * @return Zero if the context was successfully initialized, else a 53 * @return Zero if the context was successfully initialized, else a
54 * GXIO_ERR_xxx error code. 54 * GXIO_ERR_xxx error code.
55 */ 55 */
56extern int gxio_usb_host_init(gxio_usb_host_context_t * context, int usb_index, 56extern int gxio_usb_host_init(gxio_usb_host_context_t *context, int usb_index,
57 int is_ehci); 57 int is_ehci);
58 58
59/* Destroy a USB context. 59/* Destroy a USB context.
@@ -68,20 +68,20 @@ extern int gxio_usb_host_init(gxio_usb_host_context_t * context, int usb_index,
68 * @return Zero if the context was successfully destroyed, else a 68 * @return Zero if the context was successfully destroyed, else a
69 * GXIO_ERR_xxx error code. 69 * GXIO_ERR_xxx error code.
70 */ 70 */
71extern int gxio_usb_host_destroy(gxio_usb_host_context_t * context); 71extern int gxio_usb_host_destroy(gxio_usb_host_context_t *context);
72 72
73/* Retrieve the address of the shim's MMIO registers. 73/* Retrieve the address of the shim's MMIO registers.
74 * 74 *
75 * @param context Pointer to a properly initialized gxio_usb_host_context_t. 75 * @param context Pointer to a properly initialized gxio_usb_host_context_t.
76 * @return The address of the shim's MMIO registers. 76 * @return The address of the shim's MMIO registers.
77 */ 77 */
78extern void *gxio_usb_host_get_reg_start(gxio_usb_host_context_t * context); 78extern void *gxio_usb_host_get_reg_start(gxio_usb_host_context_t *context);
79 79
80/* Retrieve the length of the shim's MMIO registers. 80/* Retrieve the length of the shim's MMIO registers.
81 * 81 *
82 * @param context Pointer to a properly initialized gxio_usb_host_context_t. 82 * @param context Pointer to a properly initialized gxio_usb_host_context_t.
83 * @return The length of the shim's MMIO registers. 83 * @return The length of the shim's MMIO registers.
84 */ 84 */
85extern size_t gxio_usb_host_get_reg_len(gxio_usb_host_context_t * context); 85extern size_t gxio_usb_host_get_reg_len(gxio_usb_host_context_t *context);
86 86
87#endif /* _GXIO_USB_H_ */ 87#endif /* _GXIO_USB_H_ */
diff --git a/arch/tile/kernel/compat.c b/arch/tile/kernel/compat.c
index ed378416b86a..49120843ff96 100644
--- a/arch/tile/kernel/compat.c
+++ b/arch/tile/kernel/compat.c
@@ -84,7 +84,7 @@ COMPAT_SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned int, offset_high,
84{ 84{
85 return sys_llseek(fd, offset_high, offset_low, result, origin); 85 return sys_llseek(fd, offset_high, offset_low, result, origin);
86} 86}
87 87
88/* Provide the compat syscall number to call mapping. */ 88/* Provide the compat syscall number to call mapping. */
89#undef __SYSCALL 89#undef __SYSCALL
90#define __SYSCALL(nr, call) [nr] = (call), 90#define __SYSCALL(nr, call) [nr] = (call),
diff --git a/arch/tile/kernel/futex_64.S b/arch/tile/kernel/futex_64.S
deleted file mode 100644
index f465d1eda20f..000000000000
--- a/arch/tile/kernel/futex_64.S
+++ /dev/null
@@ -1,55 +0,0 @@
1/*
2 * Copyright 2011 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 *
14 * Atomically access user memory, but use MMU to avoid propagating
15 * kernel exceptions.
16 */
17
18#include <linux/linkage.h>
19#include <asm/errno.h>
20#include <asm/futex.h>
21#include <asm/page.h>
22#include <asm/processor.h>
23
24/*
25 * Provide a set of atomic memory operations supporting <asm/futex.h>.
26 *
27 * r0: user address to manipulate
28 * r1: new value to write, or for cmpxchg, old value to compare against
29 * r2: (cmpxchg only) new value to write
30 *
31 * Return __get_user struct, r0 with value, r1 with error.
32 */
33#define FUTEX_OP(name, ...) \
34STD_ENTRY(futex_##name) \
35 __VA_ARGS__; \
36 { \
37 move r1, zero; \
38 jrp lr \
39 }; \
40 STD_ENDPROC(futex_##name); \
41 .pushsection __ex_table,"a"; \
42 .quad 1b, get_user_fault; \
43 .popsection
44
45 .pushsection .fixup,"ax"
46get_user_fault:
47 { movei r1, -EFAULT; jrp lr }
48 ENDPROC(get_user_fault)
49 .popsection
50
51FUTEX_OP(cmpxchg, mtspr CMPEXCH_VALUE, r1; 1: cmpexch4 r0, r0, r2)
52FUTEX_OP(set, 1: exch4 r0, r0, r1)
53FUTEX_OP(add, 1: fetchadd4 r0, r0, r1)
54FUTEX_OP(or, 1: fetchor4 r0, r0, r1)
55FUTEX_OP(andn, nor r1, r1, zero; 1: fetchand4 r0, r0, r1)
diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c
index 4c34caea9dd3..74c91729a62a 100644
--- a/arch/tile/kernel/setup.c
+++ b/arch/tile/kernel/setup.c
@@ -1268,8 +1268,7 @@ static void __init validate_va(void)
1268 if ((long)VMALLOC_START >= 0) 1268 if ((long)VMALLOC_START >= 0)
1269 early_panic( 1269 early_panic(
1270 "Linux VMALLOC region below the 2GB line (%#lx)!\n" 1270 "Linux VMALLOC region below the 2GB line (%#lx)!\n"
1271 "Reconfigure the kernel with fewer NR_HUGE_VMAPS\n" 1271 "Reconfigure the kernel with smaller VMALLOC_RESERVE.\n",
1272 "or smaller VMALLOC_RESERVE.\n",
1273 VMALLOC_START); 1272 VMALLOC_START);
1274#endif 1273#endif
1275} 1274}
diff --git a/arch/tile/kernel/unaligned.c b/arch/tile/kernel/unaligned.c
index b425fb6a480d..b030b4e78845 100644
--- a/arch/tile/kernel/unaligned.c
+++ b/arch/tile/kernel/unaligned.c
@@ -551,8 +551,8 @@ static tilegx_bundle_bits jit_x1_bnezt(int ra, int broff)
551/* 551/*
552 * This function generates unalign fixup JIT. 552 * This function generates unalign fixup JIT.
553 * 553 *
554 * We fist find unalign load/store instruction's destination, source 554 * We first find unalign load/store instruction's destination, source
555 * reguisters: ra, rb and rd. and 3 scratch registers by calling 555 * registers: ra, rb and rd. and 3 scratch registers by calling
556 * find_regs(...). 3 scratch clobbers should not alias with any register 556 * find_regs(...). 3 scratch clobbers should not alias with any register
557 * used in the fault bundle. Then analyze the fault bundle to determine 557 * used in the fault bundle. Then analyze the fault bundle to determine
558 * if it's a load or store, operand width, branch or address increment etc. 558 * if it's a load or store, operand width, branch or address increment etc.
diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c
index 4c288f199453..6c0571216a9d 100644
--- a/arch/tile/mm/fault.c
+++ b/arch/tile/mm/fault.c
@@ -149,8 +149,6 @@ static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
149 pmd_k = vmalloc_sync_one(pgd, address); 149 pmd_k = vmalloc_sync_one(pgd, address);
150 if (!pmd_k) 150 if (!pmd_k)
151 return -1; 151 return -1;
152 if (pmd_huge(*pmd_k))
153 return 0; /* support TILE huge_vmap() API */
154 pte_k = pte_offset_kernel(pmd_k, address); 152 pte_k = pte_offset_kernel(pmd_k, address);
155 if (!pte_present(*pte_k)) 153 if (!pte_present(*pte_k))
156 return -1; 154 return -1;
diff --git a/arch/tile/mm/init.c b/arch/tile/mm/init.c
index 4e316deb92fd..0fa1acfac79a 100644
--- a/arch/tile/mm/init.c
+++ b/arch/tile/mm/init.c
@@ -828,10 +828,6 @@ void __init mem_init(void)
828 printk(KERN_DEBUG " PKMAP %#lx - %#lx\n", 828 printk(KERN_DEBUG " PKMAP %#lx - %#lx\n",
829 PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP) - 1); 829 PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP) - 1);
830#endif 830#endif
831#ifdef CONFIG_HUGEVMAP
832 printk(KERN_DEBUG " HUGEMAP %#lx - %#lx\n",
833 HUGE_VMAP_BASE, HUGE_VMAP_END - 1);
834#endif
835 printk(KERN_DEBUG " VMALLOC %#lx - %#lx\n", 831 printk(KERN_DEBUG " VMALLOC %#lx - %#lx\n",
836 _VMALLOC_START, _VMALLOC_END - 1); 832 _VMALLOC_START, _VMALLOC_END - 1);
837#ifdef __tilegx__ 833#ifdef __tilegx__
diff --git a/arch/tile/mm/pgtable.c b/arch/tile/mm/pgtable.c
index 2deaddf3e01f..4fd9ec0b58ed 100644
--- a/arch/tile/mm/pgtable.c
+++ b/arch/tile/mm/pgtable.c
@@ -127,8 +127,7 @@ void shatter_huge_page(unsigned long addr)
127 } 127 }
128 128
129 /* Shatter the huge page into the preallocated L2 page table. */ 129 /* Shatter the huge page into the preallocated L2 page table. */
130 pmd_populate_kernel(&init_mm, pmd, 130 pmd_populate_kernel(&init_mm, pmd, get_prealloc_pte(pmd_pfn(*pmd)));
131 get_prealloc_pte(pte_pfn(*(pte_t *)pmd)));
132 131
133#ifdef __PAGETABLE_PMD_FOLDED 132#ifdef __PAGETABLE_PMD_FOLDED
134 /* Walk every pgd on the system and update the pmd there. */ 133 /* Walk every pgd on the system and update the pmd there. */
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e241a1930c98..ee2fb9d37745 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -481,11 +481,12 @@ config X86_INTEL_LPSS
481 bool "Intel Low Power Subsystem Support" 481 bool "Intel Low Power Subsystem Support"
482 depends on ACPI 482 depends on ACPI
483 select COMMON_CLK 483 select COMMON_CLK
484 select PINCTRL
484 ---help--- 485 ---help---
485 Select to build support for Intel Low Power Subsystem such as 486 Select to build support for Intel Low Power Subsystem such as
486 found on Intel Lynxpoint PCH. Selecting this option enables 487 found on Intel Lynxpoint PCH. Selecting this option enables
487 things like clock tree (common clock framework) which are needed 488 things like clock tree (common clock framework) and pincontrol
488 by the LPSS peripheral drivers. 489 which are needed by the LPSS peripheral drivers.
489 490
490config X86_RDC321X 491config X86_RDC321X
491 bool "RDC R-321x SoC" 492 bool "RDC R-321x SoC"
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 6aef9fbc09b7..b913915e8e63 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -79,30 +79,38 @@ static inline int phys_to_machine_mapping_valid(unsigned long pfn)
79 return get_phys_to_machine(pfn) != INVALID_P2M_ENTRY; 79 return get_phys_to_machine(pfn) != INVALID_P2M_ENTRY;
80} 80}
81 81
82static inline unsigned long mfn_to_pfn(unsigned long mfn) 82static inline unsigned long mfn_to_pfn_no_overrides(unsigned long mfn)
83{ 83{
84 unsigned long pfn; 84 unsigned long pfn;
85 int ret = 0; 85 int ret;
86 86
87 if (xen_feature(XENFEAT_auto_translated_physmap)) 87 if (xen_feature(XENFEAT_auto_translated_physmap))
88 return mfn; 88 return mfn;
89 89
90 if (unlikely(mfn >= machine_to_phys_nr)) { 90 if (unlikely(mfn >= machine_to_phys_nr))
91 pfn = ~0; 91 return ~0;
92 goto try_override; 92
93 }
94 pfn = 0;
95 /* 93 /*
96 * The array access can fail (e.g., device space beyond end of RAM). 94 * The array access can fail (e.g., device space beyond end of RAM).
97 * In such cases it doesn't matter what we return (we return garbage), 95 * In such cases it doesn't matter what we return (we return garbage),
98 * but we must handle the fault without crashing! 96 * but we must handle the fault without crashing!
99 */ 97 */
100 ret = __get_user(pfn, &machine_to_phys_mapping[mfn]); 98 ret = __get_user(pfn, &machine_to_phys_mapping[mfn]);
101try_override:
102 /* ret might be < 0 if there are no entries in the m2p for mfn */
103 if (ret < 0) 99 if (ret < 0)
104 pfn = ~0; 100 return ~0;
105 else if (get_phys_to_machine(pfn) != mfn) 101
102 return pfn;
103}
104
105static inline unsigned long mfn_to_pfn(unsigned long mfn)
106{
107 unsigned long pfn;
108
109 if (xen_feature(XENFEAT_auto_translated_physmap))
110 return mfn;
111
112 pfn = mfn_to_pfn_no_overrides(mfn);
113 if (get_phys_to_machine(pfn) != mfn) {
106 /* 114 /*
107 * If this appears to be a foreign mfn (because the pfn 115 * If this appears to be a foreign mfn (because the pfn
108 * doesn't map back to the mfn), then check the local override 116 * doesn't map back to the mfn), then check the local override
@@ -111,6 +119,7 @@ try_override:
111 * m2p_find_override_pfn returns ~0 if it doesn't find anything. 119 * m2p_find_override_pfn returns ~0 if it doesn't find anything.
112 */ 120 */
113 pfn = m2p_find_override_pfn(mfn, ~0); 121 pfn = m2p_find_override_pfn(mfn, ~0);
122 }
114 123
115 /* 124 /*
116 * pfn is ~0 if there are no entries in the m2p for mfn or if the 125 * pfn is ~0 if there are no entries in the m2p for mfn or if the
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 8355c84b9729..897783b3302a 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1506,7 +1506,7 @@ static int __init init_hw_perf_events(void)
1506 err = amd_pmu_init(); 1506 err = amd_pmu_init();
1507 break; 1507 break;
1508 default: 1508 default:
1509 return 0; 1509 err = -ENOTSUPP;
1510 } 1510 }
1511 if (err != 0) { 1511 if (err != 0) {
1512 pr_cont("no PMU driver, software events only.\n"); 1512 pr_cont("no PMU driver, software events only.\n");
@@ -1883,9 +1883,9 @@ static struct pmu pmu = {
1883 1883
1884void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) 1884void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
1885{ 1885{
1886 userpg->cap_usr_time = 0; 1886 userpg->cap_user_time = 0;
1887 userpg->cap_usr_time_zero = 0; 1887 userpg->cap_user_time_zero = 0;
1888 userpg->cap_usr_rdpmc = x86_pmu.attr_rdpmc; 1888 userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc;
1889 userpg->pmc_width = x86_pmu.cntval_bits; 1889 userpg->pmc_width = x86_pmu.cntval_bits;
1890 1890
1891 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 1891 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
@@ -1894,13 +1894,13 @@ void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
1894 if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1894 if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1895 return; 1895 return;
1896 1896
1897 userpg->cap_usr_time = 1; 1897 userpg->cap_user_time = 1;
1898 userpg->time_mult = this_cpu_read(cyc2ns); 1898 userpg->time_mult = this_cpu_read(cyc2ns);
1899 userpg->time_shift = CYC2NS_SCALE_FACTOR; 1899 userpg->time_shift = CYC2NS_SCALE_FACTOR;
1900 userpg->time_offset = this_cpu_read(cyc2ns_offset) - now; 1900 userpg->time_offset = this_cpu_read(cyc2ns_offset) - now;
1901 1901
1902 if (sched_clock_stable && !check_tsc_disabled()) { 1902 if (sched_clock_stable && !check_tsc_disabled()) {
1903 userpg->cap_usr_time_zero = 1; 1903 userpg->cap_user_time_zero = 1;
1904 userpg->time_zero = this_cpu_read(cyc2ns_offset); 1904 userpg->time_zero = this_cpu_read(cyc2ns_offset);
1905 } 1905 }
1906} 1906}
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index c62d88396ad5..f31a1655d1ff 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -899,8 +899,8 @@ static __initconst const u64 atom_hw_cache_event_ids
899static struct extra_reg intel_slm_extra_regs[] __read_mostly = 899static struct extra_reg intel_slm_extra_regs[] __read_mostly =
900{ 900{
901 /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 901 /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
902 INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffff, RSP_0), 902 INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffffull, RSP_0),
903 INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x768005ffff, RSP_1), 903 INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x768005ffffull, RSP_1),
904 EVENT_EXTRA_END 904 EVENT_EXTRA_END
905}; 905};
906 906
@@ -2325,6 +2325,7 @@ __init int intel_pmu_init(void)
2325 break; 2325 break;
2326 2326
2327 case 55: /* Atom 22nm "Silvermont" */ 2327 case 55: /* Atom 22nm "Silvermont" */
2328 case 77: /* Avoton "Silvermont" */
2328 memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, 2329 memcpy(hw_cache_event_ids, slm_hw_cache_event_ids,
2329 sizeof(hw_cache_event_ids)); 2330 sizeof(hw_cache_event_ids));
2330 memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs, 2331 memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs,
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 63438aad177f..ab3ba1c1b7dd 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -584,6 +584,7 @@ struct event_constraint intel_snb_pebs_event_constraints[] = {
584 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */ 584 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
585 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ 585 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
586 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ 586 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
587 INTEL_EVENT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
587 INTEL_UEVENT_CONSTRAINT(0x02d4, 0xf), /* MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS */ 588 INTEL_UEVENT_CONSTRAINT(0x02d4, 0xf), /* MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS */
588 EVENT_CONSTRAINT_END 589 EVENT_CONSTRAINT_END
589}; 590};
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index 8ed44589b0e4..4118f9f68315 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -2706,14 +2706,14 @@ static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
2706 box->hrtimer.function = uncore_pmu_hrtimer; 2706 box->hrtimer.function = uncore_pmu_hrtimer;
2707} 2707}
2708 2708
2709struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, int cpu) 2709static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, int node)
2710{ 2710{
2711 struct intel_uncore_box *box; 2711 struct intel_uncore_box *box;
2712 int i, size; 2712 int i, size;
2713 2713
2714 size = sizeof(*box) + type->num_shared_regs * sizeof(struct intel_uncore_extra_reg); 2714 size = sizeof(*box) + type->num_shared_regs * sizeof(struct intel_uncore_extra_reg);
2715 2715
2716 box = kzalloc_node(size, GFP_KERNEL, cpu_to_node(cpu)); 2716 box = kzalloc_node(size, GFP_KERNEL, node);
2717 if (!box) 2717 if (!box)
2718 return NULL; 2718 return NULL;
2719 2719
@@ -3031,7 +3031,7 @@ static int uncore_validate_group(struct intel_uncore_pmu *pmu,
3031 struct intel_uncore_box *fake_box; 3031 struct intel_uncore_box *fake_box;
3032 int ret = -EINVAL, n; 3032 int ret = -EINVAL, n;
3033 3033
3034 fake_box = uncore_alloc_box(pmu->type, smp_processor_id()); 3034 fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE);
3035 if (!fake_box) 3035 if (!fake_box)
3036 return -ENOMEM; 3036 return -ENOMEM;
3037 3037
@@ -3294,7 +3294,7 @@ static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id
3294 } 3294 }
3295 3295
3296 type = pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; 3296 type = pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
3297 box = uncore_alloc_box(type, 0); 3297 box = uncore_alloc_box(type, NUMA_NO_NODE);
3298 if (!box) 3298 if (!box)
3299 return -ENOMEM; 3299 return -ENOMEM;
3300 3300
@@ -3499,7 +3499,7 @@ static int uncore_cpu_prepare(int cpu, int phys_id)
3499 if (pmu->func_id < 0) 3499 if (pmu->func_id < 0)
3500 pmu->func_id = j; 3500 pmu->func_id = j;
3501 3501
3502 box = uncore_alloc_box(type, cpu); 3502 box = uncore_alloc_box(type, cpu_to_node(cpu));
3503 if (!box) 3503 if (!box)
3504 return -ENOMEM; 3504 return -ENOMEM;
3505 3505
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1b69951a81e2..b077f4cc225a 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -487,21 +487,6 @@ ENDPROC(native_usergs_sysret64)
487 TRACE_IRQS_OFF 487 TRACE_IRQS_OFF
488 .endm 488 .endm
489 489
490ENTRY(save_rest)
491 PARTIAL_FRAME 1 (REST_SKIP+8)
492 movq 5*8+16(%rsp), %r11 /* save return address */
493 movq_cfi rbx, RBX+16
494 movq_cfi rbp, RBP+16
495 movq_cfi r12, R12+16
496 movq_cfi r13, R13+16
497 movq_cfi r14, R14+16
498 movq_cfi r15, R15+16
499 movq %r11, 8(%rsp) /* return address */
500 FIXUP_TOP_OF_STACK %r11, 16
501 ret
502 CFI_ENDPROC
503END(save_rest)
504
505/* save complete stack frame */ 490/* save complete stack frame */
506 .pushsection .kprobes.text, "ax" 491 .pushsection .kprobes.text, "ax"
507ENTRY(save_paranoid) 492ENTRY(save_paranoid)
diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c
index 7123b5df479d..af99f71aeb7f 100644
--- a/arch/x86/kernel/microcode_amd.c
+++ b/arch/x86/kernel/microcode_amd.c
@@ -216,6 +216,7 @@ int apply_microcode_amd(int cpu)
216 /* need to apply patch? */ 216 /* need to apply patch? */
217 if (rev >= mc_amd->hdr.patch_id) { 217 if (rev >= mc_amd->hdr.patch_id) {
218 c->microcode = rev; 218 c->microcode = rev;
219 uci->cpu_sig.rev = rev;
219 return 0; 220 return 0;
220 } 221 }
221 222
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index 563ed91e6faa..e643e744e4d8 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -352,12 +352,28 @@ static struct dmi_system_id __initdata reboot_dmi_table[] = {
352 }, 352 },
353 { /* Handle problems with rebooting on the Precision M6600. */ 353 { /* Handle problems with rebooting on the Precision M6600. */
354 .callback = set_pci_reboot, 354 .callback = set_pci_reboot,
355 .ident = "Dell OptiPlex 990", 355 .ident = "Dell Precision M6600",
356 .matches = { 356 .matches = {
357 DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), 357 DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
358 DMI_MATCH(DMI_PRODUCT_NAME, "Precision M6600"), 358 DMI_MATCH(DMI_PRODUCT_NAME, "Precision M6600"),
359 }, 359 },
360 }, 360 },
361 { /* Handle problems with rebooting on the Dell PowerEdge C6100. */
362 .callback = set_pci_reboot,
363 .ident = "Dell PowerEdge C6100",
364 .matches = {
365 DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
366 DMI_MATCH(DMI_PRODUCT_NAME, "C6100"),
367 },
368 },
369 { /* Some C6100 machines were shipped with vendor being 'Dell'. */
370 .callback = set_pci_reboot,
371 .ident = "Dell PowerEdge C6100",
372 .matches = {
373 DMI_MATCH(DMI_SYS_VENDOR, "Dell"),
374 DMI_MATCH(DMI_PRODUCT_NAME, "C6100"),
375 },
376 },
361 { } 377 { }
362}; 378};
363 379
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index aecc98a93d1b..6cacab671f9b 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -653,6 +653,7 @@ static void announce_cpu(int cpu, int apicid)
653{ 653{
654 static int current_node = -1; 654 static int current_node = -1;
655 int node = early_cpu_to_node(cpu); 655 int node = early_cpu_to_node(cpu);
656 int max_cpu_present = find_last_bit(cpumask_bits(cpu_present_mask), NR_CPUS);
656 657
657 if (system_state == SYSTEM_BOOTING) { 658 if (system_state == SYSTEM_BOOTING) {
658 if (node != current_node) { 659 if (node != current_node) {
@@ -661,7 +662,7 @@ static void announce_cpu(int cpu, int apicid)
661 current_node = node; 662 current_node = node;
662 pr_info("Booting Node %3d, Processors ", node); 663 pr_info("Booting Node %3d, Processors ", node);
663 } 664 }
664 pr_cont(" #%d%s", cpu, cpu == (nr_cpu_ids - 1) ? " OK\n" : ""); 665 pr_cont(" #%4d%s", cpu, cpu == max_cpu_present ? " OK\n" : "");
665 return; 666 return;
666 } else 667 } else
667 pr_info("Booting Node %d Processor %d APIC 0x%x\n", 668 pr_info("Booting Node %d Processor %d APIC 0x%x\n",
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 2bc1e81045b0..ddc3f3d2afdb 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -2025,6 +2025,17 @@ static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2025 return rc; 2025 return rc;
2026} 2026}
2027 2027
2028static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2029{
2030 int rc;
2031
2032 rc = em_ret_far(ctxt);
2033 if (rc != X86EMUL_CONTINUE)
2034 return rc;
2035 rsp_increment(ctxt, ctxt->src.val);
2036 return X86EMUL_CONTINUE;
2037}
2038
2028static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) 2039static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2029{ 2040{
2030 /* Save real source value, then compare EAX against destination. */ 2041 /* Save real source value, then compare EAX against destination. */
@@ -3763,7 +3774,8 @@ static const struct opcode opcode_table[256] = {
3763 G(ByteOp, group11), G(0, group11), 3774 G(ByteOp, group11), G(0, group11),
3764 /* 0xC8 - 0xCF */ 3775 /* 0xC8 - 0xCF */
3765 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave), 3776 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3766 N, I(ImplicitOps | Stack, em_ret_far), 3777 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3778 I(ImplicitOps | Stack, em_ret_far),
3767 D(ImplicitOps), DI(SrcImmByte, intn), 3779 D(ImplicitOps), DI(SrcImmByte, intn),
3768 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret), 3780 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3769 /* 0xD0 - 0xD7 */ 3781 /* 0xD0 - 0xD7 */
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 043330159179..ad75d77999d0 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -99,6 +99,7 @@ struct guest_walker {
99 pt_element_t prefetch_ptes[PTE_PREFETCH_NUM]; 99 pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
100 gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; 100 gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
101 pt_element_t __user *ptep_user[PT_MAX_FULL_LEVELS]; 101 pt_element_t __user *ptep_user[PT_MAX_FULL_LEVELS];
102 bool pte_writable[PT_MAX_FULL_LEVELS];
102 unsigned pt_access; 103 unsigned pt_access;
103 unsigned pte_access; 104 unsigned pte_access;
104 gfn_t gfn; 105 gfn_t gfn;
@@ -235,6 +236,22 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
235 if (pte == orig_pte) 236 if (pte == orig_pte)
236 continue; 237 continue;
237 238
239 /*
240 * If the slot is read-only, simply do not process the accessed
241 * and dirty bits. This is the correct thing to do if the slot
242 * is ROM, and page tables in read-as-ROM/write-as-MMIO slots
243 * are only supported if the accessed and dirty bits are already
244 * set in the ROM (so that MMIO writes are never needed).
245 *
246 * Note that NPT does not allow this at all and faults, since
247 * it always wants nested page table entries for the guest
248 * page tables to be writable. And EPT works but will simply
249 * overwrite the read-only memory to set the accessed and dirty
250 * bits.
251 */
252 if (unlikely(!walker->pte_writable[level - 1]))
253 continue;
254
238 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte); 255 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte);
239 if (ret) 256 if (ret)
240 return ret; 257 return ret;
@@ -309,7 +326,8 @@ retry_walk:
309 goto error; 326 goto error;
310 real_gfn = gpa_to_gfn(real_gfn); 327 real_gfn = gpa_to_gfn(real_gfn);
311 328
312 host_addr = gfn_to_hva(vcpu->kvm, real_gfn); 329 host_addr = gfn_to_hva_prot(vcpu->kvm, real_gfn,
330 &walker->pte_writable[walker->level - 1]);
313 if (unlikely(kvm_is_error_hva(host_addr))) 331 if (unlikely(kvm_is_error_hva(host_addr)))
314 goto error; 332 goto error;
315 333
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 1f1da43ff2a2..a1216de9ffda 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -5339,6 +5339,15 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu)
5339 return 0; 5339 return 0;
5340 } 5340 }
5341 5341
5342 /*
5343 * EPT violation happened while executing iret from NMI,
5344 * "blocked by NMI" bit has to be set before next VM entry.
5345 * There are errata that may cause this bit to not be set:
5346 * AAK134, BY25.
5347 */
5348 if (exit_qualification & INTR_INFO_UNBLOCK_NMI)
5349 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
5350
5342 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5351 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5343 trace_kvm_page_fault(gpa, exit_qualification); 5352 trace_kvm_page_fault(gpa, exit_qualification);
5344 5353
@@ -7766,6 +7775,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
7766 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 7775 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
7767 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 7776 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
7768 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 7777 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
7778 __clear_bit(VCPU_EXREG_PDPTR,
7779 (unsigned long *)&vcpu->arch.regs_avail);
7780 __clear_bit(VCPU_EXREG_PDPTR,
7781 (unsigned long *)&vcpu->arch.regs_dirty);
7769 } 7782 }
7770 7783
7771 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); 7784 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 90f6ed127096..c7e22ab29a5a 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -912,10 +912,13 @@ void __init efi_enter_virtual_mode(void)
912 912
913 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 913 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
914 md = p; 914 md = p;
915 if (!(md->attribute & EFI_MEMORY_RUNTIME) && 915 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
916 md->type != EFI_BOOT_SERVICES_CODE && 916#ifdef CONFIG_X86_64
917 md->type != EFI_BOOT_SERVICES_DATA) 917 if (md->type != EFI_BOOT_SERVICES_CODE &&
918 continue; 918 md->type != EFI_BOOT_SERVICES_DATA)
919#endif
920 continue;
921 }
919 922
920 size = md->num_pages << EFI_PAGE_SHIFT; 923 size = md->num_pages << EFI_PAGE_SHIFT;
921 end = md->phys_addr + size; 924 end = md->phys_addr + size;
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 8b901e8d782d..a61c7d5811be 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -879,7 +879,6 @@ int m2p_add_override(unsigned long mfn, struct page *page,
879 unsigned long uninitialized_var(address); 879 unsigned long uninitialized_var(address);
880 unsigned level; 880 unsigned level;
881 pte_t *ptep = NULL; 881 pte_t *ptep = NULL;
882 int ret = 0;
883 882
884 pfn = page_to_pfn(page); 883 pfn = page_to_pfn(page);
885 if (!PageHighMem(page)) { 884 if (!PageHighMem(page)) {
@@ -926,8 +925,8 @@ int m2p_add_override(unsigned long mfn, struct page *page,
926 * frontend pages while they are being shared with the backend, 925 * frontend pages while they are being shared with the backend,
927 * because mfn_to_pfn (that ends up being called by GUPF) will 926 * because mfn_to_pfn (that ends up being called by GUPF) will
928 * return the backend pfn rather than the frontend pfn. */ 927 * return the backend pfn rather than the frontend pfn. */
929 ret = __get_user(pfn, &machine_to_phys_mapping[mfn]); 928 pfn = mfn_to_pfn_no_overrides(mfn);
930 if (ret == 0 && get_phys_to_machine(pfn) == mfn) 929 if (get_phys_to_machine(pfn) == mfn)
931 set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)); 930 set_phys_to_machine(pfn, FOREIGN_FRAME(mfn));
932 931
933 return 0; 932 return 0;
@@ -942,7 +941,6 @@ int m2p_remove_override(struct page *page,
942 unsigned long uninitialized_var(address); 941 unsigned long uninitialized_var(address);
943 unsigned level; 942 unsigned level;
944 pte_t *ptep = NULL; 943 pte_t *ptep = NULL;
945 int ret = 0;
946 944
947 pfn = page_to_pfn(page); 945 pfn = page_to_pfn(page);
948 mfn = get_phys_to_machine(pfn); 946 mfn = get_phys_to_machine(pfn);
@@ -1029,8 +1027,8 @@ int m2p_remove_override(struct page *page,
1029 * the original pfn causes mfn_to_pfn(mfn) to return the frontend 1027 * the original pfn causes mfn_to_pfn(mfn) to return the frontend
1030 * pfn again. */ 1028 * pfn again. */
1031 mfn &= ~FOREIGN_FRAME_BIT; 1029 mfn &= ~FOREIGN_FRAME_BIT;
1032 ret = __get_user(pfn, &machine_to_phys_mapping[mfn]); 1030 pfn = mfn_to_pfn_no_overrides(mfn);
1033 if (ret == 0 && get_phys_to_machine(pfn) == FOREIGN_FRAME(mfn) && 1031 if (get_phys_to_machine(pfn) == FOREIGN_FRAME(mfn) &&
1034 m2p_find_override(mfn) == NULL) 1032 m2p_find_override(mfn) == NULL)
1035 set_phys_to_machine(pfn, mfn); 1033 set_phys_to_machine(pfn, mfn);
1036 1034
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 253f63fceea1..be6b86078957 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -259,6 +259,14 @@ void xen_uninit_lock_cpu(int cpu)
259} 259}
260 260
261 261
262/*
263 * Our init of PV spinlocks is split in two init functions due to us
264 * using paravirt patching and jump labels patching and having to do
265 * all of this before SMP code is invoked.
266 *
267 * The paravirt patching needs to be done _before_ the alternative asm code
268 * is started, otherwise we would not patch the core kernel code.
269 */
262void __init xen_init_spinlocks(void) 270void __init xen_init_spinlocks(void)
263{ 271{
264 272
@@ -267,12 +275,26 @@ void __init xen_init_spinlocks(void)
267 return; 275 return;
268 } 276 }
269 277
270 static_key_slow_inc(&paravirt_ticketlocks_enabled);
271
272 pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning); 278 pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
273 pv_lock_ops.unlock_kick = xen_unlock_kick; 279 pv_lock_ops.unlock_kick = xen_unlock_kick;
274} 280}
275 281
282/*
283 * While the jump_label init code needs to happend _after_ the jump labels are
284 * enabled and before SMP is started. Hence we use pre-SMP initcall level
285 * init. We cannot do it in xen_init_spinlocks as that is done before
286 * jump labels are activated.
287 */
288static __init int xen_init_spinlocks_jump(void)
289{
290 if (!xen_pvspin)
291 return 0;
292
293 static_key_slow_inc(&paravirt_ticketlocks_enabled);
294 return 0;
295}
296early_initcall(xen_init_spinlocks_jump);
297
276static __init int xen_parse_nopvspin(char *arg) 298static __init int xen_parse_nopvspin(char *arg)
277{ 299{
278 xen_pvspin = false; 300 xen_pvspin = false;
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-25 15:05:33 -0400