diff options
Diffstat (limited to 'arch')
373 files changed, 9589 insertions, 4061 deletions
diff --git a/arch/alpha/include/uapi/asm/Kbuild b/arch/alpha/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/alpha/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 9503a4be40f6..63e77e3944ce 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c | |||
| @@ -145,27 +145,24 @@ SYSCALL_DEFINE4(osf_getdirentries, unsigned int, fd, | |||
| 145 | long __user *, basep) | 145 | long __user *, basep) |
| 146 | { | 146 | { |
| 147 | int error; | 147 | int error; |
| 148 | struct file *file; | 148 | struct fd arg = fdget(fd); |
| 149 | struct osf_dirent_callback buf; | 149 | struct osf_dirent_callback buf; |
| 150 | 150 | ||
| 151 | error = -EBADF; | 151 | if (!arg.file) |
| 152 | file = fget(fd); | 152 | return -EBADF; |
| 153 | if (!file) | ||
| 154 | goto out; | ||
| 155 | 153 | ||
| 156 | buf.dirent = dirent; | 154 | buf.dirent = dirent; |
| 157 | buf.basep = basep; | 155 | buf.basep = basep; |
| 158 | buf.count = count; | 156 | buf.count = count; |
| 159 | buf.error = 0; | 157 | buf.error = 0; |
| 160 | 158 | ||
| 161 | error = vfs_readdir(file, osf_filldir, &buf); | 159 | error = vfs_readdir(arg.file, osf_filldir, &buf); |
| 162 | if (error >= 0) | 160 | if (error >= 0) |
| 163 | error = buf.error; | 161 | error = buf.error; |
| 164 | if (count != buf.count) | 162 | if (count != buf.count) |
| 165 | error = count - buf.count; | 163 | error = count - buf.count; |
| 166 | 164 | ||
| 167 | fput(file); | 165 | fdput(arg); |
| 168 | out: | ||
| 169 | return error; | 166 | return error; |
| 170 | } | 167 | } |
| 171 | 168 | ||
diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 770da51242c4..b86e57ef146b 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile | |||
| @@ -254,6 +254,7 @@ core-$(CONFIG_VFP) += arch/arm/vfp/ | |||
| 254 | # If we have a machine-specific directory, then include it in the build. | 254 | # If we have a machine-specific directory, then include it in the build. |
| 255 | core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/ | 255 | core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/ |
| 256 | core-y += arch/arm/net/ | 256 | core-y += arch/arm/net/ |
| 257 | core-y += arch/arm/crypto/ | ||
| 257 | core-y += $(machdirs) $(platdirs) | 258 | core-y += $(machdirs) $(platdirs) |
| 258 | 259 | ||
| 259 | drivers-$(CONFIG_OPROFILE) += arch/arm/oprofile/ | 260 | drivers-$(CONFIG_OPROFILE) += arch/arm/oprofile/ |
diff --git a/arch/arm/boot/dts/highbank.dts b/arch/arm/boot/dts/highbank.dts index 9fecf1ae777b..0c6fc34821f9 100644 --- a/arch/arm/boot/dts/highbank.dts +++ b/arch/arm/boot/dts/highbank.dts | |||
| @@ -121,6 +121,10 @@ | |||
| 121 | compatible = "calxeda,hb-ahci"; | 121 | compatible = "calxeda,hb-ahci"; |
| 122 | reg = <0xffe08000 0x10000>; | 122 | reg = <0xffe08000 0x10000>; |
| 123 | interrupts = <0 83 4>; | 123 | interrupts = <0 83 4>; |
| 124 | calxeda,port-phys = <&combophy5 0 &combophy0 0 | ||
| 125 | &combophy0 1 &combophy0 2 | ||
| 126 | &combophy0 3>; | ||
| 127 | dma-coherent; | ||
| 124 | }; | 128 | }; |
| 125 | 129 | ||
| 126 | sdhci@ffe0e000 { | 130 | sdhci@ffe0e000 { |
| @@ -306,5 +310,19 @@ | |||
| 306 | reg = <0xfff51000 0x1000>; | 310 | reg = <0xfff51000 0x1000>; |
| 307 | interrupts = <0 80 4 0 81 4 0 82 4>; | 311 | interrupts = <0 80 4 0 81 4 0 82 4>; |
| 308 | }; | 312 | }; |
| 313 | |||
| 314 | combophy0: combo-phy@fff58000 { | ||
| 315 | compatible = "calxeda,hb-combophy"; | ||
| 316 | #phy-cells = <1>; | ||
| 317 | reg = <0xfff58000 0x1000>; | ||
| 318 | phydev = <5>; | ||
| 319 | }; | ||
| 320 | |||
| 321 | combophy5: combo-phy@fff5d000 { | ||
| 322 | compatible = "calxeda,hb-combophy"; | ||
| 323 | #phy-cells = <1>; | ||
| 324 | reg = <0xfff5d000 0x1000>; | ||
| 325 | phydev = <31>; | ||
| 326 | }; | ||
| 309 | }; | 327 | }; |
| 310 | }; | 328 | }; |
diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi index 35e5895ba3df..f3990b04fecf 100644 --- a/arch/arm/boot/dts/imx6q.dtsi +++ b/arch/arm/boot/dts/imx6q.dtsi | |||
| @@ -400,8 +400,8 @@ | |||
| 400 | #clock-cells = <1>; | 400 | #clock-cells = <1>; |
| 401 | }; | 401 | }; |
| 402 | 402 | ||
| 403 | anatop@020c8000 { | 403 | anatop: anatop@020c8000 { |
| 404 | compatible = "fsl,imx6q-anatop"; | 404 | compatible = "fsl,imx6q-anatop", "syscon", "simple-bus"; |
| 405 | reg = <0x020c8000 0x1000>; | 405 | reg = <0x020c8000 0x1000>; |
| 406 | interrupts = <0 49 0x04 0 54 0x04 0 127 0x04>; | 406 | interrupts = <0 49 0x04 0 54 0x04 0 127 0x04>; |
| 407 | 407 | ||
| @@ -531,6 +531,11 @@ | |||
| 531 | interrupts = <0 89 0x04 0 90 0x04>; | 531 | interrupts = <0 89 0x04 0 90 0x04>; |
| 532 | }; | 532 | }; |
| 533 | 533 | ||
| 534 | gpr: iomuxc-gpr@020e0000 { | ||
| 535 | compatible = "fsl,imx6q-iomuxc-gpr", "syscon"; | ||
| 536 | reg = <0x020e0000 0x38>; | ||
| 537 | }; | ||
| 538 | |||
| 534 | iomuxc@020e0000 { | 539 | iomuxc@020e0000 { |
| 535 | compatible = "fsl,imx6q-iomuxc"; | 540 | compatible = "fsl,imx6q-iomuxc"; |
| 536 | reg = <0x020e0000 0x4000>; | 541 | reg = <0x020e0000 0x4000>; |
diff --git a/arch/arm/boot/dts/pxa910-dkb.dts b/arch/arm/boot/dts/pxa910-dkb.dts index e92be5a474e7..595492aa5053 100644 --- a/arch/arm/boot/dts/pxa910-dkb.dts +++ b/arch/arm/boot/dts/pxa910-dkb.dts | |||
| @@ -29,6 +29,143 @@ | |||
| 29 | }; | 29 | }; |
| 30 | twsi1: i2c@d4011000 { | 30 | twsi1: i2c@d4011000 { |
| 31 | status = "okay"; | 31 | status = "okay"; |
| 32 | |||
| 33 | pmic: 88pm860x@34 { | ||
| 34 | compatible = "marvell,88pm860x"; | ||
| 35 | reg = <0x34>; | ||
| 36 | interrupts = <4>; | ||
| 37 | interrupt-parent = <&intc>; | ||
| 38 | interrupt-controller; | ||
| 39 | #interrupt-cells = <1>; | ||
| 40 | |||
| 41 | marvell,88pm860x-irq-read-clr; | ||
| 42 | marvell,88pm860x-slave-addr = <0x11>; | ||
| 43 | |||
| 44 | regulators { | ||
| 45 | BUCK1 { | ||
| 46 | regulator-min-microvolt = <1000000>; | ||
| 47 | regulator-max-microvolt = <1500000>; | ||
| 48 | regulator-boot-on; | ||
| 49 | regulator-always-on; | ||
| 50 | }; | ||
| 51 | BUCK2 { | ||
| 52 | regulator-min-microvolt = <1000000>; | ||
| 53 | regulator-max-microvolt = <1500000>; | ||
| 54 | regulator-boot-on; | ||
| 55 | regulator-always-on; | ||
| 56 | }; | ||
| 57 | BUCK3 { | ||
| 58 | regulator-min-microvolt = <1000000>; | ||
| 59 | regulator-max-microvolt = <3000000>; | ||
| 60 | regulator-boot-on; | ||
| 61 | regulator-always-on; | ||
| 62 | }; | ||
| 63 | LDO1 { | ||
| 64 | regulator-min-microvolt = <1200000>; | ||
| 65 | regulator-max-microvolt = <2800000>; | ||
| 66 | regulator-boot-on; | ||
| 67 | regulator-always-on; | ||
| 68 | }; | ||
| 69 | LDO2 { | ||
| 70 | regulator-min-microvolt = <1800000>; | ||
| 71 | regulator-max-microvolt = <3300000>; | ||
| 72 | regulator-boot-on; | ||
| 73 | regulator-always-on; | ||
| 74 | }; | ||
| 75 | LDO3 { | ||
| 76 | regulator-min-microvolt = <1800000>; | ||
| 77 | regulator-max-microvolt = <3300000>; | ||
| 78 | regulator-boot-on; | ||
| 79 | regulator-always-on; | ||
| 80 | }; | ||
| 81 | LDO4 { | ||
| 82 | regulator-min-microvolt = <1800000>; | ||
| 83 | regulator-max-microvolt = <3300000>; | ||
| 84 | regulator-always-on; | ||
| 85 | }; | ||
| 86 | LDO5 { | ||
| 87 | regulator-min-microvolt = <2900000>; | ||
| 88 | regulator-max-microvolt = <3300000>; | ||
| 89 | regulator-boot-on; | ||
| 90 | regulator-always-on; | ||
| 91 | }; | ||
| 92 | LDO6 { | ||
| 93 | regulator-min-microvolt = <1800000>; | ||
| 94 | regulator-max-microvolt = <3300000>; | ||
| 95 | regulator-boot-on; | ||
| 96 | regulator-always-on; | ||
| 97 | }; | ||
| 98 | LDO7 { | ||
| 99 | regulator-min-microvolt = <1800000>; | ||
| 100 | regulator-max-microvolt = <2900000>; | ||
| 101 | regulator-boot-on; | ||
| 102 | regulator-always-on; | ||
| 103 | }; | ||
| 104 | LDO8 { | ||
| 105 | regulator-min-microvolt = <1800000>; | ||
| 106 | regulator-max-microvolt = <2900000>; | ||
| 107 | regulator-boot-on; | ||
| 108 | regulator-always-on; | ||
| 109 | }; | ||
| 110 | LDO9 { | ||
| 111 | regulator-min-microvolt = <1800000>; | ||
| 112 | regulator-max-microvolt = <3300000>; | ||
| 113 | regulator-boot-on; | ||
| 114 | regulator-always-on; | ||
| 115 | }; | ||
| 116 | LDO10 { | ||
| 117 | regulator-min-microvolt = <1200000>; | ||
| 118 | regulator-max-microvolt = <3300000>; | ||
| 119 | regulator-boot-on; | ||
| 120 | regulator-always-on; | ||
| 121 | }; | ||
| 122 | LDO12 { | ||
| 123 | regulator-min-microvolt = <1200000>; | ||
| 124 | regulator-max-microvolt = <3300000>; | ||
| 125 | regulator-always-on; | ||
| 126 | }; | ||
| 127 | LDO13 { | ||
| 128 | regulator-min-microvolt = <1200000>; | ||
| 129 | regulator-max-microvolt = <3300000>; | ||
| 130 | regulator-always-on; | ||
| 131 | }; | ||
| 132 | LDO14 { | ||
| 133 | regulator-min-microvolt = <1800000>; | ||
| 134 | regulator-max-microvolt = <3300000>; | ||
| 135 | regulator-always-on; | ||
| 136 | }; | ||
| 137 | }; | ||
| 138 | rtc { | ||
| 139 | marvell,88pm860x-vrtc = <1>; | ||
| 140 | }; | ||
| 141 | touch { | ||
| 142 | marvell,88pm860x-gpadc-prebias = <1>; | ||
| 143 | marvell,88pm860x-gpadc-slot-cycle = <1>; | ||
| 144 | marvell,88pm860x-tsi-prebias = <6>; | ||
| 145 | marvell,88pm860x-pen-prebias = <16>; | ||
| 146 | marvell,88pm860x-pen-prechg = <2>; | ||
| 147 | marvell,88pm860x-resistor-X = <300>; | ||
| 148 | }; | ||
| 149 | backlights { | ||
| 150 | backlight-0 { | ||
| 151 | marvell,88pm860x-iset = <4>; | ||
| 152 | marvell,88pm860x-pwm = <3>; | ||
| 153 | }; | ||
| 154 | backlight-2 { | ||
| 155 | }; | ||
| 156 | }; | ||
| 157 | leds { | ||
| 158 | led0-red { | ||
| 159 | marvell,88pm860x-iset = <12>; | ||
| 160 | }; | ||
| 161 | led0-green { | ||
| 162 | marvell,88pm860x-iset = <12>; | ||
| 163 | }; | ||
| 164 | led0-blue { | ||
| 165 | marvell,88pm860x-iset = <12>; | ||
| 166 | }; | ||
| 167 | }; | ||
| 168 | }; | ||
| 32 | }; | 169 | }; |
| 33 | rtc: rtc@d4010000 { | 170 | rtc: rtc@d4010000 { |
| 34 | status = "okay"; | 171 | status = "okay"; |
diff --git a/arch/arm/boot/dts/pxa910.dtsi b/arch/arm/boot/dts/pxa910.dtsi index a3be44d86bcd..825aaca33034 100644 --- a/arch/arm/boot/dts/pxa910.dtsi +++ b/arch/arm/boot/dts/pxa910.dtsi | |||
| @@ -120,6 +120,8 @@ | |||
| 120 | 120 | ||
| 121 | twsi1: i2c@d4011000 { | 121 | twsi1: i2c@d4011000 { |
| 122 | compatible = "mrvl,mmp-twsi"; | 122 | compatible = "mrvl,mmp-twsi"; |
| 123 | #address-cells = <1>; | ||
| 124 | #size-cells = <0>; | ||
| 123 | reg = <0xd4011000 0x1000>; | 125 | reg = <0xd4011000 0x1000>; |
| 124 | interrupts = <7>; | 126 | interrupts = <7>; |
| 125 | mrvl,i2c-fast-mode; | 127 | mrvl,i2c-fast-mode; |
| @@ -128,6 +130,8 @@ | |||
| 128 | 130 | ||
| 129 | twsi2: i2c@d4037000 { | 131 | twsi2: i2c@d4037000 { |
| 130 | compatible = "mrvl,mmp-twsi"; | 132 | compatible = "mrvl,mmp-twsi"; |
| 133 | #address-cells = <1>; | ||
| 134 | #size-cells = <0>; | ||
| 131 | reg = <0xd4037000 0x1000>; | 135 | reg = <0xd4037000 0x1000>; |
| 132 | interrupts = <54>; | 136 | interrupts = <54>; |
| 133 | status = "disabled"; | 137 | status = "disabled"; |
diff --git a/arch/arm/configs/sam9_l9260_defconfig b/arch/arm/configs/sam9_l9260_defconfig index ecf2531523a1..b4384af1bea6 100644 --- a/arch/arm/configs/sam9_l9260_defconfig +++ b/arch/arm/configs/sam9_l9260_defconfig | |||
| @@ -39,7 +39,7 @@ CONFIG_MTD_NAND=y | |||
| 39 | CONFIG_MTD_NAND_ATMEL=y | 39 | CONFIG_MTD_NAND_ATMEL=y |
| 40 | CONFIG_MTD_NAND_PLATFORM=y | 40 | CONFIG_MTD_NAND_PLATFORM=y |
| 41 | CONFIG_MTD_UBI=y | 41 | CONFIG_MTD_UBI=y |
| 42 | CONFIG_MTD_UBI_BEB_RESERVE=3 | 42 | CONFIG_MTD_UBI_BEB_LIMIT=25 |
| 43 | CONFIG_MTD_UBI_GLUEBI=y | 43 | CONFIG_MTD_UBI_GLUEBI=y |
| 44 | CONFIG_BLK_DEV_LOOP=y | 44 | CONFIG_BLK_DEV_LOOP=y |
| 45 | CONFIG_BLK_DEV_RAM=y | 45 | CONFIG_BLK_DEV_RAM=y |
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile new file mode 100644 index 000000000000..a2c83851bc90 --- /dev/null +++ b/arch/arm/crypto/Makefile | |||
| @@ -0,0 +1,9 @@ | |||
| 1 | # | ||
| 2 | # Arch-specific CryptoAPI modules. | ||
| 3 | # | ||
| 4 | |||
| 5 | obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o | ||
| 6 | obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o | ||
| 7 | |||
| 8 | aes-arm-y := aes-armv4.o aes_glue.o | ||
| 9 | sha1-arm-y := sha1-armv4-large.o sha1_glue.o | ||
diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S new file mode 100644 index 000000000000..e59b1d505d6c --- /dev/null +++ b/arch/arm/crypto/aes-armv4.S | |||
| @@ -0,0 +1,1112 @@ | |||
| 1 | #define __ARM_ARCH__ __LINUX_ARM_ARCH__ | ||
| 2 | @ ==================================================================== | ||
| 3 | @ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
| 4 | @ project. The module is, however, dual licensed under OpenSSL and | ||
| 5 | @ CRYPTOGAMS licenses depending on where you obtain it. For further | ||
| 6 | @ details see http://www.openssl.org/~appro/cryptogams/. | ||
| 7 | @ ==================================================================== | ||
| 8 | |||
| 9 | @ AES for ARMv4 | ||
| 10 | |||
| 11 | @ January 2007. | ||
| 12 | @ | ||
| 13 | @ Code uses single 1K S-box and is >2 times faster than code generated | ||
| 14 | @ by gcc-3.4.1. This is thanks to unique feature of ARMv4 ISA, which | ||
| 15 | @ allows to merge logical or arithmetic operation with shift or rotate | ||
| 16 | @ in one instruction and emit combined result every cycle. The module | ||
| 17 | @ is endian-neutral. The performance is ~42 cycles/byte for 128-bit | ||
| 18 | @ key [on single-issue Xscale PXA250 core]. | ||
| 19 | |||
| 20 | @ May 2007. | ||
| 21 | @ | ||
| 22 | @ AES_set_[en|de]crypt_key is added. | ||
| 23 | |||
| 24 | @ July 2010. | ||
| 25 | @ | ||
| 26 | @ Rescheduling for dual-issue pipeline resulted in 12% improvement on | ||
| 27 | @ Cortex A8 core and ~25 cycles per byte processed with 128-bit key. | ||
| 28 | |||
| 29 | @ February 2011. | ||
| 30 | @ | ||
| 31 | @ Profiler-assisted and platform-specific optimization resulted in 16% | ||
| 32 | @ improvement on Cortex A8 core and ~21.5 cycles per byte. | ||
| 33 | |||
| 34 | @ A little glue here to select the correct code below for the ARM CPU | ||
| 35 | @ that is being targetted. | ||
| 36 | |||
| 37 | .text | ||
| 38 | .code 32 | ||
| 39 | |||
| 40 | .type AES_Te,%object | ||
| 41 | .align 5 | ||
| 42 | AES_Te: | ||
| 43 | .word 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d | ||
| 44 | .word 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554 | ||
| 45 | .word 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d | ||
| 46 | .word 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a | ||
| 47 | .word 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87 | ||
| 48 | .word 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b | ||
| 49 | .word 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea | ||
| 50 | .word 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b | ||
| 51 | .word 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a | ||
| 52 | .word 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f | ||
| 53 | .word 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108 | ||
| 54 | .word 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f | ||
| 55 | .word 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e | ||
| 56 | .word 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5 | ||
| 57 | .word 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d | ||
| 58 | .word 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f | ||
| 59 | .word 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e | ||
| 60 | .word 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb | ||
| 61 | .word 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce | ||
| 62 | .word 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497 | ||
| 63 | .word 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c | ||
| 64 | .word 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed | ||
| 65 | .word 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b | ||
| 66 | .word 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a | ||
| 67 | .word 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16 | ||
| 68 | .word 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594 | ||
| 69 | .word 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81 | ||
| 70 | .word 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3 | ||
| 71 | .word 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a | ||
| 72 | .word 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504 | ||
| 73 | .word 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163 | ||
| 74 | .word 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d | ||
| 75 | .word 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f | ||
| 76 | .word 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739 | ||
| 77 | .word 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47 | ||
| 78 | .word 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395 | ||
| 79 | .word 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f | ||
| 80 | .word 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883 | ||
| 81 | .word 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c | ||
| 82 | .word 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76 | ||
| 83 | .word 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e | ||
| 84 | .word 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4 | ||
| 85 | .word 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6 | ||
| 86 | .word 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b | ||
| 87 | .word 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7 | ||
| 88 | .word 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0 | ||
| 89 | .word 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25 | ||
| 90 | .word 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818 | ||
| 91 | .word 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72 | ||
| 92 | .word 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651 | ||
| 93 | .word 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21 | ||
| 94 | .word 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85 | ||
| 95 | .word 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa | ||
| 96 | .word 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12 | ||
| 97 | .word 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0 | ||
| 98 | .word 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9 | ||
| 99 | .word 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133 | ||
| 100 | .word 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7 | ||
| 101 | .word 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920 | ||
| 102 | .word 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a | ||
| 103 | .word 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17 | ||
| 104 | .word 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8 | ||
| 105 | .word 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11 | ||
| 106 | .word 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a | ||
| 107 | @ Te4[256] | ||
| 108 | .byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 | ||
| 109 | .byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76 | ||
| 110 | .byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0 | ||
| 111 | .byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0 | ||
| 112 | .byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc | ||
| 113 | .byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15 | ||
| 114 | .byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a | ||
| 115 | .byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75 | ||
| 116 | .byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0 | ||
| 117 | .byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84 | ||
| 118 | .byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b | ||
| 119 | .byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf | ||
| 120 | .byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85 | ||
| 121 | .byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8 | ||
| 122 | .byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5 | ||
| 123 | .byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2 | ||
| 124 | .byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17 | ||
| 125 | .byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73 | ||
| 126 | .byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88 | ||
| 127 | .byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb | ||
| 128 | .byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c | ||
| 129 | .byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79 | ||
| 130 | .byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9 | ||
| 131 | .byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08 | ||
| 132 | .byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6 | ||
| 133 | .byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a | ||
| 134 | .byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e | ||
| 135 | .byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e | ||
| 136 | .byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94 | ||
| 137 | .byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf | ||
| 138 | .byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68 | ||
| 139 | .byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 | ||
| 140 | @ rcon[] | ||
| 141 | .word 0x01000000, 0x02000000, 0x04000000, 0x08000000 | ||
| 142 | .word 0x10000000, 0x20000000, 0x40000000, 0x80000000 | ||
| 143 | .word 0x1B000000, 0x36000000, 0, 0, 0, 0, 0, 0 | ||
| 144 | .size AES_Te,.-AES_Te | ||
| 145 | |||
| 146 | @ void AES_encrypt(const unsigned char *in, unsigned char *out, | ||
| 147 | @ const AES_KEY *key) { | ||
| 148 | .global AES_encrypt | ||
| 149 | .type AES_encrypt,%function | ||
| 150 | .align 5 | ||
| 151 | AES_encrypt: | ||
| 152 | sub r3,pc,#8 @ AES_encrypt | ||
| 153 | stmdb sp!,{r1,r4-r12,lr} | ||
| 154 | mov r12,r0 @ inp | ||
| 155 | mov r11,r2 | ||
| 156 | sub r10,r3,#AES_encrypt-AES_Te @ Te | ||
| 157 | #if __ARM_ARCH__<7 | ||
| 158 | ldrb r0,[r12,#3] @ load input data in endian-neutral | ||
| 159 | ldrb r4,[r12,#2] @ manner... | ||
| 160 | ldrb r5,[r12,#1] | ||
| 161 | ldrb r6,[r12,#0] | ||
| 162 | orr r0,r0,r4,lsl#8 | ||
| 163 | ldrb r1,[r12,#7] | ||
| 164 | orr r0,r0,r5,lsl#16 | ||
| 165 | ldrb r4,[r12,#6] | ||
| 166 | orr r0,r0,r6,lsl#24 | ||
| 167 | ldrb r5,[r12,#5] | ||
| 168 | ldrb r6,[r12,#4] | ||
| 169 | orr r1,r1,r4,lsl#8 | ||
| 170 | ldrb r2,[r12,#11] | ||
| 171 | orr r1,r1,r5,lsl#16 | ||
| 172 | ldrb r4,[r12,#10] | ||
| 173 | orr r1,r1,r6,lsl#24 | ||
| 174 | ldrb r5,[r12,#9] | ||
| 175 | ldrb r6,[r12,#8] | ||
| 176 | orr r2,r2,r4,lsl#8 | ||
| 177 | ldrb r3,[r12,#15] | ||
| 178 | orr r2,r2,r5,lsl#16 | ||
| 179 | ldrb r4,[r12,#14] | ||
| 180 | orr r2,r2,r6,lsl#24 | ||
| 181 | ldrb r5,[r12,#13] | ||
| 182 | ldrb r6,[r12,#12] | ||
| 183 | orr r3,r3,r4,lsl#8 | ||
| 184 | orr r3,r3,r5,lsl#16 | ||
| 185 | orr r3,r3,r6,lsl#24 | ||
| 186 | #else | ||
| 187 | ldr r0,[r12,#0] | ||
| 188 | ldr r1,[r12,#4] | ||
| 189 | ldr r2,[r12,#8] | ||
| 190 | ldr r3,[r12,#12] | ||
| 191 | #ifdef __ARMEL__ | ||
| 192 | rev r0,r0 | ||
| 193 | rev r1,r1 | ||
| 194 | rev r2,r2 | ||
| 195 | rev r3,r3 | ||
| 196 | #endif | ||
| 197 | #endif | ||
| 198 | bl _armv4_AES_encrypt | ||
| 199 | |||
| 200 | ldr r12,[sp],#4 @ pop out | ||
| 201 | #if __ARM_ARCH__>=7 | ||
| 202 | #ifdef __ARMEL__ | ||
| 203 | rev r0,r0 | ||
| 204 | rev r1,r1 | ||
| 205 | rev r2,r2 | ||
| 206 | rev r3,r3 | ||
| 207 | #endif | ||
| 208 | str r0,[r12,#0] | ||
| 209 | str r1,[r12,#4] | ||
| 210 | str r2,[r12,#8] | ||
| 211 | str r3,[r12,#12] | ||
| 212 | #else | ||
| 213 | mov r4,r0,lsr#24 @ write output in endian-neutral | ||
| 214 | mov r5,r0,lsr#16 @ manner... | ||
| 215 | mov r6,r0,lsr#8 | ||
| 216 | strb r4,[r12,#0] | ||
| 217 | strb r5,[r12,#1] | ||
| 218 | mov r4,r1,lsr#24 | ||
| 219 | strb r6,[r12,#2] | ||
| 220 | mov r5,r1,lsr#16 | ||
| 221 | strb r0,[r12,#3] | ||
| 222 | mov r6,r1,lsr#8 | ||
| 223 | strb r4,[r12,#4] | ||
| 224 | strb r5,[r12,#5] | ||
| 225 | mov r4,r2,lsr#24 | ||
| 226 | strb r6,[r12,#6] | ||
| 227 | mov r5,r2,lsr#16 | ||
| 228 | strb r1,[r12,#7] | ||
| 229 | mov r6,r2,lsr#8 | ||
| 230 | strb r4,[r12,#8] | ||
| 231 | strb r5,[r12,#9] | ||
| 232 | mov r4,r3,lsr#24 | ||
| 233 | strb r6,[r12,#10] | ||
| 234 | mov r5,r3,lsr#16 | ||
| 235 | strb r2,[r12,#11] | ||
| 236 | mov r6,r3,lsr#8 | ||
| 237 | strb r4,[r12,#12] | ||
| 238 | strb r5,[r12,#13] | ||
| 239 | strb r6,[r12,#14] | ||
| 240 | strb r3,[r12,#15] | ||
| 241 | #endif | ||
| 242 | #if __ARM_ARCH__>=5 | ||
| 243 | ldmia sp!,{r4-r12,pc} | ||
| 244 | #else | ||
| 245 | ldmia sp!,{r4-r12,lr} | ||
| 246 | tst lr,#1 | ||
| 247 | moveq pc,lr @ be binary compatible with V4, yet | ||
| 248 | .word 0xe12fff1e @ interoperable with Thumb ISA:-) | ||
| 249 | #endif | ||
| 250 | .size AES_encrypt,.-AES_encrypt | ||
| 251 | |||
| 252 | .type _armv4_AES_encrypt,%function | ||
| 253 | .align 2 | ||
| 254 | _armv4_AES_encrypt: | ||
| 255 | str lr,[sp,#-4]! @ push lr | ||
| 256 | ldmia r11!,{r4-r7} | ||
| 257 | eor r0,r0,r4 | ||
| 258 | ldr r12,[r11,#240-16] | ||
| 259 | eor r1,r1,r5 | ||
| 260 | eor r2,r2,r6 | ||
| 261 | eor r3,r3,r7 | ||
| 262 | sub r12,r12,#1 | ||
| 263 | mov lr,#255 | ||
| 264 | |||
| 265 | and r7,lr,r0 | ||
| 266 | and r8,lr,r0,lsr#8 | ||
| 267 | and r9,lr,r0,lsr#16 | ||
| 268 | mov r0,r0,lsr#24 | ||
| 269 | .Lenc_loop: | ||
| 270 | ldr r4,[r10,r7,lsl#2] @ Te3[s0>>0] | ||
| 271 | and r7,lr,r1,lsr#16 @ i0 | ||
| 272 | ldr r5,[r10,r8,lsl#2] @ Te2[s0>>8] | ||
| 273 | and r8,lr,r1 | ||
| 274 | ldr r6,[r10,r9,lsl#2] @ Te1[s0>>16] | ||
| 275 | and r9,lr,r1,lsr#8 | ||
| 276 | ldr r0,[r10,r0,lsl#2] @ Te0[s0>>24] | ||
| 277 | mov r1,r1,lsr#24 | ||
| 278 | |||
| 279 | ldr r7,[r10,r7,lsl#2] @ Te1[s1>>16] | ||
| 280 | ldr r8,[r10,r8,lsl#2] @ Te3[s1>>0] | ||
| 281 | ldr r9,[r10,r9,lsl#2] @ Te2[s1>>8] | ||
| 282 | eor r0,r0,r7,ror#8 | ||
| 283 | ldr r1,[r10,r1,lsl#2] @ Te0[s1>>24] | ||
| 284 | and r7,lr,r2,lsr#8 @ i0 | ||
| 285 | eor r5,r5,r8,ror#8 | ||
| 286 | and r8,lr,r2,lsr#16 @ i1 | ||
| 287 | eor r6,r6,r9,ror#8 | ||
| 288 | and r9,lr,r2 | ||
| 289 | ldr r7,[r10,r7,lsl#2] @ Te2[s2>>8] | ||
| 290 | eor r1,r1,r4,ror#24 | ||
| 291 | ldr r8,[r10,r8,lsl#2] @ Te1[s2>>16] | ||
| 292 | mov r2,r2,lsr#24 | ||
| 293 | |||
| 294 | ldr r9,[r10,r9,lsl#2] @ Te3[s2>>0] | ||
| 295 | eor r0,r0,r7,ror#16 | ||
| 296 | ldr r2,[r10,r2,lsl#2] @ Te0[s2>>24] | ||
| 297 | and r7,lr,r3 @ i0 | ||
| 298 | eor r1,r1,r8,ror#8 | ||
| 299 | and r8,lr,r3,lsr#8 @ i1 | ||
| 300 | eor r6,r6,r9,ror#16 | ||
| 301 | and r9,lr,r3,lsr#16 @ i2 | ||
| 302 | ldr r7,[r10,r7,lsl#2] @ Te3[s3>>0] | ||
| 303 | eor r2,r2,r5,ror#16 | ||
| 304 | ldr r8,[r10,r8,lsl#2] @ Te2[s3>>8] | ||
| 305 | mov r3,r3,lsr#24 | ||
| 306 | |||
| 307 | ldr r9,[r10,r9,lsl#2] @ Te1[s3>>16] | ||
| 308 | eor r0,r0,r7,ror#24 | ||
| 309 | ldr r7,[r11],#16 | ||
| 310 | eor r1,r1,r8,ror#16 | ||
| 311 | ldr r3,[r10,r3,lsl#2] @ Te0[s3>>24] | ||
| 312 | eor r2,r2,r9,ror#8 | ||
| 313 | ldr r4,[r11,#-12] | ||
| 314 | eor r3,r3,r6,ror#8 | ||
| 315 | |||
| 316 | ldr r5,[r11,#-8] | ||
| 317 | eor r0,r0,r7 | ||
| 318 | ldr r6,[r11,#-4] | ||
| 319 | and r7,lr,r0 | ||
| 320 | eor r1,r1,r4 | ||
| 321 | and r8,lr,r0,lsr#8 | ||
| 322 | eor r2,r2,r5 | ||
| 323 | and r9,lr,r0,lsr#16 | ||
| 324 | eor r3,r3,r6 | ||
| 325 | mov r0,r0,lsr#24 | ||
| 326 | |||
| 327 | subs r12,r12,#1 | ||
| 328 | bne .Lenc_loop | ||
| 329 | |||
| 330 | add r10,r10,#2 | ||
| 331 | |||
| 332 | ldrb r4,[r10,r7,lsl#2] @ Te4[s0>>0] | ||
| 333 | and r7,lr,r1,lsr#16 @ i0 | ||
| 334 | ldrb r5,[r10,r8,lsl#2] @ Te4[s0>>8] | ||
| 335 | and r8,lr,r1 | ||
| 336 | ldrb r6,[r10,r9,lsl#2] @ Te4[s0>>16] | ||
| 337 | and r9,lr,r1,lsr#8 | ||
| 338 | ldrb r0,[r10,r0,lsl#2] @ Te4[s0>>24] | ||
| 339 | mov r1,r1,lsr#24 | ||
| 340 | |||
| 341 | ldrb r7,[r10,r7,lsl#2] @ Te4[s1>>16] | ||
| 342 | ldrb r8,[r10,r8,lsl#2] @ Te4[s1>>0] | ||
| 343 | ldrb r9,[r10,r9,lsl#2] @ Te4[s1>>8] | ||
| 344 | eor r0,r7,r0,lsl#8 | ||
| 345 | ldrb r1,[r10,r1,lsl#2] @ Te4[s1>>24] | ||
| 346 | and r7,lr,r2,lsr#8 @ i0 | ||
| 347 | eor r5,r8,r5,lsl#8 | ||
| 348 | and r8,lr,r2,lsr#16 @ i1 | ||
| 349 | eor r6,r9,r6,lsl#8 | ||
| 350 | and r9,lr,r2 | ||
| 351 | ldrb r7,[r10,r7,lsl#2] @ Te4[s2>>8] | ||
| 352 | eor r1,r4,r1,lsl#24 | ||
| 353 | ldrb r8,[r10,r8,lsl#2] @ Te4[s2>>16] | ||
| 354 | mov r2,r2,lsr#24 | ||
| 355 | |||
| 356 | ldrb r9,[r10,r9,lsl#2] @ Te4[s2>>0] | ||
| 357 | eor r0,r7,r0,lsl#8 | ||
| 358 | ldrb r2,[r10,r2,lsl#2] @ Te4[s2>>24] | ||
| 359 | and r7,lr,r3 @ i0 | ||
| 360 | eor r1,r1,r8,lsl#16 | ||
| 361 | and r8,lr,r3,lsr#8 @ i1 | ||
| 362 | eor r6,r9,r6,lsl#8 | ||
| 363 | and r9,lr,r3,lsr#16 @ i2 | ||
| 364 | ldrb r7,[r10,r7,lsl#2] @ Te4[s3>>0] | ||
| 365 | eor r2,r5,r2,lsl#24 | ||
| 366 | ldrb r8,[r10,r8,lsl#2] @ Te4[s3>>8] | ||
| 367 | mov r3,r3,lsr#24 | ||
| 368 | |||
| 369 | ldrb r9,[r10,r9,lsl#2] @ Te4[s3>>16] | ||
| 370 | eor r0,r7,r0,lsl#8 | ||
| 371 | ldr r7,[r11,#0] | ||
| 372 | ldrb r3,[r10,r3,lsl#2] @ Te4[s3>>24] | ||
| 373 | eor r1,r1,r8,lsl#8 | ||
| 374 | ldr r4,[r11,#4] | ||
| 375 | eor r2,r2,r9,lsl#16 | ||
| 376 | ldr r5,[r11,#8] | ||
| 377 | eor r3,r6,r3,lsl#24 | ||
| 378 | ldr r6,[r11,#12] | ||
| 379 | |||
| 380 | eor r0,r0,r7 | ||
| 381 | eor r1,r1,r4 | ||
| 382 | eor r2,r2,r5 | ||
| 383 | eor r3,r3,r6 | ||
| 384 | |||
| 385 | sub r10,r10,#2 | ||
| 386 | ldr pc,[sp],#4 @ pop and return | ||
| 387 | .size _armv4_AES_encrypt,.-_armv4_AES_encrypt | ||
| 388 | |||
| 389 | .global private_AES_set_encrypt_key | ||
| 390 | .type private_AES_set_encrypt_key,%function | ||
| 391 | .align 5 | ||
| 392 | private_AES_set_encrypt_key: | ||
| 393 | _armv4_AES_set_encrypt_key: | ||
| 394 | sub r3,pc,#8 @ AES_set_encrypt_key | ||
| 395 | teq r0,#0 | ||
| 396 | moveq r0,#-1 | ||
| 397 | beq .Labrt | ||
| 398 | teq r2,#0 | ||
| 399 | moveq r0,#-1 | ||
| 400 | beq .Labrt | ||
| 401 | |||
| 402 | teq r1,#128 | ||
| 403 | beq .Lok | ||
| 404 | teq r1,#192 | ||
| 405 | beq .Lok | ||
| 406 | teq r1,#256 | ||
| 407 | movne r0,#-1 | ||
| 408 | bne .Labrt | ||
| 409 | |||
| 410 | .Lok: stmdb sp!,{r4-r12,lr} | ||
| 411 | sub r10,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024 @ Te4 | ||
| 412 | |||
| 413 | mov r12,r0 @ inp | ||
| 414 | mov lr,r1 @ bits | ||
| 415 | mov r11,r2 @ key | ||
| 416 | |||
| 417 | #if __ARM_ARCH__<7 | ||
| 418 | ldrb r0,[r12,#3] @ load input data in endian-neutral | ||
| 419 | ldrb r4,[r12,#2] @ manner... | ||
| 420 | ldrb r5,[r12,#1] | ||
| 421 | ldrb r6,[r12,#0] | ||
| 422 | orr r0,r0,r4,lsl#8 | ||
| 423 | ldrb r1,[r12,#7] | ||
| 424 | orr r0,r0,r5,lsl#16 | ||
| 425 | ldrb r4,[r12,#6] | ||
| 426 | orr r0,r0,r6,lsl#24 | ||
| 427 | ldrb r5,[r12,#5] | ||
| 428 | ldrb r6,[r12,#4] | ||
| 429 | orr r1,r1,r4,lsl#8 | ||
| 430 | ldrb r2,[r12,#11] | ||
| 431 | orr r1,r1,r5,lsl#16 | ||
| 432 | ldrb r4,[r12,#10] | ||
| 433 | orr r1,r1,r6,lsl#24 | ||
| 434 | ldrb r5,[r12,#9] | ||
| 435 | ldrb r6,[r12,#8] | ||
| 436 | orr r2,r2,r4,lsl#8 | ||
| 437 | ldrb r3,[r12,#15] | ||
| 438 | orr r2,r2,r5,lsl#16 | ||
| 439 | ldrb r4,[r12,#14] | ||
| 440 | orr r2,r2,r6,lsl#24 | ||
| 441 | ldrb r5,[r12,#13] | ||
| 442 | ldrb r6,[r12,#12] | ||
| 443 | orr r3,r3,r4,lsl#8 | ||
| 444 | str r0,[r11],#16 | ||
| 445 | orr r3,r3,r5,lsl#16 | ||
| 446 | str r1,[r11,#-12] | ||
| 447 | orr r3,r3,r6,lsl#24 | ||
| 448 | str r2,[r11,#-8] | ||
| 449 | str r3,[r11,#-4] | ||
| 450 | #else | ||
| 451 | ldr r0,[r12,#0] | ||
| 452 | ldr r1,[r12,#4] | ||
| 453 | ldr r2,[r12,#8] | ||
| 454 | ldr r3,[r12,#12] | ||
| 455 | #ifdef __ARMEL__ | ||
| 456 | rev r0,r0 | ||
| 457 | rev r1,r1 | ||
| 458 | rev r2,r2 | ||
| 459 | rev r3,r3 | ||
| 460 | #endif | ||
| 461 | str r0,[r11],#16 | ||
| 462 | str r1,[r11,#-12] | ||
| 463 | str r2,[r11,#-8] | ||
| 464 | str r3,[r11,#-4] | ||
| 465 | #endif | ||
| 466 | |||
| 467 | teq lr,#128 | ||
| 468 | bne .Lnot128 | ||
| 469 | mov r12,#10 | ||
| 470 | str r12,[r11,#240-16] | ||
| 471 | add r6,r10,#256 @ rcon | ||
| 472 | mov lr,#255 | ||
| 473 | |||
| 474 | .L128_loop: | ||
| 475 | and r5,lr,r3,lsr#24 | ||
| 476 | and r7,lr,r3,lsr#16 | ||
| 477 | ldrb r5,[r10,r5] | ||
| 478 | and r8,lr,r3,lsr#8 | ||
| 479 | ldrb r7,[r10,r7] | ||
| 480 | and r9,lr,r3 | ||
| 481 | ldrb r8,[r10,r8] | ||
| 482 | orr r5,r5,r7,lsl#24 | ||
| 483 | ldrb r9,[r10,r9] | ||
| 484 | orr r5,r5,r8,lsl#16 | ||
| 485 | ldr r4,[r6],#4 @ rcon[i++] | ||
| 486 | orr r5,r5,r9,lsl#8 | ||
| 487 | eor r5,r5,r4 | ||
| 488 | eor r0,r0,r5 @ rk[4]=rk[0]^... | ||
| 489 | eor r1,r1,r0 @ rk[5]=rk[1]^rk[4] | ||
| 490 | str r0,[r11],#16 | ||
| 491 | eor r2,r2,r1 @ rk[6]=rk[2]^rk[5] | ||
| 492 | str r1,[r11,#-12] | ||
| 493 | eor r3,r3,r2 @ rk[7]=rk[3]^rk[6] | ||
| 494 | str r2,[r11,#-8] | ||
| 495 | subs r12,r12,#1 | ||
| 496 | str r3,[r11,#-4] | ||
| 497 | bne .L128_loop | ||
| 498 | sub r2,r11,#176 | ||
| 499 | b .Ldone | ||
| 500 | |||
| 501 | .Lnot128: | ||
| 502 | #if __ARM_ARCH__<7 | ||
| 503 | ldrb r8,[r12,#19] | ||
| 504 | ldrb r4,[r12,#18] | ||
| 505 | ldrb r5,[r12,#17] | ||
| 506 | ldrb r6,[r12,#16] | ||
| 507 | orr r8,r8,r4,lsl#8 | ||
| 508 | ldrb r9,[r12,#23] | ||
| 509 | orr r8,r8,r5,lsl#16 | ||
| 510 | ldrb r4,[r12,#22] | ||
| 511 | orr r8,r8,r6,lsl#24 | ||
| 512 | ldrb r5,[r12,#21] | ||
| 513 | ldrb r6,[r12,#20] | ||
| 514 | orr r9,r9,r4,lsl#8 | ||
| 515 | orr r9,r9,r5,lsl#16 | ||
| 516 | str r8,[r11],#8 | ||
| 517 | orr r9,r9,r6,lsl#24 | ||
| 518 | str r9,[r11,#-4] | ||
| 519 | #else | ||
| 520 | ldr r8,[r12,#16] | ||
| 521 | ldr r9,[r12,#20] | ||
| 522 | #ifdef __ARMEL__ | ||
| 523 | rev r8,r8 | ||
| 524 | rev r9,r9 | ||
| 525 | #endif | ||
| 526 | str r8,[r11],#8 | ||
| 527 | str r9,[r11,#-4] | ||
| 528 | #endif | ||
| 529 | |||
| 530 | teq lr,#192 | ||
| 531 | bne .Lnot192 | ||
| 532 | mov r12,#12 | ||
| 533 | str r12,[r11,#240-24] | ||
| 534 | add r6,r10,#256 @ rcon | ||
| 535 | mov lr,#255 | ||
| 536 | mov r12,#8 | ||
| 537 | |||
| 538 | .L192_loop: | ||
| 539 | and r5,lr,r9,lsr#24 | ||
| 540 | and r7,lr,r9,lsr#16 | ||
| 541 | ldrb r5,[r10,r5] | ||
| 542 | and r8,lr,r9,lsr#8 | ||
| 543 | ldrb r7,[r10,r7] | ||
| 544 | and r9,lr,r9 | ||
| 545 | ldrb r8,[r10,r8] | ||
| 546 | orr r5,r5,r7,lsl#24 | ||
| 547 | ldrb r9,[r10,r9] | ||
| 548 | orr r5,r5,r8,lsl#16 | ||
| 549 | ldr r4,[r6],#4 @ rcon[i++] | ||
| 550 | orr r5,r5,r9,lsl#8 | ||
| 551 | eor r9,r5,r4 | ||
| 552 | eor r0,r0,r9 @ rk[6]=rk[0]^... | ||
| 553 | eor r1,r1,r0 @ rk[7]=rk[1]^rk[6] | ||
| 554 | str r0,[r11],#24 | ||
| 555 | eor r2,r2,r1 @ rk[8]=rk[2]^rk[7] | ||
| 556 | str r1,[r11,#-20] | ||
| 557 | eor r3,r3,r2 @ rk[9]=rk[3]^rk[8] | ||
| 558 | str r2,[r11,#-16] | ||
| 559 | subs r12,r12,#1 | ||
| 560 | str r3,[r11,#-12] | ||
| 561 | subeq r2,r11,#216 | ||
| 562 | beq .Ldone | ||
| 563 | |||
| 564 | ldr r7,[r11,#-32] | ||
| 565 | ldr r8,[r11,#-28] | ||
| 566 | eor r7,r7,r3 @ rk[10]=rk[4]^rk[9] | ||
| 567 | eor r9,r8,r7 @ rk[11]=rk[5]^rk[10] | ||
| 568 | str r7,[r11,#-8] | ||
| 569 | str r9,[r11,#-4] | ||
| 570 | b .L192_loop | ||
| 571 | |||
| 572 | .Lnot192: | ||
| 573 | #if __ARM_ARCH__<7 | ||
| 574 | ldrb r8,[r12,#27] | ||
| 575 | ldrb r4,[r12,#26] | ||
| 576 | ldrb r5,[r12,#25] | ||
| 577 | ldrb r6,[r12,#24] | ||
| 578 | orr r8,r8,r4,lsl#8 | ||
| 579 | ldrb r9,[r12,#31] | ||
| 580 | orr r8,r8,r5,lsl#16 | ||
| 581 | ldrb r4,[r12,#30] | ||
| 582 | orr r8,r8,r6,lsl#24 | ||
| 583 | ldrb r5,[r12,#29] | ||
| 584 | ldrb r6,[r12,#28] | ||
| 585 | orr r9,r9,r4,lsl#8 | ||
| 586 | orr r9,r9,r5,lsl#16 | ||
| 587 | str r8,[r11],#8 | ||
| 588 | orr r9,r9,r6,lsl#24 | ||
| 589 | str r9,[r11,#-4] | ||
| 590 | #else | ||
| 591 | ldr r8,[r12,#24] | ||
| 592 | ldr r9,[r12,#28] | ||
| 593 | #ifdef __ARMEL__ | ||
| 594 | rev r8,r8 | ||
| 595 | rev r9,r9 | ||
| 596 | #endif | ||
| 597 | str r8,[r11],#8 | ||
| 598 | str r9,[r11,#-4] | ||
| 599 | #endif | ||
| 600 | |||
| 601 | mov r12,#14 | ||
| 602 | str r12,[r11,#240-32] | ||
| 603 | add r6,r10,#256 @ rcon | ||
| 604 | mov lr,#255 | ||
| 605 | mov r12,#7 | ||
| 606 | |||
| 607 | .L256_loop: | ||
| 608 | and r5,lr,r9,lsr#24 | ||
| 609 | and r7,lr,r9,lsr#16 | ||
| 610 | ldrb r5,[r10,r5] | ||
| 611 | and r8,lr,r9,lsr#8 | ||
| 612 | ldrb r7,[r10,r7] | ||
| 613 | and r9,lr,r9 | ||
| 614 | ldrb r8,[r10,r8] | ||
| 615 | orr r5,r5,r7,lsl#24 | ||
| 616 | ldrb r9,[r10,r9] | ||
| 617 | orr r5,r5,r8,lsl#16 | ||
| 618 | ldr r4,[r6],#4 @ rcon[i++] | ||
| 619 | orr r5,r5,r9,lsl#8 | ||
| 620 | eor r9,r5,r4 | ||
| 621 | eor r0,r0,r9 @ rk[8]=rk[0]^... | ||
| 622 | eor r1,r1,r0 @ rk[9]=rk[1]^rk[8] | ||
| 623 | str r0,[r11],#32 | ||
| 624 | eor r2,r2,r1 @ rk[10]=rk[2]^rk[9] | ||
| 625 | str r1,[r11,#-28] | ||
| 626 | eor r3,r3,r2 @ rk[11]=rk[3]^rk[10] | ||
| 627 | str r2,[r11,#-24] | ||
| 628 | subs r12,r12,#1 | ||
| 629 | str r3,[r11,#-20] | ||
| 630 | subeq r2,r11,#256 | ||
| 631 | beq .Ldone | ||
| 632 | |||
| 633 | and r5,lr,r3 | ||
| 634 | and r7,lr,r3,lsr#8 | ||
| 635 | ldrb r5,[r10,r5] | ||
| 636 | and r8,lr,r3,lsr#16 | ||
| 637 | ldrb r7,[r10,r7] | ||
| 638 | and r9,lr,r3,lsr#24 | ||
| 639 | ldrb r8,[r10,r8] | ||
| 640 | orr r5,r5,r7,lsl#8 | ||
| 641 | ldrb r9,[r10,r9] | ||
| 642 | orr r5,r5,r8,lsl#16 | ||
| 643 | ldr r4,[r11,#-48] | ||
| 644 | orr r5,r5,r9,lsl#24 | ||
| 645 | |||
| 646 | ldr r7,[r11,#-44] | ||
| 647 | ldr r8,[r11,#-40] | ||
| 648 | eor r4,r4,r5 @ rk[12]=rk[4]^... | ||
| 649 | ldr r9,[r11,#-36] | ||
| 650 | eor r7,r7,r4 @ rk[13]=rk[5]^rk[12] | ||
| 651 | str r4,[r11,#-16] | ||
| 652 | eor r8,r8,r7 @ rk[14]=rk[6]^rk[13] | ||
| 653 | str r7,[r11,#-12] | ||
| 654 | eor r9,r9,r8 @ rk[15]=rk[7]^rk[14] | ||
| 655 | str r8,[r11,#-8] | ||
| 656 | str r9,[r11,#-4] | ||
| 657 | b .L256_loop | ||
| 658 | |||
| 659 | .Ldone: mov r0,#0 | ||
| 660 | ldmia sp!,{r4-r12,lr} | ||
| 661 | .Labrt: tst lr,#1 | ||
| 662 | moveq pc,lr @ be binary compatible with V4, yet | ||
| 663 | .word 0xe12fff1e @ interoperable with Thumb ISA:-) | ||
| 664 | .size private_AES_set_encrypt_key,.-private_AES_set_encrypt_key | ||
| 665 | |||
| 666 | .global private_AES_set_decrypt_key | ||
| 667 | .type private_AES_set_decrypt_key,%function | ||
| 668 | .align 5 | ||
| 669 | private_AES_set_decrypt_key: | ||
| 670 | str lr,[sp,#-4]! @ push lr | ||
| 671 | #if 0 | ||
| 672 | @ kernel does both of these in setkey so optimise this bit out by | ||
| 673 | @ expecting the key to already have the enc_key work done (see aes_glue.c) | ||
| 674 | bl _armv4_AES_set_encrypt_key | ||
| 675 | #else | ||
| 676 | mov r0,#0 | ||
| 677 | #endif | ||
| 678 | teq r0,#0 | ||
| 679 | ldrne lr,[sp],#4 @ pop lr | ||
| 680 | bne .Labrt | ||
| 681 | |||
| 682 | stmdb sp!,{r4-r12} | ||
| 683 | |||
| 684 | ldr r12,[r2,#240] @ AES_set_encrypt_key preserves r2, | ||
| 685 | mov r11,r2 @ which is AES_KEY *key | ||
| 686 | mov r7,r2 | ||
| 687 | add r8,r2,r12,lsl#4 | ||
| 688 | |||
| 689 | .Linv: ldr r0,[r7] | ||
| 690 | ldr r1,[r7,#4] | ||
| 691 | ldr r2,[r7,#8] | ||
| 692 | ldr r3,[r7,#12] | ||
| 693 | ldr r4,[r8] | ||
| 694 | ldr r5,[r8,#4] | ||
| 695 | ldr r6,[r8,#8] | ||
| 696 | ldr r9,[r8,#12] | ||
| 697 | str r0,[r8],#-16 | ||
| 698 | str r1,[r8,#16+4] | ||
| 699 | str r2,[r8,#16+8] | ||
| 700 | str r3,[r8,#16+12] | ||
| 701 | str r4,[r7],#16 | ||
| 702 | str r5,[r7,#-12] | ||
| 703 | str r6,[r7,#-8] | ||
| 704 | str r9,[r7,#-4] | ||
| 705 | teq r7,r8 | ||
| 706 | bne .Linv | ||
| 707 | ldr r0,[r11,#16]! @ prefetch tp1 | ||
| 708 | mov r7,#0x80 | ||
| 709 | mov r8,#0x1b | ||
| 710 | orr r7,r7,#0x8000 | ||
| 711 | orr r8,r8,#0x1b00 | ||
| 712 | orr r7,r7,r7,lsl#16 | ||
| 713 | orr r8,r8,r8,lsl#16 | ||
| 714 | sub r12,r12,#1 | ||
| 715 | mvn r9,r7 | ||
| 716 | mov r12,r12,lsl#2 @ (rounds-1)*4 | ||
| 717 | |||
| 718 | .Lmix: and r4,r0,r7 | ||
| 719 | and r1,r0,r9 | ||
| 720 | sub r4,r4,r4,lsr#7 | ||
| 721 | and r4,r4,r8 | ||
| 722 | eor r1,r4,r1,lsl#1 @ tp2 | ||
| 723 | |||
| 724 | and r4,r1,r7 | ||
| 725 | and r2,r1,r9 | ||
| 726 | sub r4,r4,r4,lsr#7 | ||
| 727 | and r4,r4,r8 | ||
| 728 | eor r2,r4,r2,lsl#1 @ tp4 | ||
| 729 | |||
| 730 | and r4,r2,r7 | ||
| 731 | and r3,r2,r9 | ||
| 732 | sub r4,r4,r4,lsr#7 | ||
| 733 | and r4,r4,r8 | ||
| 734 | eor r3,r4,r3,lsl#1 @ tp8 | ||
| 735 | |||
| 736 | eor r4,r1,r2 | ||
| 737 | eor r5,r0,r3 @ tp9 | ||
| 738 | eor r4,r4,r3 @ tpe | ||
| 739 | eor r4,r4,r1,ror#24 | ||
| 740 | eor r4,r4,r5,ror#24 @ ^= ROTATE(tpb=tp9^tp2,8) | ||
| 741 | eor r4,r4,r2,ror#16 | ||
| 742 | eor r4,r4,r5,ror#16 @ ^= ROTATE(tpd=tp9^tp4,16) | ||
| 743 | eor r4,r4,r5,ror#8 @ ^= ROTATE(tp9,24) | ||
| 744 | |||
| 745 | ldr r0,[r11,#4] @ prefetch tp1 | ||
| 746 | str r4,[r11],#4 | ||
| 747 | subs r12,r12,#1 | ||
| 748 | bne .Lmix | ||
| 749 | |||
| 750 | mov r0,#0 | ||
| 751 | #if __ARM_ARCH__>=5 | ||
| 752 | ldmia sp!,{r4-r12,pc} | ||
| 753 | #else | ||
| 754 | ldmia sp!,{r4-r12,lr} | ||
| 755 | tst lr,#1 | ||
| 756 | moveq pc,lr @ be binary compatible with V4, yet | ||
| 757 | .word 0xe12fff1e @ interoperable with Thumb ISA:-) | ||
| 758 | #endif | ||
| 759 | .size private_AES_set_decrypt_key,.-private_AES_set_decrypt_key | ||
| 760 | |||
| 761 | .type AES_Td,%object | ||
| 762 | .align 5 | ||
| 763 | AES_Td: | ||
| 764 | .word 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96 | ||
| 765 | .word 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393 | ||
| 766 | .word 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25 | ||
| 767 | .word 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f | ||
| 768 | .word 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1 | ||
| 769 | .word 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6 | ||
| 770 | .word 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da | ||
| 771 | .word 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844 | ||
| 772 | .word 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd | ||
| 773 | .word 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4 | ||
| 774 | .word 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45 | ||
| 775 | .word 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94 | ||
| 776 | .word 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7 | ||
| 777 | .word 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a | ||
| 778 | .word 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5 | ||
| 779 | .word 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c | ||
| 780 | .word 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1 | ||
| 781 | .word 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a | ||
| 782 | .word 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75 | ||
| 783 | .word 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051 | ||
| 784 | .word 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46 | ||
| 785 | .word 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff | ||
| 786 | .word 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77 | ||
| 787 | .word 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb | ||
| 788 | .word 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000 | ||
| 789 | .word 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e | ||
| 790 | .word 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927 | ||
| 791 | .word 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a | ||
| 792 | .word 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e | ||
| 793 | .word 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16 | ||
| 794 | .word 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d | ||
| 795 | .word 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8 | ||
| 796 | .word 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd | ||
| 797 | .word 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34 | ||
| 798 | .word 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163 | ||
| 799 | .word 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120 | ||
| 800 | .word 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d | ||
| 801 | .word 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0 | ||
| 802 | .word 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422 | ||
| 803 | .word 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef | ||
| 804 | .word 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36 | ||
| 805 | .word 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4 | ||
| 806 | .word 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662 | ||
| 807 | .word 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5 | ||
| 808 | .word 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3 | ||
| 809 | .word 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b | ||
| 810 | .word 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8 | ||
| 811 | .word 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6 | ||
| 812 | .word 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6 | ||
| 813 | .word 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0 | ||
| 814 | .word 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815 | ||
| 815 | .word 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f | ||
| 816 | .word 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df | ||
| 817 | .word 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f | ||
| 818 | .word 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e | ||
| 819 | .word 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713 | ||
| 820 | .word 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89 | ||
| 821 | .word 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c | ||
| 822 | .word 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf | ||
| 823 | .word 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86 | ||
| 824 | .word 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f | ||
| 825 | .word 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541 | ||
| 826 | .word 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190 | ||
| 827 | .word 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742 | ||
| 828 | @ Td4[256] | ||
| 829 | .byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38 | ||
| 830 | .byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb | ||
| 831 | .byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87 | ||
| 832 | .byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb | ||
| 833 | .byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d | ||
| 834 | .byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e | ||
| 835 | .byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2 | ||
| 836 | .byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25 | ||
| 837 | .byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16 | ||
| 838 | .byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92 | ||
| 839 | .byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda | ||
| 840 | .byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84 | ||
| 841 | .byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a | ||
| 842 | .byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06 | ||
| 843 | .byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02 | ||
| 844 | .byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b | ||
| 845 | .byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea | ||
| 846 | .byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73 | ||
| 847 | .byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85 | ||
| 848 | .byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e | ||
| 849 | .byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89 | ||
| 850 | .byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b | ||
| 851 | .byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20 | ||
| 852 | .byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4 | ||
| 853 | .byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31 | ||
| 854 | .byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f | ||
| 855 | .byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d | ||
| 856 | .byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef | ||
| 857 | .byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0 | ||
| 858 | .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 | ||
| 859 | .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 | ||
| 860 | .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d | ||
| 861 | .size AES_Td,.-AES_Td | ||
| 862 | |||
| 863 | @ void AES_decrypt(const unsigned char *in, unsigned char *out, | ||
| 864 | @ const AES_KEY *key) { | ||
| 865 | .global AES_decrypt | ||
| 866 | .type AES_decrypt,%function | ||
| 867 | .align 5 | ||
| 868 | AES_decrypt: | ||
| 869 | sub r3,pc,#8 @ AES_decrypt | ||
| 870 | stmdb sp!,{r1,r4-r12,lr} | ||
| 871 | mov r12,r0 @ inp | ||
| 872 | mov r11,r2 | ||
| 873 | sub r10,r3,#AES_decrypt-AES_Td @ Td | ||
| 874 | #if __ARM_ARCH__<7 | ||
| 875 | ldrb r0,[r12,#3] @ load input data in endian-neutral | ||
| 876 | ldrb r4,[r12,#2] @ manner... | ||
| 877 | ldrb r5,[r12,#1] | ||
| 878 | ldrb r6,[r12,#0] | ||
| 879 | orr r0,r0,r4,lsl#8 | ||
| 880 | ldrb r1,[r12,#7] | ||
| 881 | orr r0,r0,r5,lsl#16 | ||
| 882 | ldrb r4,[r12,#6] | ||
| 883 | orr r0,r0,r6,lsl#24 | ||
| 884 | ldrb r5,[r12,#5] | ||
| 885 | ldrb r6,[r12,#4] | ||
| 886 | orr r1,r1,r4,lsl#8 | ||
| 887 | ldrb r2,[r12,#11] | ||
| 888 | orr r1,r1,r5,lsl#16 | ||
| 889 | ldrb r4,[r12,#10] | ||
| 890 | orr r1,r1,r6,lsl#24 | ||
| 891 | ldrb r5,[r12,#9] | ||
| 892 | ldrb r6,[r12,#8] | ||
| 893 | orr r2,r2,r4,lsl#8 | ||
| 894 | ldrb r3,[r12,#15] | ||
| 895 | orr r2,r2,r5,lsl#16 | ||
| 896 | ldrb r4,[r12,#14] | ||
| 897 | orr r2,r2,r6,lsl#24 | ||
| 898 | ldrb r5,[r12,#13] | ||
| 899 | ldrb r6,[r12,#12] | ||
| 900 | orr r3,r3,r4,lsl#8 | ||
| 901 | orr r3,r3,r5,lsl#16 | ||
| 902 | orr r3,r3,r6,lsl#24 | ||
| 903 | #else | ||
| 904 | ldr r0,[r12,#0] | ||
| 905 | ldr r1,[r12,#4] | ||
| 906 | ldr r2,[r12,#8] | ||
| 907 | ldr r3,[r12,#12] | ||
| 908 | #ifdef __ARMEL__ | ||
| 909 | rev r0,r0 | ||
| 910 | rev r1,r1 | ||
| 911 | rev r2,r2 | ||
| 912 | rev r3,r3 | ||
| 913 | #endif | ||
| 914 | #endif | ||
| 915 | bl _armv4_AES_decrypt | ||
| 916 | |||
| 917 | ldr r12,[sp],#4 @ pop out | ||
| 918 | #if __ARM_ARCH__>=7 | ||
| 919 | #ifdef __ARMEL__ | ||
| 920 | rev r0,r0 | ||
| 921 | rev r1,r1 | ||
| 922 | rev r2,r2 | ||
| 923 | rev r3,r3 | ||
| 924 | #endif | ||
| 925 | str r0,[r12,#0] | ||
| 926 | str r1,[r12,#4] | ||
| 927 | str r2,[r12,#8] | ||
| 928 | str r3,[r12,#12] | ||
| 929 | #else | ||
| 930 | mov r4,r0,lsr#24 @ write output in endian-neutral | ||
| 931 | mov r5,r0,lsr#16 @ manner... | ||
| 932 | mov r6,r0,lsr#8 | ||
| 933 | strb r4,[r12,#0] | ||
| 934 | strb r5,[r12,#1] | ||
| 935 | mov r4,r1,lsr#24 | ||
| 936 | strb r6,[r12,#2] | ||
| 937 | mov r5,r1,lsr#16 | ||
| 938 | strb r0,[r12,#3] | ||
| 939 | mov r6,r1,lsr#8 | ||
| 940 | strb r4,[r12,#4] | ||
| 941 | strb r5,[r12,#5] | ||
| 942 | mov r4,r2,lsr#24 | ||
| 943 | strb r6,[r12,#6] | ||
| 944 | mov r5,r2,lsr#16 | ||
| 945 | strb r1,[r12,#7] | ||
| 946 | mov r6,r2,lsr#8 | ||
| 947 | strb r4,[r12,#8] | ||
| 948 | strb r5,[r12,#9] | ||
| 949 | mov r4,r3,lsr#24 | ||
| 950 | strb r6,[r12,#10] | ||
| 951 | mov r5,r3,lsr#16 | ||
| 952 | strb r2,[r12,#11] | ||
| 953 | mov r6,r3,lsr#8 | ||
| 954 | strb r4,[r12,#12] | ||
| 955 | strb r5,[r12,#13] | ||
| 956 | strb r6,[r12,#14] | ||
| 957 | strb r3,[r12,#15] | ||
| 958 | #endif | ||
| 959 | #if __ARM_ARCH__>=5 | ||
| 960 | ldmia sp!,{r4-r12,pc} | ||
| 961 | #else | ||
| 962 | ldmia sp!,{r4-r12,lr} | ||
| 963 | tst lr,#1 | ||
| 964 | moveq pc,lr @ be binary compatible with V4, yet | ||
| 965 | .word 0xe12fff1e @ interoperable with Thumb ISA:-) | ||
| 966 | #endif | ||
| 967 | .size AES_decrypt,.-AES_decrypt | ||
| 968 | |||
| 969 | .type _armv4_AES_decrypt,%function | ||
| 970 | .align 2 | ||
| 971 | _armv4_AES_decrypt: | ||
| 972 | str lr,[sp,#-4]! @ push lr | ||
| 973 | ldmia r11!,{r4-r7} | ||
| 974 | eor r0,r0,r4 | ||
| 975 | ldr r12,[r11,#240-16] | ||
| 976 | eor r1,r1,r5 | ||
| 977 | eor r2,r2,r6 | ||
| 978 | eor r3,r3,r7 | ||
| 979 | sub r12,r12,#1 | ||
| 980 | mov lr,#255 | ||
| 981 | |||
| 982 | and r7,lr,r0,lsr#16 | ||
| 983 | and r8,lr,r0,lsr#8 | ||
| 984 | and r9,lr,r0 | ||
| 985 | mov r0,r0,lsr#24 | ||
| 986 | .Ldec_loop: | ||
| 987 | ldr r4,[r10,r7,lsl#2] @ Td1[s0>>16] | ||
| 988 | and r7,lr,r1 @ i0 | ||
| 989 | ldr r5,[r10,r8,lsl#2] @ Td2[s0>>8] | ||
| 990 | and r8,lr,r1,lsr#16 | ||
| 991 | ldr r6,[r10,r9,lsl#2] @ Td3[s0>>0] | ||
| 992 | and r9,lr,r1,lsr#8 | ||
| 993 | ldr r0,[r10,r0,lsl#2] @ Td0[s0>>24] | ||
| 994 | mov r1,r1,lsr#24 | ||
| 995 | |||
| 996 | ldr r7,[r10,r7,lsl#2] @ Td3[s1>>0] | ||
| 997 | ldr r8,[r10,r8,lsl#2] @ Td1[s1>>16] | ||
| 998 | ldr r9,[r10,r9,lsl#2] @ Td2[s1>>8] | ||
| 999 | eor r0,r0,r7,ror#24 | ||
| 1000 | ldr r1,[r10,r1,lsl#2] @ Td0[s1>>24] | ||
| 1001 | and r7,lr,r2,lsr#8 @ i0 | ||
| 1002 | eor r5,r8,r5,ror#8 | ||
| 1003 | and r8,lr,r2 @ i1 | ||
| 1004 | eor r6,r9,r6,ror#8 | ||
| 1005 | and r9,lr,r2,lsr#16 | ||
| 1006 | ldr r7,[r10,r7,lsl#2] @ Td2[s2>>8] | ||
| 1007 | eor r1,r1,r4,ror#8 | ||
| 1008 | ldr r8,[r10,r8,lsl#2] @ Td3[s2>>0] | ||
| 1009 | mov r2,r2,lsr#24 | ||
| 1010 | |||
| 1011 | ldr r9,[r10,r9,lsl#2] @ Td1[s2>>16] | ||
| 1012 | eor r0,r0,r7,ror#16 | ||
| 1013 | ldr r2,[r10,r2,lsl#2] @ Td0[s2>>24] | ||
| 1014 | and r7,lr,r3,lsr#16 @ i0 | ||
| 1015 | eor r1,r1,r8,ror#24 | ||
| 1016 | and r8,lr,r3,lsr#8 @ i1 | ||
| 1017 | eor r6,r9,r6,ror#8 | ||
| 1018 | and r9,lr,r3 @ i2 | ||
| 1019 | ldr r7,[r10,r7,lsl#2] @ Td1[s3>>16] | ||
| 1020 | eor r2,r2,r5,ror#8 | ||
| 1021 | ldr r8,[r10,r8,lsl#2] @ Td2[s3>>8] | ||
| 1022 | mov r3,r3,lsr#24 | ||
| 1023 | |||
| 1024 | ldr r9,[r10,r9,lsl#2] @ Td3[s3>>0] | ||
| 1025 | eor r0,r0,r7,ror#8 | ||
| 1026 | ldr r7,[r11],#16 | ||
| 1027 | eor r1,r1,r8,ror#16 | ||
| 1028 | ldr r3,[r10,r3,lsl#2] @ Td0[s3>>24] | ||
| 1029 | eor r2,r2,r9,ror#24 | ||
| 1030 | |||
| 1031 | ldr r4,[r11,#-12] | ||
| 1032 | eor r0,r0,r7 | ||
| 1033 | ldr r5,[r11,#-8] | ||
| 1034 | eor r3,r3,r6,ror#8 | ||
| 1035 | ldr r6,[r11,#-4] | ||
| 1036 | and r7,lr,r0,lsr#16 | ||
| 1037 | eor r1,r1,r4 | ||
| 1038 | and r8,lr,r0,lsr#8 | ||
| 1039 | eor r2,r2,r5 | ||
| 1040 | and r9,lr,r0 | ||
| 1041 | eor r3,r3,r6 | ||
| 1042 | mov r0,r0,lsr#24 | ||
| 1043 | |||
| 1044 | subs r12,r12,#1 | ||
| 1045 | bne .Ldec_loop | ||
| 1046 | |||
| 1047 | add r10,r10,#1024 | ||
| 1048 | |||
| 1049 | ldr r5,[r10,#0] @ prefetch Td4 | ||
| 1050 | ldr r6,[r10,#32] | ||
| 1051 | ldr r4,[r10,#64] | ||
| 1052 | ldr r5,[r10,#96] | ||
| 1053 | ldr r6,[r10,#128] | ||
| 1054 | ldr r4,[r10,#160] | ||
| 1055 | ldr r5,[r10,#192] | ||
| 1056 | ldr r6,[r10,#224] | ||
| 1057 | |||
| 1058 | ldrb r0,[r10,r0] @ Td4[s0>>24] | ||
| 1059 | ldrb r4,[r10,r7] @ Td4[s0>>16] | ||
| 1060 | and r7,lr,r1 @ i0 | ||
| 1061 | ldrb r5,[r10,r8] @ Td4[s0>>8] | ||
| 1062 | and r8,lr,r1,lsr#16 | ||
| 1063 | ldrb r6,[r10,r9] @ Td4[s0>>0] | ||
| 1064 | and r9,lr,r1,lsr#8 | ||
| 1065 | |||
| 1066 | ldrb r7,[r10,r7] @ Td4[s1>>0] | ||
| 1067 | ldrb r1,[r10,r1,lsr#24] @ Td4[s1>>24] | ||
| 1068 | ldrb r8,[r10,r8] @ Td4[s1>>16] | ||
| 1069 | eor r0,r7,r0,lsl#24 | ||
| 1070 | ldrb r9,[r10,r9] @ Td4[s1>>8] | ||
| 1071 | eor r1,r4,r1,lsl#8 | ||
| 1072 | and r7,lr,r2,lsr#8 @ i0 | ||
| 1073 | eor r5,r5,r8,lsl#8 | ||
| 1074 | and r8,lr,r2 @ i1 | ||
| 1075 | ldrb r7,[r10,r7] @ Td4[s2>>8] | ||
| 1076 | eor r6,r6,r9,lsl#8 | ||
| 1077 | ldrb r8,[r10,r8] @ Td4[s2>>0] | ||
| 1078 | and r9,lr,r2,lsr#16 | ||
| 1079 | |||
| 1080 | ldrb r2,[r10,r2,lsr#24] @ Td4[s2>>24] | ||
| 1081 | eor r0,r0,r7,lsl#8 | ||
| 1082 | ldrb r9,[r10,r9] @ Td4[s2>>16] | ||
| 1083 | eor r1,r8,r1,lsl#16 | ||
| 1084 | and r7,lr,r3,lsr#16 @ i0 | ||
| 1085 | eor r2,r5,r2,lsl#16 | ||
| 1086 | and r8,lr,r3,lsr#8 @ i1 | ||
| 1087 | ldrb r7,[r10,r7] @ Td4[s3>>16] | ||
| 1088 | eor r6,r6,r9,lsl#16 | ||
| 1089 | ldrb r8,[r10,r8] @ Td4[s3>>8] | ||
| 1090 | and r9,lr,r3 @ i2 | ||
| 1091 | |||
| 1092 | ldrb r9,[r10,r9] @ Td4[s3>>0] | ||
| 1093 | ldrb r3,[r10,r3,lsr#24] @ Td4[s3>>24] | ||
| 1094 | eor r0,r0,r7,lsl#16 | ||
| 1095 | ldr r7,[r11,#0] | ||
| 1096 | eor r1,r1,r8,lsl#8 | ||
| 1097 | ldr r4,[r11,#4] | ||
| 1098 | eor r2,r9,r2,lsl#8 | ||
| 1099 | ldr r5,[r11,#8] | ||
| 1100 | eor r3,r6,r3,lsl#24 | ||
| 1101 | ldr r6,[r11,#12] | ||
| 1102 | |||
| 1103 | eor r0,r0,r7 | ||
| 1104 | eor r1,r1,r4 | ||
| 1105 | eor r2,r2,r5 | ||
| 1106 | eor r3,r3,r6 | ||
| 1107 | |||
| 1108 | sub r10,r10,#1024 | ||
| 1109 | ldr pc,[sp],#4 @ pop and return | ||
| 1110 | .size _armv4_AES_decrypt,.-_armv4_AES_decrypt | ||
| 1111 | .asciz "AES for ARMv4, CRYPTOGAMS by <appro@openssl.org>" | ||
| 1112 | .align 2 | ||
diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c new file mode 100644 index 000000000000..59f7877ead6a --- /dev/null +++ b/arch/arm/crypto/aes_glue.c | |||
| @@ -0,0 +1,108 @@ | |||
| 1 | /* | ||
| 2 | * Glue Code for the asm optimized version of the AES Cipher Algorithm | ||
| 3 | */ | ||
| 4 | |||
| 5 | #include <linux/module.h> | ||
| 6 | #include <linux/crypto.h> | ||
| 7 | #include <crypto/aes.h> | ||
| 8 | |||
| 9 | #define AES_MAXNR 14 | ||
| 10 | |||
| 11 | typedef struct { | ||
| 12 | unsigned int rd_key[4 *(AES_MAXNR + 1)]; | ||
| 13 | int rounds; | ||
| 14 | } AES_KEY; | ||
| 15 | |||
| 16 | struct AES_CTX { | ||
| 17 | AES_KEY enc_key; | ||
| 18 | AES_KEY dec_key; | ||
| 19 | }; | ||
| 20 | |||
| 21 | asmlinkage void AES_encrypt(const u8 *in, u8 *out, AES_KEY *ctx); | ||
| 22 | asmlinkage void AES_decrypt(const u8 *in, u8 *out, AES_KEY *ctx); | ||
| 23 | asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); | ||
| 24 | asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); | ||
| 25 | |||
| 26 | static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | ||
| 27 | { | ||
| 28 | struct AES_CTX *ctx = crypto_tfm_ctx(tfm); | ||
| 29 | AES_encrypt(src, dst, &ctx->enc_key); | ||
| 30 | } | ||
| 31 | |||
| 32 | static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | ||
| 33 | { | ||
| 34 | struct AES_CTX *ctx = crypto_tfm_ctx(tfm); | ||
| 35 | AES_decrypt(src, dst, &ctx->dec_key); | ||
| 36 | } | ||
| 37 | |||
| 38 | static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, | ||
| 39 | unsigned int key_len) | ||
| 40 | { | ||
| 41 | struct AES_CTX *ctx = crypto_tfm_ctx(tfm); | ||
| 42 | |||
| 43 | switch (key_len) { | ||
| 44 | case AES_KEYSIZE_128: | ||
| 45 | key_len = 128; | ||
| 46 | break; | ||
| 47 | case AES_KEYSIZE_192: | ||
| 48 | key_len = 192; | ||
| 49 | break; | ||
| 50 | case AES_KEYSIZE_256: | ||
| 51 | key_len = 256; | ||
| 52 | break; | ||
| 53 | default: | ||
| 54 | tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | ||
| 55 | return -EINVAL; | ||
| 56 | } | ||
| 57 | |||
| 58 | if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) { | ||
| 59 | tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | ||
| 60 | return -EINVAL; | ||
| 61 | } | ||
| 62 | /* private_AES_set_decrypt_key expects an encryption key as input */ | ||
| 63 | ctx->dec_key = ctx->enc_key; | ||
| 64 | if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) { | ||
| 65 | tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | ||
| 66 | return -EINVAL; | ||
| 67 | } | ||
| 68 | return 0; | ||
| 69 | } | ||
| 70 | |||
| 71 | static struct crypto_alg aes_alg = { | ||
| 72 | .cra_name = "aes", | ||
| 73 | .cra_driver_name = "aes-asm", | ||
| 74 | .cra_priority = 200, | ||
| 75 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER, | ||
| 76 | .cra_blocksize = AES_BLOCK_SIZE, | ||
| 77 | .cra_ctxsize = sizeof(struct AES_CTX), | ||
| 78 | .cra_module = THIS_MODULE, | ||
| 79 | .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), | ||
| 80 | .cra_u = { | ||
| 81 | .cipher = { | ||
| 82 | .cia_min_keysize = AES_MIN_KEY_SIZE, | ||
| 83 | .cia_max_keysize = AES_MAX_KEY_SIZE, | ||
| 84 | .cia_setkey = aes_set_key, | ||
| 85 | .cia_encrypt = aes_encrypt, | ||
| 86 | .cia_decrypt = aes_decrypt | ||
| 87 | } | ||
| 88 | } | ||
| 89 | }; | ||
| 90 | |||
| 91 | static int __init aes_init(void) | ||
| 92 | { | ||
| 93 | return crypto_register_alg(&aes_alg); | ||
| 94 | } | ||
| 95 | |||
| 96 | static void __exit aes_fini(void) | ||
| 97 | { | ||
| 98 | crypto_unregister_alg(&aes_alg); | ||
| 99 | } | ||
| 100 | |||
| 101 | module_init(aes_init); | ||
| 102 | module_exit(aes_fini); | ||
| 103 | |||
| 104 | MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)"); | ||
| 105 | MODULE_LICENSE("GPL"); | ||
| 106 | MODULE_ALIAS("aes"); | ||
| 107 | MODULE_ALIAS("aes-asm"); | ||
| 108 | MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>"); | ||
diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S new file mode 100644 index 000000000000..7050ab133b9d --- /dev/null +++ b/arch/arm/crypto/sha1-armv4-large.S | |||
| @@ -0,0 +1,503 @@ | |||
| 1 | #define __ARM_ARCH__ __LINUX_ARM_ARCH__ | ||
| 2 | @ ==================================================================== | ||
| 3 | @ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | ||
| 4 | @ project. The module is, however, dual licensed under OpenSSL and | ||
| 5 | @ CRYPTOGAMS licenses depending on where you obtain it. For further | ||
| 6 | @ details see http://www.openssl.org/~appro/cryptogams/. | ||
| 7 | @ ==================================================================== | ||
| 8 | |||
| 9 | @ sha1_block procedure for ARMv4. | ||
| 10 | @ | ||
| 11 | @ January 2007. | ||
| 12 | |||
| 13 | @ Size/performance trade-off | ||
| 14 | @ ==================================================================== | ||
| 15 | @ impl size in bytes comp cycles[*] measured performance | ||
| 16 | @ ==================================================================== | ||
| 17 | @ thumb 304 3212 4420 | ||
| 18 | @ armv4-small 392/+29% 1958/+64% 2250/+96% | ||
| 19 | @ armv4-compact 740/+89% 1552/+26% 1840/+22% | ||
| 20 | @ armv4-large 1420/+92% 1307/+19% 1370/+34%[***] | ||
| 21 | @ full unroll ~5100/+260% ~1260/+4% ~1300/+5% | ||
| 22 | @ ==================================================================== | ||
| 23 | @ thumb = same as 'small' but in Thumb instructions[**] and | ||
| 24 | @ with recurring code in two private functions; | ||
| 25 | @ small = detached Xload/update, loops are folded; | ||
| 26 | @ compact = detached Xload/update, 5x unroll; | ||
| 27 | @ large = interleaved Xload/update, 5x unroll; | ||
| 28 | @ full unroll = interleaved Xload/update, full unroll, estimated[!]; | ||
| 29 | @ | ||
| 30 | @ [*] Manually counted instructions in "grand" loop body. Measured | ||
| 31 | @ performance is affected by prologue and epilogue overhead, | ||
| 32 | @ i-cache availability, branch penalties, etc. | ||
| 33 | @ [**] While each Thumb instruction is twice smaller, they are not as | ||
| 34 | @ diverse as ARM ones: e.g., there are only two arithmetic | ||
| 35 | @ instructions with 3 arguments, no [fixed] rotate, addressing | ||
| 36 | @ modes are limited. As result it takes more instructions to do | ||
| 37 | @ the same job in Thumb, therefore the code is never twice as | ||
| 38 | @ small and always slower. | ||
| 39 | @ [***] which is also ~35% better than compiler generated code. Dual- | ||
| 40 | @ issue Cortex A8 core was measured to process input block in | ||
| 41 | @ ~990 cycles. | ||
| 42 | |||
| 43 | @ August 2010. | ||
| 44 | @ | ||
| 45 | @ Rescheduling for dual-issue pipeline resulted in 13% improvement on | ||
| 46 | @ Cortex A8 core and in absolute terms ~870 cycles per input block | ||
| 47 | @ [or 13.6 cycles per byte]. | ||
| 48 | |||
| 49 | @ February 2011. | ||
| 50 | @ | ||
| 51 | @ Profiler-assisted and platform-specific optimization resulted in 10% | ||
| 52 | @ improvement on Cortex A8 core and 12.2 cycles per byte. | ||
| 53 | |||
| 54 | .text | ||
| 55 | |||
| 56 | .global sha1_block_data_order | ||
| 57 | .type sha1_block_data_order,%function | ||
| 58 | |||
| 59 | .align 2 | ||
| 60 | sha1_block_data_order: | ||
| 61 | stmdb sp!,{r4-r12,lr} | ||
| 62 | add r2,r1,r2,lsl#6 @ r2 to point at the end of r1 | ||
| 63 | ldmia r0,{r3,r4,r5,r6,r7} | ||
| 64 | .Lloop: | ||
| 65 | ldr r8,.LK_00_19 | ||
| 66 | mov r14,sp | ||
| 67 | sub sp,sp,#15*4 | ||
| 68 | mov r5,r5,ror#30 | ||
| 69 | mov r6,r6,ror#30 | ||
| 70 | mov r7,r7,ror#30 @ [6] | ||
| 71 | .L_00_15: | ||
| 72 | #if __ARM_ARCH__<7 | ||
| 73 | ldrb r10,[r1,#2] | ||
| 74 | ldrb r9,[r1,#3] | ||
| 75 | ldrb r11,[r1,#1] | ||
| 76 | add r7,r8,r7,ror#2 @ E+=K_00_19 | ||
| 77 | ldrb r12,[r1],#4 | ||
| 78 | orr r9,r9,r10,lsl#8 | ||
| 79 | eor r10,r5,r6 @ F_xx_xx | ||
| 80 | orr r9,r9,r11,lsl#16 | ||
| 81 | add r7,r7,r3,ror#27 @ E+=ROR(A,27) | ||
| 82 | orr r9,r9,r12,lsl#24 | ||
| 83 | #else | ||
| 84 | ldr r9,[r1],#4 @ handles unaligned | ||
| 85 | add r7,r8,r7,ror#2 @ E+=K_00_19 | ||
| 86 | eor r10,r5,r6 @ F_xx_xx | ||
| 87 | add r7,r7,r3,ror#27 @ E+=ROR(A,27) | ||
| 88 | #ifdef __ARMEL__ | ||
| 89 | rev r9,r9 @ byte swap | ||
| 90 | #endif | ||
| 91 | #endif | ||
| 92 | and r10,r4,r10,ror#2 | ||
| 93 | add r7,r7,r9 @ E+=X[i] | ||
| 94 | eor r10,r10,r6,ror#2 @ F_00_19(B,C,D) | ||
| 95 | str r9,[r14,#-4]! | ||
| 96 | add r7,r7,r10 @ E+=F_00_19(B,C,D) | ||
| 97 | #if __ARM_ARCH__<7 | ||
| 98 | ldrb r10,[r1,#2] | ||
| 99 | ldrb r9,[r1,#3] | ||
| 100 | ldrb r11,[r1,#1] | ||
| 101 | add r6,r8,r6,ror#2 @ E+=K_00_19 | ||
| 102 | ldrb r12,[r1],#4 | ||
| 103 | orr r9,r9,r10,lsl#8 | ||
| 104 | eor r10,r4,r5 @ F_xx_xx | ||
| 105 | orr r9,r9,r11,lsl#16 | ||
| 106 | add r6,r6,r7,ror#27 @ E+=ROR(A,27) | ||
| 107 | orr r9,r9,r12,lsl#24 | ||
| 108 | #else | ||
| 109 | ldr r9,[r1],#4 @ handles unaligned | ||
| 110 | add r6,r8,r6,ror#2 @ E+=K_00_19 | ||
| 111 | eor r10,r4,r5 @ F_xx_xx | ||
| 112 | add r6,r6,r7,ror#27 @ E+=ROR(A,27) | ||
| 113 | #ifdef __ARMEL__ | ||
| 114 | rev r9,r9 @ byte swap | ||
| 115 | #endif | ||
| 116 | #endif | ||
| 117 | and r10,r3,r10,ror#2 | ||
| 118 | add r6,r6,r9 @ E+=X[i] | ||
| 119 | eor r10,r10,r5,ror#2 @ F_00_19(B,C,D) | ||
| 120 | str r9,[r14,#-4]! | ||
| 121 | add r6,r6,r10 @ E+=F_00_19(B,C,D) | ||
| 122 | #if __ARM_ARCH__<7 | ||
| 123 | ldrb r10,[r1,#2] | ||
| 124 | ldrb r9,[r1,#3] | ||
| 125 | ldrb r11,[r1,#1] | ||
| 126 | add r5,r8,r5,ror#2 @ E+=K_00_19 | ||
| 127 | ldrb r12,[r1],#4 | ||
| 128 | orr r9,r9,r10,lsl#8 | ||
| 129 | eor r10,r3,r4 @ F_xx_xx | ||
| 130 | orr r9,r9,r11,lsl#16 | ||
| 131 | add r5,r5,r6,ror#27 @ E+=ROR(A,27) | ||
| 132 | orr r9,r9,r12,lsl#24 | ||
| 133 | #else | ||
| 134 | ldr r9,[r1],#4 @ handles unaligned | ||
| 135 | add r5,r8,r5,ror#2 @ E+=K_00_19 | ||
| 136 | eor r10,r3,r4 @ F_xx_xx | ||
| 137 | add r5,r5,r6,ror#27 @ E+=ROR(A,27) | ||
| 138 | #ifdef __ARMEL__ | ||
| 139 | rev r9,r9 @ byte swap | ||
| 140 | #endif | ||
| 141 | #endif | ||
| 142 | and r10,r7,r10,ror#2 | ||
| 143 | add r5,r5,r9 @ E+=X[i] | ||
| 144 | eor r10,r10,r4,ror#2 @ F_00_19(B,C,D) | ||
| 145 | str r9,[r14,#-4]! | ||
| 146 | add r5,r5,r10 @ E+=F_00_19(B,C,D) | ||
| 147 | #if __ARM_ARCH__<7 | ||
| 148 | ldrb r10,[r1,#2] | ||
| 149 | ldrb r9,[r1,#3] | ||
| 150 | ldrb r11,[r1,#1] | ||
| 151 | add r4,r8,r4,ror#2 @ E+=K_00_19 | ||
| 152 | ldrb r12,[r1],#4 | ||
| 153 | orr r9,r9,r10,lsl#8 | ||
| 154 | eor r10,r7,r3 @ F_xx_xx | ||
| 155 | orr r9,r9,r11,lsl#16 | ||
| 156 | add r4,r4,r5,ror#27 @ E+=ROR(A,27) | ||
| 157 | orr r9,r9,r12,lsl#24 | ||
| 158 | #else | ||
| 159 | ldr r9,[r1],#4 @ handles unaligned | ||
| 160 | add r4,r8,r4,ror#2 @ E+=K_00_19 | ||
| 161 | eor r10,r7,r3 @ F_xx_xx | ||
| 162 | add r4,r4,r5,ror#27 @ E+=ROR(A,27) | ||
| 163 | #ifdef __ARMEL__ | ||
| 164 | rev r9,r9 @ byte swap | ||
| 165 | #endif | ||
| 166 | #endif | ||
| 167 | and r10,r6,r10,ror#2 | ||
| 168 | add r4,r4,r9 @ E+=X[i] | ||
| 169 | eor r10,r10,r3,ror#2 @ F_00_19(B,C,D) | ||
| 170 | str r9,[r14,#-4]! | ||
| 171 | add r4,r4,r10 @ E+=F_00_19(B,C,D) | ||
| 172 | #if __ARM_ARCH__<7 | ||
| 173 | ldrb r10,[r1,#2] | ||
| 174 | ldrb r9,[r1,#3] | ||
| 175 | ldrb r11,[r1,#1] | ||
| 176 | add r3,r8,r3,ror#2 @ E+=K_00_19 | ||
| 177 | ldrb r12,[r1],#4 | ||
| 178 | orr r9,r9,r10,lsl#8 | ||
| 179 | eor r10,r6,r7 @ F_xx_xx | ||
| 180 | orr r9,r9,r11,lsl#16 | ||
| 181 | add r3,r3,r4,ror#27 @ E+=ROR(A,27) | ||
| 182 | orr r9,r9,r12,lsl#24 | ||
| 183 | #else | ||
| 184 | ldr r9,[r1],#4 @ handles unaligned | ||
| 185 | add r3,r8,r3,ror#2 @ E+=K_00_19 | ||
| 186 | eor r10,r6,r7 @ F_xx_xx | ||
| 187 | add r3,r3,r4,ror#27 @ E+=ROR(A,27) | ||
| 188 | #ifdef __ARMEL__ | ||
| 189 | rev r9,r9 @ byte swap | ||
| 190 | #endif | ||
| 191 | #endif | ||
| 192 | and r10,r5,r10,ror#2 | ||
| 193 | add r3,r3,r9 @ E+=X[i] | ||
| 194 | eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) | ||
| 195 | str r9,[r14,#-4]! | ||
| 196 | add r3,r3,r10 @ E+=F_00_19(B,C,D) | ||
| 197 | teq r14,sp | ||
| 198 | bne .L_00_15 @ [((11+4)*5+2)*3] | ||
| 199 | #if __ARM_ARCH__<7 | ||
| 200 | ldrb r10,[r1,#2] | ||
| 201 | ldrb r9,[r1,#3] | ||
| 202 | ldrb r11,[r1,#1] | ||
| 203 | add r7,r8,r7,ror#2 @ E+=K_00_19 | ||
| 204 | ldrb r12,[r1],#4 | ||
| 205 | orr r9,r9,r10,lsl#8 | ||
| 206 | eor r10,r5,r6 @ F_xx_xx | ||
| 207 | orr r9,r9,r11,lsl#16 | ||
| 208 | add r7,r7,r3,ror#27 @ E+=ROR(A,27) | ||
| 209 | orr r9,r9,r12,lsl#24 | ||
| 210 | #else | ||
| 211 | ldr r9,[r1],#4 @ handles unaligned | ||
| 212 | add r7,r8,r7,ror#2 @ E+=K_00_19 | ||
| 213 | eor r10,r5,r6 @ F_xx_xx | ||
| 214 | add r7,r7,r3,ror#27 @ E+=ROR(A,27) | ||
| 215 | #ifdef __ARMEL__ | ||
| 216 | rev r9,r9 @ byte swap | ||
| 217 | #endif | ||
| 218 | #endif | ||
| 219 | and r10,r4,r10,ror#2 | ||
| 220 | add r7,r7,r9 @ E+=X[i] | ||
| 221 | eor r10,r10,r6,ror#2 @ F_00_19(B,C,D) | ||
| 222 | str r9,[r14,#-4]! | ||
| 223 | add r7,r7,r10 @ E+=F_00_19(B,C,D) | ||
| 224 | ldr r9,[r14,#15*4] | ||
| 225 | ldr r10,[r14,#13*4] | ||
| 226 | ldr r11,[r14,#7*4] | ||
| 227 | add r6,r8,r6,ror#2 @ E+=K_xx_xx | ||
| 228 | ldr r12,[r14,#2*4] | ||
| 229 | eor r9,r9,r10 | ||
| 230 | eor r11,r11,r12 @ 1 cycle stall | ||
| 231 | eor r10,r4,r5 @ F_xx_xx | ||
| 232 | mov r9,r9,ror#31 | ||
| 233 | add r6,r6,r7,ror#27 @ E+=ROR(A,27) | ||
| 234 | eor r9,r9,r11,ror#31 | ||
| 235 | str r9,[r14,#-4]! | ||
| 236 | and r10,r3,r10,ror#2 @ F_xx_xx | ||
| 237 | @ F_xx_xx | ||
| 238 | add r6,r6,r9 @ E+=X[i] | ||
| 239 | eor r10,r10,r5,ror#2 @ F_00_19(B,C,D) | ||
| 240 | add r6,r6,r10 @ E+=F_00_19(B,C,D) | ||
| 241 | ldr r9,[r14,#15*4] | ||
| 242 | ldr r10,[r14,#13*4] | ||
| 243 | ldr r11,[r14,#7*4] | ||
| 244 | add r5,r8,r5,ror#2 @ E+=K_xx_xx | ||
| 245 | ldr r12,[r14,#2*4] | ||
| 246 | eor r9,r9,r10 | ||
| 247 | eor r11,r11,r12 @ 1 cycle stall | ||
| 248 | eor r10,r3,r4 @ F_xx_xx | ||
| 249 | mov r9,r9,ror#31 | ||
| 250 | add r5,r5,r6,ror#27 @ E+=ROR(A,27) | ||
| 251 | eor r9,r9,r11,ror#31 | ||
| 252 | str r9,[r14,#-4]! | ||
| 253 | and r10,r7,r10,ror#2 @ F_xx_xx | ||
| 254 | @ F_xx_xx | ||
| 255 | add r5,r5,r9 @ E+=X[i] | ||
| 256 | eor r10,r10,r4,ror#2 @ F_00_19(B,C,D) | ||
| 257 | add r5,r5,r10 @ E+=F_00_19(B,C,D) | ||
| 258 | ldr r9,[r14,#15*4] | ||
| 259 | ldr r10,[r14,#13*4] | ||
| 260 | ldr r11,[r14,#7*4] | ||
| 261 | add r4,r8,r4,ror#2 @ E+=K_xx_xx | ||
| 262 | ldr r12,[r14,#2*4] | ||
| 263 | eor r9,r9,r10 | ||
| 264 | eor r11,r11,r12 @ 1 cycle stall | ||
| 265 | eor r10,r7,r3 @ F_xx_xx | ||
| 266 | mov r9,r9,ror#31 | ||
| 267 | add r4,r4,r5,ror#27 @ E+=ROR(A,27) | ||
| 268 | eor r9,r9,r11,ror#31 | ||
| 269 | str r9,[r14,#-4]! | ||
| 270 | and r10,r6,r10,ror#2 @ F_xx_xx | ||
| 271 | @ F_xx_xx | ||
| 272 | add r4,r4,r9 @ E+=X[i] | ||
| 273 | eor r10,r10,r3,ror#2 @ F_00_19(B,C,D) | ||
| 274 | add r4,r4,r10 @ E+=F_00_19(B,C,D) | ||
| 275 | ldr r9,[r14,#15*4] | ||
| 276 | ldr r10,[r14,#13*4] | ||
| 277 | ldr r11,[r14,#7*4] | ||
| 278 | add r3,r8,r3,ror#2 @ E+=K_xx_xx | ||
| 279 | ldr r12,[r14,#2*4] | ||
| 280 | eor r9,r9,r10 | ||
| 281 | eor r11,r11,r12 @ 1 cycle stall | ||
| 282 | eor r10,r6,r7 @ F_xx_xx | ||
| 283 | mov r9,r9,ror#31 | ||
| 284 | add r3,r3,r4,ror#27 @ E+=ROR(A,27) | ||
| 285 | eor r9,r9,r11,ror#31 | ||
| 286 | str r9,[r14,#-4]! | ||
| 287 | and r10,r5,r10,ror#2 @ F_xx_xx | ||
| 288 | @ F_xx_xx | ||
| 289 | add r3,r3,r9 @ E+=X[i] | ||
| 290 | eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) | ||
| 291 | add r3,r3,r10 @ E+=F_00_19(B,C,D) | ||
| 292 | |||
| 293 | ldr r8,.LK_20_39 @ [+15+16*4] | ||
| 294 | sub sp,sp,#25*4 | ||
| 295 | cmn sp,#0 @ [+3], clear carry to denote 20_39 | ||
| 296 | .L_20_39_or_60_79: | ||
| 297 | ldr r9,[r14,#15*4] | ||
| 298 | ldr r10,[r14,#13*4] | ||
| 299 | ldr r11,[r14,#7*4] | ||
| 300 | add r7,r8,r7,ror#2 @ E+=K_xx_xx | ||
| 301 | ldr r12,[r14,#2*4] | ||
| 302 | eor r9,r9,r10 | ||
| 303 | eor r11,r11,r12 @ 1 cycle stall | ||
| 304 | eor r10,r5,r6 @ F_xx_xx | ||
| 305 | mov r9,r9,ror#31 | ||
| 306 | add r7,r7,r3,ror#27 @ E+=ROR(A,27) | ||
| 307 | eor r9,r9,r11,ror#31 | ||
| 308 | str r9,[r14,#-4]! | ||
| 309 | eor r10,r4,r10,ror#2 @ F_xx_xx | ||
| 310 | @ F_xx_xx | ||
| 311 | add r7,r7,r9 @ E+=X[i] | ||
| 312 | add r7,r7,r10 @ E+=F_20_39(B,C,D) | ||
| 313 | ldr r9,[r14,#15*4] | ||
| 314 | ldr r10,[r14,#13*4] | ||
| 315 | ldr r11,[r14,#7*4] | ||
| 316 | add r6,r8,r6,ror#2 @ E+=K_xx_xx | ||
| 317 | ldr r12,[r14,#2*4] | ||
| 318 | eor r9,r9,r10 | ||
| 319 | eor r11,r11,r12 @ 1 cycle stall | ||
| 320 | eor r10,r4,r5 @ F_xx_xx | ||
| 321 | mov r9,r9,ror#31 | ||
| 322 | add r6,r6,r7,ror#27 @ E+=ROR(A,27) | ||
| 323 | eor r9,r9,r11,ror#31 | ||
| 324 | str r9,[r14,#-4]! | ||
| 325 | eor r10,r3,r10,ror#2 @ F_xx_xx | ||
| 326 | @ F_xx_xx | ||
| 327 | add r6,r6,r9 @ E+=X[i] | ||
| 328 | add r6,r6,r10 @ E+=F_20_39(B,C,D) | ||
| 329 | ldr r9,[r14,#15*4] | ||
| 330 | ldr r10,[r14,#13*4] | ||
| 331 | ldr r11,[r14,#7*4] | ||
| 332 | add r5,r8,r5,ror#2 @ E+=K_xx_xx | ||
| 333 | ldr r12,[r14,#2*4] | ||
| 334 | eor r9,r9,r10 | ||
| 335 | eor r11,r11,r12 @ 1 cycle stall | ||
| 336 | eor r10,r3,r4 @ F_xx_xx | ||
| 337 | mov r9,r9,ror#31 | ||
| 338 | add r5,r5,r6,ror#27 @ E+=ROR(A,27) | ||
| 339 | eor r9,r9,r11,ror#31 | ||
| 340 | str r9,[r14,#-4]! | ||
| 341 | eor r10,r7,r10,ror#2 @ F_xx_xx | ||
| 342 | @ F_xx_xx | ||
| 343 | add r5,r5,r9 @ E+=X[i] | ||
| 344 | add r5,r5,r10 @ E+=F_20_39(B,C,D) | ||
| 345 | ldr r9,[r14,#15*4] | ||
| 346 | ldr r10,[r14,#13*4] | ||
| 347 | ldr r11,[r14,#7*4] | ||
| 348 | add r4,r8,r4,ror#2 @ E+=K_xx_xx | ||
| 349 | ldr r12,[r14,#2*4] | ||
| 350 | eor r9,r9,r10 | ||
| 351 | eor r11,r11,r12 @ 1 cycle stall | ||
| 352 | eor r10,r7,r3 @ F_xx_xx | ||
| 353 | mov r9,r9,ror#31 | ||
| 354 | add r4,r4,r5,ror#27 @ E+=ROR(A,27) | ||
| 355 | eor r9,r9,r11,ror#31 | ||
| 356 | str r9,[r14,#-4]! | ||
| 357 | eor r10,r6,r10,ror#2 @ F_xx_xx | ||
| 358 | @ F_xx_xx | ||
| 359 | add r4,r4,r9 @ E+=X[i] | ||
| 360 | add r4,r4,r10 @ E+=F_20_39(B,C,D) | ||
| 361 | ldr r9,[r14,#15*4] | ||
| 362 | ldr r10,[r14,#13*4] | ||
| 363 | ldr r11,[r14,#7*4] | ||
| 364 | add r3,r8,r3,ror#2 @ E+=K_xx_xx | ||
| 365 | ldr r12,[r14,#2*4] | ||
| 366 | eor r9,r9,r10 | ||
| 367 | eor r11,r11,r12 @ 1 cycle stall | ||
| 368 | eor r10,r6,r7 @ F_xx_xx | ||
| 369 | mov r9,r9,ror#31 | ||
| 370 | add r3,r3,r4,ror#27 @ E+=ROR(A,27) | ||
| 371 | eor r9,r9,r11,ror#31 | ||
| 372 | str r9,[r14,#-4]! | ||
| 373 | eor r10,r5,r10,ror#2 @ F_xx_xx | ||
| 374 | @ F_xx_xx | ||
| 375 | add r3,r3,r9 @ E+=X[i] | ||
| 376 | add r3,r3,r10 @ E+=F_20_39(B,C,D) | ||
| 377 | teq r14,sp @ preserve carry | ||
| 378 | bne .L_20_39_or_60_79 @ [+((12+3)*5+2)*4] | ||
| 379 | bcs .L_done @ [+((12+3)*5+2)*4], spare 300 bytes | ||
| 380 | |||
| 381 | ldr r8,.LK_40_59 | ||
| 382 | sub sp,sp,#20*4 @ [+2] | ||
| 383 | .L_40_59: | ||
| 384 | ldr r9,[r14,#15*4] | ||
| 385 | ldr r10,[r14,#13*4] | ||
| 386 | ldr r11,[r14,#7*4] | ||
| 387 | add r7,r8,r7,ror#2 @ E+=K_xx_xx | ||
| 388 | ldr r12,[r14,#2*4] | ||
| 389 | eor r9,r9,r10 | ||
| 390 | eor r11,r11,r12 @ 1 cycle stall | ||
| 391 | eor r10,r5,r6 @ F_xx_xx | ||
| 392 | mov r9,r9,ror#31 | ||
| 393 | add r7,r7,r3,ror#27 @ E+=ROR(A,27) | ||
| 394 | eor r9,r9,r11,ror#31 | ||
| 395 | str r9,[r14,#-4]! | ||
| 396 | and r10,r4,r10,ror#2 @ F_xx_xx | ||
| 397 | and r11,r5,r6 @ F_xx_xx | ||
| 398 | add r7,r7,r9 @ E+=X[i] | ||
| 399 | add r7,r7,r10 @ E+=F_40_59(B,C,D) | ||
| 400 | add r7,r7,r11,ror#2 | ||
| 401 | ldr r9,[r14,#15*4] | ||
| 402 | ldr r10,[r14,#13*4] | ||
| 403 | ldr r11,[r14,#7*4] | ||
| 404 | add r6,r8,r6,ror#2 @ E+=K_xx_xx | ||
| 405 | ldr r12,[r14,#2*4] | ||
| 406 | eor r9,r9,r10 | ||
| 407 | eor r11,r11,r12 @ 1 cycle stall | ||
| 408 | eor r10,r4,r5 @ F_xx_xx | ||
| 409 | mov r9,r9,ror#31 | ||
| 410 | add r6,r6,r7,ror#27 @ E+=ROR(A,27) | ||
| 411 | eor r9,r9,r11,ror#31 | ||
| 412 | str r9,[r14,#-4]! | ||
| 413 | and r10,r3,r10,ror#2 @ F_xx_xx | ||
| 414 | and r11,r4,r5 @ F_xx_xx | ||
| 415 | add r6,r6,r9 @ E+=X[i] | ||
| 416 | add r6,r6,r10 @ E+=F_40_59(B,C,D) | ||
| 417 | add r6,r6,r11,ror#2 | ||
| 418 | ldr r9,[r14,#15*4] | ||
| 419 | ldr r10,[r14,#13*4] | ||
| 420 | ldr r11,[r14,#7*4] | ||
| 421 | add r5,r8,r5,ror#2 @ E+=K_xx_xx | ||
| 422 | ldr r12,[r14,#2*4] | ||
| 423 | eor r9,r9,r10 | ||
| 424 | eor r11,r11,r12 @ 1 cycle stall | ||
| 425 | eor r10,r3,r4 @ F_xx_xx | ||
| 426 | mov r9,r9,ror#31 | ||
| 427 | add r5,r5,r6,ror#27 @ E+=ROR(A,27) | ||
| 428 | eor r9,r9,r11,ror#31 | ||
| 429 | str r9,[r14,#-4]! | ||
| 430 | and r10,r7,r10,ror#2 @ F_xx_xx | ||
| 431 | and r11,r3,r4 @ F_xx_xx | ||
| 432 | add r5,r5,r9 @ E+=X[i] | ||
| 433 | add r5,r5,r10 @ E+=F_40_59(B,C,D) | ||
| 434 | add r5,r5,r11,ror#2 | ||
| 435 | ldr r9,[r14,#15*4] | ||
| 436 | ldr r10,[r14,#13*4] | ||
| 437 | ldr r11,[r14,#7*4] | ||
| 438 | add r4,r8,r4,ror#2 @ E+=K_xx_xx | ||
| 439 | ldr r12,[r14,#2*4] | ||
| 440 | eor r9,r9,r10 | ||
| 441 | eor r11,r11,r12 @ 1 cycle stall | ||
| 442 | eor r10,r7,r3 @ F_xx_xx | ||
| 443 | mov r9,r9,ror#31 | ||
| 444 | add r4,r4,r5,ror#27 @ E+=ROR(A,27) | ||
| 445 | eor r9,r9,r11,ror#31 | ||
| 446 | str r9,[r14,#-4]! | ||
| 447 | and r10,r6,r10,ror#2 @ F_xx_xx | ||
| 448 | and r11,r7,r3 @ F_xx_xx | ||
| 449 | add r4,r4,r9 @ E+=X[i] | ||
| 450 | add r4,r4,r10 @ E+=F_40_59(B,C,D) | ||
| 451 | add r4,r4,r11,ror#2 | ||
| 452 | ldr r9,[r14,#15*4] | ||
| 453 | ldr r10,[r14,#13*4] | ||
| 454 | ldr r11,[r14,#7*4] | ||
| 455 | add r3,r8,r3,ror#2 @ E+=K_xx_xx | ||
| 456 | ldr r12,[r14,#2*4] | ||
| 457 | eor r9,r9,r10 | ||
| 458 | eor r11,r11,r12 @ 1 cycle stall | ||
| 459 | eor r10,r6,r7 @ F_xx_xx | ||
| 460 | mov r9,r9,ror#31 | ||
| 461 | add r3,r3,r4,ror#27 @ E+=ROR(A,27) | ||
| 462 | eor r9,r9,r11,ror#31 | ||
| 463 | str r9,[r14,#-4]! | ||
| 464 | and r10,r5,r10,ror#2 @ F_xx_xx | ||
| 465 | and r11,r6,r7 @ F_xx_xx | ||
| 466 | add r3,r3,r9 @ E+=X[i] | ||
| 467 | add r3,r3,r10 @ E+=F_40_59(B,C,D) | ||
| 468 | add r3,r3,r11,ror#2 | ||
| 469 | teq r14,sp | ||
| 470 | bne .L_40_59 @ [+((12+5)*5+2)*4] | ||
| 471 | |||
| 472 | ldr r8,.LK_60_79 | ||
| 473 | sub sp,sp,#20*4 | ||
| 474 | cmp sp,#0 @ set carry to denote 60_79 | ||
| 475 | b .L_20_39_or_60_79 @ [+4], spare 300 bytes | ||
| 476 | .L_done: | ||
| 477 | add sp,sp,#80*4 @ "deallocate" stack frame | ||
| 478 | ldmia r0,{r8,r9,r10,r11,r12} | ||
| 479 | add r3,r8,r3 | ||
| 480 | add r4,r9,r4 | ||
| 481 | add r5,r10,r5,ror#2 | ||
| 482 | add r6,r11,r6,ror#2 | ||
| 483 | add r7,r12,r7,ror#2 | ||
| 484 | stmia r0,{r3,r4,r5,r6,r7} | ||
| 485 | teq r1,r2 | ||
| 486 | bne .Lloop @ [+18], total 1307 | ||
| 487 | |||
| 488 | #if __ARM_ARCH__>=5 | ||
| 489 | ldmia sp!,{r4-r12,pc} | ||
| 490 | #else | ||
| 491 | ldmia sp!,{r4-r12,lr} | ||
| 492 | tst lr,#1 | ||
| 493 | moveq pc,lr @ be binary compatible with V4, yet | ||
| 494 | .word 0xe12fff1e @ interoperable with Thumb ISA:-) | ||
| 495 | #endif | ||
| 496 | .align 2 | ||
| 497 | .LK_00_19: .word 0x5a827999 | ||
| 498 | .LK_20_39: .word 0x6ed9eba1 | ||
| 499 | .LK_40_59: .word 0x8f1bbcdc | ||
| 500 | .LK_60_79: .word 0xca62c1d6 | ||
| 501 | .size sha1_block_data_order,.-sha1_block_data_order | ||
| 502 | .asciz "SHA1 block transform for ARMv4, CRYPTOGAMS by <appro@openssl.org>" | ||
| 503 | .align 2 | ||
diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c new file mode 100644 index 000000000000..76cd976230bc --- /dev/null +++ b/arch/arm/crypto/sha1_glue.c | |||
| @@ -0,0 +1,179 @@ | |||
| 1 | /* | ||
| 2 | * Cryptographic API. | ||
| 3 | * Glue code for the SHA1 Secure Hash Algorithm assembler implementation | ||
| 4 | * | ||
| 5 | * This file is based on sha1_generic.c and sha1_ssse3_glue.c | ||
| 6 | * | ||
| 7 | * Copyright (c) Alan Smithee. | ||
| 8 | * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> | ||
| 9 | * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> | ||
| 10 | * Copyright (c) Mathias Krause <minipli@googlemail.com> | ||
| 11 | * | ||
| 12 | * This program is free software; you can redistribute it and/or modify it | ||
| 13 | * under the terms of the GNU General Public License as published by the Free | ||
| 14 | * Software Foundation; either version 2 of the License, or (at your option) | ||
| 15 | * any later version. | ||
| 16 | * | ||
| 17 | */ | ||
| 18 | |||
| 19 | #include <crypto/internal/hash.h> | ||
| 20 | #include <linux/init.h> | ||
| 21 | #include <linux/module.h> | ||
| 22 | #include <linux/cryptohash.h> | ||
| 23 | #include <linux/types.h> | ||
| 24 | #include <crypto/sha.h> | ||
| 25 | #include <asm/byteorder.h> | ||
| 26 | |||
| 27 | struct SHA1_CTX { | ||
| 28 | uint32_t h0,h1,h2,h3,h4; | ||
| 29 | u64 count; | ||
| 30 | u8 data[SHA1_BLOCK_SIZE]; | ||
| 31 | }; | ||
| 32 | |||
| 33 | asmlinkage void sha1_block_data_order(struct SHA1_CTX *digest, | ||
| 34 | const unsigned char *data, unsigned int rounds); | ||
| 35 | |||
| 36 | |||
| 37 | static int sha1_init(struct shash_desc *desc) | ||
| 38 | { | ||
| 39 | struct SHA1_CTX *sctx = shash_desc_ctx(desc); | ||
| 40 | memset(sctx, 0, sizeof(*sctx)); | ||
| 41 | sctx->h0 = SHA1_H0; | ||
| 42 | sctx->h1 = SHA1_H1; | ||
| 43 | sctx->h2 = SHA1_H2; | ||
| 44 | sctx->h3 = SHA1_H3; | ||
| 45 | sctx->h4 = SHA1_H4; | ||
| 46 | return 0; | ||
| 47 | } | ||
| 48 | |||
| 49 | |||
| 50 | static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, | ||
| 51 | unsigned int len, unsigned int partial) | ||
| 52 | { | ||
| 53 | unsigned int done = 0; | ||
| 54 | |||
| 55 | sctx->count += len; | ||
| 56 | |||
| 57 | if (partial) { | ||
| 58 | done = SHA1_BLOCK_SIZE - partial; | ||
| 59 | memcpy(sctx->data + partial, data, done); | ||
| 60 | sha1_block_data_order(sctx, sctx->data, 1); | ||
| 61 | } | ||
| 62 | |||
| 63 | if (len - done >= SHA1_BLOCK_SIZE) { | ||
| 64 | const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; | ||
| 65 | sha1_block_data_order(sctx, data + done, rounds); | ||
| 66 | done += rounds * SHA1_BLOCK_SIZE; | ||
| 67 | } | ||
| 68 | |||
| 69 | memcpy(sctx->data, data + done, len - done); | ||
| 70 | return 0; | ||
| 71 | } | ||
| 72 | |||
| 73 | |||
| 74 | static int sha1_update(struct shash_desc *desc, const u8 *data, | ||
| 75 | unsigned int len) | ||
| 76 | { | ||
| 77 | struct SHA1_CTX *sctx = shash_desc_ctx(desc); | ||
| 78 | unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; | ||
| 79 | int res; | ||
| 80 | |||
| 81 | /* Handle the fast case right here */ | ||
| 82 | if (partial + len < SHA1_BLOCK_SIZE) { | ||
| 83 | sctx->count += len; | ||
| 84 | memcpy(sctx->data + partial, data, len); | ||
| 85 | return 0; | ||
| 86 | } | ||
| 87 | res = __sha1_update(sctx, data, len, partial); | ||
| 88 | return res; | ||
| 89 | } | ||
| 90 | |||
| 91 | |||
| 92 | /* Add padding and return the message digest. */ | ||
| 93 | static int sha1_final(struct shash_desc *desc, u8 *out) | ||
| 94 | { | ||
| 95 | struct SHA1_CTX *sctx = shash_desc_ctx(desc); | ||
| 96 | unsigned int i, index, padlen; | ||
| 97 | __be32 *dst = (__be32 *)out; | ||
| 98 | __be64 bits; | ||
| 99 | static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; | ||
| 100 | |||
| 101 | bits = cpu_to_be64(sctx->count << 3); | ||
| 102 | |||
| 103 | /* Pad out to 56 mod 64 and append length */ | ||
| 104 | index = sctx->count % SHA1_BLOCK_SIZE; | ||
| 105 | padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); | ||
| 106 | /* We need to fill a whole block for __sha1_update() */ | ||
| 107 | if (padlen <= 56) { | ||
| 108 | sctx->count += padlen; | ||
| 109 | memcpy(sctx->data + index, padding, padlen); | ||
| 110 | } else { | ||
| 111 | __sha1_update(sctx, padding, padlen, index); | ||
| 112 | } | ||
| 113 | __sha1_update(sctx, (const u8 *)&bits, sizeof(bits), 56); | ||
| 114 | |||
| 115 | /* Store state in digest */ | ||
| 116 | for (i = 0; i < 5; i++) | ||
| 117 | dst[i] = cpu_to_be32(((u32 *)sctx)[i]); | ||
| 118 | |||
| 119 | /* Wipe context */ | ||
| 120 | memset(sctx, 0, sizeof(*sctx)); | ||
| 121 | return 0; | ||
| 122 | } | ||
| 123 | |||
| 124 | |||
| 125 | static int sha1_export(struct shash_desc *desc, void *out) | ||
| 126 | { | ||
| 127 | struct SHA1_CTX *sctx = shash_desc_ctx(desc); | ||
| 128 | memcpy(out, sctx, sizeof(*sctx)); | ||
| 129 | return 0; | ||
| 130 | } | ||
| 131 | |||
| 132 | |||
| 133 | static int sha1_import(struct shash_desc *desc, const void *in) | ||
| 134 | { | ||
| 135 | struct SHA1_CTX *sctx = shash_desc_ctx(desc); | ||
| 136 | memcpy(sctx, in, sizeof(*sctx)); | ||
| 137 | return 0; | ||
| 138 | } | ||
| 139 | |||
| 140 | |||
| 141 | static struct shash_alg alg = { | ||
| 142 | .digestsize = SHA1_DIGEST_SIZE, | ||
| 143 | .init = sha1_init, | ||
| 144 | .update = sha1_update, | ||
| 145 | .final = sha1_final, | ||
| 146 | .export = sha1_export, | ||
| 147 | .import = sha1_import, | ||
| 148 | .descsize = sizeof(struct SHA1_CTX), | ||
| 149 | .statesize = sizeof(struct SHA1_CTX), | ||
| 150 | .base = { | ||
| 151 | .cra_name = "sha1", | ||
| 152 | .cra_driver_name= "sha1-asm", | ||
| 153 | .cra_priority = 150, | ||
| 154 | .cra_flags = CRYPTO_ALG_TYPE_SHASH, | ||
| 155 | .cra_blocksize = SHA1_BLOCK_SIZE, | ||
| 156 | .cra_module = THIS_MODULE, | ||
| 157 | } | ||
| 158 | }; | ||
| 159 | |||
| 160 | |||
| 161 | static int __init sha1_mod_init(void) | ||
| 162 | { | ||
| 163 | return crypto_register_shash(&alg); | ||
| 164 | } | ||
| 165 | |||
| 166 | |||
| 167 | static void __exit sha1_mod_fini(void) | ||
| 168 | { | ||
| 169 | crypto_unregister_shash(&alg); | ||
| 170 | } | ||
| 171 | |||
| 172 | |||
| 173 | module_init(sha1_mod_init); | ||
| 174 | module_exit(sha1_mod_fini); | ||
| 175 | |||
| 176 | MODULE_LICENSE("GPL"); | ||
| 177 | MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (ARM)"); | ||
| 178 | MODULE_ALIAS("sha1"); | ||
| 179 | MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>"); | ||
diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index 05112380dc53..8dcd9c702d90 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h | |||
| @@ -44,10 +44,9 @@ | |||
| 44 | #define rmb() dsb() | 44 | #define rmb() dsb() |
| 45 | #define wmb() mb() | 45 | #define wmb() mb() |
| 46 | #else | 46 | #else |
| 47 | #include <asm/memory.h> | 47 | #define mb() barrier() |
| 48 | #define mb() do { if (arch_is_coherent()) dmb(); else barrier(); } while (0) | 48 | #define rmb() barrier() |
| 49 | #define rmb() do { if (arch_is_coherent()) dmb(); else barrier(); } while (0) | 49 | #define wmb() barrier() |
| 50 | #define wmb() do { if (arch_is_coherent()) dmb(); else barrier(); } while (0) | ||
| 51 | #endif | 50 | #endif |
| 52 | 51 | ||
| 53 | #ifndef CONFIG_SMP | 52 | #ifndef CONFIG_SMP |
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 5c44dcb0987b..23004847bb05 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h | |||
| @@ -13,6 +13,7 @@ | |||
| 13 | 13 | ||
| 14 | #define DMA_ERROR_CODE (~0) | 14 | #define DMA_ERROR_CODE (~0) |
| 15 | extern struct dma_map_ops arm_dma_ops; | 15 | extern struct dma_map_ops arm_dma_ops; |
| 16 | extern struct dma_map_ops arm_coherent_dma_ops; | ||
| 16 | 17 | ||
| 17 | static inline struct dma_map_ops *get_dma_ops(struct device *dev) | 18 | static inline struct dma_map_ops *get_dma_ops(struct device *dev) |
| 18 | { | 19 | { |
diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 5f6ddcc56452..73cf03aa981e 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h | |||
| @@ -275,14 +275,6 @@ static inline __deprecated void *bus_to_virt(unsigned long x) | |||
| 275 | #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) | 275 | #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) |
| 276 | #define virt_addr_valid(kaddr) ((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) | 276 | #define virt_addr_valid(kaddr) ((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) |
| 277 | 277 | ||
| 278 | /* | ||
| 279 | * Optional coherency support. Currently used only by selected | ||
| 280 | * Intel XSC3-based systems. | ||
| 281 | */ | ||
| 282 | #ifndef arch_is_coherent | ||
| 283 | #define arch_is_coherent() 0 | ||
| 284 | #endif | ||
| 285 | |||
| 286 | #endif | 278 | #endif |
| 287 | 279 | ||
| 288 | #include <asm-generic/memory_model.h> | 280 | #include <asm-generic/memory_model.h> |
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index ecf901902e44..812a4944e783 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h | |||
| @@ -19,7 +19,7 @@ | |||
| 19 | 19 | ||
| 20 | #ifndef CONFIG_MMU | 20 | #ifndef CONFIG_MMU |
| 21 | 21 | ||
| 22 | #include "page-nommu.h" | 22 | #include <asm/page-nommu.h> |
| 23 | 23 | ||
| 24 | #else | 24 | #else |
| 25 | 25 | ||
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 41dc31f834c3..08c12312a1f9 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h | |||
| @@ -16,7 +16,7 @@ | |||
| 16 | #ifndef CONFIG_MMU | 16 | #ifndef CONFIG_MMU |
| 17 | 17 | ||
| 18 | #include <asm-generic/4level-fixup.h> | 18 | #include <asm-generic/4level-fixup.h> |
| 19 | #include "pgtable-nommu.h" | 19 | #include <asm/pgtable-nommu.h> |
| 20 | 20 | ||
| 21 | #else | 21 | #else |
| 22 | 22 | ||
diff --git a/arch/arm/include/asm/vfpmacros.h b/arch/arm/include/asm/vfpmacros.h index 3d5fc41ae8d3..a7aadbd9a6dd 100644 --- a/arch/arm/include/asm/vfpmacros.h +++ b/arch/arm/include/asm/vfpmacros.h | |||
| @@ -5,7 +5,7 @@ | |||
| 5 | */ | 5 | */ |
| 6 | #include <asm/hwcap.h> | 6 | #include <asm/hwcap.h> |
| 7 | 7 | ||
| 8 | #include "vfp.h" | 8 | #include <asm/vfp.h> |
| 9 | 9 | ||
| 10 | @ Macros to allow building with old toolkits (with no VFP support) | 10 | @ Macros to allow building with old toolkits (with no VFP support) |
| 11 | .macro VFPFMRX, rd, sysreg, cond | 11 | .macro VFPFMRX, rd, sysreg, cond |
diff --git a/arch/arm/include/uapi/asm/Kbuild b/arch/arm/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/arm/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index aa4ffe6e5ecf..dea7a925c7e2 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c | |||
| @@ -24,6 +24,7 @@ | |||
| 24 | #include <linux/percpu.h> | 24 | #include <linux/percpu.h> |
| 25 | #include <linux/clockchips.h> | 25 | #include <linux/clockchips.h> |
| 26 | #include <linux/completion.h> | 26 | #include <linux/completion.h> |
| 27 | #include <linux/cpufreq.h> | ||
| 27 | 28 | ||
| 28 | #include <linux/atomic.h> | 29 | #include <linux/atomic.h> |
| 29 | #include <asm/smp.h> | 30 | #include <asm/smp.h> |
| @@ -650,3 +651,56 @@ int setup_profiling_timer(unsigned int multiplier) | |||
| 650 | { | 651 | { |
| 651 | return -EINVAL; | 652 | return -EINVAL; |
| 652 | } | 653 | } |
| 654 | |||
| 655 | #ifdef CONFIG_CPU_FREQ | ||
| 656 | |||
| 657 | static DEFINE_PER_CPU(unsigned long, l_p_j_ref); | ||
| 658 | static DEFINE_PER_CPU(unsigned long, l_p_j_ref_freq); | ||
| 659 | static unsigned long global_l_p_j_ref; | ||
| 660 | static unsigned long global_l_p_j_ref_freq; | ||
| 661 | |||
| 662 | static int cpufreq_callback(struct notifier_block *nb, | ||
| 663 | unsigned long val, void *data) | ||
| 664 | { | ||
| 665 | struct cpufreq_freqs *freq = data; | ||
| 666 | int cpu = freq->cpu; | ||
| 667 | |||
| 668 | if (freq->flags & CPUFREQ_CONST_LOOPS) | ||
| 669 | return NOTIFY_OK; | ||
| 670 | |||
| 671 | if (!per_cpu(l_p_j_ref, cpu)) { | ||
| 672 | per_cpu(l_p_j_ref, cpu) = | ||
| 673 | per_cpu(cpu_data, cpu).loops_per_jiffy; | ||
| 674 | per_cpu(l_p_j_ref_freq, cpu) = freq->old; | ||
| 675 | if (!global_l_p_j_ref) { | ||
| 676 | global_l_p_j_ref = loops_per_jiffy; | ||
| 677 | global_l_p_j_ref_freq = freq->old; | ||
| 678 | } | ||
| 679 | } | ||
| 680 | |||
| 681 | if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || | ||
| 682 | (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || | ||
| 683 | (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) { | ||
| 684 | loops_per_jiffy = cpufreq_scale(global_l_p_j_ref, | ||
| 685 | global_l_p_j_ref_freq, | ||
| 686 | freq->new); | ||
| 687 | per_cpu(cpu_data, cpu).loops_per_jiffy = | ||
| 688 | cpufreq_scale(per_cpu(l_p_j_ref, cpu), | ||
| 689 | per_cpu(l_p_j_ref_freq, cpu), | ||
| 690 | freq->new); | ||
| 691 | } | ||
| 692 | return NOTIFY_OK; | ||
| 693 | } | ||
| 694 | |||
| 695 | static struct notifier_block cpufreq_notifier = { | ||
| 696 | .notifier_call = cpufreq_callback, | ||
| 697 | }; | ||
| 698 | |||
| 699 | static int __init register_cpufreq_notifier(void) | ||
| 700 | { | ||
| 701 | return cpufreq_register_notifier(&cpufreq_notifier, | ||
| 702 | CPUFREQ_TRANSITION_NOTIFIER); | ||
| 703 | } | ||
| 704 | core_initcall(register_cpufreq_notifier); | ||
| 705 | |||
| 706 | #endif | ||
diff --git a/arch/arm/mach-at91/clock.c b/arch/arm/mach-at91/clock.c index 188c82971ebd..33361505c0cd 100644 --- a/arch/arm/mach-at91/clock.c +++ b/arch/arm/mach-at91/clock.c | |||
| @@ -625,7 +625,7 @@ fail: | |||
| 625 | return 0; | 625 | return 0; |
| 626 | } | 626 | } |
| 627 | 627 | ||
| 628 | static struct clk *const standard_pmc_clocks[] __initdata = { | 628 | static struct clk *const standard_pmc_clocks[] __initconst = { |
| 629 | /* four primary clocks */ | 629 | /* four primary clocks */ |
| 630 | &clk32k, | 630 | &clk32k, |
| 631 | &main_clk, | 631 | &main_clk, |
diff --git a/arch/arm/mach-davinci/board-tnetv107x-evm.c b/arch/arm/mach-davinci/board-tnetv107x-evm.c index ac4e003ad863..be3099733b1f 100644 --- a/arch/arm/mach-davinci/board-tnetv107x-evm.c +++ b/arch/arm/mach-davinci/board-tnetv107x-evm.c | |||
| @@ -88,7 +88,7 @@ static struct davinci_mmc_config mmc_config = { | |||
| 88 | .version = MMC_CTLR_VERSION_1, | 88 | .version = MMC_CTLR_VERSION_1, |
| 89 | }; | 89 | }; |
| 90 | 90 | ||
| 91 | static const short sdio1_pins[] __initdata = { | 91 | static const short sdio1_pins[] __initconst = { |
| 92 | TNETV107X_SDIO1_CLK_1, TNETV107X_SDIO1_CMD_1, | 92 | TNETV107X_SDIO1_CLK_1, TNETV107X_SDIO1_CMD_1, |
| 93 | TNETV107X_SDIO1_DATA0_1, TNETV107X_SDIO1_DATA1_1, | 93 | TNETV107X_SDIO1_DATA0_1, TNETV107X_SDIO1_DATA1_1, |
| 94 | TNETV107X_SDIO1_DATA2_1, TNETV107X_SDIO1_DATA3_1, | 94 | TNETV107X_SDIO1_DATA2_1, TNETV107X_SDIO1_DATA3_1, |
| @@ -96,12 +96,12 @@ static const short sdio1_pins[] __initdata = { | |||
| 96 | -1 | 96 | -1 |
| 97 | }; | 97 | }; |
| 98 | 98 | ||
| 99 | static const short uart1_pins[] __initdata = { | 99 | static const short uart1_pins[] __initconst = { |
| 100 | TNETV107X_UART1_RD, TNETV107X_UART1_TD, | 100 | TNETV107X_UART1_RD, TNETV107X_UART1_TD, |
| 101 | -1 | 101 | -1 |
| 102 | }; | 102 | }; |
| 103 | 103 | ||
| 104 | static const short ssp_pins[] __initdata = { | 104 | static const short ssp_pins[] __initconst = { |
| 105 | TNETV107X_SSP0_0, TNETV107X_SSP0_1, TNETV107X_SSP0_2, | 105 | TNETV107X_SSP0_0, TNETV107X_SSP0_1, TNETV107X_SSP0_2, |
| 106 | TNETV107X_SSP1_0, TNETV107X_SSP1_1, TNETV107X_SSP1_2, | 106 | TNETV107X_SSP1_0, TNETV107X_SSP1_1, TNETV107X_SSP1_2, |
| 107 | TNETV107X_SSP1_3, -1 | 107 | TNETV107X_SSP1_3, -1 |
diff --git a/arch/arm/mach-davinci/da830.c b/arch/arm/mach-davinci/da830.c index deee5c2da754..510648e0394b 100644 --- a/arch/arm/mach-davinci/da830.c +++ b/arch/arm/mach-davinci/da830.c | |||
| @@ -838,7 +838,7 @@ static const struct mux_config da830_pins[] = { | |||
| 838 | #endif | 838 | #endif |
| 839 | }; | 839 | }; |
| 840 | 840 | ||
| 841 | const short da830_emif25_pins[] __initdata = { | 841 | const short da830_emif25_pins[] __initconst = { |
| 842 | DA830_EMA_D_0, DA830_EMA_D_1, DA830_EMA_D_2, DA830_EMA_D_3, | 842 | DA830_EMA_D_0, DA830_EMA_D_1, DA830_EMA_D_2, DA830_EMA_D_3, |
| 843 | DA830_EMA_D_4, DA830_EMA_D_5, DA830_EMA_D_6, DA830_EMA_D_7, | 843 | DA830_EMA_D_4, DA830_EMA_D_5, DA830_EMA_D_6, DA830_EMA_D_7, |
| 844 | DA830_EMA_D_8, DA830_EMA_D_9, DA830_EMA_D_10, DA830_EMA_D_11, | 844 | DA830_EMA_D_8, DA830_EMA_D_9, DA830_EMA_D_10, DA830_EMA_D_11, |
| @@ -853,19 +853,19 @@ const short da830_emif25_pins[] __initdata = { | |||
| 853 | -1 | 853 | -1 |
| 854 | }; | 854 | }; |
| 855 | 855 | ||
| 856 | const short da830_spi0_pins[] __initdata = { | 856 | const short da830_spi0_pins[] __initconst = { |
| 857 | DA830_SPI0_SOMI_0, DA830_SPI0_SIMO_0, DA830_SPI0_CLK, DA830_NSPI0_ENA, | 857 | DA830_SPI0_SOMI_0, DA830_SPI0_SIMO_0, DA830_SPI0_CLK, DA830_NSPI0_ENA, |
| 858 | DA830_NSPI0_SCS_0, | 858 | DA830_NSPI0_SCS_0, |
| 859 | -1 | 859 | -1 |
| 860 | }; | 860 | }; |
| 861 | 861 | ||
| 862 | const short da830_spi1_pins[] __initdata = { | 862 | const short da830_spi1_pins[] __initconst = { |
| 863 | DA830_SPI1_SOMI_0, DA830_SPI1_SIMO_0, DA830_SPI1_CLK, DA830_NSPI1_ENA, | 863 | DA830_SPI1_SOMI_0, DA830_SPI1_SIMO_0, DA830_SPI1_CLK, DA830_NSPI1_ENA, |
| 864 | DA830_NSPI1_SCS_0, | 864 | DA830_NSPI1_SCS_0, |
| 865 | -1 | 865 | -1 |
| 866 | }; | 866 | }; |
| 867 | 867 | ||
| 868 | const short da830_mmc_sd_pins[] __initdata = { | 868 | const short da830_mmc_sd_pins[] __initconst = { |
| 869 | DA830_MMCSD_DAT_0, DA830_MMCSD_DAT_1, DA830_MMCSD_DAT_2, | 869 | DA830_MMCSD_DAT_0, DA830_MMCSD_DAT_1, DA830_MMCSD_DAT_2, |
| 870 | DA830_MMCSD_DAT_3, DA830_MMCSD_DAT_4, DA830_MMCSD_DAT_5, | 870 | DA830_MMCSD_DAT_3, DA830_MMCSD_DAT_4, DA830_MMCSD_DAT_5, |
| 871 | DA830_MMCSD_DAT_6, DA830_MMCSD_DAT_7, DA830_MMCSD_CLK, | 871 | DA830_MMCSD_DAT_6, DA830_MMCSD_DAT_7, DA830_MMCSD_CLK, |
| @@ -873,32 +873,32 @@ const short da830_mmc_sd_pins[] __initdata = { | |||
| 873 | -1 | 873 | -1 |
| 874 | }; | 874 | }; |
| 875 | 875 | ||
| 876 | const short da830_uart0_pins[] __initdata = { | 876 | const short da830_uart0_pins[] __initconst = { |
| 877 | DA830_NUART0_CTS, DA830_NUART0_RTS, DA830_UART0_RXD, DA830_UART0_TXD, | 877 | DA830_NUART0_CTS, DA830_NUART0_RTS, DA830_UART0_RXD, DA830_UART0_TXD, |
| 878 | -1 | 878 | -1 |
| 879 | }; | 879 | }; |
| 880 | 880 | ||
| 881 | const short da830_uart1_pins[] __initdata = { | 881 | const short da830_uart1_pins[] __initconst = { |
| 882 | DA830_UART1_RXD, DA830_UART1_TXD, | 882 | DA830_UART1_RXD, DA830_UART1_TXD, |
| 883 | -1 | 883 | -1 |
| 884 | }; | 884 | }; |
| 885 | 885 | ||
| 886 | const short da830_uart2_pins[] __initdata = { | 886 | const short da830_uart2_pins[] __initconst = { |
| 887 | DA830_UART2_RXD, DA830_UART2_TXD, | 887 | DA830_UART2_RXD, DA830_UART2_TXD, |
| 888 | -1 | 888 | -1 |
| 889 | }; | 889 | }; |
| 890 | 890 | ||
| 891 | const short da830_usb20_pins[] __initdata = { | 891 | const short da830_usb20_pins[] __initconst = { |
| 892 | DA830_USB0_DRVVBUS, DA830_USB_REFCLKIN, | 892 | DA830_USB0_DRVVBUS, DA830_USB_REFCLKIN, |
| 893 | -1 | 893 | -1 |
| 894 | }; | 894 | }; |
| 895 | 895 | ||
| 896 | const short da830_usb11_pins[] __initdata = { | 896 | const short da830_usb11_pins[] __initconst = { |
| 897 | DA830_USB_REFCLKIN, | 897 | DA830_USB_REFCLKIN, |
| 898 | -1 | 898 | -1 |
| 899 | }; | 899 | }; |
| 900 | 900 | ||
| 901 | const short da830_uhpi_pins[] __initdata = { | 901 | const short da830_uhpi_pins[] __initconst = { |
| 902 | DA830_UHPI_HD_0, DA830_UHPI_HD_1, DA830_UHPI_HD_2, DA830_UHPI_HD_3, | 902 | DA830_UHPI_HD_0, DA830_UHPI_HD_1, DA830_UHPI_HD_2, DA830_UHPI_HD_3, |
| 903 | DA830_UHPI_HD_4, DA830_UHPI_HD_5, DA830_UHPI_HD_6, DA830_UHPI_HD_7, | 903 | DA830_UHPI_HD_4, DA830_UHPI_HD_5, DA830_UHPI_HD_6, DA830_UHPI_HD_7, |
| 904 | DA830_UHPI_HD_8, DA830_UHPI_HD_9, DA830_UHPI_HD_10, DA830_UHPI_HD_11, | 904 | DA830_UHPI_HD_8, DA830_UHPI_HD_9, DA830_UHPI_HD_10, DA830_UHPI_HD_11, |
| @@ -909,14 +909,14 @@ const short da830_uhpi_pins[] __initdata = { | |||
| 909 | -1 | 909 | -1 |
| 910 | }; | 910 | }; |
| 911 | 911 | ||
| 912 | const short da830_cpgmac_pins[] __initdata = { | 912 | const short da830_cpgmac_pins[] __initconst = { |
| 913 | DA830_RMII_TXD_0, DA830_RMII_TXD_1, DA830_RMII_TXEN, DA830_RMII_CRS_DV, | 913 | DA830_RMII_TXD_0, DA830_RMII_TXD_1, DA830_RMII_TXEN, DA830_RMII_CRS_DV, |
| 914 | DA830_RMII_RXD_0, DA830_RMII_RXD_1, DA830_RMII_RXER, DA830_MDIO_CLK, | 914 | DA830_RMII_RXD_0, DA830_RMII_RXD_1, DA830_RMII_RXER, DA830_MDIO_CLK, |
| 915 | DA830_MDIO_D, | 915 | DA830_MDIO_D, |
| 916 | -1 | 916 | -1 |
| 917 | }; | 917 | }; |
| 918 | 918 | ||
| 919 | const short da830_emif3c_pins[] __initdata = { | 919 | const short da830_emif3c_pins[] __initconst = { |
| 920 | DA830_EMB_SDCKE, DA830_EMB_CLK_GLUE, DA830_EMB_CLK, DA830_NEMB_CS_0, | 920 | DA830_EMB_SDCKE, DA830_EMB_CLK_GLUE, DA830_EMB_CLK, DA830_NEMB_CS_0, |
| 921 | DA830_NEMB_CAS, DA830_NEMB_RAS, DA830_NEMB_WE, DA830_EMB_BA_1, | 921 | DA830_NEMB_CAS, DA830_NEMB_RAS, DA830_NEMB_WE, DA830_EMB_BA_1, |
| 922 | DA830_EMB_BA_0, DA830_EMB_A_0, DA830_EMB_A_1, DA830_EMB_A_2, | 922 | DA830_EMB_BA_0, DA830_EMB_A_0, DA830_EMB_A_1, DA830_EMB_A_2, |
| @@ -935,7 +935,7 @@ const short da830_emif3c_pins[] __initdata = { | |||
| 935 | -1 | 935 | -1 |
| 936 | }; | 936 | }; |
| 937 | 937 | ||
| 938 | const short da830_mcasp0_pins[] __initdata = { | 938 | const short da830_mcasp0_pins[] __initconst = { |
| 939 | DA830_AHCLKX0, DA830_ACLKX0, DA830_AFSX0, | 939 | DA830_AHCLKX0, DA830_ACLKX0, DA830_AFSX0, |
| 940 | DA830_AHCLKR0, DA830_ACLKR0, DA830_AFSR0, DA830_AMUTE0, | 940 | DA830_AHCLKR0, DA830_ACLKR0, DA830_AFSR0, DA830_AMUTE0, |
| 941 | DA830_AXR0_0, DA830_AXR0_1, DA830_AXR0_2, DA830_AXR0_3, | 941 | DA830_AXR0_0, DA830_AXR0_1, DA830_AXR0_2, DA830_AXR0_3, |
| @@ -945,7 +945,7 @@ const short da830_mcasp0_pins[] __initdata = { | |||
| 945 | -1 | 945 | -1 |
| 946 | }; | 946 | }; |
| 947 | 947 | ||
| 948 | const short da830_mcasp1_pins[] __initdata = { | 948 | const short da830_mcasp1_pins[] __initconst = { |
| 949 | DA830_AHCLKX1, DA830_ACLKX1, DA830_AFSX1, | 949 | DA830_AHCLKX1, DA830_ACLKX1, DA830_AFSX1, |
| 950 | DA830_AHCLKR1, DA830_ACLKR1, DA830_AFSR1, DA830_AMUTE1, | 950 | DA830_AHCLKR1, DA830_ACLKR1, DA830_AFSR1, DA830_AMUTE1, |
| 951 | DA830_AXR1_0, DA830_AXR1_1, DA830_AXR1_2, DA830_AXR1_3, | 951 | DA830_AXR1_0, DA830_AXR1_1, DA830_AXR1_2, DA830_AXR1_3, |
| @@ -954,24 +954,24 @@ const short da830_mcasp1_pins[] __initdata = { | |||
| 954 | -1 | 954 | -1 |
| 955 | }; | 955 | }; |
| 956 | 956 | ||
| 957 | const short da830_mcasp2_pins[] __initdata = { | 957 | const short da830_mcasp2_pins[] __initconst = { |
| 958 | DA830_AHCLKX2, DA830_ACLKX2, DA830_AFSX2, | 958 | DA830_AHCLKX2, DA830_ACLKX2, DA830_AFSX2, |
| 959 | DA830_AHCLKR2, DA830_ACLKR2, DA830_AFSR2, DA830_AMUTE2, | 959 | DA830_AHCLKR2, DA830_ACLKR2, DA830_AFSR2, DA830_AMUTE2, |
| 960 | DA830_AXR2_0, DA830_AXR2_1, DA830_AXR2_2, DA830_AXR2_3, | 960 | DA830_AXR2_0, DA830_AXR2_1, DA830_AXR2_2, DA830_AXR2_3, |
| 961 | -1 | 961 | -1 |
| 962 | }; | 962 | }; |
| 963 | 963 | ||
| 964 | const short da830_i2c0_pins[] __initdata = { | 964 | const short da830_i2c0_pins[] __initconst = { |
| 965 | DA830_I2C0_SDA, DA830_I2C0_SCL, | 965 | DA830_I2C0_SDA, DA830_I2C0_SCL, |
| 966 | -1 | 966 | -1 |
| 967 | }; | 967 | }; |
| 968 | 968 | ||
| 969 | const short da830_i2c1_pins[] __initdata = { | 969 | const short da830_i2c1_pins[] __initconst = { |
| 970 | DA830_I2C1_SCL, DA830_I2C1_SDA, | 970 | DA830_I2C1_SCL, DA830_I2C1_SDA, |
| 971 | -1 | 971 | -1 |
| 972 | }; | 972 | }; |
| 973 | 973 | ||
| 974 | const short da830_lcdcntl_pins[] __initdata = { | 974 | const short da830_lcdcntl_pins[] __initconst = { |
| 975 | DA830_LCD_D_0, DA830_LCD_D_1, DA830_LCD_D_2, DA830_LCD_D_3, | 975 | DA830_LCD_D_0, DA830_LCD_D_1, DA830_LCD_D_2, DA830_LCD_D_3, |
| 976 | DA830_LCD_D_4, DA830_LCD_D_5, DA830_LCD_D_6, DA830_LCD_D_7, | 976 | DA830_LCD_D_4, DA830_LCD_D_5, DA830_LCD_D_6, DA830_LCD_D_7, |
| 977 | DA830_LCD_D_8, DA830_LCD_D_9, DA830_LCD_D_10, DA830_LCD_D_11, | 977 | DA830_LCD_D_8, DA830_LCD_D_9, DA830_LCD_D_10, DA830_LCD_D_11, |
| @@ -981,34 +981,34 @@ const short da830_lcdcntl_pins[] __initdata = { | |||
| 981 | -1 | 981 | -1 |
| 982 | }; | 982 | }; |
| 983 | 983 | ||
| 984 | const short da830_pwm_pins[] __initdata = { | 984 | const short da830_pwm_pins[] __initconst = { |
| 985 | DA830_ECAP0_APWM0, DA830_ECAP1_APWM1, DA830_EPWM0B, DA830_EPWM0A, | 985 | DA830_ECAP0_APWM0, DA830_ECAP1_APWM1, DA830_EPWM0B, DA830_EPWM0A, |
| 986 | DA830_EPWMSYNCI, DA830_EPWMSYNC0, DA830_ECAP2_APWM2, DA830_EHRPWMGLUETZ, | 986 | DA830_EPWMSYNCI, DA830_EPWMSYNC0, DA830_ECAP2_APWM2, DA830_EHRPWMGLUETZ, |
| 987 | DA830_EPWM2B, DA830_EPWM2A, DA830_EPWM1B, DA830_EPWM1A, | 987 | DA830_EPWM2B, DA830_EPWM2A, DA830_EPWM1B, DA830_EPWM1A, |
| 988 | -1 | 988 | -1 |
| 989 | }; | 989 | }; |
| 990 | 990 | ||
| 991 | const short da830_ecap0_pins[] __initdata = { | 991 | const short da830_ecap0_pins[] __initconst = { |
| 992 | DA830_ECAP0_APWM0, | 992 | DA830_ECAP0_APWM0, |
| 993 | -1 | 993 | -1 |
| 994 | }; | 994 | }; |
| 995 | 995 | ||
| 996 | const short da830_ecap1_pins[] __initdata = { | 996 | const short da830_ecap1_pins[] __initconst = { |
| 997 | DA830_ECAP1_APWM1, | 997 | DA830_ECAP1_APWM1, |
| 998 | -1 | 998 | -1 |
| 999 | }; | 999 | }; |
| 1000 | 1000 | ||
| 1001 | const short da830_ecap2_pins[] __initdata = { | 1001 | const short da830_ecap2_pins[] __initconst = { |
| 1002 | DA830_ECAP2_APWM2, | 1002 | DA830_ECAP2_APWM2, |
| 1003 | -1 | 1003 | -1 |
| 1004 | }; | 1004 | }; |
| 1005 | 1005 | ||
| 1006 | const short da830_eqep0_pins[] __initdata = { | 1006 | const short da830_eqep0_pins[] __initconst = { |
| 1007 | DA830_EQEP0I, DA830_EQEP0S, DA830_EQEP0A, DA830_EQEP0B, | 1007 | DA830_EQEP0I, DA830_EQEP0S, DA830_EQEP0A, DA830_EQEP0B, |
| 1008 | -1 | 1008 | -1 |
| 1009 | }; | 1009 | }; |
| 1010 | 1010 | ||
| 1011 | const short da830_eqep1_pins[] __initdata = { | 1011 | const short da830_eqep1_pins[] __initconst = { |
| 1012 | DA830_EQEP1I, DA830_EQEP1S, DA830_EQEP1A, DA830_EQEP1B, | 1012 | DA830_EQEP1I, DA830_EQEP1S, DA830_EQEP1A, DA830_EQEP1B, |
| 1013 | -1 | 1013 | -1 |
| 1014 | }; | 1014 | }; |
diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c index b44dc844e15e..6676dee7104e 100644 --- a/arch/arm/mach-davinci/da850.c +++ b/arch/arm/mach-davinci/da850.c | |||
| @@ -576,17 +576,17 @@ static const struct mux_config da850_pins[] = { | |||
| 576 | #endif | 576 | #endif |
| 577 | }; | 577 | }; |
| 578 | 578 | ||
| 579 | const short da850_i2c0_pins[] __initdata = { | 579 | const short da850_i2c0_pins[] __initconst = { |
| 580 | DA850_I2C0_SDA, DA850_I2C0_SCL, | 580 | DA850_I2C0_SDA, DA850_I2C0_SCL, |
| 581 | -1 | 581 | -1 |
| 582 | }; | 582 | }; |
| 583 | 583 | ||
| 584 | const short da850_i2c1_pins[] __initdata = { | 584 | const short da850_i2c1_pins[] __initconst = { |
| 585 | DA850_I2C1_SCL, DA850_I2C1_SDA, | 585 | DA850_I2C1_SCL, DA850_I2C1_SDA, |
| 586 | -1 | 586 | -1 |
| 587 | }; | 587 | }; |
| 588 | 588 | ||
| 589 | const short da850_lcdcntl_pins[] __initdata = { | 589 | const short da850_lcdcntl_pins[] __initconst = { |
| 590 | DA850_LCD_D_0, DA850_LCD_D_1, DA850_LCD_D_2, DA850_LCD_D_3, | 590 | DA850_LCD_D_0, DA850_LCD_D_1, DA850_LCD_D_2, DA850_LCD_D_3, |
| 591 | DA850_LCD_D_4, DA850_LCD_D_5, DA850_LCD_D_6, DA850_LCD_D_7, | 591 | DA850_LCD_D_4, DA850_LCD_D_5, DA850_LCD_D_6, DA850_LCD_D_7, |
| 592 | DA850_LCD_D_8, DA850_LCD_D_9, DA850_LCD_D_10, DA850_LCD_D_11, | 592 | DA850_LCD_D_8, DA850_LCD_D_9, DA850_LCD_D_10, DA850_LCD_D_11, |
diff --git a/arch/arm/mach-highbank/highbank.c b/arch/arm/mach-highbank/highbank.c index af1da34ccf9d..40e36a50304c 100644 --- a/arch/arm/mach-highbank/highbank.c +++ b/arch/arm/mach-highbank/highbank.c | |||
| @@ -15,6 +15,7 @@ | |||
| 15 | */ | 15 | */ |
| 16 | #include <linux/clk.h> | 16 | #include <linux/clk.h> |
| 17 | #include <linux/clkdev.h> | 17 | #include <linux/clkdev.h> |
| 18 | #include <linux/dma-mapping.h> | ||
| 18 | #include <linux/io.h> | 19 | #include <linux/io.h> |
| 19 | #include <linux/irq.h> | 20 | #include <linux/irq.h> |
| 20 | #include <linux/irqdomain.h> | 21 | #include <linux/irqdomain.h> |
| @@ -23,6 +24,7 @@ | |||
| 23 | #include <linux/of_platform.h> | 24 | #include <linux/of_platform.h> |
| 24 | #include <linux/of_address.h> | 25 | #include <linux/of_address.h> |
| 25 | #include <linux/smp.h> | 26 | #include <linux/smp.h> |
| 27 | #include <linux/amba/bus.h> | ||
| 26 | 28 | ||
| 27 | #include <asm/cacheflush.h> | 29 | #include <asm/cacheflush.h> |
| 28 | #include <asm/smp_plat.h> | 30 | #include <asm/smp_plat.h> |
| @@ -149,11 +151,61 @@ static void highbank_power_off(void) | |||
| 149 | cpu_do_idle(); | 151 | cpu_do_idle(); |
| 150 | } | 152 | } |
| 151 | 153 | ||
| 154 | static int highbank_platform_notifier(struct notifier_block *nb, | ||
| 155 | unsigned long event, void *__dev) | ||
| 156 | { | ||
| 157 | struct resource *res; | ||
| 158 | int reg = -1; | ||
| 159 | struct device *dev = __dev; | ||
| 160 | |||
| 161 | if (event != BUS_NOTIFY_ADD_DEVICE) | ||
| 162 | return NOTIFY_DONE; | ||
| 163 | |||
| 164 | if (of_device_is_compatible(dev->of_node, "calxeda,hb-ahci")) | ||
| 165 | reg = 0xc; | ||
| 166 | else if (of_device_is_compatible(dev->of_node, "calxeda,hb-sdhci")) | ||
| 167 | reg = 0x18; | ||
| 168 | else if (of_device_is_compatible(dev->of_node, "arm,pl330")) | ||
| 169 | reg = 0x20; | ||
| 170 | else if (of_device_is_compatible(dev->of_node, "calxeda,hb-xgmac")) { | ||
| 171 | res = platform_get_resource(to_platform_device(dev), | ||
| 172 | IORESOURCE_MEM, 0); | ||
| 173 | if (res) { | ||
| 174 | if (res->start == 0xfff50000) | ||
| 175 | reg = 0; | ||
| 176 | else if (res->start == 0xfff51000) | ||
| 177 | reg = 4; | ||
| 178 | } | ||
| 179 | } | ||
| 180 | |||
| 181 | if (reg < 0) | ||
| 182 | return NOTIFY_DONE; | ||
| 183 | |||
| 184 | if (of_property_read_bool(dev->of_node, "dma-coherent")) { | ||
| 185 | writel(0xff31, sregs_base + reg); | ||
| 186 | set_dma_ops(dev, &arm_coherent_dma_ops); | ||
| 187 | } else | ||
| 188 | writel(0, sregs_base + reg); | ||
| 189 | |||
| 190 | return NOTIFY_OK; | ||
| 191 | } | ||
| 192 | |||
| 193 | static struct notifier_block highbank_amba_nb = { | ||
| 194 | .notifier_call = highbank_platform_notifier, | ||
| 195 | }; | ||
| 196 | |||
| 197 | static struct notifier_block highbank_platform_nb = { | ||
| 198 | .notifier_call = highbank_platform_notifier, | ||
| 199 | }; | ||
| 200 | |||
| 152 | static void __init highbank_init(void) | 201 | static void __init highbank_init(void) |
| 153 | { | 202 | { |
| 154 | pm_power_off = highbank_power_off; | 203 | pm_power_off = highbank_power_off; |
| 155 | highbank_pm_init(); | 204 | highbank_pm_init(); |
| 156 | 205 | ||
| 206 | bus_register_notifier(&platform_bus_type, &highbank_platform_nb); | ||
| 207 | bus_register_notifier(&amba_bustype, &highbank_amba_nb); | ||
| 208 | |||
| 157 | of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); | 209 | of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); |
| 158 | } | 210 | } |
| 159 | 211 | ||
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig index 3a2042fb9712..32197c117afe 100644 --- a/arch/arm/mach-imx/Kconfig +++ b/arch/arm/mach-imx/Kconfig | |||
| @@ -758,7 +758,7 @@ config SOC_IMX6Q | |||
| 758 | select HAVE_IMX_MMDC | 758 | select HAVE_IMX_MMDC |
| 759 | select HAVE_IMX_SRC | 759 | select HAVE_IMX_SRC |
| 760 | select HAVE_SMP | 760 | select HAVE_SMP |
| 761 | select MFD_ANATOP | 761 | select MFD_SYSCON |
| 762 | select PINCTRL | 762 | select PINCTRL |
| 763 | select PINCTRL_IMX6Q | 763 | select PINCTRL_IMX6Q |
| 764 | 764 | ||
diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c index 36979d3dfe34..47c91f7185d2 100644 --- a/arch/arm/mach-imx/mach-imx6q.c +++ b/arch/arm/mach-imx/mach-imx6q.c | |||
| @@ -23,8 +23,9 @@ | |||
| 23 | #include <linux/of_irq.h> | 23 | #include <linux/of_irq.h> |
| 24 | #include <linux/of_platform.h> | 24 | #include <linux/of_platform.h> |
| 25 | #include <linux/phy.h> | 25 | #include <linux/phy.h> |
| 26 | #include <linux/regmap.h> | ||
| 26 | #include <linux/micrel_phy.h> | 27 | #include <linux/micrel_phy.h> |
| 27 | #include <linux/mfd/anatop.h> | 28 | #include <linux/mfd/syscon.h> |
| 28 | #include <asm/cpuidle.h> | 29 | #include <asm/cpuidle.h> |
| 29 | #include <asm/smp_twd.h> | 30 | #include <asm/smp_twd.h> |
| 30 | #include <asm/hardware/cache-l2x0.h> | 31 | #include <asm/hardware/cache-l2x0.h> |
| @@ -118,20 +119,7 @@ static void __init imx6q_sabrelite_init(void) | |||
| 118 | 119 | ||
| 119 | static void __init imx6q_usb_init(void) | 120 | static void __init imx6q_usb_init(void) |
| 120 | { | 121 | { |
| 121 | struct device_node *np; | 122 | struct regmap *anatop; |
| 122 | struct platform_device *pdev = NULL; | ||
| 123 | struct anatop *adata = NULL; | ||
| 124 | |||
| 125 | np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-anatop"); | ||
| 126 | if (np) | ||
| 127 | pdev = of_find_device_by_node(np); | ||
| 128 | if (pdev) | ||
| 129 | adata = platform_get_drvdata(pdev); | ||
| 130 | if (!adata) { | ||
| 131 | if (np) | ||
| 132 | of_node_put(np); | ||
| 133 | return; | ||
| 134 | } | ||
| 135 | 123 | ||
| 136 | #define HW_ANADIG_USB1_CHRG_DETECT 0x000001b0 | 124 | #define HW_ANADIG_USB1_CHRG_DETECT 0x000001b0 |
| 137 | #define HW_ANADIG_USB2_CHRG_DETECT 0x00000210 | 125 | #define HW_ANADIG_USB2_CHRG_DETECT 0x00000210 |
| @@ -139,20 +127,21 @@ static void __init imx6q_usb_init(void) | |||
| 139 | #define BM_ANADIG_USB_CHRG_DETECT_EN_B 0x00100000 | 127 | #define BM_ANADIG_USB_CHRG_DETECT_EN_B 0x00100000 |
| 140 | #define BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B 0x00080000 | 128 | #define BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B 0x00080000 |
| 141 | 129 | ||
| 142 | /* | 130 | anatop = syscon_regmap_lookup_by_compatible("fsl,imx6q-anatop"); |
| 143 | * The external charger detector needs to be disabled, | 131 | if (!IS_ERR(anatop)) { |
| 144 | * or the signal at DP will be poor | 132 | /* |
| 145 | */ | 133 | * The external charger detector needs to be disabled, |
| 146 | anatop_write_reg(adata, HW_ANADIG_USB1_CHRG_DETECT, | 134 | * or the signal at DP will be poor |
| 147 | BM_ANADIG_USB_CHRG_DETECT_EN_B | 135 | */ |
| 148 | | BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B, | 136 | regmap_write(anatop, HW_ANADIG_USB1_CHRG_DETECT, |
| 149 | ~0); | 137 | BM_ANADIG_USB_CHRG_DETECT_EN_B |
| 150 | anatop_write_reg(adata, HW_ANADIG_USB2_CHRG_DETECT, | 138 | | BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B); |
| 151 | BM_ANADIG_USB_CHRG_DETECT_EN_B | | 139 | regmap_write(anatop, HW_ANADIG_USB2_CHRG_DETECT, |
| 152 | BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B, | 140 | BM_ANADIG_USB_CHRG_DETECT_EN_B | |
| 153 | ~0); | 141 | BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B); |
| 154 | 142 | } else { | |
| 155 | of_node_put(np); | 143 | pr_warn("failed to find fsl,imx6q-anatop regmap\n"); |
| 144 | } | ||
| 156 | } | 145 | } |
| 157 | 146 | ||
| 158 | static void __init imx6q_init_machine(void) | 147 | static void __init imx6q_init_machine(void) |
diff --git a/arch/arm/mach-msm/board-qsd8x50.c b/arch/arm/mach-msm/board-qsd8x50.c index a344a373928b..2448fcf09eb1 100644 --- a/arch/arm/mach-msm/board-qsd8x50.c +++ b/arch/arm/mach-msm/board-qsd8x50.c | |||
| @@ -37,8 +37,8 @@ | |||
| 37 | #include "devices.h" | 37 | #include "devices.h" |
| 38 | #include "common.h" | 38 | #include "common.h" |
| 39 | 39 | ||
| 40 | static const resource_size_t qsd8x50_surf_smc91x_base __initdata = 0x70000300; | 40 | static const resource_size_t qsd8x50_surf_smc91x_base __initconst = 0x70000300; |
| 41 | static const unsigned qsd8x50_surf_smc91x_gpio __initdata = 156; | 41 | static const unsigned qsd8x50_surf_smc91x_gpio __initconst = 156; |
| 42 | 42 | ||
| 43 | /* Leave smc91x resources empty here, as we'll fill them in | 43 | /* Leave smc91x resources empty here, as we'll fill them in |
| 44 | * at run-time: they vary from board to board, and the true | 44 | * at run-time: they vary from board to board, and the true |
diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c index af1ed7d24a1f..e470c6e50acd 100644 --- a/arch/arm/mach-omap2/display.c +++ b/arch/arm/mach-omap2/display.c | |||
| @@ -76,14 +76,14 @@ struct omap_dss_hwmod_data { | |||
| 76 | const int id; | 76 | const int id; |
| 77 | }; | 77 | }; |
| 78 | 78 | ||
| 79 | static const struct omap_dss_hwmod_data omap2_dss_hwmod_data[] __initdata = { | 79 | static const struct omap_dss_hwmod_data omap2_dss_hwmod_data[] __initconst = { |
| 80 | { "dss_core", "omapdss_dss", -1 }, | 80 | { "dss_core", "omapdss_dss", -1 }, |
| 81 | { "dss_dispc", "omapdss_dispc", -1 }, | 81 | { "dss_dispc", "omapdss_dispc", -1 }, |
| 82 | { "dss_rfbi", "omapdss_rfbi", -1 }, | 82 | { "dss_rfbi", "omapdss_rfbi", -1 }, |
| 83 | { "dss_venc", "omapdss_venc", -1 }, | 83 | { "dss_venc", "omapdss_venc", -1 }, |
| 84 | }; | 84 | }; |
| 85 | 85 | ||
| 86 | static const struct omap_dss_hwmod_data omap3_dss_hwmod_data[] __initdata = { | 86 | static const struct omap_dss_hwmod_data omap3_dss_hwmod_data[] __initconst = { |
| 87 | { "dss_core", "omapdss_dss", -1 }, | 87 | { "dss_core", "omapdss_dss", -1 }, |
| 88 | { "dss_dispc", "omapdss_dispc", -1 }, | 88 | { "dss_dispc", "omapdss_dispc", -1 }, |
| 89 | { "dss_rfbi", "omapdss_rfbi", -1 }, | 89 | { "dss_rfbi", "omapdss_rfbi", -1 }, |
| @@ -91,7 +91,7 @@ static const struct omap_dss_hwmod_data omap3_dss_hwmod_data[] __initdata = { | |||
| 91 | { "dss_dsi1", "omapdss_dsi", 0 }, | 91 | { "dss_dsi1", "omapdss_dsi", 0 }, |
| 92 | }; | 92 | }; |
| 93 | 93 | ||
| 94 | static const struct omap_dss_hwmod_data omap4_dss_hwmod_data[] __initdata = { | 94 | static const struct omap_dss_hwmod_data omap4_dss_hwmod_data[] __initconst = { |
| 95 | { "dss_core", "omapdss_dss", -1 }, | 95 | { "dss_core", "omapdss_dss", -1 }, |
| 96 | { "dss_dispc", "omapdss_dispc", -1 }, | 96 | { "dss_dispc", "omapdss_dispc", -1 }, |
| 97 | { "dss_rfbi", "omapdss_rfbi", -1 }, | 97 | { "dss_rfbi", "omapdss_rfbi", -1 }, |
diff --git a/arch/arm/mach-omap2/usb-host.c b/arch/arm/mach-omap2/usb-host.c index ac95daaa4702..3c434498e12e 100644 --- a/arch/arm/mach-omap2/usb-host.c +++ b/arch/arm/mach-omap2/usb-host.c | |||
| @@ -33,10 +33,12 @@ | |||
| 33 | #ifdef CONFIG_MFD_OMAP_USB_HOST | 33 | #ifdef CONFIG_MFD_OMAP_USB_HOST |
| 34 | 34 | ||
| 35 | #define OMAP_USBHS_DEVICE "usbhs_omap" | 35 | #define OMAP_USBHS_DEVICE "usbhs_omap" |
| 36 | #define OMAP_USBTLL_DEVICE "usbhs_tll" | ||
| 36 | #define USBHS_UHH_HWMODNAME "usb_host_hs" | 37 | #define USBHS_UHH_HWMODNAME "usb_host_hs" |
| 37 | #define USBHS_TLL_HWMODNAME "usb_tll_hs" | 38 | #define USBHS_TLL_HWMODNAME "usb_tll_hs" |
| 38 | 39 | ||
| 39 | static struct usbhs_omap_platform_data usbhs_data; | 40 | static struct usbhs_omap_platform_data usbhs_data; |
| 41 | static struct usbtll_omap_platform_data usbtll_data; | ||
| 40 | static struct ehci_hcd_omap_platform_data ehci_data; | 42 | static struct ehci_hcd_omap_platform_data ehci_data; |
| 41 | static struct ohci_hcd_omap_platform_data ohci_data; | 43 | static struct ohci_hcd_omap_platform_data ohci_data; |
| 42 | 44 | ||
| @@ -485,13 +487,14 @@ void __init setup_4430ohci_io_mux(const enum usbhs_omap_port_mode *port_mode) | |||
| 485 | 487 | ||
| 486 | void __init usbhs_init(const struct usbhs_omap_board_data *pdata) | 488 | void __init usbhs_init(const struct usbhs_omap_board_data *pdata) |
| 487 | { | 489 | { |
| 488 | struct omap_hwmod *oh[2]; | 490 | struct omap_hwmod *uhh_hwm, *tll_hwm; |
| 489 | struct platform_device *pdev; | 491 | struct platform_device *pdev; |
| 490 | int bus_id = -1; | 492 | int bus_id = -1; |
| 491 | int i; | 493 | int i; |
| 492 | 494 | ||
| 493 | for (i = 0; i < OMAP3_HS_USB_PORTS; i++) { | 495 | for (i = 0; i < OMAP3_HS_USB_PORTS; i++) { |
| 494 | usbhs_data.port_mode[i] = pdata->port_mode[i]; | 496 | usbhs_data.port_mode[i] = pdata->port_mode[i]; |
| 497 | usbtll_data.port_mode[i] = pdata->port_mode[i]; | ||
| 495 | ohci_data.port_mode[i] = pdata->port_mode[i]; | 498 | ohci_data.port_mode[i] = pdata->port_mode[i]; |
| 496 | ehci_data.port_mode[i] = pdata->port_mode[i]; | 499 | ehci_data.port_mode[i] = pdata->port_mode[i]; |
| 497 | ehci_data.reset_gpio_port[i] = pdata->reset_gpio_port[i]; | 500 | ehci_data.reset_gpio_port[i] = pdata->reset_gpio_port[i]; |
| @@ -510,25 +513,35 @@ void __init usbhs_init(const struct usbhs_omap_board_data *pdata) | |||
| 510 | setup_4430ohci_io_mux(pdata->port_mode); | 513 | setup_4430ohci_io_mux(pdata->port_mode); |
| 511 | } | 514 | } |
| 512 | 515 | ||
| 513 | oh[0] = omap_hwmod_lookup(USBHS_UHH_HWMODNAME); | 516 | uhh_hwm = omap_hwmod_lookup(USBHS_UHH_HWMODNAME); |
| 514 | if (!oh[0]) { | 517 | if (!uhh_hwm) { |
| 515 | pr_err("Could not look up %s\n", USBHS_UHH_HWMODNAME); | 518 | pr_err("Could not look up %s\n", USBHS_UHH_HWMODNAME); |
| 516 | return; | 519 | return; |
| 517 | } | 520 | } |
| 518 | 521 | ||
| 519 | oh[1] = omap_hwmod_lookup(USBHS_TLL_HWMODNAME); | 522 | tll_hwm = omap_hwmod_lookup(USBHS_TLL_HWMODNAME); |
| 520 | if (!oh[1]) { | 523 | if (!tll_hwm) { |
| 521 | pr_err("Could not look up %s\n", USBHS_TLL_HWMODNAME); | 524 | pr_err("Could not look up %s\n", USBHS_TLL_HWMODNAME); |
| 522 | return; | 525 | return; |
| 523 | } | 526 | } |
| 524 | 527 | ||
| 525 | pdev = omap_device_build_ss(OMAP_USBHS_DEVICE, bus_id, oh, 2, | 528 | pdev = omap_device_build(OMAP_USBTLL_DEVICE, bus_id, tll_hwm, |
| 526 | (void *)&usbhs_data, sizeof(usbhs_data), | 529 | &usbtll_data, sizeof(usbtll_data), |
| 527 | omap_uhhtll_latency, | 530 | omap_uhhtll_latency, |
| 528 | ARRAY_SIZE(omap_uhhtll_latency), false); | 531 | ARRAY_SIZE(omap_uhhtll_latency), false); |
| 529 | if (IS_ERR(pdev)) { | 532 | if (IS_ERR(pdev)) { |
| 530 | pr_err("Could not build hwmod devices %s,%s\n", | 533 | pr_err("Could not build hwmod device %s\n", |
| 531 | USBHS_UHH_HWMODNAME, USBHS_TLL_HWMODNAME); | 534 | USBHS_TLL_HWMODNAME); |
| 535 | return; | ||
| 536 | } | ||
| 537 | |||
| 538 | pdev = omap_device_build(OMAP_USBHS_DEVICE, bus_id, uhh_hwm, | ||
| 539 | &usbhs_data, sizeof(usbhs_data), | ||
| 540 | omap_uhhtll_latency, | ||
| 541 | ARRAY_SIZE(omap_uhhtll_latency), false); | ||
| 542 | if (IS_ERR(pdev)) { | ||
| 543 | pr_err("Could not build hwmod devices %s\n", | ||
| 544 | USBHS_UHH_HWMODNAME); | ||
| 532 | return; | 545 | return; |
| 533 | } | 546 | } |
| 534 | } | 547 | } |
diff --git a/arch/arm/mach-shmobile/Makefile b/arch/arm/mach-shmobile/Makefile index 0df5ae6740c6..fe2c97c179d1 100644 --- a/arch/arm/mach-shmobile/Makefile +++ b/arch/arm/mach-shmobile/Makefile | |||
| @@ -3,7 +3,7 @@ | |||
| 3 | # | 3 | # |
| 4 | 4 | ||
| 5 | # Common objects | 5 | # Common objects |
| 6 | obj-y := timer.o console.o clock.o common.o | 6 | obj-y := timer.o console.o clock.o |
| 7 | 7 | ||
| 8 | # CPU objects | 8 | # CPU objects |
| 9 | obj-$(CONFIG_ARCH_SH7367) += setup-sh7367.o clock-sh7367.o intc-sh7367.o | 9 | obj-$(CONFIG_ARCH_SH7367) += setup-sh7367.o clock-sh7367.o intc-sh7367.o |
diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c index bc3b5da59e25..790dc68c4312 100644 --- a/arch/arm/mach-shmobile/board-ap4evb.c +++ b/arch/arm/mach-shmobile/board-ap4evb.c | |||
| @@ -1231,6 +1231,15 @@ static struct i2c_board_info i2c1_devices[] = { | |||
| 1231 | #define USCCR1 IOMEM(0xE6058144) | 1231 | #define USCCR1 IOMEM(0xE6058144) |
| 1232 | static void __init ap4evb_init(void) | 1232 | static void __init ap4evb_init(void) |
| 1233 | { | 1233 | { |
| 1234 | struct pm_domain_device domain_devices[] = { | ||
| 1235 | { "A4LC", &lcdc1_device, }, | ||
| 1236 | { "A4LC", &lcdc_device, }, | ||
| 1237 | { "A4MP", &fsi_device, }, | ||
| 1238 | { "A3SP", &sh_mmcif_device, }, | ||
| 1239 | { "A3SP", &sdhi0_device, }, | ||
| 1240 | { "A3SP", &sdhi1_device, }, | ||
| 1241 | { "A4R", &ceu_device, }, | ||
| 1242 | }; | ||
| 1234 | u32 srcr4; | 1243 | u32 srcr4; |
| 1235 | struct clk *clk; | 1244 | struct clk *clk; |
| 1236 | 1245 | ||
| @@ -1463,14 +1472,8 @@ static void __init ap4evb_init(void) | |||
| 1463 | 1472 | ||
| 1464 | platform_add_devices(ap4evb_devices, ARRAY_SIZE(ap4evb_devices)); | 1473 | platform_add_devices(ap4evb_devices, ARRAY_SIZE(ap4evb_devices)); |
| 1465 | 1474 | ||
| 1466 | rmobile_add_device_to_domain(&sh7372_pd_a4lc, &lcdc1_device); | 1475 | rmobile_add_devices_to_domains(domain_devices, |
| 1467 | rmobile_add_device_to_domain(&sh7372_pd_a4lc, &lcdc_device); | 1476 | ARRAY_SIZE(domain_devices)); |
| 1468 | rmobile_add_device_to_domain(&sh7372_pd_a4mp, &fsi_device); | ||
| 1469 | |||
| 1470 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sh_mmcif_device); | ||
| 1471 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi0_device); | ||
| 1472 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi1_device); | ||
| 1473 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &ceu_device); | ||
| 1474 | 1477 | ||
| 1475 | hdmi_init_pm_clock(); | 1478 | hdmi_init_pm_clock(); |
| 1476 | fsi_init_pm_clock(); | 1479 | fsi_init_pm_clock(); |
| @@ -1485,6 +1488,6 @@ MACHINE_START(AP4EVB, "ap4evb") | |||
| 1485 | .init_irq = sh7372_init_irq, | 1488 | .init_irq = sh7372_init_irq, |
| 1486 | .handle_irq = shmobile_handle_irq_intc, | 1489 | .handle_irq = shmobile_handle_irq_intc, |
| 1487 | .init_machine = ap4evb_init, | 1490 | .init_machine = ap4evb_init, |
| 1488 | .init_late = shmobile_init_late, | 1491 | .init_late = sh7372_pm_init_late, |
| 1489 | .timer = &shmobile_timer, | 1492 | .timer = &shmobile_timer, |
| 1490 | MACHINE_END | 1493 | MACHINE_END |
diff --git a/arch/arm/mach-shmobile/board-armadillo800eva.c b/arch/arm/mach-shmobile/board-armadillo800eva.c index c6593d394273..2912eab3b967 100644 --- a/arch/arm/mach-shmobile/board-armadillo800eva.c +++ b/arch/arm/mach-shmobile/board-armadillo800eva.c | |||
| @@ -1209,10 +1209,10 @@ static void __init eva_init(void) | |||
| 1209 | 1209 | ||
| 1210 | eva_clock_init(); | 1210 | eva_clock_init(); |
| 1211 | 1211 | ||
| 1212 | rmobile_add_device_to_domain(&r8a7740_pd_a4lc, &lcdc0_device); | 1212 | rmobile_add_device_to_domain("A4LC", &lcdc0_device); |
| 1213 | rmobile_add_device_to_domain(&r8a7740_pd_a4lc, &hdmi_lcdc_device); | 1213 | rmobile_add_device_to_domain("A4LC", &hdmi_lcdc_device); |
| 1214 | if (usb) | 1214 | if (usb) |
| 1215 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, usb); | 1215 | rmobile_add_device_to_domain("A3SP", usb); |
| 1216 | } | 1216 | } |
| 1217 | 1217 | ||
| 1218 | static void __init eva_earlytimer_init(void) | 1218 | static void __init eva_earlytimer_init(void) |
diff --git a/arch/arm/mach-shmobile/board-mackerel.c b/arch/arm/mach-shmobile/board-mackerel.c index 62783b5d8813..0c27c810cf99 100644 --- a/arch/arm/mach-shmobile/board-mackerel.c +++ b/arch/arm/mach-shmobile/board-mackerel.c | |||
| @@ -1412,6 +1412,22 @@ static struct i2c_board_info i2c1_devices[] = { | |||
| 1412 | #define USCCR1 IOMEM(0xE6058144) | 1412 | #define USCCR1 IOMEM(0xE6058144) |
| 1413 | static void __init mackerel_init(void) | 1413 | static void __init mackerel_init(void) |
| 1414 | { | 1414 | { |
| 1415 | struct pm_domain_device domain_devices[] = { | ||
| 1416 | { "A4LC", &lcdc_device, }, | ||
| 1417 | { "A4LC", &hdmi_lcdc_device, }, | ||
| 1418 | { "A4LC", &meram_device, }, | ||
| 1419 | { "A4MP", &fsi_device, }, | ||
| 1420 | { "A3SP", &usbhs0_device, }, | ||
| 1421 | { "A3SP", &usbhs1_device, }, | ||
| 1422 | { "A3SP", &nand_flash_device, }, | ||
| 1423 | { "A3SP", &sh_mmcif_device, }, | ||
| 1424 | { "A3SP", &sdhi0_device, }, | ||
| 1425 | #if !defined(CONFIG_MMC_SH_MMCIF) && !defined(CONFIG_MMC_SH_MMCIF_MODULE) | ||
| 1426 | { "A3SP", &sdhi1_device, }, | ||
| 1427 | #endif | ||
| 1428 | { "A3SP", &sdhi2_device, }, | ||
| 1429 | { "A4R", &ceu_device, }, | ||
| 1430 | }; | ||
| 1415 | u32 srcr4; | 1431 | u32 srcr4; |
| 1416 | struct clk *clk; | 1432 | struct clk *clk; |
| 1417 | 1433 | ||
| @@ -1626,20 +1642,8 @@ static void __init mackerel_init(void) | |||
| 1626 | 1642 | ||
| 1627 | platform_add_devices(mackerel_devices, ARRAY_SIZE(mackerel_devices)); | 1643 | platform_add_devices(mackerel_devices, ARRAY_SIZE(mackerel_devices)); |
| 1628 | 1644 | ||
| 1629 | rmobile_add_device_to_domain(&sh7372_pd_a4lc, &lcdc_device); | 1645 | rmobile_add_devices_to_domains(domain_devices, |
| 1630 | rmobile_add_device_to_domain(&sh7372_pd_a4lc, &hdmi_lcdc_device); | 1646 | ARRAY_SIZE(domain_devices)); |
| 1631 | rmobile_add_device_to_domain(&sh7372_pd_a4lc, &meram_device); | ||
| 1632 | rmobile_add_device_to_domain(&sh7372_pd_a4mp, &fsi_device); | ||
| 1633 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usbhs0_device); | ||
| 1634 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usbhs1_device); | ||
| 1635 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &nand_flash_device); | ||
| 1636 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sh_mmcif_device); | ||
| 1637 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi0_device); | ||
| 1638 | #if !defined(CONFIG_MMC_SH_MMCIF) && !defined(CONFIG_MMC_SH_MMCIF_MODULE) | ||
| 1639 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi1_device); | ||
| 1640 | #endif | ||
| 1641 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi2_device); | ||
| 1642 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &ceu_device); | ||
| 1643 | 1647 | ||
| 1644 | hdmi_init_pm_clock(); | 1648 | hdmi_init_pm_clock(); |
| 1645 | sh7372_pm_init(); | 1649 | sh7372_pm_init(); |
| @@ -1653,6 +1657,6 @@ MACHINE_START(MACKEREL, "mackerel") | |||
| 1653 | .init_irq = sh7372_init_irq, | 1657 | .init_irq = sh7372_init_irq, |
| 1654 | .handle_irq = shmobile_handle_irq_intc, | 1658 | .handle_irq = shmobile_handle_irq_intc, |
| 1655 | .init_machine = mackerel_init, | 1659 | .init_machine = mackerel_init, |
| 1656 | .init_late = shmobile_init_late, | 1660 | .init_late = sh7372_pm_init_late, |
| 1657 | .timer = &shmobile_timer, | 1661 | .timer = &shmobile_timer, |
| 1658 | MACHINE_END | 1662 | MACHINE_END |
diff --git a/arch/arm/mach-shmobile/common.c b/arch/arm/mach-shmobile/common.c deleted file mode 100644 index 608aba9d60d7..000000000000 --- a/arch/arm/mach-shmobile/common.c +++ /dev/null | |||
| @@ -1,24 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * This program is free software; you can redistribute it and/or modify | ||
| 3 | * it under the terms of the GNU General Public License as published by | ||
| 4 | * the Free Software Foundation; version 2 of the License. | ||
| 5 | * | ||
| 6 | * This program is distributed in the hope that it will be useful, | ||
| 7 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 8 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 9 | * GNU General Public License for more details. | ||
| 10 | * | ||
| 11 | * You should have received a copy of the GNU General Public License | ||
| 12 | * along with this program; if not, write to the Free Software | ||
| 13 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
| 14 | * | ||
| 15 | */ | ||
| 16 | #include <linux/kernel.h> | ||
| 17 | #include <linux/init.h> | ||
| 18 | #include <mach/common.h> | ||
| 19 | |||
| 20 | void __init shmobile_init_late(void) | ||
| 21 | { | ||
| 22 | shmobile_suspend_init(); | ||
| 23 | shmobile_cpuidle_init(); | ||
| 24 | } | ||
diff --git a/arch/arm/mach-shmobile/cpuidle.c b/arch/arm/mach-shmobile/cpuidle.c index 7b541e911ab4..9e050268cde4 100644 --- a/arch/arm/mach-shmobile/cpuidle.c +++ b/arch/arm/mach-shmobile/cpuidle.c | |||
| @@ -16,51 +16,38 @@ | |||
| 16 | #include <asm/cpuidle.h> | 16 | #include <asm/cpuidle.h> |
| 17 | #include <asm/io.h> | 17 | #include <asm/io.h> |
| 18 | 18 | ||
| 19 | static void shmobile_enter_wfi(void) | 19 | int shmobile_enter_wfi(struct cpuidle_device *dev, struct cpuidle_driver *drv, |
| 20 | int index) | ||
| 20 | { | 21 | { |
| 21 | cpu_do_idle(); | 22 | cpu_do_idle(); |
| 22 | } | 23 | return 0; |
| 23 | |||
| 24 | void (*shmobile_cpuidle_modes[CPUIDLE_STATE_MAX])(void) = { | ||
| 25 | shmobile_enter_wfi, /* regular sleep mode */ | ||
| 26 | }; | ||
| 27 | |||
| 28 | static int shmobile_cpuidle_enter(struct cpuidle_device *dev, | ||
| 29 | struct cpuidle_driver *drv, | ||
| 30 | int index) | ||
| 31 | { | ||
| 32 | shmobile_cpuidle_modes[index](); | ||
| 33 | |||
| 34 | return index; | ||
| 35 | } | 24 | } |
| 36 | 25 | ||
| 37 | static struct cpuidle_device shmobile_cpuidle_dev; | 26 | static struct cpuidle_device shmobile_cpuidle_dev; |
| 38 | static struct cpuidle_driver shmobile_cpuidle_driver = { | 27 | static struct cpuidle_driver shmobile_cpuidle_default_driver = { |
| 39 | .name = "shmobile_cpuidle", | 28 | .name = "shmobile_cpuidle", |
| 40 | .owner = THIS_MODULE, | 29 | .owner = THIS_MODULE, |
| 41 | .en_core_tk_irqen = 1, | 30 | .en_core_tk_irqen = 1, |
| 42 | .states[0] = ARM_CPUIDLE_WFI_STATE, | 31 | .states[0] = ARM_CPUIDLE_WFI_STATE, |
| 32 | .states[0].enter = shmobile_enter_wfi, | ||
| 43 | .safe_state_index = 0, /* C1 */ | 33 | .safe_state_index = 0, /* C1 */ |
| 44 | .state_count = 1, | 34 | .state_count = 1, |
| 45 | }; | 35 | }; |
| 46 | 36 | ||
| 47 | void (*shmobile_cpuidle_setup)(struct cpuidle_driver *drv); | 37 | static struct cpuidle_driver *cpuidle_drv = &shmobile_cpuidle_default_driver; |
| 38 | |||
| 39 | void shmobile_cpuidle_set_driver(struct cpuidle_driver *drv) | ||
| 40 | { | ||
| 41 | cpuidle_drv = drv; | ||
| 42 | } | ||
| 48 | 43 | ||
| 49 | int shmobile_cpuidle_init(void) | 44 | int shmobile_cpuidle_init(void) |
| 50 | { | 45 | { |
| 51 | struct cpuidle_device *dev = &shmobile_cpuidle_dev; | 46 | struct cpuidle_device *dev = &shmobile_cpuidle_dev; |
| 52 | struct cpuidle_driver *drv = &shmobile_cpuidle_driver; | ||
| 53 | int i; | ||
| 54 | |||
| 55 | for (i = 0; i < CPUIDLE_STATE_MAX; i++) | ||
| 56 | drv->states[i].enter = shmobile_cpuidle_enter; | ||
| 57 | |||
| 58 | if (shmobile_cpuidle_setup) | ||
| 59 | shmobile_cpuidle_setup(drv); | ||
| 60 | 47 | ||
| 61 | cpuidle_register_driver(drv); | 48 | cpuidle_register_driver(cpuidle_drv); |
| 62 | 49 | ||
| 63 | dev->state_count = drv->state_count; | 50 | dev->state_count = cpuidle_drv->state_count; |
| 64 | cpuidle_register_device(dev); | 51 | cpuidle_register_device(dev); |
| 65 | 52 | ||
| 66 | return 0; | 53 | return 0; |
diff --git a/arch/arm/mach-shmobile/include/mach/common.h b/arch/arm/mach-shmobile/include/mach/common.h index f80f9c549393..ed77ab8c9143 100644 --- a/arch/arm/mach-shmobile/include/mach/common.h +++ b/arch/arm/mach-shmobile/include/mach/common.h | |||
| @@ -13,8 +13,10 @@ extern int shmobile_clk_init(void); | |||
| 13 | extern void shmobile_handle_irq_intc(struct pt_regs *); | 13 | extern void shmobile_handle_irq_intc(struct pt_regs *); |
| 14 | extern struct platform_suspend_ops shmobile_suspend_ops; | 14 | extern struct platform_suspend_ops shmobile_suspend_ops; |
| 15 | struct cpuidle_driver; | 15 | struct cpuidle_driver; |
| 16 | extern void (*shmobile_cpuidle_modes[])(void); | 16 | struct cpuidle_device; |
| 17 | extern void (*shmobile_cpuidle_setup)(struct cpuidle_driver *drv); | 17 | extern int shmobile_enter_wfi(struct cpuidle_device *dev, |
| 18 | struct cpuidle_driver *drv, int index); | ||
| 19 | extern void shmobile_cpuidle_set_driver(struct cpuidle_driver *drv); | ||
| 18 | 20 | ||
| 19 | extern void sh7367_init_irq(void); | 21 | extern void sh7367_init_irq(void); |
| 20 | extern void sh7367_map_io(void); | 22 | extern void sh7367_map_io(void); |
| @@ -75,8 +77,6 @@ extern void r8a7740_meram_workaround(void); | |||
| 75 | 77 | ||
| 76 | extern void r8a7779_register_twd(void); | 78 | extern void r8a7779_register_twd(void); |
| 77 | 79 | ||
| 78 | extern void shmobile_init_late(void); | ||
| 79 | |||
| 80 | #ifdef CONFIG_SUSPEND | 80 | #ifdef CONFIG_SUSPEND |
| 81 | int shmobile_suspend_init(void); | 81 | int shmobile_suspend_init(void); |
| 82 | #else | 82 | #else |
| @@ -100,4 +100,10 @@ static inline int shmobile_cpu_is_dead(unsigned int cpu) { return 1; } | |||
| 100 | 100 | ||
| 101 | extern void shmobile_smp_init_cpus(unsigned int ncores); | 101 | extern void shmobile_smp_init_cpus(unsigned int ncores); |
| 102 | 102 | ||
| 103 | static inline void shmobile_init_late(void) | ||
| 104 | { | ||
| 105 | shmobile_suspend_init(); | ||
| 106 | shmobile_cpuidle_init(); | ||
| 107 | } | ||
| 108 | |||
| 103 | #endif /* __ARCH_MACH_COMMON_H */ | 109 | #endif /* __ARCH_MACH_COMMON_H */ |
diff --git a/arch/arm/mach-shmobile/include/mach/pm-rmobile.h b/arch/arm/mach-shmobile/include/mach/pm-rmobile.h index 5a402840fe28..690553a06887 100644 --- a/arch/arm/mach-shmobile/include/mach/pm-rmobile.h +++ b/arch/arm/mach-shmobile/include/mach/pm-rmobile.h | |||
| @@ -12,6 +12,8 @@ | |||
| 12 | 12 | ||
| 13 | #include <linux/pm_domain.h> | 13 | #include <linux/pm_domain.h> |
| 14 | 14 | ||
| 15 | #define DEFAULT_DEV_LATENCY_NS 250000 | ||
| 16 | |||
| 15 | struct platform_device; | 17 | struct platform_device; |
| 16 | 18 | ||
| 17 | struct rmobile_pm_domain { | 19 | struct rmobile_pm_domain { |
| @@ -29,16 +31,33 @@ struct rmobile_pm_domain *to_rmobile_pd(struct generic_pm_domain *d) | |||
| 29 | return container_of(d, struct rmobile_pm_domain, genpd); | 31 | return container_of(d, struct rmobile_pm_domain, genpd); |
| 30 | } | 32 | } |
| 31 | 33 | ||
| 34 | struct pm_domain_device { | ||
| 35 | const char *domain_name; | ||
| 36 | struct platform_device *pdev; | ||
| 37 | }; | ||
| 38 | |||
| 32 | #ifdef CONFIG_PM | 39 | #ifdef CONFIG_PM |
| 33 | extern void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd); | 40 | extern void rmobile_init_domains(struct rmobile_pm_domain domains[], int num); |
| 34 | extern void rmobile_add_device_to_domain(struct rmobile_pm_domain *rmobile_pd, | 41 | extern void rmobile_add_device_to_domain_td(const char *domain_name, |
| 35 | struct platform_device *pdev); | 42 | struct platform_device *pdev, |
| 36 | extern void rmobile_pm_add_subdomain(struct rmobile_pm_domain *rmobile_pd, | 43 | struct gpd_timing_data *td); |
| 37 | struct rmobile_pm_domain *rmobile_sd); | 44 | |
| 45 | static inline void rmobile_add_device_to_domain(const char *domain_name, | ||
| 46 | struct platform_device *pdev) | ||
| 47 | { | ||
| 48 | rmobile_add_device_to_domain_td(domain_name, pdev, NULL); | ||
| 49 | } | ||
| 50 | |||
| 51 | extern void rmobile_add_devices_to_domains(struct pm_domain_device data[], | ||
| 52 | int size); | ||
| 38 | #else | 53 | #else |
| 39 | #define rmobile_init_pm_domain(pd) do { } while (0) | 54 | |
| 40 | #define rmobile_add_device_to_domain(pd, pdev) do { } while (0) | 55 | #define rmobile_init_domains(domains, num) do { } while (0) |
| 41 | #define rmobile_pm_add_subdomain(pd, sd) do { } while (0) | 56 | #define rmobile_add_device_to_domain_td(name, pdev, td) do { } while (0) |
| 57 | #define rmobile_add_device_to_domain(name, pdev) do { } while (0) | ||
| 58 | |||
| 59 | static inline void rmobile_add_devices_to_domains(struct pm_domain_device d[], | ||
| 60 | int size) {} | ||
| 42 | #endif /* CONFIG_PM */ | 61 | #endif /* CONFIG_PM */ |
| 43 | 62 | ||
| 44 | #endif /* PM_RMOBILE_H */ | 63 | #endif /* PM_RMOBILE_H */ |
diff --git a/arch/arm/mach-shmobile/include/mach/r8a7740.h b/arch/arm/mach-shmobile/include/mach/r8a7740.h index 7143147780df..59d252f4cf97 100644 --- a/arch/arm/mach-shmobile/include/mach/r8a7740.h +++ b/arch/arm/mach-shmobile/include/mach/r8a7740.h | |||
| @@ -607,9 +607,9 @@ enum { | |||
| 607 | }; | 607 | }; |
| 608 | 608 | ||
| 609 | #ifdef CONFIG_PM | 609 | #ifdef CONFIG_PM |
| 610 | extern struct rmobile_pm_domain r8a7740_pd_a4s; | 610 | extern void __init r8a7740_init_pm_domains(void); |
| 611 | extern struct rmobile_pm_domain r8a7740_pd_a3sp; | 611 | #else |
| 612 | extern struct rmobile_pm_domain r8a7740_pd_a4lc; | 612 | static inline void r8a7740_init_pm_domains(void) {} |
| 613 | #endif /* CONFIG_PM */ | 613 | #endif /* CONFIG_PM */ |
| 614 | 614 | ||
| 615 | #endif /* __ASM_R8A7740_H__ */ | 615 | #endif /* __ASM_R8A7740_H__ */ |
diff --git a/arch/arm/mach-shmobile/include/mach/r8a7779.h b/arch/arm/mach-shmobile/include/mach/r8a7779.h index f504c5e81b47..499f52d2a4a1 100644 --- a/arch/arm/mach-shmobile/include/mach/r8a7779.h +++ b/arch/arm/mach-shmobile/include/mach/r8a7779.h | |||
| @@ -347,17 +347,9 @@ extern int r8a7779_sysc_power_down(struct r8a7779_pm_ch *r8a7779_ch); | |||
| 347 | extern int r8a7779_sysc_power_up(struct r8a7779_pm_ch *r8a7779_ch); | 347 | extern int r8a7779_sysc_power_up(struct r8a7779_pm_ch *r8a7779_ch); |
| 348 | 348 | ||
| 349 | #ifdef CONFIG_PM | 349 | #ifdef CONFIG_PM |
| 350 | extern struct r8a7779_pm_domain r8a7779_sh4a; | 350 | extern void __init r8a7779_init_pm_domains(void); |
| 351 | extern struct r8a7779_pm_domain r8a7779_sgx; | ||
| 352 | extern struct r8a7779_pm_domain r8a7779_vdp1; | ||
| 353 | extern struct r8a7779_pm_domain r8a7779_impx3; | ||
| 354 | |||
| 355 | extern void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd); | ||
| 356 | extern void r8a7779_add_device_to_domain(struct r8a7779_pm_domain *r8a7779_pd, | ||
| 357 | struct platform_device *pdev); | ||
| 358 | #else | 351 | #else |
| 359 | #define r8a7779_init_pm_domain(pd) do { } while (0) | 352 | static inline void r8a7779_init_pm_domains(void) {} |
| 360 | #define r8a7779_add_device_to_domain(pd, pdev) do { } while (0) | ||
| 361 | #endif /* CONFIG_PM */ | 353 | #endif /* CONFIG_PM */ |
| 362 | 354 | ||
| 363 | extern struct smp_operations r8a7779_smp_ops; | 355 | extern struct smp_operations r8a7779_smp_ops; |
diff --git a/arch/arm/mach-shmobile/include/mach/sh7372.h b/arch/arm/mach-shmobile/include/mach/sh7372.h index b59048e6d8fd..eb98b45c5089 100644 --- a/arch/arm/mach-shmobile/include/mach/sh7372.h +++ b/arch/arm/mach-shmobile/include/mach/sh7372.h | |||
| @@ -478,21 +478,17 @@ extern struct clk sh7372_fsibck_clk; | |||
| 478 | extern struct clk sh7372_fsidiva_clk; | 478 | extern struct clk sh7372_fsidiva_clk; |
| 479 | extern struct clk sh7372_fsidivb_clk; | 479 | extern struct clk sh7372_fsidivb_clk; |
| 480 | 480 | ||
| 481 | #ifdef CONFIG_PM | ||
| 482 | extern struct rmobile_pm_domain sh7372_pd_a4lc; | ||
| 483 | extern struct rmobile_pm_domain sh7372_pd_a4mp; | ||
| 484 | extern struct rmobile_pm_domain sh7372_pd_d4; | ||
| 485 | extern struct rmobile_pm_domain sh7372_pd_a4r; | ||
| 486 | extern struct rmobile_pm_domain sh7372_pd_a3rv; | ||
| 487 | extern struct rmobile_pm_domain sh7372_pd_a3ri; | ||
| 488 | extern struct rmobile_pm_domain sh7372_pd_a4s; | ||
| 489 | extern struct rmobile_pm_domain sh7372_pd_a3sp; | ||
| 490 | extern struct rmobile_pm_domain sh7372_pd_a3sg; | ||
| 491 | #endif /* CONFIG_PM */ | ||
| 492 | |||
| 493 | extern void sh7372_intcs_suspend(void); | 481 | extern void sh7372_intcs_suspend(void); |
| 494 | extern void sh7372_intcs_resume(void); | 482 | extern void sh7372_intcs_resume(void); |
| 495 | extern void sh7372_intca_suspend(void); | 483 | extern void sh7372_intca_suspend(void); |
| 496 | extern void sh7372_intca_resume(void); | 484 | extern void sh7372_intca_resume(void); |
| 497 | 485 | ||
| 486 | #ifdef CONFIG_PM | ||
| 487 | extern void __init sh7372_init_pm_domains(void); | ||
| 488 | #else | ||
| 489 | static inline void sh7372_init_pm_domains(void) {} | ||
| 490 | #endif | ||
| 491 | |||
| 492 | extern void __init sh7372_pm_init_late(void); | ||
| 493 | |||
| 498 | #endif /* __ASM_SH7372_H__ */ | 494 | #endif /* __ASM_SH7372_H__ */ |
diff --git a/arch/arm/mach-shmobile/pm-r8a7740.c b/arch/arm/mach-shmobile/pm-r8a7740.c index 893504d012a6..21e5316d2d88 100644 --- a/arch/arm/mach-shmobile/pm-r8a7740.c +++ b/arch/arm/mach-shmobile/pm-r8a7740.c | |||
| @@ -21,14 +21,6 @@ static int r8a7740_pd_a4s_suspend(void) | |||
| 21 | return -EBUSY; | 21 | return -EBUSY; |
| 22 | } | 22 | } |
| 23 | 23 | ||
| 24 | struct rmobile_pm_domain r8a7740_pd_a4s = { | ||
| 25 | .genpd.name = "A4S", | ||
| 26 | .bit_shift = 10, | ||
| 27 | .gov = &pm_domain_always_on_gov, | ||
| 28 | .no_debug = true, | ||
| 29 | .suspend = r8a7740_pd_a4s_suspend, | ||
| 30 | }; | ||
| 31 | |||
| 32 | static int r8a7740_pd_a3sp_suspend(void) | 24 | static int r8a7740_pd_a3sp_suspend(void) |
| 33 | { | 25 | { |
| 34 | /* | 26 | /* |
| @@ -38,17 +30,31 @@ static int r8a7740_pd_a3sp_suspend(void) | |||
| 38 | return console_suspend_enabled ? 0 : -EBUSY; | 30 | return console_suspend_enabled ? 0 : -EBUSY; |
| 39 | } | 31 | } |
| 40 | 32 | ||
| 41 | struct rmobile_pm_domain r8a7740_pd_a3sp = { | 33 | static struct rmobile_pm_domain r8a7740_pm_domains[] = { |
| 42 | .genpd.name = "A3SP", | 34 | { |
| 43 | .bit_shift = 11, | 35 | .genpd.name = "A4S", |
| 44 | .gov = &pm_domain_always_on_gov, | 36 | .bit_shift = 10, |
| 45 | .no_debug = true, | 37 | .gov = &pm_domain_always_on_gov, |
| 46 | .suspend = r8a7740_pd_a3sp_suspend, | 38 | .no_debug = true, |
| 39 | .suspend = r8a7740_pd_a4s_suspend, | ||
| 40 | }, | ||
| 41 | { | ||
| 42 | .genpd.name = "A3SP", | ||
| 43 | .bit_shift = 11, | ||
| 44 | .gov = &pm_domain_always_on_gov, | ||
| 45 | .no_debug = true, | ||
| 46 | .suspend = r8a7740_pd_a3sp_suspend, | ||
| 47 | }, | ||
| 48 | { | ||
| 49 | .genpd.name = "A4LC", | ||
| 50 | .bit_shift = 1, | ||
| 51 | }, | ||
| 47 | }; | 52 | }; |
| 48 | 53 | ||
| 49 | struct rmobile_pm_domain r8a7740_pd_a4lc = { | 54 | void __init r8a7740_init_pm_domains(void) |
| 50 | .genpd.name = "A4LC", | 55 | { |
| 51 | .bit_shift = 1, | 56 | rmobile_init_domains(r8a7740_pm_domains, ARRAY_SIZE(r8a7740_pm_domains)); |
| 52 | }; | 57 | pm_genpd_add_subdomain_names("A4S", "A3SP"); |
| 58 | } | ||
| 53 | 59 | ||
| 54 | #endif /* CONFIG_PM */ | 60 | #endif /* CONFIG_PM */ |
diff --git a/arch/arm/mach-shmobile/pm-r8a7779.c b/arch/arm/mach-shmobile/pm-r8a7779.c index a18a4ae16d2b..d50a8e9b94a4 100644 --- a/arch/arm/mach-shmobile/pm-r8a7779.c +++ b/arch/arm/mach-shmobile/pm-r8a7779.c | |||
| @@ -183,7 +183,7 @@ static bool pd_active_wakeup(struct device *dev) | |||
| 183 | return true; | 183 | return true; |
| 184 | } | 184 | } |
| 185 | 185 | ||
| 186 | void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd) | 186 | static void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd) |
| 187 | { | 187 | { |
| 188 | struct generic_pm_domain *genpd = &r8a7779_pd->genpd; | 188 | struct generic_pm_domain *genpd = &r8a7779_pd->genpd; |
| 189 | 189 | ||
| @@ -199,43 +199,44 @@ void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd) | |||
| 199 | pd_power_up(&r8a7779_pd->genpd); | 199 | pd_power_up(&r8a7779_pd->genpd); |
| 200 | } | 200 | } |
| 201 | 201 | ||
| 202 | void r8a7779_add_device_to_domain(struct r8a7779_pm_domain *r8a7779_pd, | 202 | static struct r8a7779_pm_domain r8a7779_pm_domains[] = { |
| 203 | struct platform_device *pdev) | 203 | { |
| 204 | { | 204 | .genpd.name = "SH4A", |
| 205 | struct device *dev = &pdev->dev; | 205 | .ch = { |
| 206 | 206 | .chan_offs = 0x80, /* PWRSR1 .. PWRER1 */ | |
| 207 | pm_genpd_add_device(&r8a7779_pd->genpd, dev); | 207 | .isr_bit = 16, /* SH4A */ |
| 208 | if (pm_clk_no_clocks(dev)) | 208 | }, |
| 209 | pm_clk_add(dev, NULL); | 209 | }, |
| 210 | } | 210 | { |
| 211 | 211 | .genpd.name = "SGX", | |
| 212 | struct r8a7779_pm_domain r8a7779_sh4a = { | 212 | .ch = { |
| 213 | .ch = { | 213 | .chan_offs = 0xc0, /* PWRSR2 .. PWRER2 */ |
| 214 | .chan_offs = 0x80, /* PWRSR1 .. PWRER1 */ | 214 | .isr_bit = 20, /* SGX */ |
| 215 | .isr_bit = 16, /* SH4A */ | 215 | }, |
| 216 | } | 216 | }, |
| 217 | }; | 217 | { |
| 218 | 218 | .genpd.name = "VDP1", | |
| 219 | struct r8a7779_pm_domain r8a7779_sgx = { | 219 | .ch = { |
| 220 | .ch = { | 220 | .chan_offs = 0x100, /* PWRSR3 .. PWRER3 */ |
| 221 | .chan_offs = 0xc0, /* PWRSR2 .. PWRER2 */ | 221 | .isr_bit = 21, /* VDP */ |
| 222 | .isr_bit = 20, /* SGX */ | 222 | }, |
| 223 | } | 223 | }, |
| 224 | { | ||
| 225 | .genpd.name = "IMPX3", | ||
| 226 | .ch = { | ||
| 227 | .chan_offs = 0x140, /* PWRSR4 .. PWRER4 */ | ||
| 228 | .isr_bit = 24, /* IMP */ | ||
| 229 | }, | ||
| 230 | }, | ||
| 224 | }; | 231 | }; |
| 225 | 232 | ||
| 226 | struct r8a7779_pm_domain r8a7779_vdp1 = { | 233 | void __init r8a7779_init_pm_domains(void) |
| 227 | .ch = { | 234 | { |
| 228 | .chan_offs = 0x100, /* PWRSR3 .. PWRER3 */ | 235 | int j; |
| 229 | .isr_bit = 21, /* VDP */ | ||
| 230 | } | ||
| 231 | }; | ||
| 232 | 236 | ||
| 233 | struct r8a7779_pm_domain r8a7779_impx3 = { | 237 | for (j = 0; j < ARRAY_SIZE(r8a7779_pm_domains); j++) |
| 234 | .ch = { | 238 | r8a7779_init_pm_domain(&r8a7779_pm_domains[j]); |
| 235 | .chan_offs = 0x140, /* PWRSR4 .. PWRER4 */ | 239 | } |
| 236 | .isr_bit = 24, /* IMP */ | ||
| 237 | } | ||
| 238 | }; | ||
| 239 | 240 | ||
| 240 | #endif /* CONFIG_PM */ | 241 | #endif /* CONFIG_PM */ |
| 241 | 242 | ||
diff --git a/arch/arm/mach-shmobile/pm-rmobile.c b/arch/arm/mach-shmobile/pm-rmobile.c index 32e177275e47..1fc05d9453d0 100644 --- a/arch/arm/mach-shmobile/pm-rmobile.c +++ b/arch/arm/mach-shmobile/pm-rmobile.c | |||
| @@ -134,7 +134,7 @@ static int rmobile_pd_start_dev(struct device *dev) | |||
| 134 | return ret; | 134 | return ret; |
| 135 | } | 135 | } |
| 136 | 136 | ||
| 137 | void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd) | 137 | static void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd) |
| 138 | { | 138 | { |
| 139 | struct generic_pm_domain *genpd = &rmobile_pd->genpd; | 139 | struct generic_pm_domain *genpd = &rmobile_pd->genpd; |
| 140 | struct dev_power_governor *gov = rmobile_pd->gov; | 140 | struct dev_power_governor *gov = rmobile_pd->gov; |
| @@ -149,19 +149,38 @@ void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd) | |||
| 149 | __rmobile_pd_power_up(rmobile_pd, false); | 149 | __rmobile_pd_power_up(rmobile_pd, false); |
| 150 | } | 150 | } |
| 151 | 151 | ||
| 152 | void rmobile_add_device_to_domain(struct rmobile_pm_domain *rmobile_pd, | 152 | void rmobile_init_domains(struct rmobile_pm_domain domains[], int num) |
| 153 | struct platform_device *pdev) | 153 | { |
| 154 | int j; | ||
| 155 | |||
| 156 | for (j = 0; j < num; j++) | ||
| 157 | rmobile_init_pm_domain(&domains[j]); | ||
| 158 | } | ||
| 159 | |||
| 160 | void rmobile_add_device_to_domain_td(const char *domain_name, | ||
| 161 | struct platform_device *pdev, | ||
| 162 | struct gpd_timing_data *td) | ||
| 154 | { | 163 | { |
| 155 | struct device *dev = &pdev->dev; | 164 | struct device *dev = &pdev->dev; |
| 156 | 165 | ||
| 157 | pm_genpd_add_device(&rmobile_pd->genpd, dev); | 166 | __pm_genpd_name_add_device(domain_name, dev, td); |
| 158 | if (pm_clk_no_clocks(dev)) | 167 | if (pm_clk_no_clocks(dev)) |
| 159 | pm_clk_add(dev, NULL); | 168 | pm_clk_add(dev, NULL); |
| 160 | } | 169 | } |
| 161 | 170 | ||
| 162 | void rmobile_pm_add_subdomain(struct rmobile_pm_domain *rmobile_pd, | 171 | void rmobile_add_devices_to_domains(struct pm_domain_device data[], |
| 163 | struct rmobile_pm_domain *rmobile_sd) | 172 | int size) |
| 164 | { | 173 | { |
| 165 | pm_genpd_add_subdomain(&rmobile_pd->genpd, &rmobile_sd->genpd); | 174 | struct gpd_timing_data latencies = { |
| 175 | .stop_latency_ns = DEFAULT_DEV_LATENCY_NS, | ||
| 176 | .start_latency_ns = DEFAULT_DEV_LATENCY_NS, | ||
| 177 | .save_state_latency_ns = DEFAULT_DEV_LATENCY_NS, | ||
| 178 | .restore_state_latency_ns = DEFAULT_DEV_LATENCY_NS, | ||
| 179 | }; | ||
| 180 | int j; | ||
| 181 | |||
| 182 | for (j = 0; j < size; j++) | ||
| 183 | rmobile_add_device_to_domain_td(data[j].domain_name, | ||
| 184 | data[j].pdev, &latencies); | ||
| 166 | } | 185 | } |
| 167 | #endif /* CONFIG_PM */ | 186 | #endif /* CONFIG_PM */ |
diff --git a/arch/arm/mach-shmobile/pm-sh7372.c b/arch/arm/mach-shmobile/pm-sh7372.c index 162121842a2b..a0826a48dd08 100644 --- a/arch/arm/mach-shmobile/pm-sh7372.c +++ b/arch/arm/mach-shmobile/pm-sh7372.c | |||
| @@ -21,6 +21,7 @@ | |||
| 21 | #include <linux/irq.h> | 21 | #include <linux/irq.h> |
| 22 | #include <linux/bitrev.h> | 22 | #include <linux/bitrev.h> |
| 23 | #include <linux/console.h> | 23 | #include <linux/console.h> |
| 24 | #include <asm/cpuidle.h> | ||
| 24 | #include <asm/io.h> | 25 | #include <asm/io.h> |
| 25 | #include <asm/tlbflush.h> | 26 | #include <asm/tlbflush.h> |
| 26 | #include <asm/suspend.h> | 27 | #include <asm/suspend.h> |
| @@ -72,20 +73,7 @@ | |||
| 72 | 73 | ||
| 73 | #ifdef CONFIG_PM | 74 | #ifdef CONFIG_PM |
| 74 | 75 | ||
| 75 | struct rmobile_pm_domain sh7372_pd_a4lc = { | 76 | #define PM_DOMAIN_ON_OFF_LATENCY_NS 250000 |
| 76 | .genpd.name = "A4LC", | ||
| 77 | .bit_shift = 1, | ||
| 78 | }; | ||
| 79 | |||
| 80 | struct rmobile_pm_domain sh7372_pd_a4mp = { | ||
| 81 | .genpd.name = "A4MP", | ||
| 82 | .bit_shift = 2, | ||
| 83 | }; | ||
| 84 | |||
| 85 | struct rmobile_pm_domain sh7372_pd_d4 = { | ||
| 86 | .genpd.name = "D4", | ||
| 87 | .bit_shift = 3, | ||
| 88 | }; | ||
| 89 | 77 | ||
| 90 | static int sh7372_a4r_pd_suspend(void) | 78 | static int sh7372_a4r_pd_suspend(void) |
| 91 | { | 79 | { |
| @@ -94,39 +82,25 @@ static int sh7372_a4r_pd_suspend(void) | |||
| 94 | return 0; | 82 | return 0; |
| 95 | } | 83 | } |
| 96 | 84 | ||
| 97 | struct rmobile_pm_domain sh7372_pd_a4r = { | 85 | static bool a4s_suspend_ready; |
| 98 | .genpd.name = "A4R", | ||
| 99 | .bit_shift = 5, | ||
| 100 | .suspend = sh7372_a4r_pd_suspend, | ||
| 101 | .resume = sh7372_intcs_resume, | ||
| 102 | }; | ||
| 103 | 86 | ||
| 104 | struct rmobile_pm_domain sh7372_pd_a3rv = { | 87 | static int sh7372_a4s_pd_suspend(void) |
| 105 | .genpd.name = "A3RV", | ||
| 106 | .bit_shift = 6, | ||
| 107 | }; | ||
| 108 | |||
| 109 | struct rmobile_pm_domain sh7372_pd_a3ri = { | ||
| 110 | .genpd.name = "A3RI", | ||
| 111 | .bit_shift = 8, | ||
| 112 | }; | ||
| 113 | |||
| 114 | static int sh7372_pd_a4s_suspend(void) | ||
| 115 | { | 88 | { |
| 116 | /* | 89 | /* |
| 117 | * The A4S domain contains the CPU core and therefore it should | 90 | * The A4S domain contains the CPU core and therefore it should |
| 118 | * only be turned off if the CPU is in use. | 91 | * only be turned off if the CPU is not in use. This may happen |
| 92 | * during system suspend, when SYSC is going to be used for generating | ||
| 93 | * resume signals and a4s_suspend_ready is set to let | ||
| 94 | * sh7372_enter_suspend() know that it can turn A4S off. | ||
| 119 | */ | 95 | */ |
| 96 | a4s_suspend_ready = true; | ||
| 120 | return -EBUSY; | 97 | return -EBUSY; |
| 121 | } | 98 | } |
| 122 | 99 | ||
| 123 | struct rmobile_pm_domain sh7372_pd_a4s = { | 100 | static void sh7372_a4s_pd_resume(void) |
| 124 | .genpd.name = "A4S", | 101 | { |
| 125 | .bit_shift = 10, | 102 | a4s_suspend_ready = false; |
| 126 | .gov = &pm_domain_always_on_gov, | 103 | } |
| 127 | .no_debug = true, | ||
| 128 | .suspend = sh7372_pd_a4s_suspend, | ||
| 129 | }; | ||
| 130 | 104 | ||
| 131 | static int sh7372_a3sp_pd_suspend(void) | 105 | static int sh7372_a3sp_pd_suspend(void) |
| 132 | { | 106 | { |
| @@ -137,18 +111,80 @@ static int sh7372_a3sp_pd_suspend(void) | |||
| 137 | return console_suspend_enabled ? 0 : -EBUSY; | 111 | return console_suspend_enabled ? 0 : -EBUSY; |
| 138 | } | 112 | } |
| 139 | 113 | ||
| 140 | struct rmobile_pm_domain sh7372_pd_a3sp = { | 114 | static struct rmobile_pm_domain sh7372_pm_domains[] = { |
| 141 | .genpd.name = "A3SP", | 115 | { |
| 142 | .bit_shift = 11, | 116 | .genpd.name = "A4LC", |
| 143 | .gov = &pm_domain_always_on_gov, | 117 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, |
| 144 | .no_debug = true, | 118 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, |
| 145 | .suspend = sh7372_a3sp_pd_suspend, | 119 | .bit_shift = 1, |
| 120 | }, | ||
| 121 | { | ||
| 122 | .genpd.name = "A4MP", | ||
| 123 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 124 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 125 | .bit_shift = 2, | ||
| 126 | }, | ||
| 127 | { | ||
| 128 | .genpd.name = "D4", | ||
| 129 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 130 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 131 | .bit_shift = 3, | ||
| 132 | }, | ||
| 133 | { | ||
| 134 | .genpd.name = "A4R", | ||
| 135 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 136 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 137 | .bit_shift = 5, | ||
| 138 | .suspend = sh7372_a4r_pd_suspend, | ||
| 139 | .resume = sh7372_intcs_resume, | ||
| 140 | }, | ||
| 141 | { | ||
| 142 | .genpd.name = "A3RV", | ||
| 143 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 144 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 145 | .bit_shift = 6, | ||
| 146 | }, | ||
| 147 | { | ||
| 148 | .genpd.name = "A3RI", | ||
| 149 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 150 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 151 | .bit_shift = 8, | ||
| 152 | }, | ||
| 153 | { | ||
| 154 | .genpd.name = "A4S", | ||
| 155 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 156 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 157 | .bit_shift = 10, | ||
| 158 | .gov = &pm_domain_always_on_gov, | ||
| 159 | .no_debug = true, | ||
| 160 | .suspend = sh7372_a4s_pd_suspend, | ||
| 161 | .resume = sh7372_a4s_pd_resume, | ||
| 162 | }, | ||
| 163 | { | ||
| 164 | .genpd.name = "A3SP", | ||
| 165 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 166 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 167 | .bit_shift = 11, | ||
| 168 | .gov = &pm_domain_always_on_gov, | ||
| 169 | .no_debug = true, | ||
| 170 | .suspend = sh7372_a3sp_pd_suspend, | ||
| 171 | }, | ||
| 172 | { | ||
| 173 | .genpd.name = "A3SG", | ||
| 174 | .genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 175 | .genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS, | ||
| 176 | .bit_shift = 13, | ||
| 177 | }, | ||
| 146 | }; | 178 | }; |
| 147 | 179 | ||
| 148 | struct rmobile_pm_domain sh7372_pd_a3sg = { | 180 | void __init sh7372_init_pm_domains(void) |
| 149 | .genpd.name = "A3SG", | 181 | { |
| 150 | .bit_shift = 13, | 182 | rmobile_init_domains(sh7372_pm_domains, ARRAY_SIZE(sh7372_pm_domains)); |
| 151 | }; | 183 | pm_genpd_add_subdomain_names("A4LC", "A3RV"); |
| 184 | pm_genpd_add_subdomain_names("A4R", "A4LC"); | ||
| 185 | pm_genpd_add_subdomain_names("A4S", "A3SG"); | ||
| 186 | pm_genpd_add_subdomain_names("A4S", "A3SP"); | ||
| 187 | } | ||
| 152 | 188 | ||
| 153 | #endif /* CONFIG_PM */ | 189 | #endif /* CONFIG_PM */ |
| 154 | 190 | ||
| @@ -304,6 +340,21 @@ static void sh7372_enter_a3sm_common(int pllc0_on) | |||
| 304 | sh7372_set_reset_vector(__pa(sh7372_resume_core_standby_sysc)); | 340 | sh7372_set_reset_vector(__pa(sh7372_resume_core_standby_sysc)); |
| 305 | sh7372_enter_sysc(pllc0_on, 1 << 12); | 341 | sh7372_enter_sysc(pllc0_on, 1 << 12); |
| 306 | } | 342 | } |
| 343 | |||
| 344 | static void sh7372_enter_a4s_common(int pllc0_on) | ||
| 345 | { | ||
| 346 | sh7372_intca_suspend(); | ||
| 347 | sh7372_set_reset_vector(SMFRAM); | ||
| 348 | sh7372_enter_sysc(pllc0_on, 1 << 10); | ||
| 349 | sh7372_intca_resume(); | ||
| 350 | } | ||
| 351 | |||
| 352 | static void sh7372_pm_setup_smfram(void) | ||
| 353 | { | ||
| 354 | memcpy((void *)SMFRAM, sh7372_resume_core_standby_sysc, 0x100); | ||
| 355 | } | ||
| 356 | #else | ||
| 357 | static inline void sh7372_pm_setup_smfram(void) {} | ||
| 307 | #endif /* CONFIG_SUSPEND || CONFIG_CPU_IDLE */ | 358 | #endif /* CONFIG_SUSPEND || CONFIG_CPU_IDLE */ |
| 308 | 359 | ||
| 309 | #ifdef CONFIG_CPU_IDLE | 360 | #ifdef CONFIG_CPU_IDLE |
| @@ -313,7 +364,8 @@ static int sh7372_do_idle_core_standby(unsigned long unused) | |||
| 313 | return 0; | 364 | return 0; |
| 314 | } | 365 | } |
| 315 | 366 | ||
| 316 | static void sh7372_enter_core_standby(void) | 367 | static int sh7372_enter_core_standby(struct cpuidle_device *dev, |
| 368 | struct cpuidle_driver *drv, int index) | ||
| 317 | { | 369 | { |
| 318 | sh7372_set_reset_vector(__pa(sh7372_resume_core_standby_sysc)); | 370 | sh7372_set_reset_vector(__pa(sh7372_resume_core_standby_sysc)); |
| 319 | 371 | ||
| @@ -324,83 +376,102 @@ static void sh7372_enter_core_standby(void) | |||
| 324 | 376 | ||
| 325 | /* disable reset vector translation */ | 377 | /* disable reset vector translation */ |
| 326 | __raw_writel(0, SBAR); | 378 | __raw_writel(0, SBAR); |
| 379 | |||
| 380 | return 1; | ||
| 327 | } | 381 | } |
| 328 | 382 | ||
| 329 | static void sh7372_enter_a3sm_pll_on(void) | 383 | static int sh7372_enter_a3sm_pll_on(struct cpuidle_device *dev, |
| 384 | struct cpuidle_driver *drv, int index) | ||
| 330 | { | 385 | { |
| 331 | sh7372_enter_a3sm_common(1); | 386 | sh7372_enter_a3sm_common(1); |
| 387 | return 2; | ||
| 332 | } | 388 | } |
| 333 | 389 | ||
| 334 | static void sh7372_enter_a3sm_pll_off(void) | 390 | static int sh7372_enter_a3sm_pll_off(struct cpuidle_device *dev, |
| 391 | struct cpuidle_driver *drv, int index) | ||
| 335 | { | 392 | { |
| 336 | sh7372_enter_a3sm_common(0); | 393 | sh7372_enter_a3sm_common(0); |
| 394 | return 3; | ||
| 337 | } | 395 | } |
| 338 | 396 | ||
| 339 | static void sh7372_cpuidle_setup(struct cpuidle_driver *drv) | 397 | static int sh7372_enter_a4s(struct cpuidle_device *dev, |
| 398 | struct cpuidle_driver *drv, int index) | ||
| 340 | { | 399 | { |
| 341 | struct cpuidle_state *state = &drv->states[drv->state_count]; | 400 | unsigned long msk, msk2; |
| 342 | 401 | ||
| 343 | snprintf(state->name, CPUIDLE_NAME_LEN, "C2"); | 402 | if (!sh7372_sysc_valid(&msk, &msk2)) |
| 344 | strncpy(state->desc, "Core Standby Mode", CPUIDLE_DESC_LEN); | 403 | return sh7372_enter_a3sm_pll_off(dev, drv, index); |
| 345 | state->exit_latency = 10; | 404 | |
| 346 | state->target_residency = 20 + 10; | 405 | sh7372_setup_sysc(msk, msk2); |
| 347 | state->flags = CPUIDLE_FLAG_TIME_VALID; | 406 | sh7372_enter_a4s_common(0); |
| 348 | shmobile_cpuidle_modes[drv->state_count] = sh7372_enter_core_standby; | 407 | return 4; |
| 349 | drv->state_count++; | ||
| 350 | |||
| 351 | state = &drv->states[drv->state_count]; | ||
| 352 | snprintf(state->name, CPUIDLE_NAME_LEN, "C3"); | ||
| 353 | strncpy(state->desc, "A3SM PLL ON", CPUIDLE_DESC_LEN); | ||
| 354 | state->exit_latency = 20; | ||
| 355 | state->target_residency = 30 + 20; | ||
| 356 | state->flags = CPUIDLE_FLAG_TIME_VALID; | ||
| 357 | shmobile_cpuidle_modes[drv->state_count] = sh7372_enter_a3sm_pll_on; | ||
| 358 | drv->state_count++; | ||
| 359 | |||
| 360 | state = &drv->states[drv->state_count]; | ||
| 361 | snprintf(state->name, CPUIDLE_NAME_LEN, "C4"); | ||
| 362 | strncpy(state->desc, "A3SM PLL OFF", CPUIDLE_DESC_LEN); | ||
| 363 | state->exit_latency = 120; | ||
| 364 | state->target_residency = 30 + 120; | ||
| 365 | state->flags = CPUIDLE_FLAG_TIME_VALID; | ||
| 366 | shmobile_cpuidle_modes[drv->state_count] = sh7372_enter_a3sm_pll_off; | ||
| 367 | drv->state_count++; | ||
| 368 | } | 408 | } |
| 369 | 409 | ||
| 410 | static struct cpuidle_driver sh7372_cpuidle_driver = { | ||
| 411 | .name = "sh7372_cpuidle", | ||
| 412 | .owner = THIS_MODULE, | ||
| 413 | .en_core_tk_irqen = 1, | ||
| 414 | .state_count = 5, | ||
| 415 | .safe_state_index = 0, /* C1 */ | ||
| 416 | .states[0] = ARM_CPUIDLE_WFI_STATE, | ||
| 417 | .states[0].enter = shmobile_enter_wfi, | ||
| 418 | .states[1] = { | ||
| 419 | .name = "C2", | ||
| 420 | .desc = "Core Standby Mode", | ||
| 421 | .exit_latency = 10, | ||
| 422 | .target_residency = 20 + 10, | ||
| 423 | .flags = CPUIDLE_FLAG_TIME_VALID, | ||
| 424 | .enter = sh7372_enter_core_standby, | ||
| 425 | }, | ||
| 426 | .states[2] = { | ||
| 427 | .name = "C3", | ||
| 428 | .desc = "A3SM PLL ON", | ||
| 429 | .exit_latency = 20, | ||
| 430 | .target_residency = 30 + 20, | ||
| 431 | .flags = CPUIDLE_FLAG_TIME_VALID, | ||
| 432 | .enter = sh7372_enter_a3sm_pll_on, | ||
| 433 | }, | ||
| 434 | .states[3] = { | ||
| 435 | .name = "C4", | ||
| 436 | .desc = "A3SM PLL OFF", | ||
| 437 | .exit_latency = 120, | ||
| 438 | .target_residency = 30 + 120, | ||
| 439 | .flags = CPUIDLE_FLAG_TIME_VALID, | ||
| 440 | .enter = sh7372_enter_a3sm_pll_off, | ||
| 441 | }, | ||
| 442 | .states[4] = { | ||
| 443 | .name = "C5", | ||
| 444 | .desc = "A4S PLL OFF", | ||
| 445 | .exit_latency = 240, | ||
| 446 | .target_residency = 30 + 240, | ||
| 447 | .flags = CPUIDLE_FLAG_TIME_VALID, | ||
| 448 | .enter = sh7372_enter_a4s, | ||
| 449 | .disabled = true, | ||
| 450 | }, | ||
| 451 | }; | ||
| 452 | |||
| 370 | static void sh7372_cpuidle_init(void) | 453 | static void sh7372_cpuidle_init(void) |
| 371 | { | 454 | { |
| 372 | shmobile_cpuidle_setup = sh7372_cpuidle_setup; | 455 | shmobile_cpuidle_set_driver(&sh7372_cpuidle_driver); |
| 373 | } | 456 | } |
| 374 | #else | 457 | #else |
| 375 | static void sh7372_cpuidle_init(void) {} | 458 | static void sh7372_cpuidle_init(void) {} |
| 376 | #endif | 459 | #endif |
| 377 | 460 | ||
| 378 | #ifdef CONFIG_SUSPEND | 461 | #ifdef CONFIG_SUSPEND |
| 379 | static void sh7372_enter_a4s_common(int pllc0_on) | ||
| 380 | { | ||
| 381 | sh7372_intca_suspend(); | ||
| 382 | memcpy((void *)SMFRAM, sh7372_resume_core_standby_sysc, 0x100); | ||
| 383 | sh7372_set_reset_vector(SMFRAM); | ||
| 384 | sh7372_enter_sysc(pllc0_on, 1 << 10); | ||
| 385 | sh7372_intca_resume(); | ||
| 386 | } | ||
| 387 | |||
| 388 | static int sh7372_enter_suspend(suspend_state_t suspend_state) | 462 | static int sh7372_enter_suspend(suspend_state_t suspend_state) |
| 389 | { | 463 | { |
| 390 | unsigned long msk, msk2; | 464 | unsigned long msk, msk2; |
| 391 | 465 | ||
| 392 | /* check active clocks to determine potential wakeup sources */ | 466 | /* check active clocks to determine potential wakeup sources */ |
| 393 | if (sh7372_sysc_valid(&msk, &msk2)) { | 467 | if (sh7372_sysc_valid(&msk, &msk2) && a4s_suspend_ready) { |
| 394 | if (!console_suspend_enabled && | 468 | /* convert INTC mask/sense to SYSC mask/sense */ |
| 395 | sh7372_pd_a4s.genpd.status == GPD_STATE_POWER_OFF) { | 469 | sh7372_setup_sysc(msk, msk2); |
| 396 | /* convert INTC mask/sense to SYSC mask/sense */ | 470 | |
| 397 | sh7372_setup_sysc(msk, msk2); | 471 | /* enter A4S sleep with PLLC0 off */ |
| 398 | 472 | pr_debug("entering A4S\n"); | |
| 399 | /* enter A4S sleep with PLLC0 off */ | 473 | sh7372_enter_a4s_common(0); |
| 400 | pr_debug("entering A4S\n"); | 474 | return 0; |
| 401 | sh7372_enter_a4s_common(0); | ||
| 402 | return 0; | ||
| 403 | } | ||
| 404 | } | 475 | } |
| 405 | 476 | ||
| 406 | /* default to enter A3SM sleep with PLLC0 off */ | 477 | /* default to enter A3SM sleep with PLLC0 off */ |
| @@ -426,7 +497,7 @@ static int sh7372_pm_notifier_fn(struct notifier_block *notifier, | |||
| 426 | * executed during system suspend and resume, respectively, so | 497 | * executed during system suspend and resume, respectively, so |
| 427 | * that those functions don't crash while accessing the INTCS. | 498 | * that those functions don't crash while accessing the INTCS. |
| 428 | */ | 499 | */ |
| 429 | pm_genpd_poweron(&sh7372_pd_a4r.genpd); | 500 | pm_genpd_name_poweron("A4R"); |
| 430 | break; | 501 | break; |
| 431 | case PM_POST_SUSPEND: | 502 | case PM_POST_SUSPEND: |
| 432 | pm_genpd_poweroff_unused(); | 503 | pm_genpd_poweroff_unused(); |
| @@ -455,6 +526,14 @@ void __init sh7372_pm_init(void) | |||
| 455 | /* do not convert A3SM, A3SP, A3SG, A4R power down into A4S */ | 526 | /* do not convert A3SM, A3SP, A3SG, A4R power down into A4S */ |
| 456 | __raw_writel(0, PDNSEL); | 527 | __raw_writel(0, PDNSEL); |
| 457 | 528 | ||
| 529 | sh7372_pm_setup_smfram(); | ||
| 530 | |||
| 458 | sh7372_suspend_init(); | 531 | sh7372_suspend_init(); |
| 459 | sh7372_cpuidle_init(); | 532 | sh7372_cpuidle_init(); |
| 460 | } | 533 | } |
| 534 | |||
| 535 | void __init sh7372_pm_init_late(void) | ||
| 536 | { | ||
| 537 | shmobile_init_late(); | ||
| 538 | pm_genpd_name_attach_cpuidle("A4S", 4); | ||
| 539 | } | ||
diff --git a/arch/arm/mach-shmobile/setup-r8a7740.c b/arch/arm/mach-shmobile/setup-r8a7740.c index 78948a9dba0e..11bb1d984197 100644 --- a/arch/arm/mach-shmobile/setup-r8a7740.c +++ b/arch/arm/mach-shmobile/setup-r8a7740.c | |||
| @@ -673,12 +673,7 @@ void __init r8a7740_add_standard_devices(void) | |||
| 673 | r8a7740_i2c_workaround(&i2c0_device); | 673 | r8a7740_i2c_workaround(&i2c0_device); |
| 674 | r8a7740_i2c_workaround(&i2c1_device); | 674 | r8a7740_i2c_workaround(&i2c1_device); |
| 675 | 675 | ||
| 676 | /* PM domain */ | 676 | r8a7740_init_pm_domains(); |
| 677 | rmobile_init_pm_domain(&r8a7740_pd_a4s); | ||
| 678 | rmobile_init_pm_domain(&r8a7740_pd_a3sp); | ||
| 679 | rmobile_init_pm_domain(&r8a7740_pd_a4lc); | ||
| 680 | |||
| 681 | rmobile_pm_add_subdomain(&r8a7740_pd_a4s, &r8a7740_pd_a3sp); | ||
| 682 | 677 | ||
| 683 | /* add devices */ | 678 | /* add devices */ |
| 684 | platform_add_devices(r8a7740_early_devices, | 679 | platform_add_devices(r8a7740_early_devices, |
| @@ -688,16 +683,16 @@ void __init r8a7740_add_standard_devices(void) | |||
| 688 | 683 | ||
| 689 | /* add devices to PM domain */ | 684 | /* add devices to PM domain */ |
| 690 | 685 | ||
| 691 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif0_device); | 686 | rmobile_add_device_to_domain("A3SP", &scif0_device); |
| 692 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif1_device); | 687 | rmobile_add_device_to_domain("A3SP", &scif1_device); |
| 693 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif2_device); | 688 | rmobile_add_device_to_domain("A3SP", &scif2_device); |
| 694 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif3_device); | 689 | rmobile_add_device_to_domain("A3SP", &scif3_device); |
| 695 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif4_device); | 690 | rmobile_add_device_to_domain("A3SP", &scif4_device); |
| 696 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif5_device); | 691 | rmobile_add_device_to_domain("A3SP", &scif5_device); |
| 697 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif6_device); | 692 | rmobile_add_device_to_domain("A3SP", &scif6_device); |
| 698 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scif7_device); | 693 | rmobile_add_device_to_domain("A3SP", &scif7_device); |
| 699 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &scifb_device); | 694 | rmobile_add_device_to_domain("A3SP", &scifb_device); |
| 700 | rmobile_add_device_to_domain(&r8a7740_pd_a3sp, &i2c1_device); | 695 | rmobile_add_device_to_domain("A3SP", &i2c1_device); |
| 701 | } | 696 | } |
| 702 | 697 | ||
| 703 | static void __init r8a7740_earlytimer_init(void) | 698 | static void __init r8a7740_earlytimer_init(void) |
diff --git a/arch/arm/mach-shmobile/setup-r8a7779.c b/arch/arm/mach-shmobile/setup-r8a7779.c index e98e46f6cf55..2917668f0091 100644 --- a/arch/arm/mach-shmobile/setup-r8a7779.c +++ b/arch/arm/mach-shmobile/setup-r8a7779.c | |||
| @@ -251,10 +251,7 @@ void __init r8a7779_add_standard_devices(void) | |||
| 251 | #endif | 251 | #endif |
| 252 | r8a7779_pm_init(); | 252 | r8a7779_pm_init(); |
| 253 | 253 | ||
| 254 | r8a7779_init_pm_domain(&r8a7779_sh4a); | 254 | r8a7779_init_pm_domains(); |
| 255 | r8a7779_init_pm_domain(&r8a7779_sgx); | ||
| 256 | r8a7779_init_pm_domain(&r8a7779_vdp1); | ||
| 257 | r8a7779_init_pm_domain(&r8a7779_impx3); | ||
| 258 | 255 | ||
| 259 | platform_add_devices(r8a7779_early_devices, | 256 | platform_add_devices(r8a7779_early_devices, |
| 260 | ARRAY_SIZE(r8a7779_early_devices)); | 257 | ARRAY_SIZE(r8a7779_early_devices)); |
diff --git a/arch/arm/mach-shmobile/setup-sh7372.c b/arch/arm/mach-shmobile/setup-sh7372.c index 838a87be1d5c..a07954fbcd22 100644 --- a/arch/arm/mach-shmobile/setup-sh7372.c +++ b/arch/arm/mach-shmobile/setup-sh7372.c | |||
| @@ -1001,21 +1001,34 @@ static struct platform_device *sh7372_late_devices[] __initdata = { | |||
| 1001 | 1001 | ||
| 1002 | void __init sh7372_add_standard_devices(void) | 1002 | void __init sh7372_add_standard_devices(void) |
| 1003 | { | 1003 | { |
| 1004 | rmobile_init_pm_domain(&sh7372_pd_a4lc); | 1004 | struct pm_domain_device domain_devices[] = { |
| 1005 | rmobile_init_pm_domain(&sh7372_pd_a4mp); | 1005 | { "A3RV", &vpu_device, }, |
| 1006 | rmobile_init_pm_domain(&sh7372_pd_d4); | 1006 | { "A4MP", &spu0_device, }, |
| 1007 | rmobile_init_pm_domain(&sh7372_pd_a4r); | 1007 | { "A4MP", &spu1_device, }, |
| 1008 | rmobile_init_pm_domain(&sh7372_pd_a3rv); | 1008 | { "A3SP", &scif0_device, }, |
| 1009 | rmobile_init_pm_domain(&sh7372_pd_a3ri); | 1009 | { "A3SP", &scif1_device, }, |
| 1010 | rmobile_init_pm_domain(&sh7372_pd_a4s); | 1010 | { "A3SP", &scif2_device, }, |
| 1011 | rmobile_init_pm_domain(&sh7372_pd_a3sp); | 1011 | { "A3SP", &scif3_device, }, |
| 1012 | rmobile_init_pm_domain(&sh7372_pd_a3sg); | 1012 | { "A3SP", &scif4_device, }, |
| 1013 | 1013 | { "A3SP", &scif5_device, }, | |
| 1014 | rmobile_pm_add_subdomain(&sh7372_pd_a4lc, &sh7372_pd_a3rv); | 1014 | { "A3SP", &scif6_device, }, |
| 1015 | rmobile_pm_add_subdomain(&sh7372_pd_a4r, &sh7372_pd_a4lc); | 1015 | { "A3SP", &iic1_device, }, |
| 1016 | 1016 | { "A3SP", &dma0_device, }, | |
| 1017 | rmobile_pm_add_subdomain(&sh7372_pd_a4s, &sh7372_pd_a3sg); | 1017 | { "A3SP", &dma1_device, }, |
| 1018 | rmobile_pm_add_subdomain(&sh7372_pd_a4s, &sh7372_pd_a3sp); | 1018 | { "A3SP", &dma2_device, }, |
| 1019 | { "A3SP", &usb_dma0_device, }, | ||
| 1020 | { "A3SP", &usb_dma1_device, }, | ||
| 1021 | { "A4R", &iic0_device, }, | ||
| 1022 | { "A4R", &veu0_device, }, | ||
| 1023 | { "A4R", &veu1_device, }, | ||
| 1024 | { "A4R", &veu2_device, }, | ||
| 1025 | { "A4R", &veu3_device, }, | ||
| 1026 | { "A4R", &jpu_device, }, | ||
| 1027 | { "A4R", &tmu00_device, }, | ||
| 1028 | { "A4R", &tmu01_device, }, | ||
| 1029 | }; | ||
| 1030 | |||
| 1031 | sh7372_init_pm_domains(); | ||
| 1019 | 1032 | ||
| 1020 | platform_add_devices(sh7372_early_devices, | 1033 | platform_add_devices(sh7372_early_devices, |
| 1021 | ARRAY_SIZE(sh7372_early_devices)); | 1034 | ARRAY_SIZE(sh7372_early_devices)); |
| @@ -1023,30 +1036,8 @@ void __init sh7372_add_standard_devices(void) | |||
| 1023 | platform_add_devices(sh7372_late_devices, | 1036 | platform_add_devices(sh7372_late_devices, |
| 1024 | ARRAY_SIZE(sh7372_late_devices)); | 1037 | ARRAY_SIZE(sh7372_late_devices)); |
| 1025 | 1038 | ||
| 1026 | rmobile_add_device_to_domain(&sh7372_pd_a3rv, &vpu_device); | 1039 | rmobile_add_devices_to_domains(domain_devices, |
| 1027 | rmobile_add_device_to_domain(&sh7372_pd_a4mp, &spu0_device); | 1040 | ARRAY_SIZE(domain_devices)); |
| 1028 | rmobile_add_device_to_domain(&sh7372_pd_a4mp, &spu1_device); | ||
| 1029 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif0_device); | ||
| 1030 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif1_device); | ||
| 1031 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif2_device); | ||
| 1032 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif3_device); | ||
| 1033 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif4_device); | ||
| 1034 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif5_device); | ||
| 1035 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif6_device); | ||
| 1036 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &iic1_device); | ||
| 1037 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &dma0_device); | ||
| 1038 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &dma1_device); | ||
| 1039 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &dma2_device); | ||
| 1040 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usb_dma0_device); | ||
| 1041 | rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usb_dma1_device); | ||
| 1042 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &iic0_device); | ||
| 1043 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu0_device); | ||
| 1044 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu1_device); | ||
| 1045 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu2_device); | ||
| 1046 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu3_device); | ||
| 1047 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &jpu_device); | ||
| 1048 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &tmu00_device); | ||
| 1049 | rmobile_add_device_to_domain(&sh7372_pd_a4r, &tmu01_device); | ||
| 1050 | } | 1041 | } |
| 1051 | 1042 | ||
| 1052 | static void __init sh7372_earlytimer_init(void) | 1043 | static void __init sh7372_earlytimer_init(void) |
diff --git a/arch/arm/mach-u300/i2c.c b/arch/arm/mach-u300/i2c.c index 0d4620ed853c..96800aa1316d 100644 --- a/arch/arm/mach-u300/i2c.c +++ b/arch/arm/mach-u300/i2c.c | |||
| @@ -9,7 +9,7 @@ | |||
| 9 | */ | 9 | */ |
| 10 | #include <linux/kernel.h> | 10 | #include <linux/kernel.h> |
| 11 | #include <linux/i2c.h> | 11 | #include <linux/i2c.h> |
| 12 | #include <linux/mfd/abx500.h> | 12 | #include <linux/mfd/ab3100.h> |
| 13 | #include <linux/regulator/machine.h> | 13 | #include <linux/regulator/machine.h> |
| 14 | #include <linux/amba/bus.h> | 14 | #include <linux/amba/bus.h> |
| 15 | #include <mach/irqs.h> | 15 | #include <mach/irqs.h> |
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 13f555d62491..477a2d23ddf1 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c | |||
| @@ -73,11 +73,18 @@ static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, | |||
| 73 | unsigned long offset, size_t size, enum dma_data_direction dir, | 73 | unsigned long offset, size_t size, enum dma_data_direction dir, |
| 74 | struct dma_attrs *attrs) | 74 | struct dma_attrs *attrs) |
| 75 | { | 75 | { |
| 76 | if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | 76 | if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) |
| 77 | __dma_page_cpu_to_dev(page, offset, size, dir); | 77 | __dma_page_cpu_to_dev(page, offset, size, dir); |
| 78 | return pfn_to_dma(dev, page_to_pfn(page)) + offset; | 78 | return pfn_to_dma(dev, page_to_pfn(page)) + offset; |
| 79 | } | 79 | } |
| 80 | 80 | ||
| 81 | static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, | ||
| 82 | unsigned long offset, size_t size, enum dma_data_direction dir, | ||
| 83 | struct dma_attrs *attrs) | ||
| 84 | { | ||
| 85 | return pfn_to_dma(dev, page_to_pfn(page)) + offset; | ||
| 86 | } | ||
| 87 | |||
| 81 | /** | 88 | /** |
| 82 | * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() | 89 | * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() |
| 83 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices | 90 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices |
| @@ -96,7 +103,7 @@ static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, | |||
| 96 | size_t size, enum dma_data_direction dir, | 103 | size_t size, enum dma_data_direction dir, |
| 97 | struct dma_attrs *attrs) | 104 | struct dma_attrs *attrs) |
| 98 | { | 105 | { |
| 99 | if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | 106 | if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) |
| 100 | __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), | 107 | __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), |
| 101 | handle & ~PAGE_MASK, size, dir); | 108 | handle & ~PAGE_MASK, size, dir); |
| 102 | } | 109 | } |
| @@ -106,8 +113,7 @@ static void arm_dma_sync_single_for_cpu(struct device *dev, | |||
| 106 | { | 113 | { |
| 107 | unsigned int offset = handle & (PAGE_SIZE - 1); | 114 | unsigned int offset = handle & (PAGE_SIZE - 1); |
| 108 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); | 115 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); |
| 109 | if (!arch_is_coherent()) | 116 | __dma_page_dev_to_cpu(page, offset, size, dir); |
| 110 | __dma_page_dev_to_cpu(page, offset, size, dir); | ||
| 111 | } | 117 | } |
| 112 | 118 | ||
| 113 | static void arm_dma_sync_single_for_device(struct device *dev, | 119 | static void arm_dma_sync_single_for_device(struct device *dev, |
| @@ -115,8 +121,7 @@ static void arm_dma_sync_single_for_device(struct device *dev, | |||
| 115 | { | 121 | { |
| 116 | unsigned int offset = handle & (PAGE_SIZE - 1); | 122 | unsigned int offset = handle & (PAGE_SIZE - 1); |
| 117 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); | 123 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); |
| 118 | if (!arch_is_coherent()) | 124 | __dma_page_cpu_to_dev(page, offset, size, dir); |
| 119 | __dma_page_cpu_to_dev(page, offset, size, dir); | ||
| 120 | } | 125 | } |
| 121 | 126 | ||
| 122 | static int arm_dma_set_mask(struct device *dev, u64 dma_mask); | 127 | static int arm_dma_set_mask(struct device *dev, u64 dma_mask); |
| @@ -138,6 +143,22 @@ struct dma_map_ops arm_dma_ops = { | |||
| 138 | }; | 143 | }; |
| 139 | EXPORT_SYMBOL(arm_dma_ops); | 144 | EXPORT_SYMBOL(arm_dma_ops); |
| 140 | 145 | ||
| 146 | static void *arm_coherent_dma_alloc(struct device *dev, size_t size, | ||
| 147 | dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); | ||
| 148 | static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, | ||
| 149 | dma_addr_t handle, struct dma_attrs *attrs); | ||
| 150 | |||
| 151 | struct dma_map_ops arm_coherent_dma_ops = { | ||
| 152 | .alloc = arm_coherent_dma_alloc, | ||
| 153 | .free = arm_coherent_dma_free, | ||
| 154 | .mmap = arm_dma_mmap, | ||
| 155 | .get_sgtable = arm_dma_get_sgtable, | ||
| 156 | .map_page = arm_coherent_dma_map_page, | ||
| 157 | .map_sg = arm_dma_map_sg, | ||
| 158 | .set_dma_mask = arm_dma_set_mask, | ||
| 159 | }; | ||
| 160 | EXPORT_SYMBOL(arm_coherent_dma_ops); | ||
| 161 | |||
| 141 | static u64 get_coherent_dma_mask(struct device *dev) | 162 | static u64 get_coherent_dma_mask(struct device *dev) |
| 142 | { | 163 | { |
| 143 | u64 mask = (u64)arm_dma_limit; | 164 | u64 mask = (u64)arm_dma_limit; |
| @@ -586,7 +607,7 @@ static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, | |||
| 586 | 607 | ||
| 587 | 608 | ||
| 588 | static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, | 609 | static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, |
| 589 | gfp_t gfp, pgprot_t prot, const void *caller) | 610 | gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller) |
| 590 | { | 611 | { |
| 591 | u64 mask = get_coherent_dma_mask(dev); | 612 | u64 mask = get_coherent_dma_mask(dev); |
| 592 | struct page *page; | 613 | struct page *page; |
| @@ -619,7 +640,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, | |||
| 619 | *handle = DMA_ERROR_CODE; | 640 | *handle = DMA_ERROR_CODE; |
| 620 | size = PAGE_ALIGN(size); | 641 | size = PAGE_ALIGN(size); |
| 621 | 642 | ||
| 622 | if (arch_is_coherent() || nommu()) | 643 | if (is_coherent || nommu()) |
| 623 | addr = __alloc_simple_buffer(dev, size, gfp, &page); | 644 | addr = __alloc_simple_buffer(dev, size, gfp, &page); |
| 624 | else if (gfp & GFP_ATOMIC) | 645 | else if (gfp & GFP_ATOMIC) |
| 625 | addr = __alloc_from_pool(size, &page); | 646 | addr = __alloc_from_pool(size, &page); |
| @@ -647,7 +668,20 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, | |||
| 647 | if (dma_alloc_from_coherent(dev, size, handle, &memory)) | 668 | if (dma_alloc_from_coherent(dev, size, handle, &memory)) |
| 648 | return memory; | 669 | return memory; |
| 649 | 670 | ||
| 650 | return __dma_alloc(dev, size, handle, gfp, prot, | 671 | return __dma_alloc(dev, size, handle, gfp, prot, false, |
| 672 | __builtin_return_address(0)); | ||
| 673 | } | ||
| 674 | |||
| 675 | static void *arm_coherent_dma_alloc(struct device *dev, size_t size, | ||
| 676 | dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) | ||
| 677 | { | ||
| 678 | pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); | ||
| 679 | void *memory; | ||
| 680 | |||
| 681 | if (dma_alloc_from_coherent(dev, size, handle, &memory)) | ||
| 682 | return memory; | ||
| 683 | |||
| 684 | return __dma_alloc(dev, size, handle, gfp, prot, true, | ||
| 651 | __builtin_return_address(0)); | 685 | __builtin_return_address(0)); |
| 652 | } | 686 | } |
| 653 | 687 | ||
| @@ -684,8 +718,9 @@ int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, | |||
| 684 | /* | 718 | /* |
| 685 | * Free a buffer as defined by the above mapping. | 719 | * Free a buffer as defined by the above mapping. |
| 686 | */ | 720 | */ |
| 687 | void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, | 721 | static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, |
| 688 | dma_addr_t handle, struct dma_attrs *attrs) | 722 | dma_addr_t handle, struct dma_attrs *attrs, |
| 723 | bool is_coherent) | ||
| 689 | { | 724 | { |
| 690 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); | 725 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); |
| 691 | 726 | ||
| @@ -694,7 +729,7 @@ void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, | |||
| 694 | 729 | ||
| 695 | size = PAGE_ALIGN(size); | 730 | size = PAGE_ALIGN(size); |
| 696 | 731 | ||
| 697 | if (arch_is_coherent() || nommu()) { | 732 | if (is_coherent || nommu()) { |
| 698 | __dma_free_buffer(page, size); | 733 | __dma_free_buffer(page, size); |
| 699 | } else if (__free_from_pool(cpu_addr, size)) { | 734 | } else if (__free_from_pool(cpu_addr, size)) { |
| 700 | return; | 735 | return; |
| @@ -710,6 +745,18 @@ void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, | |||
| 710 | } | 745 | } |
| 711 | } | 746 | } |
| 712 | 747 | ||
| 748 | void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, | ||
| 749 | dma_addr_t handle, struct dma_attrs *attrs) | ||
| 750 | { | ||
| 751 | __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); | ||
| 752 | } | ||
| 753 | |||
| 754 | static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, | ||
| 755 | dma_addr_t handle, struct dma_attrs *attrs) | ||
| 756 | { | ||
| 757 | __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); | ||
| 758 | } | ||
| 759 | |||
| 713 | int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, | 760 | int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, |
| 714 | void *cpu_addr, dma_addr_t handle, size_t size, | 761 | void *cpu_addr, dma_addr_t handle, size_t size, |
| 715 | struct dma_attrs *attrs) | 762 | struct dma_attrs *attrs) |
| @@ -1012,11 +1059,12 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t | |||
| 1012 | if (!pages[i]) | 1059 | if (!pages[i]) |
| 1013 | goto error; | 1060 | goto error; |
| 1014 | 1061 | ||
| 1015 | if (order) | 1062 | if (order) { |
| 1016 | split_page(pages[i], order); | 1063 | split_page(pages[i], order); |
| 1017 | j = 1 << order; | 1064 | j = 1 << order; |
| 1018 | while (--j) | 1065 | while (--j) |
| 1019 | pages[i + j] = pages[i] + j; | 1066 | pages[i + j] = pages[i] + j; |
| 1067 | } | ||
| 1020 | 1068 | ||
| 1021 | __dma_clear_buffer(pages[i], PAGE_SIZE << order); | 1069 | __dma_clear_buffer(pages[i], PAGE_SIZE << order); |
| 1022 | i += 1 << order; | 1070 | i += 1 << order; |
| @@ -1303,7 +1351,8 @@ static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, | |||
| 1303 | */ | 1351 | */ |
| 1304 | static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, | 1352 | static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, |
| 1305 | size_t size, dma_addr_t *handle, | 1353 | size_t size, dma_addr_t *handle, |
| 1306 | enum dma_data_direction dir, struct dma_attrs *attrs) | 1354 | enum dma_data_direction dir, struct dma_attrs *attrs, |
| 1355 | bool is_coherent) | ||
| 1307 | { | 1356 | { |
| 1308 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | 1357 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; |
| 1309 | dma_addr_t iova, iova_base; | 1358 | dma_addr_t iova, iova_base; |
| @@ -1322,8 +1371,8 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, | |||
| 1322 | phys_addr_t phys = page_to_phys(sg_page(s)); | 1371 | phys_addr_t phys = page_to_phys(sg_page(s)); |
| 1323 | unsigned int len = PAGE_ALIGN(s->offset + s->length); | 1372 | unsigned int len = PAGE_ALIGN(s->offset + s->length); |
| 1324 | 1373 | ||
| 1325 | if (!arch_is_coherent() && | 1374 | if (!is_coherent && |
| 1326 | !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | 1375 | !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) |
| 1327 | __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); | 1376 | __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); |
| 1328 | 1377 | ||
| 1329 | ret = iommu_map(mapping->domain, iova, phys, len, 0); | 1378 | ret = iommu_map(mapping->domain, iova, phys, len, 0); |
| @@ -1341,20 +1390,9 @@ fail: | |||
| 1341 | return ret; | 1390 | return ret; |
| 1342 | } | 1391 | } |
| 1343 | 1392 | ||
| 1344 | /** | 1393 | static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, |
| 1345 | * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA | 1394 | enum dma_data_direction dir, struct dma_attrs *attrs, |
| 1346 | * @dev: valid struct device pointer | 1395 | bool is_coherent) |
| 1347 | * @sg: list of buffers | ||
| 1348 | * @nents: number of buffers to map | ||
| 1349 | * @dir: DMA transfer direction | ||
| 1350 | * | ||
| 1351 | * Map a set of buffers described by scatterlist in streaming mode for DMA. | ||
| 1352 | * The scatter gather list elements are merged together (if possible) and | ||
| 1353 | * tagged with the appropriate dma address and length. They are obtained via | ||
| 1354 | * sg_dma_{address,length}. | ||
| 1355 | */ | ||
| 1356 | int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, | ||
| 1357 | enum dma_data_direction dir, struct dma_attrs *attrs) | ||
| 1358 | { | 1396 | { |
| 1359 | struct scatterlist *s = sg, *dma = sg, *start = sg; | 1397 | struct scatterlist *s = sg, *dma = sg, *start = sg; |
| 1360 | int i, count = 0; | 1398 | int i, count = 0; |
| @@ -1370,7 +1408,7 @@ int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, | |||
| 1370 | 1408 | ||
| 1371 | if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { | 1409 | if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { |
| 1372 | if (__map_sg_chunk(dev, start, size, &dma->dma_address, | 1410 | if (__map_sg_chunk(dev, start, size, &dma->dma_address, |
| 1373 | dir, attrs) < 0) | 1411 | dir, attrs, is_coherent) < 0) |
| 1374 | goto bad_mapping; | 1412 | goto bad_mapping; |
| 1375 | 1413 | ||
| 1376 | dma->dma_address += offset; | 1414 | dma->dma_address += offset; |
| @@ -1383,7 +1421,8 @@ int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, | |||
| 1383 | } | 1421 | } |
| 1384 | size += s->length; | 1422 | size += s->length; |
| 1385 | } | 1423 | } |
| 1386 | if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs) < 0) | 1424 | if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, |
| 1425 | is_coherent) < 0) | ||
| 1387 | goto bad_mapping; | 1426 | goto bad_mapping; |
| 1388 | 1427 | ||
| 1389 | dma->dma_address += offset; | 1428 | dma->dma_address += offset; |
| @@ -1398,17 +1437,44 @@ bad_mapping: | |||
| 1398 | } | 1437 | } |
| 1399 | 1438 | ||
| 1400 | /** | 1439 | /** |
| 1401 | * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg | 1440 | * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA |
| 1402 | * @dev: valid struct device pointer | 1441 | * @dev: valid struct device pointer |
| 1403 | * @sg: list of buffers | 1442 | * @sg: list of buffers |
| 1404 | * @nents: number of buffers to unmap (same as was passed to dma_map_sg) | 1443 | * @nents: number of buffers to map |
| 1405 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | 1444 | * @dir: DMA transfer direction |
| 1406 | * | 1445 | * |
| 1407 | * Unmap a set of streaming mode DMA translations. Again, CPU access | 1446 | * Map a set of i/o coherent buffers described by scatterlist in streaming |
| 1408 | * rules concerning calls here are the same as for dma_unmap_single(). | 1447 | * mode for DMA. The scatter gather list elements are merged together (if |
| 1448 | * possible) and tagged with the appropriate dma address and length. They are | ||
| 1449 | * obtained via sg_dma_{address,length}. | ||
| 1409 | */ | 1450 | */ |
| 1410 | void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, | 1451 | int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, |
| 1411 | enum dma_data_direction dir, struct dma_attrs *attrs) | 1452 | int nents, enum dma_data_direction dir, struct dma_attrs *attrs) |
| 1453 | { | ||
| 1454 | return __iommu_map_sg(dev, sg, nents, dir, attrs, true); | ||
| 1455 | } | ||
| 1456 | |||
| 1457 | /** | ||
| 1458 | * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA | ||
| 1459 | * @dev: valid struct device pointer | ||
| 1460 | * @sg: list of buffers | ||
| 1461 | * @nents: number of buffers to map | ||
| 1462 | * @dir: DMA transfer direction | ||
| 1463 | * | ||
| 1464 | * Map a set of buffers described by scatterlist in streaming mode for DMA. | ||
| 1465 | * The scatter gather list elements are merged together (if possible) and | ||
| 1466 | * tagged with the appropriate dma address and length. They are obtained via | ||
| 1467 | * sg_dma_{address,length}. | ||
| 1468 | */ | ||
| 1469 | int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, | ||
| 1470 | int nents, enum dma_data_direction dir, struct dma_attrs *attrs) | ||
| 1471 | { | ||
| 1472 | return __iommu_map_sg(dev, sg, nents, dir, attrs, false); | ||
| 1473 | } | ||
| 1474 | |||
| 1475 | static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, | ||
| 1476 | int nents, enum dma_data_direction dir, struct dma_attrs *attrs, | ||
| 1477 | bool is_coherent) | ||
| 1412 | { | 1478 | { |
| 1413 | struct scatterlist *s; | 1479 | struct scatterlist *s; |
| 1414 | int i; | 1480 | int i; |
| @@ -1417,7 +1483,7 @@ void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, | |||
| 1417 | if (sg_dma_len(s)) | 1483 | if (sg_dma_len(s)) |
| 1418 | __iommu_remove_mapping(dev, sg_dma_address(s), | 1484 | __iommu_remove_mapping(dev, sg_dma_address(s), |
| 1419 | sg_dma_len(s)); | 1485 | sg_dma_len(s)); |
| 1420 | if (!arch_is_coherent() && | 1486 | if (!is_coherent && |
| 1421 | !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | 1487 | !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) |
| 1422 | __dma_page_dev_to_cpu(sg_page(s), s->offset, | 1488 | __dma_page_dev_to_cpu(sg_page(s), s->offset, |
| 1423 | s->length, dir); | 1489 | s->length, dir); |
| @@ -1425,6 +1491,38 @@ void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, | |||
| 1425 | } | 1491 | } |
| 1426 | 1492 | ||
| 1427 | /** | 1493 | /** |
| 1494 | * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg | ||
| 1495 | * @dev: valid struct device pointer | ||
| 1496 | * @sg: list of buffers | ||
| 1497 | * @nents: number of buffers to unmap (same as was passed to dma_map_sg) | ||
| 1498 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | ||
| 1499 | * | ||
| 1500 | * Unmap a set of streaming mode DMA translations. Again, CPU access | ||
| 1501 | * rules concerning calls here are the same as for dma_unmap_single(). | ||
| 1502 | */ | ||
| 1503 | void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, | ||
| 1504 | int nents, enum dma_data_direction dir, struct dma_attrs *attrs) | ||
| 1505 | { | ||
| 1506 | __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); | ||
| 1507 | } | ||
| 1508 | |||
| 1509 | /** | ||
| 1510 | * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg | ||
| 1511 | * @dev: valid struct device pointer | ||
| 1512 | * @sg: list of buffers | ||
| 1513 | * @nents: number of buffers to unmap (same as was passed to dma_map_sg) | ||
| 1514 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | ||
| 1515 | * | ||
| 1516 | * Unmap a set of streaming mode DMA translations. Again, CPU access | ||
| 1517 | * rules concerning calls here are the same as for dma_unmap_single(). | ||
| 1518 | */ | ||
| 1519 | void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, | ||
| 1520 | enum dma_data_direction dir, struct dma_attrs *attrs) | ||
| 1521 | { | ||
| 1522 | __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); | ||
| 1523 | } | ||
| 1524 | |||
| 1525 | /** | ||
| 1428 | * arm_iommu_sync_sg_for_cpu | 1526 | * arm_iommu_sync_sg_for_cpu |
| 1429 | * @dev: valid struct device pointer | 1527 | * @dev: valid struct device pointer |
| 1430 | * @sg: list of buffers | 1528 | * @sg: list of buffers |
| @@ -1438,8 +1536,7 @@ void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, | |||
| 1438 | int i; | 1536 | int i; |
| 1439 | 1537 | ||
| 1440 | for_each_sg(sg, s, nents, i) | 1538 | for_each_sg(sg, s, nents, i) |
| 1441 | if (!arch_is_coherent()) | 1539 | __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); |
| 1442 | __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); | ||
| 1443 | 1540 | ||
| 1444 | } | 1541 | } |
| 1445 | 1542 | ||
| @@ -1457,22 +1554,21 @@ void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, | |||
| 1457 | int i; | 1554 | int i; |
| 1458 | 1555 | ||
| 1459 | for_each_sg(sg, s, nents, i) | 1556 | for_each_sg(sg, s, nents, i) |
| 1460 | if (!arch_is_coherent()) | 1557 | __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); |
| 1461 | __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); | ||
| 1462 | } | 1558 | } |
| 1463 | 1559 | ||
| 1464 | 1560 | ||
| 1465 | /** | 1561 | /** |
| 1466 | * arm_iommu_map_page | 1562 | * arm_coherent_iommu_map_page |
| 1467 | * @dev: valid struct device pointer | 1563 | * @dev: valid struct device pointer |
| 1468 | * @page: page that buffer resides in | 1564 | * @page: page that buffer resides in |
| 1469 | * @offset: offset into page for start of buffer | 1565 | * @offset: offset into page for start of buffer |
| 1470 | * @size: size of buffer to map | 1566 | * @size: size of buffer to map |
| 1471 | * @dir: DMA transfer direction | 1567 | * @dir: DMA transfer direction |
| 1472 | * | 1568 | * |
| 1473 | * IOMMU aware version of arm_dma_map_page() | 1569 | * Coherent IOMMU aware version of arm_dma_map_page() |
| 1474 | */ | 1570 | */ |
| 1475 | static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, | 1571 | static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, |
| 1476 | unsigned long offset, size_t size, enum dma_data_direction dir, | 1572 | unsigned long offset, size_t size, enum dma_data_direction dir, |
| 1477 | struct dma_attrs *attrs) | 1573 | struct dma_attrs *attrs) |
| 1478 | { | 1574 | { |
| @@ -1480,9 +1576,6 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, | |||
| 1480 | dma_addr_t dma_addr; | 1576 | dma_addr_t dma_addr; |
| 1481 | int ret, len = PAGE_ALIGN(size + offset); | 1577 | int ret, len = PAGE_ALIGN(size + offset); |
| 1482 | 1578 | ||
| 1483 | if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | ||
| 1484 | __dma_page_cpu_to_dev(page, offset, size, dir); | ||
| 1485 | |||
| 1486 | dma_addr = __alloc_iova(mapping, len); | 1579 | dma_addr = __alloc_iova(mapping, len); |
| 1487 | if (dma_addr == DMA_ERROR_CODE) | 1580 | if (dma_addr == DMA_ERROR_CODE) |
| 1488 | return dma_addr; | 1581 | return dma_addr; |
| @@ -1498,6 +1591,51 @@ fail: | |||
| 1498 | } | 1591 | } |
| 1499 | 1592 | ||
| 1500 | /** | 1593 | /** |
| 1594 | * arm_iommu_map_page | ||
| 1595 | * @dev: valid struct device pointer | ||
| 1596 | * @page: page that buffer resides in | ||
| 1597 | * @offset: offset into page for start of buffer | ||
| 1598 | * @size: size of buffer to map | ||
| 1599 | * @dir: DMA transfer direction | ||
| 1600 | * | ||
| 1601 | * IOMMU aware version of arm_dma_map_page() | ||
| 1602 | */ | ||
| 1603 | static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, | ||
| 1604 | unsigned long offset, size_t size, enum dma_data_direction dir, | ||
| 1605 | struct dma_attrs *attrs) | ||
| 1606 | { | ||
| 1607 | if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | ||
| 1608 | __dma_page_cpu_to_dev(page, offset, size, dir); | ||
| 1609 | |||
| 1610 | return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); | ||
| 1611 | } | ||
| 1612 | |||
| 1613 | /** | ||
| 1614 | * arm_coherent_iommu_unmap_page | ||
| 1615 | * @dev: valid struct device pointer | ||
| 1616 | * @handle: DMA address of buffer | ||
| 1617 | * @size: size of buffer (same as passed to dma_map_page) | ||
| 1618 | * @dir: DMA transfer direction (same as passed to dma_map_page) | ||
| 1619 | * | ||
| 1620 | * Coherent IOMMU aware version of arm_dma_unmap_page() | ||
| 1621 | */ | ||
| 1622 | static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, | ||
| 1623 | size_t size, enum dma_data_direction dir, | ||
| 1624 | struct dma_attrs *attrs) | ||
| 1625 | { | ||
| 1626 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | ||
| 1627 | dma_addr_t iova = handle & PAGE_MASK; | ||
| 1628 | int offset = handle & ~PAGE_MASK; | ||
| 1629 | int len = PAGE_ALIGN(size + offset); | ||
| 1630 | |||
| 1631 | if (!iova) | ||
| 1632 | return; | ||
| 1633 | |||
| 1634 | iommu_unmap(mapping->domain, iova, len); | ||
| 1635 | __free_iova(mapping, iova, len); | ||
| 1636 | } | ||
| 1637 | |||
| 1638 | /** | ||
| 1501 | * arm_iommu_unmap_page | 1639 | * arm_iommu_unmap_page |
| 1502 | * @dev: valid struct device pointer | 1640 | * @dev: valid struct device pointer |
| 1503 | * @handle: DMA address of buffer | 1641 | * @handle: DMA address of buffer |
| @@ -1519,7 +1657,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, | |||
| 1519 | if (!iova) | 1657 | if (!iova) |
| 1520 | return; | 1658 | return; |
| 1521 | 1659 | ||
| 1522 | if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) | 1660 | if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) |
| 1523 | __dma_page_dev_to_cpu(page, offset, size, dir); | 1661 | __dma_page_dev_to_cpu(page, offset, size, dir); |
| 1524 | 1662 | ||
| 1525 | iommu_unmap(mapping->domain, iova, len); | 1663 | iommu_unmap(mapping->domain, iova, len); |
| @@ -1537,8 +1675,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev, | |||
| 1537 | if (!iova) | 1675 | if (!iova) |
| 1538 | return; | 1676 | return; |
| 1539 | 1677 | ||
| 1540 | if (!arch_is_coherent()) | 1678 | __dma_page_dev_to_cpu(page, offset, size, dir); |
| 1541 | __dma_page_dev_to_cpu(page, offset, size, dir); | ||
| 1542 | } | 1679 | } |
| 1543 | 1680 | ||
| 1544 | static void arm_iommu_sync_single_for_device(struct device *dev, | 1681 | static void arm_iommu_sync_single_for_device(struct device *dev, |
| @@ -1572,6 +1709,19 @@ struct dma_map_ops iommu_ops = { | |||
| 1572 | .sync_sg_for_device = arm_iommu_sync_sg_for_device, | 1709 | .sync_sg_for_device = arm_iommu_sync_sg_for_device, |
| 1573 | }; | 1710 | }; |
| 1574 | 1711 | ||
| 1712 | struct dma_map_ops iommu_coherent_ops = { | ||
| 1713 | .alloc = arm_iommu_alloc_attrs, | ||
| 1714 | .free = arm_iommu_free_attrs, | ||
| 1715 | .mmap = arm_iommu_mmap_attrs, | ||
| 1716 | .get_sgtable = arm_iommu_get_sgtable, | ||
| 1717 | |||
| 1718 | .map_page = arm_coherent_iommu_map_page, | ||
| 1719 | .unmap_page = arm_coherent_iommu_unmap_page, | ||
| 1720 | |||
| 1721 | .map_sg = arm_coherent_iommu_map_sg, | ||
| 1722 | .unmap_sg = arm_coherent_iommu_unmap_sg, | ||
| 1723 | }; | ||
| 1724 | |||
| 1575 | /** | 1725 | /** |
| 1576 | * arm_iommu_create_mapping | 1726 | * arm_iommu_create_mapping |
| 1577 | * @bus: pointer to the bus holding the client device (for IOMMU calls) | 1727 | * @bus: pointer to the bus holding the client device (for IOMMU calls) |
| @@ -1665,7 +1815,7 @@ int arm_iommu_attach_device(struct device *dev, | |||
| 1665 | dev->archdata.mapping = mapping; | 1815 | dev->archdata.mapping = mapping; |
| 1666 | set_dma_ops(dev, &iommu_ops); | 1816 | set_dma_ops(dev, &iommu_ops); |
| 1667 | 1817 | ||
| 1668 | pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev)); | 1818 | pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); |
| 1669 | return 0; | 1819 | return 0; |
| 1670 | } | 1820 | } |
| 1671 | 1821 | ||
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 18144e6a3115..941dfb9e9a78 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c | |||
| @@ -423,17 +423,6 @@ static void __init build_mem_type_table(void) | |||
| 423 | vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; | 423 | vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; |
| 424 | 424 | ||
| 425 | /* | 425 | /* |
| 426 | * Enable CPU-specific coherency if supported. | ||
| 427 | * (Only available on XSC3 at the moment.) | ||
| 428 | */ | ||
| 429 | if (arch_is_coherent() && cpu_is_xsc3()) { | ||
| 430 | mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; | ||
| 431 | mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; | ||
| 432 | mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED; | ||
| 433 | mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; | ||
| 434 | mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; | ||
| 435 | } | ||
| 436 | /* | ||
| 437 | * ARMv6 and above have extended page tables. | 426 | * ARMv6 and above have extended page tables. |
| 438 | */ | 427 | */ |
| 439 | if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { | 428 | if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { |
diff --git a/arch/arm/plat-omap/include/plat/usb.h b/arch/arm/plat-omap/include/plat/usb.h index bd20588c356b..87ee140fefaa 100644 --- a/arch/arm/plat-omap/include/plat/usb.h +++ b/arch/arm/plat-omap/include/plat/usb.h | |||
| @@ -4,6 +4,7 @@ | |||
| 4 | #define __ASM_ARCH_OMAP_USB_H | 4 | #define __ASM_ARCH_OMAP_USB_H |
| 5 | 5 | ||
| 6 | #include <linux/io.h> | 6 | #include <linux/io.h> |
| 7 | #include <linux/platform_device.h> | ||
| 7 | #include <linux/usb/musb.h> | 8 | #include <linux/usb/musb.h> |
| 8 | 9 | ||
| 9 | #define OMAP3_HS_USB_PORTS 3 | 10 | #define OMAP3_HS_USB_PORTS 3 |
| @@ -63,6 +64,10 @@ struct usbhs_omap_platform_data { | |||
| 63 | struct ehci_hcd_omap_platform_data *ehci_data; | 64 | struct ehci_hcd_omap_platform_data *ehci_data; |
| 64 | struct ohci_hcd_omap_platform_data *ohci_data; | 65 | struct ohci_hcd_omap_platform_data *ohci_data; |
| 65 | }; | 66 | }; |
| 67 | |||
| 68 | struct usbtll_omap_platform_data { | ||
| 69 | enum usbhs_omap_port_mode port_mode[OMAP3_HS_USB_PORTS]; | ||
| 70 | }; | ||
| 66 | /*-------------------------------------------------------------------------*/ | 71 | /*-------------------------------------------------------------------------*/ |
| 67 | 72 | ||
| 68 | struct omap_musb_board_data { | 73 | struct omap_musb_board_data { |
| @@ -81,6 +86,8 @@ enum musb_interface {MUSB_INTERFACE_ULPI, MUSB_INTERFACE_UTMI}; | |||
| 81 | extern void usb_musb_init(struct omap_musb_board_data *board_data); | 86 | extern void usb_musb_init(struct omap_musb_board_data *board_data); |
| 82 | 87 | ||
| 83 | extern void usbhs_init(const struct usbhs_omap_board_data *pdata); | 88 | extern void usbhs_init(const struct usbhs_omap_board_data *pdata); |
| 89 | extern int omap_tll_enable(void); | ||
| 90 | extern int omap_tll_disable(void); | ||
| 84 | 91 | ||
| 85 | extern int omap4430_phy_power(struct device *dev, int ID, int on); | 92 | extern int omap4430_phy_power(struct device *dev, int ID, int on); |
| 86 | extern int omap4430_phy_set_clk(struct device *dev, int on); | 93 | extern int omap4430_phy_set_clk(struct device *dev, int on); |
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h index a670a33ad736..37e610dc084e 100644 --- a/arch/arm64/include/asm/compat.h +++ b/arch/arm64/include/asm/compat.h | |||
| @@ -55,6 +55,7 @@ typedef s64 compat_s64; | |||
| 55 | typedef u32 compat_uint_t; | 55 | typedef u32 compat_uint_t; |
| 56 | typedef u32 compat_ulong_t; | 56 | typedef u32 compat_ulong_t; |
| 57 | typedef u64 compat_u64; | 57 | typedef u64 compat_u64; |
| 58 | typedef u32 compat_uptr_t; | ||
| 58 | 59 | ||
| 59 | struct compat_timespec { | 60 | struct compat_timespec { |
| 60 | compat_time_t tv_sec; | 61 | compat_time_t tv_sec; |
| @@ -130,6 +131,64 @@ typedef u32 compat_old_sigset_t; | |||
| 130 | 131 | ||
| 131 | typedef u32 compat_sigset_word; | 132 | typedef u32 compat_sigset_word; |
| 132 | 133 | ||
| 134 | typedef union compat_sigval { | ||
| 135 | compat_int_t sival_int; | ||
| 136 | compat_uptr_t sival_ptr; | ||
| 137 | } compat_sigval_t; | ||
| 138 | |||
| 139 | typedef struct compat_siginfo { | ||
| 140 | int si_signo; | ||
| 141 | int si_errno; | ||
| 142 | int si_code; | ||
| 143 | |||
| 144 | union { | ||
| 145 | /* The padding is the same size as AArch64. */ | ||
| 146 | int _pad[128/sizeof(int) - 3]; | ||
| 147 | |||
| 148 | /* kill() */ | ||
| 149 | struct { | ||
| 150 | compat_pid_t _pid; /* sender's pid */ | ||
| 151 | __compat_uid32_t _uid; /* sender's uid */ | ||
| 152 | } _kill; | ||
| 153 | |||
| 154 | /* POSIX.1b timers */ | ||
| 155 | struct { | ||
| 156 | compat_timer_t _tid; /* timer id */ | ||
| 157 | int _overrun; /* overrun count */ | ||
| 158 | compat_sigval_t _sigval; /* same as below */ | ||
| 159 | int _sys_private; /* not to be passed to user */ | ||
| 160 | } _timer; | ||
| 161 | |||
| 162 | /* POSIX.1b signals */ | ||
| 163 | struct { | ||
| 164 | compat_pid_t _pid; /* sender's pid */ | ||
| 165 | __compat_uid32_t _uid; /* sender's uid */ | ||
| 166 | compat_sigval_t _sigval; | ||
| 167 | } _rt; | ||
| 168 | |||
| 169 | /* SIGCHLD */ | ||
| 170 | struct { | ||
| 171 | compat_pid_t _pid; /* which child */ | ||
| 172 | __compat_uid32_t _uid; /* sender's uid */ | ||
| 173 | int _status; /* exit code */ | ||
| 174 | compat_clock_t _utime; | ||
| 175 | compat_clock_t _stime; | ||
| 176 | } _sigchld; | ||
| 177 | |||
| 178 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 179 | struct { | ||
| 180 | compat_uptr_t _addr; /* faulting insn/memory ref. */ | ||
| 181 | short _addr_lsb; /* LSB of the reported address */ | ||
| 182 | } _sigfault; | ||
| 183 | |||
| 184 | /* SIGPOLL */ | ||
| 185 | struct { | ||
| 186 | compat_long_t _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 187 | int _fd; | ||
| 188 | } _sigpoll; | ||
| 189 | } _sifields; | ||
| 190 | } compat_siginfo_t; | ||
| 191 | |||
| 133 | #define COMPAT_OFF_T_MAX 0x7fffffff | 192 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 134 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 193 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 135 | 194 | ||
| @@ -139,7 +198,6 @@ typedef u32 compat_sigset_word; | |||
| 139 | * as pointers because the syscall entry code will have | 198 | * as pointers because the syscall entry code will have |
| 140 | * appropriately converted them already. | 199 | * appropriately converted them already. |
| 141 | */ | 200 | */ |
| 142 | typedef u32 compat_uptr_t; | ||
| 143 | 201 | ||
| 144 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 202 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 145 | { | 203 | { |
diff --git a/arch/arm64/include/uapi/asm/Kbuild b/arch/arm64/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/arm64/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index ac74c2f261e3..0790a87a4346 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c | |||
| @@ -30,59 +30,6 @@ | |||
| 30 | #include <asm/uaccess.h> | 30 | #include <asm/uaccess.h> |
| 31 | #include <asm/unistd.h> | 31 | #include <asm/unistd.h> |
| 32 | 32 | ||
| 33 | typedef struct compat_siginfo { | ||
| 34 | int si_signo; | ||
| 35 | int si_errno; | ||
| 36 | int si_code; | ||
| 37 | |||
| 38 | union { | ||
| 39 | /* The padding is the same size as AArch64. */ | ||
| 40 | int _pad[SI_PAD_SIZE]; | ||
| 41 | |||
| 42 | /* kill() */ | ||
| 43 | struct { | ||
| 44 | compat_pid_t _pid; /* sender's pid */ | ||
| 45 | __compat_uid32_t _uid; /* sender's uid */ | ||
| 46 | } _kill; | ||
| 47 | |||
| 48 | /* POSIX.1b timers */ | ||
| 49 | struct { | ||
| 50 | compat_timer_t _tid; /* timer id */ | ||
| 51 | int _overrun; /* overrun count */ | ||
| 52 | compat_sigval_t _sigval; /* same as below */ | ||
| 53 | int _sys_private; /* not to be passed to user */ | ||
| 54 | } _timer; | ||
| 55 | |||
| 56 | /* POSIX.1b signals */ | ||
| 57 | struct { | ||
| 58 | compat_pid_t _pid; /* sender's pid */ | ||
| 59 | __compat_uid32_t _uid; /* sender's uid */ | ||
| 60 | compat_sigval_t _sigval; | ||
| 61 | } _rt; | ||
| 62 | |||
| 63 | /* SIGCHLD */ | ||
| 64 | struct { | ||
| 65 | compat_pid_t _pid; /* which child */ | ||
| 66 | __compat_uid32_t _uid; /* sender's uid */ | ||
| 67 | int _status; /* exit code */ | ||
| 68 | compat_clock_t _utime; | ||
| 69 | compat_clock_t _stime; | ||
| 70 | } _sigchld; | ||
| 71 | |||
| 72 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 73 | struct { | ||
| 74 | compat_uptr_t _addr; /* faulting insn/memory ref. */ | ||
| 75 | short _addr_lsb; /* LSB of the reported address */ | ||
| 76 | } _sigfault; | ||
| 77 | |||
| 78 | /* SIGPOLL */ | ||
| 79 | struct { | ||
| 80 | compat_long_t _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 81 | int _fd; | ||
| 82 | } _sigpoll; | ||
| 83 | } _sifields; | ||
| 84 | } compat_siginfo_t; | ||
| 85 | |||
| 86 | struct compat_sigaction { | 33 | struct compat_sigaction { |
| 87 | compat_uptr_t sa_handler; | 34 | compat_uptr_t sa_handler; |
| 88 | compat_ulong_t sa_flags; | 35 | compat_ulong_t sa_flags; |
diff --git a/arch/avr32/include/asm/elf.h b/arch/avr32/include/asm/elf.h index 3b3159b710d4..e2c328739808 100644 --- a/arch/avr32/include/asm/elf.h +++ b/arch/avr32/include/asm/elf.h | |||
| @@ -102,6 +102,7 @@ typedef struct user_fpu_struct elf_fpregset_t; | |||
| 102 | 102 | ||
| 103 | #define ELF_PLATFORM (NULL) | 103 | #define ELF_PLATFORM (NULL) |
| 104 | 104 | ||
| 105 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT) | 105 | #define SET_PERSONALITY(ex) \ |
| 106 | set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK))) | ||
| 106 | 107 | ||
| 107 | #endif /* __ASM_AVR32_ELF_H */ | 108 | #endif /* __ASM_AVR32_ELF_H */ |
diff --git a/arch/avr32/include/uapi/asm/Kbuild b/arch/avr32/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/avr32/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/blackfin/include/asm/elf.h b/arch/blackfin/include/asm/elf.h index e6c6812a9abd..14bc98ff668f 100644 --- a/arch/blackfin/include/asm/elf.h +++ b/arch/blackfin/include/asm/elf.h | |||
| @@ -132,6 +132,7 @@ do { \ | |||
| 132 | 132 | ||
| 133 | #define ELF_PLATFORM (NULL) | 133 | #define ELF_PLATFORM (NULL) |
| 134 | 134 | ||
| 135 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 135 | #define SET_PERSONALITY(ex) \ |
| 136 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 136 | 137 | ||
| 137 | #endif | 138 | #endif |
diff --git a/arch/blackfin/include/uapi/asm/Kbuild b/arch/blackfin/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/blackfin/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/c6x/include/asm/elf.h b/arch/c6x/include/asm/elf.h index f4552db20b4a..32b997126adf 100644 --- a/arch/c6x/include/asm/elf.h +++ b/arch/c6x/include/asm/elf.h | |||
| @@ -77,7 +77,8 @@ do { \ | |||
| 77 | 77 | ||
| 78 | #define ELF_PLATFORM (NULL) | 78 | #define ELF_PLATFORM (NULL) |
| 79 | 79 | ||
| 80 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 80 | #define SET_PERSONALITY(ex) \ |
| 81 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 81 | 82 | ||
| 82 | /* C6X specific section types */ | 83 | /* C6X specific section types */ |
| 83 | #define SHT_C6000_UNWIND 0x70000001 | 84 | #define SHT_C6000_UNWIND 0x70000001 |
diff --git a/arch/c6x/include/uapi/asm/Kbuild b/arch/c6x/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/c6x/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig index e92215428a37..72bd5ae50a89 100644 --- a/arch/cris/Kconfig +++ b/arch/cris/Kconfig | |||
| @@ -138,11 +138,6 @@ config CRIS_MACH_ARTPEC3 | |||
| 138 | 138 | ||
| 139 | endchoice | 139 | endchoice |
| 140 | 140 | ||
| 141 | config ETRAX_VCS_SIM | ||
| 142 | bool "VCS Simulator" | ||
| 143 | help | ||
| 144 | Setup hardware to be run in the VCS simulator. | ||
| 145 | |||
| 146 | config ETRAX_ARCH_V10 | 141 | config ETRAX_ARCH_V10 |
| 147 | bool | 142 | bool |
| 148 | default y if ETRAX100LX || ETRAX100LX_V2 | 143 | default y if ETRAX100LX || ETRAX100LX_V2 |
diff --git a/arch/cris/Makefile b/arch/cris/Makefile index 29c2ceb38a76..39dc7d00083e 100644 --- a/arch/cris/Makefile +++ b/arch/cris/Makefile | |||
| @@ -23,7 +23,9 @@ mach-$(CONFIG_ETRAXFS) := fs | |||
| 23 | 23 | ||
| 24 | ifneq ($(arch-y),) | 24 | ifneq ($(arch-y),) |
| 25 | SARCH := arch-$(arch-y) | 25 | SARCH := arch-$(arch-y) |
| 26 | inc := -Iarch/cris/include/$(SARCH) | 26 | inc := -Iarch/cris/include/uapi/$(SARCH) |
| 27 | inc += -Iarch/cris/include/$(SARCH) | ||
| 28 | inc += -Iarch/cris/include/uapi/$(SARCH)/arch | ||
| 27 | inc += -Iarch/cris/include/$(SARCH)/arch | 29 | inc += -Iarch/cris/include/$(SARCH)/arch |
| 28 | else | 30 | else |
| 29 | SARCH := | 31 | SARCH := |
diff --git a/arch/cris/arch-v32/drivers/axisflashmap.c b/arch/cris/arch-v32/drivers/axisflashmap.c index b34438e026be..1b6ad6247204 100644 --- a/arch/cris/arch-v32/drivers/axisflashmap.c +++ b/arch/cris/arch-v32/drivers/axisflashmap.c | |||
| @@ -329,7 +329,6 @@ static int __init init_axis_flash(void) | |||
| 329 | } | 329 | } |
| 330 | #endif | 330 | #endif |
| 331 | 331 | ||
| 332 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 333 | main_mtd = flash_probe(); | 332 | main_mtd = flash_probe(); |
| 334 | if (main_mtd) | 333 | if (main_mtd) |
| 335 | printk(KERN_INFO "%s: 0x%08x bytes of NOR flash memory.\n", | 334 | printk(KERN_INFO "%s: 0x%08x bytes of NOR flash memory.\n", |
| @@ -603,34 +602,7 @@ static int __init init_axis_flash(void) | |||
| 603 | "partition %d\n", part); | 602 | "partition %d\n", part); |
| 604 | } | 603 | } |
| 605 | } | 604 | } |
| 606 | #endif /* CONFIG_EXTRAX_VCS_SIM */ | ||
| 607 | 605 | ||
| 608 | #ifdef CONFIG_ETRAX_VCS_SIM | ||
| 609 | /* For simulator, always use a RAM partition. | ||
| 610 | * The rootfs will be found after the kernel in RAM, | ||
| 611 | * with romfs_start and romfs_end indicating location and size. | ||
| 612 | */ | ||
| 613 | struct mtd_info *mtd_ram; | ||
| 614 | |||
| 615 | mtd_ram = kmalloc(sizeof(struct mtd_info), GFP_KERNEL); | ||
| 616 | if (!mtd_ram) { | ||
| 617 | panic("axisflashmap: Couldn't allocate memory for " | ||
| 618 | "mtd_info!\n"); | ||
| 619 | } | ||
| 620 | |||
| 621 | printk(KERN_INFO "axisflashmap: Adding RAM partition for romfs, " | ||
| 622 | "at %u, size %u\n", | ||
| 623 | (unsigned) romfs_start, (unsigned) romfs_length); | ||
| 624 | |||
| 625 | err = mtdram_init_device(mtd_ram, (void *)romfs_start, | ||
| 626 | romfs_length, "romfs"); | ||
| 627 | if (err) { | ||
| 628 | panic("axisflashmap: Could not initialize MTD RAM " | ||
| 629 | "device!\n"); | ||
| 630 | } | ||
| 631 | #endif /* CONFIG_EXTRAX_VCS_SIM */ | ||
| 632 | |||
| 633 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 634 | if (aux_mtd) { | 606 | if (aux_mtd) { |
| 635 | aux_partition.size = aux_mtd->size; | 607 | aux_partition.size = aux_mtd->size; |
| 636 | err = mtd_device_register(aux_mtd, &aux_partition, 1); | 608 | err = mtd_device_register(aux_mtd, &aux_partition, 1); |
| @@ -639,7 +611,6 @@ static int __init init_axis_flash(void) | |||
| 639 | "aux mtd device!\n"); | 611 | "aux mtd device!\n"); |
| 640 | 612 | ||
| 641 | } | 613 | } |
| 642 | #endif /* CONFIG_EXTRAX_VCS_SIM */ | ||
| 643 | 614 | ||
| 644 | return err; | 615 | return err; |
| 645 | } | 616 | } |
diff --git a/arch/cris/arch-v32/drivers/pci/bios.c b/arch/cris/arch-v32/drivers/pci/bios.c index 5b1ee82f63c5..e3dfc72d0cfd 100644 --- a/arch/cris/arch-v32/drivers/pci/bios.c +++ b/arch/cris/arch-v32/drivers/pci/bios.c | |||
| @@ -97,28 +97,3 @@ int pcibios_enable_device(struct pci_dev *dev, int mask) | |||
| 97 | pcibios_enable_irq(dev); | 97 | pcibios_enable_irq(dev); |
| 98 | return 0; | 98 | return 0; |
| 99 | } | 99 | } |
| 100 | |||
| 101 | int pcibios_assign_resources(void) | ||
| 102 | { | ||
| 103 | struct pci_dev *dev = NULL; | ||
| 104 | int idx; | ||
| 105 | struct resource *r; | ||
| 106 | |||
| 107 | while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) { | ||
| 108 | int class = dev->class >> 8; | ||
| 109 | |||
| 110 | /* Don't touch classless devices and host bridges */ | ||
| 111 | if (!class || class == PCI_CLASS_BRIDGE_HOST) | ||
| 112 | continue; | ||
| 113 | |||
| 114 | for(idx=0; idx<6; idx++) { | ||
| 115 | r = &dev->resource[idx]; | ||
| 116 | |||
| 117 | if (!r->start && r->end) | ||
| 118 | pci_assign_resource(dev, idx); | ||
| 119 | } | ||
| 120 | } | ||
| 121 | return 0; | ||
| 122 | } | ||
| 123 | |||
| 124 | EXPORT_SYMBOL(pcibios_assign_resources); | ||
diff --git a/arch/cris/arch-v32/kernel/head.S b/arch/cris/arch-v32/kernel/head.S index 5d502b9ab56d..51e34165ece7 100644 --- a/arch/cris/arch-v32/kernel/head.S +++ b/arch/cris/arch-v32/kernel/head.S | |||
| @@ -36,13 +36,6 @@ | |||
| 36 | .global nand_boot | 36 | .global nand_boot |
| 37 | .global swapper_pg_dir | 37 | .global swapper_pg_dir |
| 38 | 38 | ||
| 39 | ;; Dummy section to make it bootable with current VCS simulator | ||
| 40 | #ifdef CONFIG_ETRAX_VCS_SIM | ||
| 41 | .section ".boot", "ax" | ||
| 42 | ba tstart | ||
| 43 | nop | ||
| 44 | #endif | ||
| 45 | |||
| 46 | .text | 39 | .text |
| 47 | tstart: | 40 | tstart: |
| 48 | ;; This is the entry point of the kernel. The CPU is currently in | 41 | ;; This is the entry point of the kernel. The CPU is currently in |
| @@ -75,17 +68,10 @@ secondary_cpu_entry: /* Entry point for secondary CPUs */ | |||
| 75 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4) \ | 68 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4) \ |
| 76 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_d, 5) \ | 69 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_d, 5) \ |
| 77 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0 | 70 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0 |
| 78 | #elif !defined(CONFIG_ETRAX_VCS_SIM) | ||
| 79 | move.d REG_FIELD(mmu, rw_mm_kbase_hi, base_e, 8) \ | ||
| 80 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4) \ | ||
| 81 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0 | ||
| 82 | #else | 71 | #else |
| 83 | ;; Map the virtual DRAM to the RW eprom area at address 0. | ||
| 84 | ;; Also map 0xa for the hook calls, | ||
| 85 | move.d REG_FIELD(mmu, rw_mm_kbase_hi, base_e, 8) \ | 72 | move.d REG_FIELD(mmu, rw_mm_kbase_hi, base_e, 8) \ |
| 86 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4) \ | 73 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4) \ |
| 87 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb) \ | 74 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0 |
| 88 | | REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0xa), $r0 | ||
| 89 | #endif | 75 | #endif |
| 90 | 76 | ||
| 91 | ;; Temporary map of 0x40 -> 0x40 and 0x00 -> 0x00. | 77 | ;; Temporary map of 0x40 -> 0x40 and 0x00 -> 0x00. |
| @@ -126,27 +112,6 @@ secondary_cpu_entry: /* Entry point for secondary CPUs */ | |||
| 126 | | REG_STATE(mmu, rw_mm_cfg, seg_2, page) \ | 112 | | REG_STATE(mmu, rw_mm_cfg, seg_2, page) \ |
| 127 | | REG_STATE(mmu, rw_mm_cfg, seg_1, page) \ | 113 | | REG_STATE(mmu, rw_mm_cfg, seg_1, page) \ |
| 128 | | REG_STATE(mmu, rw_mm_cfg, seg_0, linear), $r2 | 114 | | REG_STATE(mmu, rw_mm_cfg, seg_0, linear), $r2 |
| 129 | #elif !defined(CONFIG_ETRAX_VCS_SIM) | ||
| 130 | move.d REG_STATE(mmu, rw_mm_cfg, we, on) \ | ||
| 131 | | REG_STATE(mmu, rw_mm_cfg, acc, on) \ | ||
| 132 | | REG_STATE(mmu, rw_mm_cfg, ex, on) \ | ||
| 133 | | REG_STATE(mmu, rw_mm_cfg, inv, on) \ | ||
| 134 | | REG_STATE(mmu, rw_mm_cfg, seg_f, linear) \ | ||
| 135 | | REG_STATE(mmu, rw_mm_cfg, seg_e, linear) \ | ||
| 136 | | REG_STATE(mmu, rw_mm_cfg, seg_d, page) \ | ||
| 137 | | REG_STATE(mmu, rw_mm_cfg, seg_c, linear) \ | ||
| 138 | | REG_STATE(mmu, rw_mm_cfg, seg_b, linear) \ | ||
| 139 | | REG_STATE(mmu, rw_mm_cfg, seg_a, page) \ | ||
| 140 | | REG_STATE(mmu, rw_mm_cfg, seg_9, page) \ | ||
| 141 | | REG_STATE(mmu, rw_mm_cfg, seg_8, page) \ | ||
| 142 | | REG_STATE(mmu, rw_mm_cfg, seg_7, page) \ | ||
| 143 | | REG_STATE(mmu, rw_mm_cfg, seg_6, page) \ | ||
| 144 | | REG_STATE(mmu, rw_mm_cfg, seg_5, page) \ | ||
| 145 | | REG_STATE(mmu, rw_mm_cfg, seg_4, linear) \ | ||
| 146 | | REG_STATE(mmu, rw_mm_cfg, seg_3, page) \ | ||
| 147 | | REG_STATE(mmu, rw_mm_cfg, seg_2, page) \ | ||
| 148 | | REG_STATE(mmu, rw_mm_cfg, seg_1, page) \ | ||
| 149 | | REG_STATE(mmu, rw_mm_cfg, seg_0, linear), $r2 | ||
| 150 | #else | 115 | #else |
| 151 | move.d REG_STATE(mmu, rw_mm_cfg, we, on) \ | 116 | move.d REG_STATE(mmu, rw_mm_cfg, we, on) \ |
| 152 | | REG_STATE(mmu, rw_mm_cfg, acc, on) \ | 117 | | REG_STATE(mmu, rw_mm_cfg, acc, on) \ |
| @@ -157,7 +122,7 @@ secondary_cpu_entry: /* Entry point for secondary CPUs */ | |||
| 157 | | REG_STATE(mmu, rw_mm_cfg, seg_d, page) \ | 122 | | REG_STATE(mmu, rw_mm_cfg, seg_d, page) \ |
| 158 | | REG_STATE(mmu, rw_mm_cfg, seg_c, linear) \ | 123 | | REG_STATE(mmu, rw_mm_cfg, seg_c, linear) \ |
| 159 | | REG_STATE(mmu, rw_mm_cfg, seg_b, linear) \ | 124 | | REG_STATE(mmu, rw_mm_cfg, seg_b, linear) \ |
| 160 | | REG_STATE(mmu, rw_mm_cfg, seg_a, linear) \ | 125 | | REG_STATE(mmu, rw_mm_cfg, seg_a, page) \ |
| 161 | | REG_STATE(mmu, rw_mm_cfg, seg_9, page) \ | 126 | | REG_STATE(mmu, rw_mm_cfg, seg_9, page) \ |
| 162 | | REG_STATE(mmu, rw_mm_cfg, seg_8, page) \ | 127 | | REG_STATE(mmu, rw_mm_cfg, seg_8, page) \ |
| 163 | | REG_STATE(mmu, rw_mm_cfg, seg_7, page) \ | 128 | | REG_STATE(mmu, rw_mm_cfg, seg_7, page) \ |
| @@ -226,7 +191,6 @@ master_cpu: | |||
| 226 | move.d secondary_cpu_entry, $r1 | 191 | move.d secondary_cpu_entry, $r1 |
| 227 | move.d $r1, [$r0] | 192 | move.d $r1, [$r0] |
| 228 | #endif | 193 | #endif |
| 229 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 230 | ; Check if starting from DRAM (network->RAM boot or unpacked | 194 | ; Check if starting from DRAM (network->RAM boot or unpacked |
| 231 | ; compressed kernel), or directly from flash. | 195 | ; compressed kernel), or directly from flash. |
| 232 | lapcq ., $r0 | 196 | lapcq ., $r0 |
| @@ -234,7 +198,6 @@ master_cpu: | |||
| 234 | cmp.d 0x10000, $r0 ; Arbitrary, something above this code. | 198 | cmp.d 0x10000, $r0 ; Arbitrary, something above this code. |
| 235 | blo _inflash0 | 199 | blo _inflash0 |
| 236 | nop | 200 | nop |
| 237 | #endif | ||
| 238 | 201 | ||
| 239 | jump _inram ; Jump to cached RAM. | 202 | jump _inram ; Jump to cached RAM. |
| 240 | nop | 203 | nop |
| @@ -326,7 +289,6 @@ move_cramfs: | |||
| 326 | move.d romfs_length, $r1 | 289 | move.d romfs_length, $r1 |
| 327 | move.d $r0, [$r1] | 290 | move.d $r0, [$r1] |
| 328 | 291 | ||
| 329 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 330 | ;; The kernel could have been unpacked to DRAM by the loader, but | 292 | ;; The kernel could have been unpacked to DRAM by the loader, but |
| 331 | ;; the cramfs image could still be in the flash immediately | 293 | ;; the cramfs image could still be in the flash immediately |
| 332 | ;; following the compressed kernel image. The loader passes the address | 294 | ;; following the compressed kernel image. The loader passes the address |
| @@ -335,10 +297,6 @@ move_cramfs: | |||
| 335 | cmp.d 0x0ffffff8, $r9 | 297 | cmp.d 0x0ffffff8, $r9 |
| 336 | bhs _no_romfs_in_flash ; R9 points outside the flash area. | 298 | bhs _no_romfs_in_flash ; R9 points outside the flash area. |
| 337 | nop | 299 | nop |
| 338 | #else | ||
| 339 | ba _no_romfs_in_flash | ||
| 340 | nop | ||
| 341 | #endif | ||
| 342 | ;; cramfs rootfs might to be in flash. Check for it. | 300 | ;; cramfs rootfs might to be in flash. Check for it. |
| 343 | move.d [$r9], $r0 ; cramfs_super.magic | 301 | move.d [$r9], $r0 ; cramfs_super.magic |
| 344 | cmp.d CRAMFS_MAGIC, $r0 | 302 | cmp.d CRAMFS_MAGIC, $r0 |
| @@ -396,7 +354,6 @@ _no_romfs_in_flash: | |||
| 396 | move.d romfs_length, $r3 | 354 | move.d romfs_length, $r3 |
| 397 | move.d $r2, [$r3] ; store size at romfs_length | 355 | move.d $r2, [$r3] ; store size at romfs_length |
| 398 | 356 | ||
| 399 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 400 | add.d $r2, $r0 ; copy from end and downwards | 357 | add.d $r2, $r0 ; copy from end and downwards |
| 401 | add.d $r2, $r1 | 358 | add.d $r2, $r1 |
| 402 | 359 | ||
| @@ -410,7 +367,6 @@ _no_romfs_in_flash: | |||
| 410 | subq 1, $r2 | 367 | subq 1, $r2 |
| 411 | bne 1b | 368 | bne 1b |
| 412 | nop | 369 | nop |
| 413 | #endif | ||
| 414 | 370 | ||
| 415 | 4: | 371 | 4: |
| 416 | ;; BSS move done. | 372 | ;; BSS move done. |
| @@ -455,7 +411,6 @@ no_command_line: | |||
| 455 | move.d etrax_irv, $r1 ; Set the exception base register and pointer. | 411 | move.d etrax_irv, $r1 ; Set the exception base register and pointer. |
| 456 | move.d $r0, [$r1] | 412 | move.d $r0, [$r1] |
| 457 | 413 | ||
| 458 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 459 | ;; Clear the BSS region from _bss_start to _end. | 414 | ;; Clear the BSS region from _bss_start to _end. |
| 460 | move.d __bss_start, $r0 | 415 | move.d __bss_start, $r0 |
| 461 | move.d _end, $r1 | 416 | move.d _end, $r1 |
| @@ -463,15 +418,6 @@ no_command_line: | |||
| 463 | cmp.d $r1, $r0 | 418 | cmp.d $r1, $r0 |
| 464 | blo 1b | 419 | blo 1b |
| 465 | nop | 420 | nop |
| 466 | #endif | ||
| 467 | |||
| 468 | #ifdef CONFIG_ETRAX_VCS_SIM | ||
| 469 | /* Set the watchdog timeout to something big. Will be removed when */ | ||
| 470 | /* watchdog can be disabled with command line option */ | ||
| 471 | move.d 0x7fffffff, $r10 | ||
| 472 | jsr CPU_WATCHDOG_TIMEOUT | ||
| 473 | nop | ||
| 474 | #endif | ||
| 475 | 421 | ||
| 476 | ; Initialize registers to increase determinism | 422 | ; Initialize registers to increase determinism |
| 477 | move.d __bss_start, $r0 | 423 | move.d __bss_start, $r0 |
diff --git a/arch/cris/arch-v32/kernel/kgdb.c b/arch/cris/arch-v32/kernel/kgdb.c index 8c1d35cdf00a..b06813aeb120 100644 --- a/arch/cris/arch-v32/kernel/kgdb.c +++ b/arch/cris/arch-v32/kernel/kgdb.c | |||
| @@ -381,23 +381,9 @@ static int read_register(char regno, unsigned int *valptr); | |||
| 381 | /* Serial port, reads one character. ETRAX 100 specific. from debugport.c */ | 381 | /* Serial port, reads one character. ETRAX 100 specific. from debugport.c */ |
| 382 | int getDebugChar(void); | 382 | int getDebugChar(void); |
| 383 | 383 | ||
| 384 | #ifdef CONFIG_ETRAX_VCS_SIM | ||
| 385 | int getDebugChar(void) | ||
| 386 | { | ||
| 387 | return socketread(); | ||
| 388 | } | ||
| 389 | #endif | ||
| 390 | |||
| 391 | /* Serial port, writes one character. ETRAX 100 specific. from debugport.c */ | 384 | /* Serial port, writes one character. ETRAX 100 specific. from debugport.c */ |
| 392 | void putDebugChar(int val); | 385 | void putDebugChar(int val); |
| 393 | 386 | ||
| 394 | #ifdef CONFIG_ETRAX_VCS_SIM | ||
| 395 | void putDebugChar(int val) | ||
| 396 | { | ||
| 397 | socketwrite((char *)&val, 1); | ||
| 398 | } | ||
| 399 | #endif | ||
| 400 | |||
| 401 | /* Returns the integer equivalent of a hexadecimal character. */ | 387 | /* Returns the integer equivalent of a hexadecimal character. */ |
| 402 | static int hex(char ch); | 388 | static int hex(char ch); |
| 403 | 389 | ||
diff --git a/arch/cris/arch-v32/mach-a3/Makefile b/arch/cris/arch-v32/mach-a3/Makefile index 41fa6a6893a9..d366e0891988 100644 --- a/arch/cris/arch-v32/mach-a3/Makefile +++ b/arch/cris/arch-v32/mach-a3/Makefile | |||
| @@ -1,10 +1,8 @@ | |||
| 1 | # $Id: Makefile,v 1.3 2007/03/13 11:57:46 starvik Exp $ | ||
| 2 | # | 1 | # |
| 3 | # Makefile for the linux kernel. | 2 | # Makefile for the linux kernel. |
| 4 | # | 3 | # |
| 5 | 4 | ||
| 6 | obj-y := dma.o pinmux.o io.o arbiter.o | 5 | obj-y := dma.o pinmux.o io.o arbiter.o |
| 7 | obj-$(CONFIG_ETRAX_VCS_SIM) += vcs_hook.o | ||
| 8 | obj-$(CONFIG_CPU_FREQ) += cpufreq.o | 6 | obj-$(CONFIG_CPU_FREQ) += cpufreq.o |
| 9 | 7 | ||
| 10 | clean: | 8 | clean: |
diff --git a/arch/cris/arch-v32/mach-a3/vcs_hook.c b/arch/cris/arch-v32/mach-a3/vcs_hook.c deleted file mode 100644 index 58b1a5469fd7..000000000000 --- a/arch/cris/arch-v32/mach-a3/vcs_hook.c +++ /dev/null | |||
| @@ -1,103 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * Simulator hook mechanism | ||
| 3 | */ | ||
| 4 | |||
| 5 | #include "vcs_hook.h" | ||
| 6 | #include <asm/io.h> | ||
| 7 | #include <stdarg.h> | ||
| 8 | |||
| 9 | #define HOOK_TRIG_ADDR 0xb7000000 | ||
| 10 | #define HOOK_MEM_BASE_ADDR 0xce000000 | ||
| 11 | |||
| 12 | static volatile unsigned *hook_base; | ||
| 13 | |||
| 14 | #define HOOK_DATA(offset) hook_base[offset] | ||
| 15 | #define VHOOK_DATA(offset) hook_base[offset] | ||
| 16 | #define HOOK_TRIG(funcid) \ | ||
| 17 | do { \ | ||
| 18 | *((unsigned *) HOOK_TRIG_ADDR) = funcid; \ | ||
| 19 | } while (0) | ||
| 20 | #define HOOK_DATA_BYTE(offset) ((unsigned char *)hook_base)[offset] | ||
| 21 | |||
| 22 | static void hook_init(void) | ||
| 23 | { | ||
| 24 | static int first = 1; | ||
| 25 | if (first) { | ||
| 26 | first = 0; | ||
| 27 | hook_base = ioremap(HOOK_MEM_BASE_ADDR, 8192); | ||
| 28 | } | ||
| 29 | } | ||
| 30 | |||
| 31 | static unsigned hook_trig(unsigned id) | ||
| 32 | { | ||
| 33 | unsigned ret; | ||
| 34 | |||
| 35 | /* preempt_disable(); */ | ||
| 36 | |||
| 37 | /* Dummy read from mem to make sure data has propagated to memory | ||
| 38 | * before trigging */ | ||
| 39 | ret = *hook_base; | ||
| 40 | |||
| 41 | /* trigger hook */ | ||
| 42 | HOOK_TRIG(id); | ||
| 43 | |||
| 44 | /* wait for call to finish */ | ||
| 45 | while (VHOOK_DATA(0) > 0) ; | ||
| 46 | |||
| 47 | /* extract return value */ | ||
| 48 | |||
| 49 | ret = VHOOK_DATA(1); | ||
| 50 | |||
| 51 | return ret; | ||
| 52 | } | ||
| 53 | |||
| 54 | int hook_call(unsigned id, unsigned pcnt, ...) | ||
| 55 | { | ||
| 56 | va_list ap; | ||
| 57 | int i; | ||
| 58 | unsigned ret; | ||
| 59 | |||
| 60 | hook_init(); | ||
| 61 | |||
| 62 | HOOK_DATA(0) = id; | ||
| 63 | |||
| 64 | va_start(ap, pcnt); | ||
| 65 | for (i = 1; i <= pcnt; i++) | ||
| 66 | HOOK_DATA(i) = va_arg(ap, unsigned); | ||
| 67 | va_end(ap); | ||
| 68 | |||
| 69 | ret = hook_trig(id); | ||
| 70 | |||
| 71 | return ret; | ||
| 72 | } | ||
| 73 | |||
| 74 | int hook_call_str(unsigned id, unsigned size, const char *str) | ||
| 75 | { | ||
| 76 | int i; | ||
| 77 | unsigned ret; | ||
| 78 | |||
| 79 | hook_init(); | ||
| 80 | |||
| 81 | HOOK_DATA(0) = id; | ||
| 82 | HOOK_DATA(1) = size; | ||
| 83 | |||
| 84 | for (i = 0; i < size; i++) | ||
| 85 | HOOK_DATA_BYTE(8 + i) = str[i]; | ||
| 86 | HOOK_DATA_BYTE(8 + i) = 0; | ||
| 87 | |||
| 88 | ret = hook_trig(id); | ||
| 89 | |||
| 90 | return ret; | ||
| 91 | } | ||
| 92 | |||
| 93 | void print_str(const char *str) | ||
| 94 | { | ||
| 95 | int i; | ||
| 96 | /* find null at end of string */ | ||
| 97 | for (i = 1; str[i]; i++) ; | ||
| 98 | hook_call(hook_print_str, i, str); | ||
| 99 | } | ||
| 100 | |||
| 101 | void CPU_WATCHDOG_TIMEOUT(unsigned t) | ||
| 102 | { | ||
| 103 | } | ||
diff --git a/arch/cris/arch-v32/mach-a3/vcs_hook.h b/arch/cris/arch-v32/mach-a3/vcs_hook.h deleted file mode 100644 index 8b73d0e8392d..000000000000 --- a/arch/cris/arch-v32/mach-a3/vcs_hook.h +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * Simulator hook call mechanism | ||
| 3 | */ | ||
| 4 | |||
| 5 | #ifndef __hook_h__ | ||
| 6 | #define __hook_h__ | ||
| 7 | |||
| 8 | int hook_call(unsigned id, unsigned pcnt, ...); | ||
| 9 | int hook_call_str(unsigned id, unsigned size, const char *str); | ||
| 10 | |||
| 11 | enum hook_ids { | ||
| 12 | hook_debug_on = 1, | ||
| 13 | hook_debug_off, | ||
| 14 | hook_stop_sim_ok, | ||
| 15 | hook_stop_sim_fail, | ||
| 16 | hook_alloc_shared, | ||
| 17 | hook_ptr_shared, | ||
| 18 | hook_free_shared, | ||
| 19 | hook_file2shared, | ||
| 20 | hook_cmp_shared, | ||
| 21 | hook_print_params, | ||
| 22 | hook_sim_time, | ||
| 23 | hook_stop_sim, | ||
| 24 | hook_kick_dog, | ||
| 25 | hook_dog_timeout, | ||
| 26 | hook_rand, | ||
| 27 | hook_srand, | ||
| 28 | hook_rand_range, | ||
| 29 | hook_print_str, | ||
| 30 | hook_print_hex, | ||
| 31 | hook_cmp_offset_shared, | ||
| 32 | hook_fill_random_shared, | ||
| 33 | hook_alloc_random_data, | ||
| 34 | hook_calloc_random_data, | ||
| 35 | hook_print_int, | ||
| 36 | hook_print_uint, | ||
| 37 | hook_fputc, | ||
| 38 | hook_init_fd, | ||
| 39 | hook_sbrk, | ||
| 40 | hook_print_context_descr, | ||
| 41 | hook_print_data_descr, | ||
| 42 | hook_print_group_descr, | ||
| 43 | hook_fill_shared, | ||
| 44 | hook_sl_srand, | ||
| 45 | hook_sl_rand_irange, | ||
| 46 | hook_sl_rand_urange, | ||
| 47 | hook_sl_sh_malloc_aligned, | ||
| 48 | hook_sl_sh_calloc_aligned, | ||
| 49 | hook_sl_sh_alloc_random_data, | ||
| 50 | hook_sl_sh_file2mem, | ||
| 51 | hook_sl_vera_mbox_handle, | ||
| 52 | hook_sl_vera_mbox_put, | ||
| 53 | hook_sl_vera_mbox_get, | ||
| 54 | hook_sl_system, | ||
| 55 | hook_sl_sh_hexdump | ||
| 56 | }; | ||
| 57 | |||
| 58 | #endif | ||
diff --git a/arch/cris/arch-v32/mach-fs/Makefile b/arch/cris/arch-v32/mach-fs/Makefile index 41fa6a6893a9..d366e0891988 100644 --- a/arch/cris/arch-v32/mach-fs/Makefile +++ b/arch/cris/arch-v32/mach-fs/Makefile | |||
| @@ -1,10 +1,8 @@ | |||
| 1 | # $Id: Makefile,v 1.3 2007/03/13 11:57:46 starvik Exp $ | ||
| 2 | # | 1 | # |
| 3 | # Makefile for the linux kernel. | 2 | # Makefile for the linux kernel. |
| 4 | # | 3 | # |
| 5 | 4 | ||
| 6 | obj-y := dma.o pinmux.o io.o arbiter.o | 5 | obj-y := dma.o pinmux.o io.o arbiter.o |
| 7 | obj-$(CONFIG_ETRAX_VCS_SIM) += vcs_hook.o | ||
| 8 | obj-$(CONFIG_CPU_FREQ) += cpufreq.o | 6 | obj-$(CONFIG_CPU_FREQ) += cpufreq.o |
| 9 | 7 | ||
| 10 | clean: | 8 | clean: |
diff --git a/arch/cris/arch-v32/mach-fs/vcs_hook.c b/arch/cris/arch-v32/mach-fs/vcs_hook.c deleted file mode 100644 index b11594ae0cb6..000000000000 --- a/arch/cris/arch-v32/mach-fs/vcs_hook.c +++ /dev/null | |||
| @@ -1,100 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * Call simulator hook. This is the part running in the | ||
| 3 | * simulated program. | ||
| 4 | */ | ||
| 5 | |||
| 6 | #include "vcs_hook.h" | ||
| 7 | #include <stdarg.h> | ||
| 8 | #include <arch-v32/hwregs/reg_map.h> | ||
| 9 | #include <arch-v32/hwregs/intr_vect_defs.h> | ||
| 10 | |||
| 11 | #define HOOK_TRIG_ADDR 0xb7000000 /* hook cvlog model reg address */ | ||
| 12 | #define HOOK_MEM_BASE_ADDR 0xa0000000 /* csp4 (shared mem) base addr */ | ||
| 13 | |||
| 14 | #define HOOK_DATA(offset) ((unsigned *)HOOK_MEM_BASE_ADDR)[offset] | ||
| 15 | #define VHOOK_DATA(offset) ((volatile unsigned *)HOOK_MEM_BASE_ADDR)[offset] | ||
| 16 | #define HOOK_TRIG(funcid) \ | ||
| 17 | do { \ | ||
| 18 | *((unsigned *) HOOK_TRIG_ADDR) = funcid; \ | ||
| 19 | } while (0) | ||
| 20 | #define HOOK_DATA_BYTE(offset) ((unsigned char *)HOOK_MEM_BASE_ADDR)[offset] | ||
| 21 | |||
| 22 | int hook_call(unsigned id, unsigned pcnt, ...) | ||
| 23 | { | ||
| 24 | va_list ap; | ||
| 25 | unsigned i; | ||
| 26 | unsigned ret; | ||
| 27 | #ifdef USING_SOS | ||
| 28 | PREEMPT_OFF_SAVE(); | ||
| 29 | #endif | ||
| 30 | |||
| 31 | /* pass parameters */ | ||
| 32 | HOOK_DATA(0) = id; | ||
| 33 | |||
| 34 | /* Have to make hook_print_str a special case since we call with a | ||
| 35 | * parameter of byte type. Should perhaps be a separate | ||
| 36 | * hook_call. */ | ||
| 37 | |||
| 38 | if (id == hook_print_str) { | ||
| 39 | int i; | ||
| 40 | char *str; | ||
| 41 | |||
| 42 | HOOK_DATA(1) = pcnt; | ||
| 43 | |||
| 44 | va_start(ap, pcnt); | ||
| 45 | str = (char *)va_arg(ap, unsigned); | ||
| 46 | |||
| 47 | for (i = 0; i != pcnt; i++) | ||
| 48 | HOOK_DATA_BYTE(8 + i) = str[i]; | ||
| 49 | |||
| 50 | HOOK_DATA_BYTE(8 + i) = 0; /* null byte */ | ||
| 51 | } else { | ||
| 52 | va_start(ap, pcnt); | ||
| 53 | for (i = 1; i <= pcnt; i++) | ||
| 54 | HOOK_DATA(i) = va_arg(ap, unsigned); | ||
| 55 | va_end(ap); | ||
| 56 | } | ||
| 57 | |||
| 58 | /* read from mem to make sure data has propagated to memory before | ||
| 59 | * trigging */ | ||
| 60 | ret = *((volatile unsigned *)HOOK_MEM_BASE_ADDR); | ||
| 61 | |||
| 62 | /* trigger hook */ | ||
| 63 | HOOK_TRIG(id); | ||
| 64 | |||
| 65 | /* wait for call to finish */ | ||
| 66 | while (VHOOK_DATA(0) > 0) ; | ||
| 67 | |||
| 68 | /* extract return value */ | ||
| 69 | |||
| 70 | ret = VHOOK_DATA(1); | ||
| 71 | |||
| 72 | #ifdef USING_SOS | ||
| 73 | PREEMPT_RESTORE(); | ||
| 74 | #endif | ||
| 75 | return ret; | ||
| 76 | } | ||
| 77 | |||
| 78 | unsigned hook_buf(unsigned i) | ||
| 79 | { | ||
| 80 | return (HOOK_DATA(i)); | ||
| 81 | } | ||
| 82 | |||
| 83 | void print_str(const char *str) | ||
| 84 | { | ||
| 85 | int i; | ||
| 86 | /* find null at end of string */ | ||
| 87 | for (i = 1; str[i]; i++) ; | ||
| 88 | hook_call(hook_print_str, i, str); | ||
| 89 | } | ||
| 90 | |||
| 91 | void CPU_KICK_DOG(void) | ||
| 92 | { | ||
| 93 | (void)hook_call(hook_kick_dog, 0); | ||
| 94 | } | ||
| 95 | |||
| 96 | void CPU_WATCHDOG_TIMEOUT(unsigned t) | ||
| 97 | { | ||
| 98 | (void)hook_call(hook_dog_timeout, 1, t); | ||
| 99 | } | ||
| 100 | |||
diff --git a/arch/cris/arch-v32/mach-fs/vcs_hook.h b/arch/cris/arch-v32/mach-fs/vcs_hook.h deleted file mode 100644 index c000b9fece41..000000000000 --- a/arch/cris/arch-v32/mach-fs/vcs_hook.h +++ /dev/null | |||
| @@ -1,42 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * Call simulator hook functions | ||
| 3 | */ | ||
| 4 | |||
| 5 | #ifndef HOOK_H | ||
| 6 | #define HOOK_H | ||
| 7 | |||
| 8 | int hook_call(unsigned id, unsigned pcnt, ...); | ||
| 9 | |||
| 10 | enum hook_ids { | ||
| 11 | hook_debug_on = 1, | ||
| 12 | hook_debug_off, | ||
| 13 | hook_stop_sim_ok, | ||
| 14 | hook_stop_sim_fail, | ||
| 15 | hook_alloc_shared, | ||
| 16 | hook_ptr_shared, | ||
| 17 | hook_free_shared, | ||
| 18 | hook_file2shared, | ||
| 19 | hook_cmp_shared, | ||
| 20 | hook_print_params, | ||
| 21 | hook_sim_time, | ||
| 22 | hook_stop_sim, | ||
| 23 | hook_kick_dog, | ||
| 24 | hook_dog_timeout, | ||
| 25 | hook_rand, | ||
| 26 | hook_srand, | ||
| 27 | hook_rand_range, | ||
| 28 | hook_print_str, | ||
| 29 | hook_print_hex, | ||
| 30 | hook_cmp_offset_shared, | ||
| 31 | hook_fill_random_shared, | ||
| 32 | hook_alloc_random_data, | ||
| 33 | hook_calloc_random_data, | ||
| 34 | hook_print_int, | ||
| 35 | hook_print_uint, | ||
| 36 | hook_fputc, | ||
| 37 | hook_init_fd, | ||
| 38 | hook_sbrk | ||
| 39 | |||
| 40 | }; | ||
| 41 | |||
| 42 | #endif | ||
diff --git a/arch/cris/arch-v32/mm/init.c b/arch/cris/arch-v32/mm/init.c index 0768bc409ca8..3deca5253d91 100644 --- a/arch/cris/arch-v32/mm/init.c +++ b/arch/cris/arch-v32/mm/init.c | |||
| @@ -73,11 +73,7 @@ void __init cris_mmu_init(void) | |||
| 73 | #endif | 73 | #endif |
| 74 | REG_STATE(mmu, rw_mm_cfg, seg_c, linear) | | 74 | REG_STATE(mmu, rw_mm_cfg, seg_c, linear) | |
| 75 | REG_STATE(mmu, rw_mm_cfg, seg_b, linear) | | 75 | REG_STATE(mmu, rw_mm_cfg, seg_b, linear) | |
| 76 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 77 | REG_STATE(mmu, rw_mm_cfg, seg_a, page) | | 76 | REG_STATE(mmu, rw_mm_cfg, seg_a, page) | |
| 78 | #else | ||
| 79 | REG_STATE(mmu, rw_mm_cfg, seg_a, linear) | | ||
| 80 | #endif | ||
| 81 | REG_STATE(mmu, rw_mm_cfg, seg_9, page) | | 77 | REG_STATE(mmu, rw_mm_cfg, seg_9, page) | |
| 82 | REG_STATE(mmu, rw_mm_cfg, seg_8, page) | | 78 | REG_STATE(mmu, rw_mm_cfg, seg_8, page) | |
| 83 | REG_STATE(mmu, rw_mm_cfg, seg_7, page) | | 79 | REG_STATE(mmu, rw_mm_cfg, seg_7, page) | |
| @@ -100,11 +96,7 @@ void __init cris_mmu_init(void) | |||
| 100 | #endif | 96 | #endif |
| 101 | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 0x4) | | 97 | REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 0x4) | |
| 102 | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb) | | 98 | REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb) | |
| 103 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 104 | REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0x0) | | 99 | REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0x0) | |
| 105 | #else | ||
| 106 | REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0xa) | | ||
| 107 | #endif | ||
| 108 | REG_FIELD(mmu, rw_mm_kbase_hi, base_9, 0x0) | | 100 | REG_FIELD(mmu, rw_mm_kbase_hi, base_9, 0x0) | |
| 109 | REG_FIELD(mmu, rw_mm_kbase_hi, base_8, 0x0)); | 101 | REG_FIELD(mmu, rw_mm_kbase_hi, base_8, 0x0)); |
| 110 | 102 | ||
diff --git a/arch/cris/include/arch-v10/arch/sv_addr_ag.h b/arch/cris/include/arch-v10/arch/sv_addr_ag.h index e4a6b68b8982..5517f04153a4 100644 --- a/arch/cris/include/arch-v10/arch/sv_addr_ag.h +++ b/arch/cris/include/arch-v10/arch/sv_addr_ag.h | |||
| @@ -114,7 +114,7 @@ | |||
| 114 | 114 | ||
| 115 | /*------------------------------------------------------------*/ | 115 | /*------------------------------------------------------------*/ |
| 116 | 116 | ||
| 117 | #include "sv_addr.agh" | 117 | #include <arch/sv_addr.agh> |
| 118 | 118 | ||
| 119 | #if __test_sv_addr__ | 119 | #if __test_sv_addr__ |
| 120 | /* IO_MASK( R_BUS_CONFIG , CE ) */ | 120 | /* IO_MASK( R_BUS_CONFIG , CE ) */ |
diff --git a/arch/cris/include/arch-v10/arch/svinto.h b/arch/cris/include/arch-v10/arch/svinto.h index 0881a1af7cee..da5c15272652 100644 --- a/arch/cris/include/arch-v10/arch/svinto.h +++ b/arch/cris/include/arch-v10/arch/svinto.h | |||
| @@ -1,7 +1,7 @@ | |||
| 1 | #ifndef _ASM_CRIS_SVINTO_H | 1 | #ifndef _ASM_CRIS_SVINTO_H |
| 2 | #define _ASM_CRIS_SVINTO_H | 2 | #define _ASM_CRIS_SVINTO_H |
| 3 | 3 | ||
| 4 | #include "sv_addr_ag.h" | 4 | #include <arch/sv_addr_ag.h> |
| 5 | 5 | ||
| 6 | extern unsigned int genconfig_shadow; /* defined and set in head.S */ | 6 | extern unsigned int genconfig_shadow; /* defined and set in head.S */ |
| 7 | 7 | ||
diff --git a/arch/cris/include/arch-v32/arch/dma.h b/arch/cris/include/arch-v32/arch/dma.h index 61906153a9af..6f92f4f23f28 100644 --- a/arch/cris/include/arch-v32/arch/dma.h +++ b/arch/cris/include/arch-v32/arch/dma.h | |||
| @@ -1 +1 @@ | |||
| #include "mach/dma.h" | #include <mach/dma.h> | ||
diff --git a/arch/cris/include/arch-v32/arch/hwregs/dma.h b/arch/cris/include/arch-v32/arch/hwregs/dma.h index 3ce322b5c731..52bf67907f28 100644 --- a/arch/cris/include/arch-v32/arch/hwregs/dma.h +++ b/arch/cris/include/arch-v32/arch/hwregs/dma.h | |||
| @@ -7,7 +7,7 @@ | |||
| 7 | #define dma_h | 7 | #define dma_h |
| 8 | 8 | ||
| 9 | /* registers */ /* Really needed, since both are listed in sw.list? */ | 9 | /* registers */ /* Really needed, since both are listed in sw.list? */ |
| 10 | #include "dma_defs.h" | 10 | #include <arch/hwregs/dma_defs.h> |
| 11 | 11 | ||
| 12 | 12 | ||
| 13 | /* descriptors */ | 13 | /* descriptors */ |
diff --git a/arch/cris/include/arch-v32/arch/page.h b/arch/cris/include/arch-v32/arch/page.h index 20f1b4806bfe..e5b5aab52de8 100644 --- a/arch/cris/include/arch-v32/arch/page.h +++ b/arch/cris/include/arch-v32/arch/page.h | |||
| @@ -11,13 +11,8 @@ | |||
| 11 | * selected bit it's possible to convert between KSEG_x and 0x40000000 where the | 11 | * selected bit it's possible to convert between KSEG_x and 0x40000000 where the |
| 12 | * DRAM really resides. DRAM is virtually at 0xc. | 12 | * DRAM really resides. DRAM is virtually at 0xc. |
| 13 | */ | 13 | */ |
| 14 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 15 | #define __pa(x) ((unsigned long)(x) & 0x7fffffff) | 14 | #define __pa(x) ((unsigned long)(x) & 0x7fffffff) |
| 16 | #define __va(x) ((void *)((unsigned long)(x) | 0x80000000)) | 15 | #define __va(x) ((void *)((unsigned long)(x) | 0x80000000)) |
| 17 | #else | ||
| 18 | #define __pa(x) ((unsigned long)(x) & 0x3fffffff) | ||
| 19 | #define __va(x) ((void *)((unsigned long)(x) | 0xc0000000)) | ||
| 20 | #endif | ||
| 21 | 16 | ||
| 22 | #define VM_STACK_DEFAULT_FLAGS (VM_READ | VM_WRITE | \ | 17 | #define VM_STACK_DEFAULT_FLAGS (VM_READ | VM_WRITE | \ |
| 23 | VM_MAYREAD | VM_MAYWRITE) | 18 | VM_MAYREAD | VM_MAYWRITE) |
diff --git a/arch/cris/include/arch-v32/arch/processor.h b/arch/cris/include/arch-v32/arch/processor.h index 9603c907fbc4..a024b7d32fed 100644 --- a/arch/cris/include/arch-v32/arch/processor.h +++ b/arch/cris/include/arch-v32/arch/processor.h | |||
| @@ -21,13 +21,9 @@ struct thread_struct { | |||
| 21 | 21 | ||
| 22 | /* | 22 | /* |
| 23 | * User-space process size. This is hardcoded into a few places, so don't | 23 | * User-space process size. This is hardcoded into a few places, so don't |
| 24 | * changed it unless everything's clear! | 24 | * change it unless everything's clear! |
| 25 | */ | 25 | */ |
| 26 | #ifndef CONFIG_ETRAX_VCS_SIM | ||
| 27 | #define TASK_SIZE (0xB0000000UL) | 26 | #define TASK_SIZE (0xB0000000UL) |
| 28 | #else | ||
| 29 | #define TASK_SIZE (0xA0000000UL) | ||
| 30 | #endif | ||
| 31 | 27 | ||
| 32 | /* CCS I=1, enable interrupts. */ | 28 | /* CCS I=1, enable interrupts. */ |
| 33 | #define INIT_THREAD { 0, 0, (1 << I_CCS_BITNR) } | 29 | #define INIT_THREAD { 0, 0, (1 << I_CCS_BITNR) } |
diff --git a/arch/cris/include/arch-v32/mach-fs/mach/startup.inc b/arch/cris/include/arch-v32/mach-fs/mach/startup.inc index dd1abbdcbc7a..96c3b0fb62c1 100644 --- a/arch/cris/include/arch-v32/mach-fs/mach/startup.inc +++ b/arch/cris/include/arch-v32/mach-fs/mach/startup.inc | |||
| @@ -71,12 +71,6 @@ | |||
| 71 | move.d REG_ADDR(bif_core, regi_bif_core, rw_grp4_cfg), $r0 | 71 | move.d REG_ADDR(bif_core, regi_bif_core, rw_grp4_cfg), $r0 |
| 72 | move.d CONFIG_ETRAX_MEM_GRP4_CONFIG, $r1 | 72 | move.d CONFIG_ETRAX_MEM_GRP4_CONFIG, $r1 |
| 73 | move.d $r1, [$r0] | 73 | move.d $r1, [$r0] |
| 74 | #ifdef CONFIG_ETRAX_VCS_SIM | ||
| 75 | ;; Set up minimal flash waitstates | ||
| 76 | move.d 0, $r10 | ||
| 77 | move.d REG_ADDR(bif_core, regi_bif_core, rw_grp1_cfg), $r11 | ||
| 78 | move.d $r10, [$r11] | ||
| 79 | #endif | ||
| 80 | .endm | 74 | .endm |
| 81 | 75 | ||
| 82 | #endif | 76 | #endif |
diff --git a/arch/cris/include/asm/elf.h b/arch/cris/include/asm/elf.h index 8a3d8e2b33c1..8182f2dc89d0 100644 --- a/arch/cris/include/asm/elf.h +++ b/arch/cris/include/asm/elf.h | |||
| @@ -86,6 +86,7 @@ typedef unsigned long elf_fpregset_t; | |||
| 86 | 86 | ||
| 87 | #define ELF_PLATFORM (NULL) | 87 | #define ELF_PLATFORM (NULL) |
| 88 | 88 | ||
| 89 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 89 | #define SET_PERSONALITY(ex) \ |
| 90 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 90 | 91 | ||
| 91 | #endif | 92 | #endif |
diff --git a/arch/cris/include/asm/pci.h b/arch/cris/include/asm/pci.h index 9f1cd56da28c..146da904cdd8 100644 --- a/arch/cris/include/asm/pci.h +++ b/arch/cris/include/asm/pci.h | |||
| @@ -19,7 +19,6 @@ extern unsigned long pci_mem_start; | |||
| 19 | 19 | ||
| 20 | void pcibios_config_init(void); | 20 | void pcibios_config_init(void); |
| 21 | struct pci_bus * pcibios_scan_root(int bus); | 21 | struct pci_bus * pcibios_scan_root(int bus); |
| 22 | int pcibios_assign_resources(void); | ||
| 23 | 22 | ||
| 24 | void pcibios_set_master(struct pci_dev *dev); | 23 | void pcibios_set_master(struct pci_dev *dev); |
| 25 | void pcibios_penalize_isa_irq(int irq); | 24 | void pcibios_penalize_isa_irq(int irq); |
diff --git a/arch/cris/include/uapi/arch-v10/arch/Kbuild b/arch/cris/include/uapi/arch-v10/arch/Kbuild new file mode 100644 index 000000000000..aafaa5aa54d4 --- /dev/null +++ b/arch/cris/include/uapi/arch-v10/arch/Kbuild | |||
| @@ -0,0 +1 @@ | |||
| # UAPI Header export list | |||
diff --git a/arch/cris/include/uapi/arch-v32/arch/Kbuild b/arch/cris/include/uapi/arch-v32/arch/Kbuild new file mode 100644 index 000000000000..aafaa5aa54d4 --- /dev/null +++ b/arch/cris/include/uapi/arch-v32/arch/Kbuild | |||
| @@ -0,0 +1 @@ | |||
| # UAPI Header export list | |||
diff --git a/arch/cris/include/uapi/asm/Kbuild b/arch/cris/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..f50236ae9ca3 --- /dev/null +++ b/arch/cris/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,5 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
| 4 | header-y += arch-v10/ | ||
| 5 | header-y += arch-v32/ | ||
diff --git a/arch/frv/include/asm/elf.h b/arch/frv/include/asm/elf.h index c3819804a74b..9ccbc80f0b11 100644 --- a/arch/frv/include/asm/elf.h +++ b/arch/frv/include/asm/elf.h | |||
| @@ -137,6 +137,7 @@ do { \ | |||
| 137 | 137 | ||
| 138 | #define ELF_PLATFORM (NULL) | 138 | #define ELF_PLATFORM (NULL) |
| 139 | 139 | ||
| 140 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 140 | #define SET_PERSONALITY(ex) \ |
| 141 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 141 | 142 | ||
| 142 | #endif | 143 | #endif |
diff --git a/arch/frv/include/uapi/asm/Kbuild b/arch/frv/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/frv/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/frv/kernel/pm.c b/arch/frv/kernel/pm.c index 5fa3889d858b..0b579927439d 100644 --- a/arch/frv/kernel/pm.c +++ b/arch/frv/kernel/pm.c | |||
| @@ -153,23 +153,22 @@ static int user_atoi(char __user *ubuf, size_t len) | |||
| 153 | static int sysctl_pm_do_suspend(ctl_table *ctl, int write, | 153 | static int sysctl_pm_do_suspend(ctl_table *ctl, int write, |
| 154 | void __user *buffer, size_t *lenp, loff_t *fpos) | 154 | void __user *buffer, size_t *lenp, loff_t *fpos) |
| 155 | { | 155 | { |
| 156 | int retval, mode; | 156 | int mode; |
| 157 | 157 | ||
| 158 | if (*lenp <= 0) | 158 | if (*lenp <= 0) |
| 159 | return -EIO; | 159 | return -EIO; |
| 160 | 160 | ||
| 161 | mode = user_atoi(buffer, *lenp); | 161 | mode = user_atoi(buffer, *lenp); |
| 162 | if ((mode != 1) && (mode != 5)) | 162 | switch (mode) { |
| 163 | return -EINVAL; | 163 | case 1: |
| 164 | return pm_do_suspend(); | ||
| 164 | 165 | ||
| 165 | if (retval == 0) { | 166 | case 5: |
| 166 | if (mode == 5) | 167 | return pm_do_bus_sleep(); |
| 167 | retval = pm_do_bus_sleep(); | ||
| 168 | else | ||
| 169 | retval = pm_do_suspend(); | ||
| 170 | } | ||
| 171 | 168 | ||
| 172 | return retval; | 169 | default: |
| 170 | return -EINVAL; | ||
| 171 | } | ||
| 173 | } | 172 | } |
| 174 | 173 | ||
| 175 | static int try_set_cmode(int new_cmode) | 174 | static int try_set_cmode(int new_cmode) |
diff --git a/arch/frv/kernel/setup.c b/arch/frv/kernel/setup.c index 75cf7f4b2fa8..1f1e5efb3385 100644 --- a/arch/frv/kernel/setup.c +++ b/arch/frv/kernel/setup.c | |||
| @@ -184,7 +184,7 @@ static struct clock_cmode __pminitdata clock_cmodes_fr555[16] = { | |||
| 184 | [6] = { _x1, _x1_5, _x1_5, _x4_5, _x0_375 }, | 184 | [6] = { _x1, _x1_5, _x1_5, _x4_5, _x0_375 }, |
| 185 | }; | 185 | }; |
| 186 | 186 | ||
| 187 | static const struct clock_cmode __pminitdata *clock_cmodes; | 187 | static const struct clock_cmode __pminitconst *clock_cmodes; |
| 188 | static int __pminitdata clock_doubled; | 188 | static int __pminitdata clock_doubled; |
| 189 | 189 | ||
| 190 | static struct uart_port __pminitdata __frv_uart0 = { | 190 | static struct uart_port __pminitdata __frv_uart0 = { |
diff --git a/arch/frv/mb93090-mb00/pci-irq.c b/arch/frv/mb93090-mb00/pci-irq.c index 20f6497b2cd5..c677b9d81d30 100644 --- a/arch/frv/mb93090-mb00/pci-irq.c +++ b/arch/frv/mb93090-mb00/pci-irq.c | |||
| @@ -28,7 +28,7 @@ | |||
| 28 | * | 28 | * |
| 29 | */ | 29 | */ |
| 30 | 30 | ||
| 31 | static const uint8_t __initdata pci_bus0_irq_routing[32][4] = { | 31 | static const uint8_t __initconst pci_bus0_irq_routing[32][4] = { |
| 32 | [0 ] = { IRQ_FPGA_MB86943_PCI_INTA }, | 32 | [0 ] = { IRQ_FPGA_MB86943_PCI_INTA }, |
| 33 | [16] = { IRQ_FPGA_RTL8029_INTA }, | 33 | [16] = { IRQ_FPGA_RTL8029_INTA }, |
| 34 | [17] = { IRQ_FPGA_PCI_INTC, IRQ_FPGA_PCI_INTD, IRQ_FPGA_PCI_INTA, IRQ_FPGA_PCI_INTB }, | 34 | [17] = { IRQ_FPGA_PCI_INTC, IRQ_FPGA_PCI_INTD, IRQ_FPGA_PCI_INTA, IRQ_FPGA_PCI_INTB }, |
diff --git a/arch/h8300/include/asm/elf.h b/arch/h8300/include/asm/elf.h index c24fa250d653..41193c396bff 100644 --- a/arch/h8300/include/asm/elf.h +++ b/arch/h8300/include/asm/elf.h | |||
| @@ -54,7 +54,8 @@ typedef unsigned long elf_fpregset_t; | |||
| 54 | 54 | ||
| 55 | #define ELF_PLATFORM (NULL) | 55 | #define ELF_PLATFORM (NULL) |
| 56 | 56 | ||
| 57 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 57 | #define SET_PERSONALITY(ex) \ |
| 58 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 58 | 59 | ||
| 59 | #define R_H8_NONE 0 | 60 | #define R_H8_NONE 0 |
| 60 | #define R_H8_DIR32 1 | 61 | #define R_H8_DIR32 1 |
diff --git a/arch/h8300/include/uapi/asm/Kbuild b/arch/h8300/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/h8300/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/h8300/kernel/sys_h8300.c b/arch/h8300/kernel/sys_h8300.c index aaf5e5a48f93..4bdc7311784e 100644 --- a/arch/h8300/kernel/sys_h8300.c +++ b/arch/h8300/kernel/sys_h8300.c | |||
| @@ -51,6 +51,7 @@ asmlinkage void syscall_print(void *dummy,...) | |||
| 51 | * Do a system call from kernel instead of calling sys_execve so we | 51 | * Do a system call from kernel instead of calling sys_execve so we |
| 52 | * end up with proper pt_regs. | 52 | * end up with proper pt_regs. |
| 53 | */ | 53 | */ |
| 54 | asmlinkage | ||
| 54 | int kernel_execve(const char *filename, | 55 | int kernel_execve(const char *filename, |
| 55 | const char *const argv[], | 56 | const char *const argv[], |
| 56 | const char *const envp[]) | 57 | const char *const envp[]) |
diff --git a/arch/h8300/kernel/timer/itu.c b/arch/h8300/kernel/timer/itu.c index a2ae5e952137..0a8b5cd5bf38 100644 --- a/arch/h8300/kernel/timer/itu.c +++ b/arch/h8300/kernel/timer/itu.c | |||
| @@ -62,7 +62,7 @@ static struct irqaction itu_irq = { | |||
| 62 | .flags = IRQF_DISABLED | IRQF_TIMER, | 62 | .flags = IRQF_DISABLED | IRQF_TIMER, |
| 63 | }; | 63 | }; |
| 64 | 64 | ||
| 65 | static const int __initdata divide_rate[] = {1, 2, 4, 8}; | 65 | static const int __initconst divide_rate[] = {1, 2, 4, 8}; |
| 66 | 66 | ||
| 67 | void __init h8300_timer_setup(void) | 67 | void __init h8300_timer_setup(void) |
| 68 | { | 68 | { |
diff --git a/arch/h8300/kernel/timer/timer16.c b/arch/h8300/kernel/timer/timer16.c index ae0d38161139..462d9f581719 100644 --- a/arch/h8300/kernel/timer/timer16.c +++ b/arch/h8300/kernel/timer/timer16.c | |||
| @@ -57,7 +57,7 @@ static struct irqaction timer16_irq = { | |||
| 57 | .flags = IRQF_DISABLED | IRQF_TIMER, | 57 | .flags = IRQF_DISABLED | IRQF_TIMER, |
| 58 | }; | 58 | }; |
| 59 | 59 | ||
| 60 | static const int __initdata divide_rate[] = {1, 2, 4, 8}; | 60 | static const int __initconst divide_rate[] = {1, 2, 4, 8}; |
| 61 | 61 | ||
| 62 | void __init h8300_timer_setup(void) | 62 | void __init h8300_timer_setup(void) |
| 63 | { | 63 | { |
diff --git a/arch/h8300/kernel/timer/timer8.c b/arch/h8300/kernel/timer/timer8.c index 7a1533fad47d..505f3415b40f 100644 --- a/arch/h8300/kernel/timer/timer8.c +++ b/arch/h8300/kernel/timer/timer8.c | |||
| @@ -77,7 +77,7 @@ static struct irqaction timer8_irq = { | |||
| 77 | .flags = IRQF_DISABLED | IRQF_TIMER, | 77 | .flags = IRQF_DISABLED | IRQF_TIMER, |
| 78 | }; | 78 | }; |
| 79 | 79 | ||
| 80 | static const int __initdata divide_rate[] = {8, 64, 8192}; | 80 | static const int __initconst divide_rate[] = {8, 64, 8192}; |
| 81 | 81 | ||
| 82 | void __init h8300_timer_setup(void) | 82 | void __init h8300_timer_setup(void) |
| 83 | { | 83 | { |
diff --git a/arch/h8300/kernel/timer/tpu.c b/arch/h8300/kernel/timer/tpu.c index 2193a2e2859a..0350f6204ecf 100644 --- a/arch/h8300/kernel/timer/tpu.c +++ b/arch/h8300/kernel/timer/tpu.c | |||
| @@ -66,7 +66,7 @@ static struct irqaction tpu_irq = { | |||
| 66 | .flags = IRQF_DISABLED | IRQF_TIMER, | 66 | .flags = IRQF_DISABLED | IRQF_TIMER, |
| 67 | }; | 67 | }; |
| 68 | 68 | ||
| 69 | static const int __initdata divide_rate[] = { | 69 | static const int __initconst divide_rate[] = { |
| 70 | #if CONFIG_H8300_TPU_CH == 0 | 70 | #if CONFIG_H8300_TPU_CH == 0 |
| 71 | 1,4,16,64,0,0,0,0, | 71 | 1,4,16,64,0,0,0,0, |
| 72 | #elif (CONFIG_H8300_TPU_CH == 1) || (CONFIG_H8300_TPU_CH == 5) | 72 | #elif (CONFIG_H8300_TPU_CH == 1) || (CONFIG_H8300_TPU_CH == 5) |
diff --git a/arch/h8300/platform/h8300h/irq.c b/arch/h8300/platform/h8300h/irq.c index bc4f51bceef5..0a50353e09d5 100644 --- a/arch/h8300/platform/h8300h/irq.c +++ b/arch/h8300/platform/h8300h/irq.c | |||
| @@ -14,14 +14,14 @@ | |||
| 14 | #include <asm/gpio-internal.h> | 14 | #include <asm/gpio-internal.h> |
| 15 | #include <asm/regs306x.h> | 15 | #include <asm/regs306x.h> |
| 16 | 16 | ||
| 17 | const int __initdata h8300_saved_vectors[] = { | 17 | const int __initconst h8300_saved_vectors[] = { |
| 18 | #if defined(CONFIG_GDB_DEBUG) | 18 | #if defined(CONFIG_GDB_DEBUG) |
| 19 | TRAP3_VEC, /* TRAPA #3 is GDB breakpoint */ | 19 | TRAP3_VEC, /* TRAPA #3 is GDB breakpoint */ |
| 20 | #endif | 20 | #endif |
| 21 | -1, | 21 | -1, |
| 22 | }; | 22 | }; |
| 23 | 23 | ||
| 24 | const h8300_vector __initdata h8300_trap_table[] = { | 24 | const h8300_vector __initconst h8300_trap_table[] = { |
| 25 | 0, 0, 0, 0, 0, 0, 0, 0, | 25 | 0, 0, 0, 0, 0, 0, 0, 0, |
| 26 | system_call, | 26 | system_call, |
| 27 | 0, | 27 | 0, |
diff --git a/arch/h8300/platform/h8s/irq.c b/arch/h8300/platform/h8s/irq.c index 7b5f29febc07..f3a5511c16b1 100644 --- a/arch/h8300/platform/h8s/irq.c +++ b/arch/h8300/platform/h8s/irq.c | |||
| @@ -18,7 +18,7 @@ | |||
| 18 | #include <asm/regs267x.h> | 18 | #include <asm/regs267x.h> |
| 19 | 19 | ||
| 20 | /* saved vector list */ | 20 | /* saved vector list */ |
| 21 | const int __initdata h8300_saved_vectors[]={ | 21 | const int __initconst h8300_saved_vectors[] = { |
| 22 | #if defined(CONFIG_GDB_DEBUG) | 22 | #if defined(CONFIG_GDB_DEBUG) |
| 23 | TRACE_VEC, | 23 | TRACE_VEC, |
| 24 | TRAP3_VEC, | 24 | TRAP3_VEC, |
| @@ -27,7 +27,7 @@ const int __initdata h8300_saved_vectors[]={ | |||
| 27 | }; | 27 | }; |
| 28 | 28 | ||
| 29 | /* trap entry table */ | 29 | /* trap entry table */ |
| 30 | const H8300_VECTOR __initdata h8300_trap_table[] = { | 30 | const H8300_VECTOR __initconst h8300_trap_table[] = { |
| 31 | 0,0,0,0,0, | 31 | 0,0,0,0,0, |
| 32 | trace_break, /* TRACE */ | 32 | trace_break, /* TRACE */ |
| 33 | 0,0, | 33 | 0,0, |
diff --git a/arch/hexagon/include/asm/elf.h b/arch/hexagon/include/asm/elf.h index 37976a0d3650..82b499621e05 100644 --- a/arch/hexagon/include/asm/elf.h +++ b/arch/hexagon/include/asm/elf.h | |||
| @@ -217,7 +217,8 @@ do { \ | |||
| 217 | #define ELF_PLATFORM (NULL) | 217 | #define ELF_PLATFORM (NULL) |
| 218 | 218 | ||
| 219 | #ifdef __KERNEL__ | 219 | #ifdef __KERNEL__ |
| 220 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 220 | #define SET_PERSONALITY(ex) \ |
| 221 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 221 | #endif | 222 | #endif |
| 222 | 223 | ||
| 223 | #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 | 224 | #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 |
diff --git a/arch/hexagon/include/uapi/asm/Kbuild b/arch/hexagon/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/hexagon/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/ia64/include/asm/xen/interface.h b/arch/ia64/include/asm/xen/interface.h index 09d5f7fd9db1..3d52a5bbd857 100644 --- a/arch/ia64/include/asm/xen/interface.h +++ b/arch/ia64/include/asm/xen/interface.h | |||
| @@ -67,6 +67,10 @@ | |||
| 67 | #define set_xen_guest_handle(hnd, val) do { (hnd).p = val; } while (0) | 67 | #define set_xen_guest_handle(hnd, val) do { (hnd).p = val; } while (0) |
| 68 | 68 | ||
| 69 | #ifndef __ASSEMBLY__ | 69 | #ifndef __ASSEMBLY__ |
| 70 | /* Explicitly size integers that represent pfns in the public interface | ||
| 71 | * with Xen so that we could have one ABI that works for 32 and 64 bit | ||
| 72 | * guests. */ | ||
| 73 | typedef unsigned long xen_pfn_t; | ||
| 70 | /* Guest handles for primitive C types. */ | 74 | /* Guest handles for primitive C types. */ |
| 71 | __DEFINE_GUEST_HANDLE(uchar, unsigned char); | 75 | __DEFINE_GUEST_HANDLE(uchar, unsigned char); |
| 72 | __DEFINE_GUEST_HANDLE(uint, unsigned int); | 76 | __DEFINE_GUEST_HANDLE(uint, unsigned int); |
| @@ -79,7 +83,6 @@ DEFINE_GUEST_HANDLE(void); | |||
| 79 | DEFINE_GUEST_HANDLE(uint64_t); | 83 | DEFINE_GUEST_HANDLE(uint64_t); |
| 80 | DEFINE_GUEST_HANDLE(uint32_t); | 84 | DEFINE_GUEST_HANDLE(uint32_t); |
| 81 | 85 | ||
| 82 | typedef unsigned long xen_pfn_t; | ||
| 83 | DEFINE_GUEST_HANDLE(xen_pfn_t); | 86 | DEFINE_GUEST_HANDLE(xen_pfn_t); |
| 84 | #define PRI_xen_pfn "lx" | 87 | #define PRI_xen_pfn "lx" |
| 85 | #endif | 88 | #endif |
| @@ -265,6 +268,8 @@ typedef struct xen_callback xen_callback_t; | |||
| 265 | 268 | ||
| 266 | #endif /* !__ASSEMBLY__ */ | 269 | #endif /* !__ASSEMBLY__ */ |
| 267 | 270 | ||
| 271 | #include <asm/pvclock-abi.h> | ||
| 272 | |||
| 268 | /* Size of the shared_info area (this is not related to page size). */ | 273 | /* Size of the shared_info area (this is not related to page size). */ |
| 269 | #define XSI_SHIFT 14 | 274 | #define XSI_SHIFT 14 |
| 270 | #define XSI_SIZE (1 << XSI_SHIFT) | 275 | #define XSI_SIZE (1 << XSI_SHIFT) |
diff --git a/arch/ia64/include/uapi/asm/Kbuild b/arch/ia64/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/ia64/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index 5a5c22245dee..f388b4e18a37 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c | |||
| @@ -2306,7 +2306,7 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t | |||
| 2306 | * partially initialize the vma for the sampling buffer | 2306 | * partially initialize the vma for the sampling buffer |
| 2307 | */ | 2307 | */ |
| 2308 | vma->vm_mm = mm; | 2308 | vma->vm_mm = mm; |
| 2309 | vma->vm_file = filp; | 2309 | vma->vm_file = get_file(filp); |
| 2310 | vma->vm_flags = VM_READ| VM_MAYREAD |VM_RESERVED; | 2310 | vma->vm_flags = VM_READ| VM_MAYREAD |VM_RESERVED; |
| 2311 | vma->vm_page_prot = PAGE_READONLY; /* XXX may need to change */ | 2311 | vma->vm_page_prot = PAGE_READONLY; /* XXX may need to change */ |
| 2312 | 2312 | ||
| @@ -2345,8 +2345,6 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t | |||
| 2345 | goto error; | 2345 | goto error; |
| 2346 | } | 2346 | } |
| 2347 | 2347 | ||
| 2348 | get_file(filp); | ||
| 2349 | |||
| 2350 | /* | 2348 | /* |
| 2351 | * now insert the vma in the vm list for the process, must be | 2349 | * now insert the vma in the vm list for the process, must be |
| 2352 | * done with mmap lock held | 2350 | * done with mmap lock held |
| @@ -4782,7 +4780,7 @@ recheck: | |||
| 4782 | asmlinkage long | 4780 | asmlinkage long |
| 4783 | sys_perfmonctl (int fd, int cmd, void __user *arg, int count) | 4781 | sys_perfmonctl (int fd, int cmd, void __user *arg, int count) |
| 4784 | { | 4782 | { |
| 4785 | struct file *file = NULL; | 4783 | struct fd f = {NULL, 0}; |
| 4786 | pfm_context_t *ctx = NULL; | 4784 | pfm_context_t *ctx = NULL; |
| 4787 | unsigned long flags = 0UL; | 4785 | unsigned long flags = 0UL; |
| 4788 | void *args_k = NULL; | 4786 | void *args_k = NULL; |
| @@ -4879,17 +4877,17 @@ restart_args: | |||
| 4879 | 4877 | ||
| 4880 | ret = -EBADF; | 4878 | ret = -EBADF; |
| 4881 | 4879 | ||
| 4882 | file = fget(fd); | 4880 | f = fdget(fd); |
| 4883 | if (unlikely(file == NULL)) { | 4881 | if (unlikely(f.file == NULL)) { |
| 4884 | DPRINT(("invalid fd %d\n", fd)); | 4882 | DPRINT(("invalid fd %d\n", fd)); |
| 4885 | goto error_args; | 4883 | goto error_args; |
| 4886 | } | 4884 | } |
| 4887 | if (unlikely(PFM_IS_FILE(file) == 0)) { | 4885 | if (unlikely(PFM_IS_FILE(f.file) == 0)) { |
| 4888 | DPRINT(("fd %d not related to perfmon\n", fd)); | 4886 | DPRINT(("fd %d not related to perfmon\n", fd)); |
| 4889 | goto error_args; | 4887 | goto error_args; |
| 4890 | } | 4888 | } |
| 4891 | 4889 | ||
| 4892 | ctx = file->private_data; | 4890 | ctx = f.file->private_data; |
| 4893 | if (unlikely(ctx == NULL)) { | 4891 | if (unlikely(ctx == NULL)) { |
| 4894 | DPRINT(("no context for fd %d\n", fd)); | 4892 | DPRINT(("no context for fd %d\n", fd)); |
| 4895 | goto error_args; | 4893 | goto error_args; |
| @@ -4919,8 +4917,8 @@ abort_locked: | |||
| 4919 | if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT; | 4917 | if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT; |
| 4920 | 4918 | ||
| 4921 | error_args: | 4919 | error_args: |
| 4922 | if (file) | 4920 | if (f.file) |
| 4923 | fput(file); | 4921 | fdput(f); |
| 4924 | 4922 | ||
| 4925 | kfree(args_k); | 4923 | kfree(args_k); |
| 4926 | 4924 | ||
diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c index bd77cb507c1c..8b3a9c0e771d 100644 --- a/arch/ia64/kvm/kvm-ia64.c +++ b/arch/ia64/kvm/kvm-ia64.c | |||
| @@ -924,6 +924,16 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
| 924 | return 0; | 924 | return 0; |
| 925 | } | 925 | } |
| 926 | 926 | ||
| 927 | int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event) | ||
| 928 | { | ||
| 929 | if (!irqchip_in_kernel(kvm)) | ||
| 930 | return -ENXIO; | ||
| 931 | |||
| 932 | irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, | ||
| 933 | irq_event->irq, irq_event->level); | ||
| 934 | return 0; | ||
| 935 | } | ||
| 936 | |||
| 927 | long kvm_arch_vm_ioctl(struct file *filp, | 937 | long kvm_arch_vm_ioctl(struct file *filp, |
| 928 | unsigned int ioctl, unsigned long arg) | 938 | unsigned int ioctl, unsigned long arg) |
| 929 | { | 939 | { |
| @@ -963,29 +973,6 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
| 963 | goto out; | 973 | goto out; |
| 964 | } | 974 | } |
| 965 | break; | 975 | break; |
| 966 | case KVM_IRQ_LINE_STATUS: | ||
| 967 | case KVM_IRQ_LINE: { | ||
| 968 | struct kvm_irq_level irq_event; | ||
| 969 | |||
| 970 | r = -EFAULT; | ||
| 971 | if (copy_from_user(&irq_event, argp, sizeof irq_event)) | ||
| 972 | goto out; | ||
| 973 | r = -ENXIO; | ||
| 974 | if (irqchip_in_kernel(kvm)) { | ||
| 975 | __s32 status; | ||
| 976 | status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, | ||
| 977 | irq_event.irq, irq_event.level); | ||
| 978 | if (ioctl == KVM_IRQ_LINE_STATUS) { | ||
| 979 | r = -EFAULT; | ||
| 980 | irq_event.status = status; | ||
| 981 | if (copy_to_user(argp, &irq_event, | ||
| 982 | sizeof irq_event)) | ||
| 983 | goto out; | ||
| 984 | } | ||
| 985 | r = 0; | ||
| 986 | } | ||
| 987 | break; | ||
| 988 | } | ||
| 989 | case KVM_GET_IRQCHIP: { | 976 | case KVM_GET_IRQCHIP: { |
| 990 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ | 977 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ |
| 991 | struct kvm_irqchip chip; | 978 | struct kvm_irqchip chip; |
| @@ -1626,11 +1613,17 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, | |||
| 1626 | return; | 1613 | return; |
| 1627 | } | 1614 | } |
| 1628 | 1615 | ||
| 1629 | void kvm_arch_flush_shadow(struct kvm *kvm) | 1616 | void kvm_arch_flush_shadow_all(struct kvm *kvm) |
| 1630 | { | 1617 | { |
| 1631 | kvm_flush_remote_tlbs(kvm); | 1618 | kvm_flush_remote_tlbs(kvm); |
| 1632 | } | 1619 | } |
| 1633 | 1620 | ||
| 1621 | void kvm_arch_flush_shadow_memslot(struct kvm *kvm, | ||
| 1622 | struct kvm_memory_slot *slot) | ||
| 1623 | { | ||
| 1624 | kvm_arch_flush_shadow_all(); | ||
| 1625 | } | ||
| 1626 | |||
| 1634 | long kvm_arch_dev_ioctl(struct file *filp, | 1627 | long kvm_arch_dev_ioctl(struct file *filp, |
| 1635 | unsigned int ioctl, unsigned long arg) | 1628 | unsigned int ioctl, unsigned long arg) |
| 1636 | { | 1629 | { |
diff --git a/arch/ia64/xen/irq_xen.c b/arch/ia64/xen/irq_xen.c index 3bb12230721f..01f479ee1c43 100644 --- a/arch/ia64/xen/irq_xen.c +++ b/arch/ia64/xen/irq_xen.c | |||
| @@ -433,7 +433,7 @@ xen_resend_irq(unsigned int vector) | |||
| 433 | (void)resend_irq_on_evtchn(vector); | 433 | (void)resend_irq_on_evtchn(vector); |
| 434 | } | 434 | } |
| 435 | 435 | ||
| 436 | const struct pv_irq_ops xen_irq_ops __initdata = { | 436 | const struct pv_irq_ops xen_irq_ops __initconst = { |
| 437 | .register_ipi = xen_register_ipi, | 437 | .register_ipi = xen_register_ipi, |
| 438 | 438 | ||
| 439 | .assign_irq_vector = xen_assign_irq_vector, | 439 | .assign_irq_vector = xen_assign_irq_vector, |
diff --git a/arch/ia64/xen/irq_xen.h b/arch/ia64/xen/irq_xen.h index 26110f330c87..1778517b90fe 100644 --- a/arch/ia64/xen/irq_xen.h +++ b/arch/ia64/xen/irq_xen.h | |||
| @@ -27,7 +27,7 @@ extern void (*late_time_init)(void); | |||
| 27 | extern char xen_event_callback; | 27 | extern char xen_event_callback; |
| 28 | void __init xen_init_IRQ(void); | 28 | void __init xen_init_IRQ(void); |
| 29 | 29 | ||
| 30 | extern const struct pv_irq_ops xen_irq_ops __initdata; | 30 | extern const struct pv_irq_ops xen_irq_ops __initconst; |
| 31 | extern void xen_smp_intr_init(void); | 31 | extern void xen_smp_intr_init(void); |
| 32 | extern void xen_send_ipi(int cpu, int vec); | 32 | extern void xen_send_ipi(int cpu, int vec); |
| 33 | 33 | ||
diff --git a/arch/m32r/include/asm/elf.h b/arch/m32r/include/asm/elf.h index b8da7d0574d2..70896161c636 100644 --- a/arch/m32r/include/asm/elf.h +++ b/arch/m32r/include/asm/elf.h | |||
| @@ -128,6 +128,7 @@ typedef elf_fpreg_t elf_fpregset_t; | |||
| 128 | intent than poking at uname or /proc/cpuinfo. */ | 128 | intent than poking at uname or /proc/cpuinfo. */ |
| 129 | #define ELF_PLATFORM (NULL) | 129 | #define ELF_PLATFORM (NULL) |
| 130 | 130 | ||
| 131 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 131 | #define SET_PERSONALITY(ex) \ |
| 132 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 132 | 133 | ||
| 133 | #endif /* _ASM_M32R__ELF_H */ | 134 | #endif /* _ASM_M32R__ELF_H */ |
diff --git a/arch/m32r/include/uapi/asm/Kbuild b/arch/m32r/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/m32r/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/m68k/include/asm/cacheflush.h b/arch/m68k/include/asm/cacheflush.h index a70d7319630a..4fc738209bd1 100644 --- a/arch/m68k/include/asm/cacheflush.h +++ b/arch/m68k/include/asm/cacheflush.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef __uClinux__ | 1 | #ifdef __uClinux__ |
| 2 | #include "cacheflush_no.h" | 2 | #include <asm/cacheflush_no.h> |
| 3 | #else | 3 | #else |
| 4 | #include "cacheflush_mm.h" | 4 | #include <asm/cacheflush_mm.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/m68k/include/asm/elf.h b/arch/m68k/include/asm/elf.h index e9b7cda59744..f83c1d0a87cf 100644 --- a/arch/m68k/include/asm/elf.h +++ b/arch/m68k/include/asm/elf.h | |||
| @@ -113,6 +113,7 @@ typedef struct user_m68kfp_struct elf_fpregset_t; | |||
| 113 | 113 | ||
| 114 | #define ELF_PLATFORM (NULL) | 114 | #define ELF_PLATFORM (NULL) |
| 115 | 115 | ||
| 116 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 116 | #define SET_PERSONALITY(ex) \ |
| 117 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 117 | 118 | ||
| 118 | #endif | 119 | #endif |
diff --git a/arch/m68k/include/asm/io.h b/arch/m68k/include/asm/io.h index c7210ba184ea..c70cc9155003 100644 --- a/arch/m68k/include/asm/io.h +++ b/arch/m68k/include/asm/io.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef __uClinux__ | 1 | #ifdef __uClinux__ |
| 2 | #include "io_no.h" | 2 | #include <asm/io_no.h> |
| 3 | #else | 3 | #else |
| 4 | #include "io_mm.h" | 4 | #include <asm/io_mm.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/m68k/include/asm/m68360.h b/arch/m68k/include/asm/m68360.h index eb7d39ef2855..4664180a3ab3 100644 --- a/arch/m68k/include/asm/m68360.h +++ b/arch/m68k/include/asm/m68360.h | |||
| @@ -1,7 +1,7 @@ | |||
| 1 | #include "m68360_regs.h" | 1 | #include <asm/m68360_regs.h> |
| 2 | #include "m68360_pram.h" | 2 | #include <asm/m68360_pram.h> |
| 3 | #include "m68360_quicc.h" | 3 | #include <asm/m68360_quicc.h> |
| 4 | #include "m68360_enet.h" | 4 | #include <asm/m68360_enet.h> |
| 5 | 5 | ||
| 6 | #ifdef CONFIG_M68360 | 6 | #ifdef CONFIG_M68360 |
| 7 | 7 | ||
diff --git a/arch/m68k/include/asm/m68360_enet.h b/arch/m68k/include/asm/m68360_enet.h index c36f4d059203..4d04037c78a2 100644 --- a/arch/m68k/include/asm/m68360_enet.h +++ b/arch/m68k/include/asm/m68360_enet.h | |||
| @@ -10,7 +10,7 @@ | |||
| 10 | #ifndef __ETHER_H | 10 | #ifndef __ETHER_H |
| 11 | #define __ETHER_H | 11 | #define __ETHER_H |
| 12 | 12 | ||
| 13 | #include "quicc_simple.h" | 13 | #include <asm/quicc_simple.h> |
| 14 | 14 | ||
| 15 | /* | 15 | /* |
| 16 | * transmit BD's | 16 | * transmit BD's |
diff --git a/arch/m68k/include/asm/page.h b/arch/m68k/include/asm/page.h index 98baa82a8615..7c360dac00b7 100644 --- a/arch/m68k/include/asm/page.h +++ b/arch/m68k/include/asm/page.h | |||
| @@ -43,9 +43,9 @@ extern unsigned long _ramend; | |||
| 43 | #endif /* !__ASSEMBLY__ */ | 43 | #endif /* !__ASSEMBLY__ */ |
| 44 | 44 | ||
| 45 | #ifdef CONFIG_MMU | 45 | #ifdef CONFIG_MMU |
| 46 | #include "page_mm.h" | 46 | #include <asm/page_mm.h> |
| 47 | #else | 47 | #else |
| 48 | #include "page_no.h" | 48 | #include <asm/page_no.h> |
| 49 | #endif | 49 | #endif |
| 50 | 50 | ||
| 51 | #include <asm-generic/getorder.h> | 51 | #include <asm-generic/getorder.h> |
diff --git a/arch/m68k/include/asm/pgtable.h b/arch/m68k/include/asm/pgtable.h index ee6759eb445a..a3d733b524d2 100644 --- a/arch/m68k/include/asm/pgtable.h +++ b/arch/m68k/include/asm/pgtable.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef __uClinux__ | 1 | #ifdef __uClinux__ |
| 2 | #include "pgtable_no.h" | 2 | #include <asm/pgtable_no.h> |
| 3 | #else | 3 | #else |
| 4 | #include "pgtable_mm.h" | 4 | #include <asm/pgtable_mm.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/m68k/include/asm/q40_master.h b/arch/m68k/include/asm/q40_master.h index 3907a09d4fca..fc5b36278d04 100644 --- a/arch/m68k/include/asm/q40_master.h +++ b/arch/m68k/include/asm/q40_master.h | |||
| @@ -60,7 +60,7 @@ | |||
| 60 | #define Q40_RTC_WRITE 128 | 60 | #define Q40_RTC_WRITE 128 |
| 61 | 61 | ||
| 62 | /* define some Q40 specific ints */ | 62 | /* define some Q40 specific ints */ |
| 63 | #include "q40ints.h" | 63 | #include <asm/q40ints.h> |
| 64 | 64 | ||
| 65 | /* misc defs */ | 65 | /* misc defs */ |
| 66 | #define DAC_LEFT ((unsigned char *)0xff008000) | 66 | #define DAC_LEFT ((unsigned char *)0xff008000) |
diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h index 38f92dbb9a45..639c731568b0 100644 --- a/arch/m68k/include/asm/uaccess.h +++ b/arch/m68k/include/asm/uaccess.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef __uClinux__ | 1 | #ifdef __uClinux__ |
| 2 | #include "uaccess_no.h" | 2 | #include <asm/uaccess_no.h> |
| 3 | #else | 3 | #else |
| 4 | #include "uaccess_mm.h" | 4 | #include <asm/uaccess_mm.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/m68k/include/uapi/asm/Kbuild b/arch/m68k/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/m68k/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/microblaze/include/asm/elf.h b/arch/microblaze/include/asm/elf.h index 834849f59ae8..640ddd4b6a9b 100644 --- a/arch/microblaze/include/asm/elf.h +++ b/arch/microblaze/include/asm/elf.h | |||
| @@ -116,7 +116,8 @@ do { \ | |||
| 116 | } while (0) | 116 | } while (0) |
| 117 | 117 | ||
| 118 | #ifdef __KERNEL__ | 118 | #ifdef __KERNEL__ |
| 119 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT) | 119 | #define SET_PERSONALITY(ex) \ |
| 120 | set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK))) | ||
| 120 | #endif | 121 | #endif |
| 121 | 122 | ||
| 122 | #endif /* __uClinux__ */ | 123 | #endif /* __uClinux__ */ |
diff --git a/arch/microblaze/include/asm/mmu_context.h b/arch/microblaze/include/asm/mmu_context.h index 24eab1674d3e..0ccd8c402cd9 100644 --- a/arch/microblaze/include/asm/mmu_context.h +++ b/arch/microblaze/include/asm/mmu_context.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_MMU | 1 | #ifdef CONFIG_MMU |
| 2 | # include "mmu_context_mm.h" | 2 | # include <asm/mmu_context_mm.h> |
| 3 | #else | 3 | #else |
| 4 | # include <asm-generic/mmu_context.h> | 4 | # include <asm-generic/mmu_context.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/microblaze/include/uapi/asm/Kbuild b/arch/microblaze/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/microblaze/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/mips/bcm63xx/boards/board_bcm963xx.c b/arch/mips/bcm63xx/boards/board_bcm963xx.c index feb05258a4d1..dd18e4b761a8 100644 --- a/arch/mips/bcm63xx/boards/board_bcm963xx.c +++ b/arch/mips/bcm63xx/boards/board_bcm963xx.c | |||
| @@ -632,7 +632,7 @@ static struct board_info __initdata board_DWVS0 = { | |||
| 632 | /* | 632 | /* |
| 633 | * all boards | 633 | * all boards |
| 634 | */ | 634 | */ |
| 635 | static const struct board_info __initdata *bcm963xx_boards[] = { | 635 | static const struct board_info __initconst *bcm963xx_boards[] = { |
| 636 | #ifdef CONFIG_BCM63XX_CPU_6328 | 636 | #ifdef CONFIG_BCM63XX_CPU_6328 |
| 637 | &board_96328avng, | 637 | &board_96328avng, |
| 638 | #endif | 638 | #endif |
diff --git a/arch/mips/include/asm/compat-signal.h b/arch/mips/include/asm/compat-signal.h index 368a99e5c3e1..6599a901b63e 100644 --- a/arch/mips/include/asm/compat-signal.h +++ b/arch/mips/include/asm/compat-signal.h | |||
| @@ -10,68 +10,6 @@ | |||
| 10 | 10 | ||
| 11 | #include <asm/uaccess.h> | 11 | #include <asm/uaccess.h> |
| 12 | 12 | ||
| 13 | #define SI_PAD_SIZE32 ((SI_MAX_SIZE/sizeof(int)) - 3) | ||
| 14 | |||
| 15 | typedef struct compat_siginfo { | ||
| 16 | int si_signo; | ||
| 17 | int si_code; | ||
| 18 | int si_errno; | ||
| 19 | |||
| 20 | union { | ||
| 21 | int _pad[SI_PAD_SIZE32]; | ||
| 22 | |||
| 23 | /* kill() */ | ||
| 24 | struct { | ||
| 25 | compat_pid_t _pid; /* sender's pid */ | ||
| 26 | compat_uid_t _uid; /* sender's uid */ | ||
| 27 | } _kill; | ||
| 28 | |||
| 29 | /* SIGCHLD */ | ||
| 30 | struct { | ||
| 31 | compat_pid_t _pid; /* which child */ | ||
| 32 | compat_uid_t _uid; /* sender's uid */ | ||
| 33 | int _status; /* exit code */ | ||
| 34 | compat_clock_t _utime; | ||
| 35 | compat_clock_t _stime; | ||
| 36 | } _sigchld; | ||
| 37 | |||
| 38 | /* IRIX SIGCHLD */ | ||
| 39 | struct { | ||
| 40 | compat_pid_t _pid; /* which child */ | ||
| 41 | compat_clock_t _utime; | ||
| 42 | int _status; /* exit code */ | ||
| 43 | compat_clock_t _stime; | ||
| 44 | } _irix_sigchld; | ||
| 45 | |||
| 46 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 47 | struct { | ||
| 48 | s32 _addr; /* faulting insn/memory ref. */ | ||
| 49 | } _sigfault; | ||
| 50 | |||
| 51 | /* SIGPOLL, SIGXFSZ (To do ...) */ | ||
| 52 | struct { | ||
| 53 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 54 | int _fd; | ||
| 55 | } _sigpoll; | ||
| 56 | |||
| 57 | /* POSIX.1b timers */ | ||
| 58 | struct { | ||
| 59 | timer_t _tid; /* timer id */ | ||
| 60 | int _overrun; /* overrun count */ | ||
| 61 | compat_sigval_t _sigval;/* same as below */ | ||
| 62 | int _sys_private; /* not to be passed to user */ | ||
| 63 | } _timer; | ||
| 64 | |||
| 65 | /* POSIX.1b signals */ | ||
| 66 | struct { | ||
| 67 | compat_pid_t _pid; /* sender's pid */ | ||
| 68 | compat_uid_t _uid; /* sender's uid */ | ||
| 69 | compat_sigval_t _sigval; | ||
| 70 | } _rt; | ||
| 71 | |||
| 72 | } _sifields; | ||
| 73 | } compat_siginfo_t; | ||
| 74 | |||
| 75 | static inline int __copy_conv_sigset_to_user(compat_sigset_t __user *d, | 13 | static inline int __copy_conv_sigset_to_user(compat_sigset_t __user *d, |
| 76 | const sigset_t *s) | 14 | const sigset_t *s) |
| 77 | { | 15 | { |
diff --git a/arch/mips/include/asm/compat.h b/arch/mips/include/asm/compat.h index b77df0366ee6..58277e0e9cd4 100644 --- a/arch/mips/include/asm/compat.h +++ b/arch/mips/include/asm/compat.h | |||
| @@ -43,6 +43,7 @@ typedef s64 compat_s64; | |||
| 43 | typedef u32 compat_uint_t; | 43 | typedef u32 compat_uint_t; |
| 44 | typedef u32 compat_ulong_t; | 44 | typedef u32 compat_ulong_t; |
| 45 | typedef u64 compat_u64; | 45 | typedef u64 compat_u64; |
| 46 | typedef u32 compat_uptr_t; | ||
| 46 | 47 | ||
| 47 | struct compat_timespec { | 48 | struct compat_timespec { |
| 48 | compat_time_t tv_sec; | 49 | compat_time_t tv_sec; |
| @@ -124,6 +125,73 @@ typedef u32 compat_old_sigset_t; /* at least 32 bits */ | |||
| 124 | 125 | ||
| 125 | typedef u32 compat_sigset_word; | 126 | typedef u32 compat_sigset_word; |
| 126 | 127 | ||
| 128 | typedef union compat_sigval { | ||
| 129 | compat_int_t sival_int; | ||
| 130 | compat_uptr_t sival_ptr; | ||
| 131 | } compat_sigval_t; | ||
| 132 | |||
| 133 | #define SI_PAD_SIZE32 (128/sizeof(int) - 3) | ||
| 134 | |||
| 135 | typedef struct compat_siginfo { | ||
| 136 | int si_signo; | ||
| 137 | int si_code; | ||
| 138 | int si_errno; | ||
| 139 | |||
| 140 | union { | ||
| 141 | int _pad[SI_PAD_SIZE32]; | ||
| 142 | |||
| 143 | /* kill() */ | ||
| 144 | struct { | ||
| 145 | compat_pid_t _pid; /* sender's pid */ | ||
| 146 | __compat_uid_t _uid; /* sender's uid */ | ||
| 147 | } _kill; | ||
| 148 | |||
| 149 | /* SIGCHLD */ | ||
| 150 | struct { | ||
| 151 | compat_pid_t _pid; /* which child */ | ||
| 152 | __compat_uid_t _uid; /* sender's uid */ | ||
| 153 | int _status; /* exit code */ | ||
| 154 | compat_clock_t _utime; | ||
| 155 | compat_clock_t _stime; | ||
| 156 | } _sigchld; | ||
| 157 | |||
| 158 | /* IRIX SIGCHLD */ | ||
| 159 | struct { | ||
| 160 | compat_pid_t _pid; /* which child */ | ||
| 161 | compat_clock_t _utime; | ||
| 162 | int _status; /* exit code */ | ||
| 163 | compat_clock_t _stime; | ||
| 164 | } _irix_sigchld; | ||
| 165 | |||
| 166 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 167 | struct { | ||
| 168 | s32 _addr; /* faulting insn/memory ref. */ | ||
| 169 | } _sigfault; | ||
| 170 | |||
| 171 | /* SIGPOLL, SIGXFSZ (To do ...) */ | ||
| 172 | struct { | ||
| 173 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 174 | int _fd; | ||
| 175 | } _sigpoll; | ||
| 176 | |||
| 177 | /* POSIX.1b timers */ | ||
| 178 | struct { | ||
| 179 | timer_t _tid; /* timer id */ | ||
| 180 | int _overrun; /* overrun count */ | ||
| 181 | compat_sigval_t _sigval;/* same as below */ | ||
| 182 | int _sys_private; /* not to be passed to user */ | ||
| 183 | } _timer; | ||
| 184 | |||
| 185 | /* POSIX.1b signals */ | ||
| 186 | struct { | ||
| 187 | compat_pid_t _pid; /* sender's pid */ | ||
| 188 | __compat_uid_t _uid; /* sender's uid */ | ||
| 189 | compat_sigval_t _sigval; | ||
| 190 | } _rt; | ||
| 191 | |||
| 192 | } _sifields; | ||
| 193 | } compat_siginfo_t; | ||
| 194 | |||
| 127 | #define COMPAT_OFF_T_MAX 0x7fffffff | 195 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 128 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 196 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 129 | 197 | ||
| @@ -133,7 +201,6 @@ typedef u32 compat_sigset_word; | |||
| 133 | * as pointers because the syscall entry code will have | 201 | * as pointers because the syscall entry code will have |
| 134 | * appropriately converted them already. | 202 | * appropriately converted them already. |
| 135 | */ | 203 | */ |
| 136 | typedef u32 compat_uptr_t; | ||
| 137 | 204 | ||
| 138 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 205 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 139 | { | 206 | { |
diff --git a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h index 9203d90e610c..03a54df5fb86 100644 --- a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h +++ b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h | |||
| @@ -1,7 +1,7 @@ | |||
| 1 | #ifndef BCM63XX_IO_H_ | 1 | #ifndef BCM63XX_IO_H_ |
| 2 | #define BCM63XX_IO_H_ | 2 | #define BCM63XX_IO_H_ |
| 3 | 3 | ||
| 4 | #include "bcm63xx_cpu.h" | 4 | #include <asm/mach-bcm63xx/bcm63xx_cpu.h> |
| 5 | 5 | ||
| 6 | /* | 6 | /* |
| 7 | * Physical memory map, RAM is mapped at 0x0. | 7 | * Physical memory map, RAM is mapped at 0x0. |
diff --git a/arch/mips/include/asm/mach-pnx833x/gpio.h b/arch/mips/include/asm/mach-pnx833x/gpio.h index ed3a88da70f6..f192acf4a8af 100644 --- a/arch/mips/include/asm/mach-pnx833x/gpio.h +++ b/arch/mips/include/asm/mach-pnx833x/gpio.h | |||
| @@ -30,7 +30,7 @@ | |||
| 30 | - including locking between different uses | 30 | - including locking between different uses |
| 31 | */ | 31 | */ |
| 32 | 32 | ||
| 33 | #include "pnx833x.h" | 33 | #include <asm/mach-pnx833x/pnx833x.h> |
| 34 | 34 | ||
| 35 | #define SET_REG_BIT(reg, bit) do { (reg |= (1 << (bit))); } while (0) | 35 | #define SET_REG_BIT(reg, bit) do { (reg |= (1 << (bit))); } while (0) |
| 36 | #define CLEAR_REG_BIT(reg, bit) do { (reg &= ~(1 << (bit))); } while (0) | 36 | #define CLEAR_REG_BIT(reg, bit) do { (reg &= ~(1 << (bit))); } while (0) |
diff --git a/arch/mips/include/asm/octeon/cvmx-asm.h b/arch/mips/include/asm/octeon/cvmx-asm.h index 5de5de95311b..31eacc24b775 100644 --- a/arch/mips/include/asm/octeon/cvmx-asm.h +++ b/arch/mips/include/asm/octeon/cvmx-asm.h | |||
| @@ -32,7 +32,7 @@ | |||
| 32 | #ifndef __CVMX_ASM_H__ | 32 | #ifndef __CVMX_ASM_H__ |
| 33 | #define __CVMX_ASM_H__ | 33 | #define __CVMX_ASM_H__ |
| 34 | 34 | ||
| 35 | #include "octeon-model.h" | 35 | #include <asm/octeon/octeon-model.h> |
| 36 | 36 | ||
| 37 | /* other useful stuff */ | 37 | /* other useful stuff */ |
| 38 | #define CVMX_SYNC asm volatile ("sync" : : : "memory") | 38 | #define CVMX_SYNC asm volatile ("sync" : : : "memory") |
diff --git a/arch/mips/include/asm/octeon/cvmx-cmd-queue.h b/arch/mips/include/asm/octeon/cvmx-cmd-queue.h index 614653b686a0..fed91125317f 100644 --- a/arch/mips/include/asm/octeon/cvmx-cmd-queue.h +++ b/arch/mips/include/asm/octeon/cvmx-cmd-queue.h | |||
| @@ -76,7 +76,7 @@ | |||
| 76 | 76 | ||
| 77 | #include <linux/prefetch.h> | 77 | #include <linux/prefetch.h> |
| 78 | 78 | ||
| 79 | #include "cvmx-fpa.h" | 79 | #include <asm/octeon/cvmx-fpa.h> |
| 80 | /** | 80 | /** |
| 81 | * By default we disable the max depth support. Most programs | 81 | * By default we disable the max depth support. Most programs |
| 82 | * don't use it and it slows down the command queue processing | 82 | * don't use it and it slows down the command queue processing |
diff --git a/arch/mips/include/asm/octeon/cvmx-fpa.h b/arch/mips/include/asm/octeon/cvmx-fpa.h index 1f04f9658736..541a1ae02b6f 100644 --- a/arch/mips/include/asm/octeon/cvmx-fpa.h +++ b/arch/mips/include/asm/octeon/cvmx-fpa.h | |||
| @@ -36,8 +36,8 @@ | |||
| 36 | #ifndef __CVMX_FPA_H__ | 36 | #ifndef __CVMX_FPA_H__ |
| 37 | #define __CVMX_FPA_H__ | 37 | #define __CVMX_FPA_H__ |
| 38 | 38 | ||
| 39 | #include "cvmx-address.h" | 39 | #include <asm/octeon/cvmx-address.h> |
| 40 | #include "cvmx-fpa-defs.h" | 40 | #include <asm/octeon/cvmx-fpa-defs.h> |
| 41 | 41 | ||
| 42 | #define CVMX_FPA_NUM_POOLS 8 | 42 | #define CVMX_FPA_NUM_POOLS 8 |
| 43 | #define CVMX_FPA_MIN_BLOCK_SIZE 128 | 43 | #define CVMX_FPA_MIN_BLOCK_SIZE 128 |
diff --git a/arch/mips/include/asm/octeon/cvmx-helper-board.h b/arch/mips/include/asm/octeon/cvmx-helper-board.h index 88527fa835c9..442f508eaac9 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-board.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-board.h | |||
| @@ -34,7 +34,7 @@ | |||
| 34 | #ifndef __CVMX_HELPER_BOARD_H__ | 34 | #ifndef __CVMX_HELPER_BOARD_H__ |
| 35 | #define __CVMX_HELPER_BOARD_H__ | 35 | #define __CVMX_HELPER_BOARD_H__ |
| 36 | 36 | ||
| 37 | #include "cvmx-helper.h" | 37 | #include <asm/octeon/cvmx-helper.h> |
| 38 | 38 | ||
| 39 | typedef enum { | 39 | typedef enum { |
| 40 | set_phy_link_flags_autoneg = 0x1, | 40 | set_phy_link_flags_autoneg = 0x1, |
diff --git a/arch/mips/include/asm/octeon/cvmx-helper.h b/arch/mips/include/asm/octeon/cvmx-helper.h index 0ac6b9f412be..691c8142cd4f 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper.h +++ b/arch/mips/include/asm/octeon/cvmx-helper.h | |||
| @@ -34,9 +34,9 @@ | |||
| 34 | #ifndef __CVMX_HELPER_H__ | 34 | #ifndef __CVMX_HELPER_H__ |
| 35 | #define __CVMX_HELPER_H__ | 35 | #define __CVMX_HELPER_H__ |
| 36 | 36 | ||
| 37 | #include "cvmx-config.h" | 37 | #include <asm/octeon/cvmx-config.h> |
| 38 | #include "cvmx-fpa.h" | 38 | #include <asm/octeon/cvmx-fpa.h> |
| 39 | #include "cvmx-wqe.h" | 39 | #include <asm/octeon/cvmx-wqe.h> |
| 40 | 40 | ||
| 41 | typedef enum { | 41 | typedef enum { |
| 42 | CVMX_HELPER_INTERFACE_MODE_DISABLED, | 42 | CVMX_HELPER_INTERFACE_MODE_DISABLED, |
| @@ -62,13 +62,13 @@ typedef union { | |||
| 62 | } cvmx_helper_link_info_t; | 62 | } cvmx_helper_link_info_t; |
| 63 | 63 | ||
| 64 | #include <asm/octeon/cvmx-helper-errata.h> | 64 | #include <asm/octeon/cvmx-helper-errata.h> |
| 65 | #include "cvmx-helper-loop.h" | 65 | #include <asm/octeon/cvmx-helper-loop.h> |
| 66 | #include "cvmx-helper-npi.h" | 66 | #include <asm/octeon/cvmx-helper-npi.h> |
| 67 | #include "cvmx-helper-rgmii.h" | 67 | #include <asm/octeon/cvmx-helper-rgmii.h> |
| 68 | #include "cvmx-helper-sgmii.h" | 68 | #include <asm/octeon/cvmx-helper-sgmii.h> |
| 69 | #include "cvmx-helper-spi.h" | 69 | #include <asm/octeon/cvmx-helper-spi.h> |
| 70 | #include "cvmx-helper-util.h" | 70 | #include <asm/octeon/cvmx-helper-util.h> |
| 71 | #include "cvmx-helper-xaui.h" | 71 | #include <asm/octeon/cvmx-helper-xaui.h> |
| 72 | 72 | ||
| 73 | /** | 73 | /** |
| 74 | * cvmx_override_pko_queue_priority(int ipd_port, uint64_t | 74 | * cvmx_override_pko_queue_priority(int ipd_port, uint64_t |
diff --git a/arch/mips/include/asm/octeon/cvmx-mdio.h b/arch/mips/include/asm/octeon/cvmx-mdio.h index d88ab8d8e37d..6f0cd182cec8 100644 --- a/arch/mips/include/asm/octeon/cvmx-mdio.h +++ b/arch/mips/include/asm/octeon/cvmx-mdio.h | |||
| @@ -35,7 +35,7 @@ | |||
| 35 | #ifndef __CVMX_MIO_H__ | 35 | #ifndef __CVMX_MIO_H__ |
| 36 | #define __CVMX_MIO_H__ | 36 | #define __CVMX_MIO_H__ |
| 37 | 37 | ||
| 38 | #include "cvmx-smix-defs.h" | 38 | #include <asm/octeon/cvmx-smix-defs.h> |
| 39 | 39 | ||
| 40 | /** | 40 | /** |
| 41 | * PHY register 0 from the 802.3 spec | 41 | * PHY register 0 from the 802.3 spec |
diff --git a/arch/mips/include/asm/octeon/cvmx-pip.h b/arch/mips/include/asm/octeon/cvmx-pip.h index 78dbce8f2c5e..9e739a640855 100644 --- a/arch/mips/include/asm/octeon/cvmx-pip.h +++ b/arch/mips/include/asm/octeon/cvmx-pip.h | |||
| @@ -33,9 +33,9 @@ | |||
| 33 | #ifndef __CVMX_PIP_H__ | 33 | #ifndef __CVMX_PIP_H__ |
| 34 | #define __CVMX_PIP_H__ | 34 | #define __CVMX_PIP_H__ |
| 35 | 35 | ||
| 36 | #include "cvmx-wqe.h" | 36 | #include <asm/octeon/cvmx-wqe.h> |
| 37 | #include "cvmx-fpa.h" | 37 | #include <asm/octeon/cvmx-fpa.h> |
| 38 | #include "cvmx-pip-defs.h" | 38 | #include <asm/octeon/cvmx-pip-defs.h> |
| 39 | 39 | ||
| 40 | #define CVMX_PIP_NUM_INPUT_PORTS 40 | 40 | #define CVMX_PIP_NUM_INPUT_PORTS 40 |
| 41 | #define CVMX_PIP_NUM_WATCHERS 4 | 41 | #define CVMX_PIP_NUM_WATCHERS 4 |
diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h index de3412aada5d..c6daeedf1f81 100644 --- a/arch/mips/include/asm/octeon/cvmx-pko.h +++ b/arch/mips/include/asm/octeon/cvmx-pko.h | |||
| @@ -58,10 +58,10 @@ | |||
| 58 | #ifndef __CVMX_PKO_H__ | 58 | #ifndef __CVMX_PKO_H__ |
| 59 | #define __CVMX_PKO_H__ | 59 | #define __CVMX_PKO_H__ |
| 60 | 60 | ||
| 61 | #include "cvmx-fpa.h" | 61 | #include <asm/octeon/cvmx-fpa.h> |
| 62 | #include "cvmx-pow.h" | 62 | #include <asm/octeon/cvmx-pow.h> |
| 63 | #include "cvmx-cmd-queue.h" | 63 | #include <asm/octeon/cvmx-cmd-queue.h> |
| 64 | #include "cvmx-pko-defs.h" | 64 | #include <asm/octeon/cvmx-pko-defs.h> |
| 65 | 65 | ||
| 66 | /* Adjust the command buffer size by 1 word so that in the case of using only | 66 | /* Adjust the command buffer size by 1 word so that in the case of using only |
| 67 | * two word PKO commands no command words stradle buffers. The useful values | 67 | * two word PKO commands no command words stradle buffers. The useful values |
diff --git a/arch/mips/include/asm/octeon/cvmx-pow.h b/arch/mips/include/asm/octeon/cvmx-pow.h index 999aefe3274c..92742b241a51 100644 --- a/arch/mips/include/asm/octeon/cvmx-pow.h +++ b/arch/mips/include/asm/octeon/cvmx-pow.h | |||
| @@ -53,8 +53,8 @@ | |||
| 53 | 53 | ||
| 54 | #include <asm/octeon/cvmx-pow-defs.h> | 54 | #include <asm/octeon/cvmx-pow-defs.h> |
| 55 | 55 | ||
| 56 | #include "cvmx-scratch.h" | 56 | #include <asm/octeon/cvmx-scratch.h> |
| 57 | #include "cvmx-wqe.h" | 57 | #include <asm/octeon/cvmx-wqe.h> |
| 58 | 58 | ||
| 59 | /* Default to having all POW constancy checks turned on */ | 59 | /* Default to having all POW constancy checks turned on */ |
| 60 | #ifndef CVMX_ENABLE_POW_CHECKS | 60 | #ifndef CVMX_ENABLE_POW_CHECKS |
diff --git a/arch/mips/include/asm/octeon/cvmx-spi.h b/arch/mips/include/asm/octeon/cvmx-spi.h index e814648953a5..3bf53b537bcf 100644 --- a/arch/mips/include/asm/octeon/cvmx-spi.h +++ b/arch/mips/include/asm/octeon/cvmx-spi.h | |||
| @@ -32,7 +32,7 @@ | |||
| 32 | #ifndef __CVMX_SPI_H__ | 32 | #ifndef __CVMX_SPI_H__ |
| 33 | #define __CVMX_SPI_H__ | 33 | #define __CVMX_SPI_H__ |
| 34 | 34 | ||
| 35 | #include "cvmx-gmxx-defs.h" | 35 | #include <asm/octeon/cvmx-gmxx-defs.h> |
| 36 | 36 | ||
| 37 | /* CSR typedefs have been moved to cvmx-csr-*.h */ | 37 | /* CSR typedefs have been moved to cvmx-csr-*.h */ |
| 38 | 38 | ||
diff --git a/arch/mips/include/asm/octeon/cvmx-spinlock.h b/arch/mips/include/asm/octeon/cvmx-spinlock.h index 2fbf0871df11..a672abb1bc4f 100644 --- a/arch/mips/include/asm/octeon/cvmx-spinlock.h +++ b/arch/mips/include/asm/octeon/cvmx-spinlock.h | |||
| @@ -35,7 +35,7 @@ | |||
| 35 | #ifndef __CVMX_SPINLOCK_H__ | 35 | #ifndef __CVMX_SPINLOCK_H__ |
| 36 | #define __CVMX_SPINLOCK_H__ | 36 | #define __CVMX_SPINLOCK_H__ |
| 37 | 37 | ||
| 38 | #include "cvmx-asm.h" | 38 | #include <asm/octeon/cvmx-asm.h> |
| 39 | 39 | ||
| 40 | /* Spinlocks for Octeon */ | 40 | /* Spinlocks for Octeon */ |
| 41 | 41 | ||
diff --git a/arch/mips/include/asm/octeon/cvmx-wqe.h b/arch/mips/include/asm/octeon/cvmx-wqe.h index 653610953d28..df762389e271 100644 --- a/arch/mips/include/asm/octeon/cvmx-wqe.h +++ b/arch/mips/include/asm/octeon/cvmx-wqe.h | |||
| @@ -40,7 +40,7 @@ | |||
| 40 | #ifndef __CVMX_WQE_H__ | 40 | #ifndef __CVMX_WQE_H__ |
| 41 | #define __CVMX_WQE_H__ | 41 | #define __CVMX_WQE_H__ |
| 42 | 42 | ||
| 43 | #include "cvmx-packet.h" | 43 | #include <asm/octeon/cvmx-packet.h> |
| 44 | 44 | ||
| 45 | 45 | ||
| 46 | #define OCT_TAG_TYPE_STRING(x) \ | 46 | #define OCT_TAG_TYPE_STRING(x) \ |
diff --git a/arch/mips/include/asm/octeon/cvmx.h b/arch/mips/include/asm/octeon/cvmx.h index 740be97a3251..db58beab6cb2 100644 --- a/arch/mips/include/asm/octeon/cvmx.h +++ b/arch/mips/include/asm/octeon/cvmx.h | |||
| @@ -52,24 +52,24 @@ enum cvmx_mips_space { | |||
| 52 | #define CVMX_ADD_IO_SEG(add) CVMX_ADD_SEG(CVMX_IO_SEG, (add)) | 52 | #define CVMX_ADD_IO_SEG(add) CVMX_ADD_SEG(CVMX_IO_SEG, (add)) |
| 53 | #endif | 53 | #endif |
| 54 | 54 | ||
| 55 | #include "cvmx-asm.h" | 55 | #include <asm/octeon/cvmx-asm.h> |
| 56 | #include "cvmx-packet.h" | 56 | #include <asm/octeon/cvmx-packet.h> |
| 57 | #include "cvmx-sysinfo.h" | 57 | #include <asm/octeon/cvmx-sysinfo.h> |
| 58 | 58 | ||
| 59 | #include "cvmx-ciu-defs.h" | 59 | #include <asm/octeon/cvmx-ciu-defs.h> |
| 60 | #include "cvmx-gpio-defs.h" | 60 | #include <asm/octeon/cvmx-gpio-defs.h> |
| 61 | #include "cvmx-iob-defs.h" | 61 | #include <asm/octeon/cvmx-iob-defs.h> |
| 62 | #include "cvmx-ipd-defs.h" | 62 | #include <asm/octeon/cvmx-ipd-defs.h> |
| 63 | #include "cvmx-l2c-defs.h" | 63 | #include <asm/octeon/cvmx-l2c-defs.h> |
| 64 | #include "cvmx-l2d-defs.h" | 64 | #include <asm/octeon/cvmx-l2d-defs.h> |
| 65 | #include "cvmx-l2t-defs.h" | 65 | #include <asm/octeon/cvmx-l2t-defs.h> |
| 66 | #include "cvmx-led-defs.h" | 66 | #include <asm/octeon/cvmx-led-defs.h> |
| 67 | #include "cvmx-mio-defs.h" | 67 | #include <asm/octeon/cvmx-mio-defs.h> |
| 68 | #include "cvmx-pow-defs.h" | 68 | #include <asm/octeon/cvmx-pow-defs.h> |
| 69 | 69 | ||
| 70 | #include "cvmx-bootinfo.h" | 70 | #include <asm/octeon/cvmx-bootinfo.h> |
| 71 | #include "cvmx-bootmem.h" | 71 | #include <asm/octeon/cvmx-bootmem.h> |
| 72 | #include "cvmx-l2c.h" | 72 | #include <asm/octeon/cvmx-l2c.h> |
| 73 | 73 | ||
| 74 | #ifndef CVMX_ENABLE_DEBUG_PRINTS | 74 | #ifndef CVMX_ENABLE_DEBUG_PRINTS |
| 75 | #define CVMX_ENABLE_DEBUG_PRINTS 1 | 75 | #define CVMX_ENABLE_DEBUG_PRINTS 1 |
diff --git a/arch/mips/include/asm/octeon/octeon-model.h b/arch/mips/include/asm/octeon/octeon-model.h index 4e338a4d9424..23b895cb260b 100644 --- a/arch/mips/include/asm/octeon/octeon-model.h +++ b/arch/mips/include/asm/octeon/octeon-model.h | |||
| @@ -313,6 +313,6 @@ static inline int __octeon_is_model_runtime__(uint32_t model) | |||
| 313 | const char *octeon_model_get_string(uint32_t chip_id); | 313 | const char *octeon_model_get_string(uint32_t chip_id); |
| 314 | const char *octeon_model_get_string_buffer(uint32_t chip_id, char *buffer); | 314 | const char *octeon_model_get_string_buffer(uint32_t chip_id, char *buffer); |
| 315 | 315 | ||
| 316 | #include "octeon-feature.h" | 316 | #include <asm/octeon/octeon-feature.h> |
| 317 | 317 | ||
| 318 | #endif /* __OCTEON_MODEL_H__ */ | 318 | #endif /* __OCTEON_MODEL_H__ */ |
diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h index 1e2486e23573..c4a1b31966bb 100644 --- a/arch/mips/include/asm/octeon/octeon.h +++ b/arch/mips/include/asm/octeon/octeon.h | |||
| @@ -8,7 +8,7 @@ | |||
| 8 | #ifndef __ASM_OCTEON_OCTEON_H | 8 | #ifndef __ASM_OCTEON_OCTEON_H |
| 9 | #define __ASM_OCTEON_OCTEON_H | 9 | #define __ASM_OCTEON_OCTEON_H |
| 10 | 10 | ||
| 11 | #include "cvmx.h" | 11 | #include <asm/octeon/cvmx.h> |
| 12 | 12 | ||
| 13 | extern uint64_t octeon_bootmem_alloc_range_phys(uint64_t size, | 13 | extern uint64_t octeon_bootmem_alloc_range_phys(uint64_t size, |
| 14 | uint64_t alignment, | 14 | uint64_t alignment, |
diff --git a/arch/mips/include/asm/sibyte/bcm1480_int.h b/arch/mips/include/asm/sibyte/bcm1480_int.h index 6109557c14e9..fffb224d2297 100644 --- a/arch/mips/include/asm/sibyte/bcm1480_int.h +++ b/arch/mips/include/asm/sibyte/bcm1480_int.h | |||
| @@ -34,7 +34,7 @@ | |||
| 34 | #ifndef _BCM1480_INT_H | 34 | #ifndef _BCM1480_INT_H |
| 35 | #define _BCM1480_INT_H | 35 | #define _BCM1480_INT_H |
| 36 | 36 | ||
| 37 | #include "sb1250_defs.h" | 37 | #include <asm/sibyte/sb1250_defs.h> |
| 38 | 38 | ||
| 39 | /* ********************************************************************* | 39 | /* ********************************************************************* |
| 40 | * Interrupt Mapper Constants | 40 | * Interrupt Mapper Constants |
diff --git a/arch/mips/include/asm/sibyte/bcm1480_l2c.h b/arch/mips/include/asm/sibyte/bcm1480_l2c.h index fd75817f7ac4..725d38cb9d1c 100644 --- a/arch/mips/include/asm/sibyte/bcm1480_l2c.h +++ b/arch/mips/include/asm/sibyte/bcm1480_l2c.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _BCM1480_L2C_H | 33 | #ifndef _BCM1480_L2C_H |
| 34 | #define _BCM1480_L2C_H | 34 | #define _BCM1480_L2C_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* | 38 | /* |
| 39 | * Format of level 2 cache management address (Table 55) | 39 | * Format of level 2 cache management address (Table 55) |
diff --git a/arch/mips/include/asm/sibyte/bcm1480_mc.h b/arch/mips/include/asm/sibyte/bcm1480_mc.h index f26a41a82b59..4307a758e3bf 100644 --- a/arch/mips/include/asm/sibyte/bcm1480_mc.h +++ b/arch/mips/include/asm/sibyte/bcm1480_mc.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _BCM1480_MC_H | 33 | #ifndef _BCM1480_MC_H |
| 34 | #define _BCM1480_MC_H | 34 | #define _BCM1480_MC_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* | 38 | /* |
| 39 | * Memory Channel Configuration Register (Table 81) | 39 | * Memory Channel Configuration Register (Table 81) |
diff --git a/arch/mips/include/asm/sibyte/bcm1480_regs.h b/arch/mips/include/asm/sibyte/bcm1480_regs.h index b4077bb72611..84d168ddfebb 100644 --- a/arch/mips/include/asm/sibyte/bcm1480_regs.h +++ b/arch/mips/include/asm/sibyte/bcm1480_regs.h | |||
| @@ -32,14 +32,14 @@ | |||
| 32 | #ifndef _BCM1480_REGS_H | 32 | #ifndef _BCM1480_REGS_H |
| 33 | #define _BCM1480_REGS_H | 33 | #define _BCM1480_REGS_H |
| 34 | 34 | ||
| 35 | #include "sb1250_defs.h" | 35 | #include <asm/sibyte/sb1250_defs.h> |
| 36 | 36 | ||
| 37 | /* ********************************************************************* | 37 | /* ********************************************************************* |
| 38 | * Pull in the BCM1250's registers since a great deal of the 1480's | 38 | * Pull in the BCM1250's registers since a great deal of the 1480's |
| 39 | * functions are the same as the BCM1250. | 39 | * functions are the same as the BCM1250. |
| 40 | ********************************************************************* */ | 40 | ********************************************************************* */ |
| 41 | 41 | ||
| 42 | #include "sb1250_regs.h" | 42 | #include <asm/sibyte/sb1250_regs.h> |
| 43 | 43 | ||
| 44 | 44 | ||
| 45 | /* ********************************************************************* | 45 | /* ********************************************************************* |
diff --git a/arch/mips/include/asm/sibyte/bcm1480_scd.h b/arch/mips/include/asm/sibyte/bcm1480_scd.h index 25ef24cbb92a..2af3706b9648 100644 --- a/arch/mips/include/asm/sibyte/bcm1480_scd.h +++ b/arch/mips/include/asm/sibyte/bcm1480_scd.h | |||
| @@ -32,13 +32,13 @@ | |||
| 32 | #ifndef _BCM1480_SCD_H | 32 | #ifndef _BCM1480_SCD_H |
| 33 | #define _BCM1480_SCD_H | 33 | #define _BCM1480_SCD_H |
| 34 | 34 | ||
| 35 | #include "sb1250_defs.h" | 35 | #include <asm/sibyte/sb1250_defs.h> |
| 36 | 36 | ||
| 37 | /* ********************************************************************* | 37 | /* ********************************************************************* |
| 38 | * Pull in the BCM1250's SCD since lots of stuff is the same. | 38 | * Pull in the BCM1250's SCD since lots of stuff is the same. |
| 39 | ********************************************************************* */ | 39 | ********************************************************************* */ |
| 40 | 40 | ||
| 41 | #include "sb1250_scd.h" | 41 | #include <asm/sibyte/sb1250_scd.h> |
| 42 | 42 | ||
| 43 | /* ********************************************************************* | 43 | /* ********************************************************************* |
| 44 | * Some general notes: | 44 | * Some general notes: |
diff --git a/arch/mips/include/asm/sibyte/sb1250_dma.h b/arch/mips/include/asm/sibyte/sb1250_dma.h index bad56171d747..6c44dfb52878 100644 --- a/arch/mips/include/asm/sibyte/sb1250_dma.h +++ b/arch/mips/include/asm/sibyte/sb1250_dma.h | |||
| @@ -36,7 +36,7 @@ | |||
| 36 | #define _SB1250_DMA_H | 36 | #define _SB1250_DMA_H |
| 37 | 37 | ||
| 38 | 38 | ||
| 39 | #include "sb1250_defs.h" | 39 | #include <asm/sibyte/sb1250_defs.h> |
| 40 | 40 | ||
| 41 | /* ********************************************************************* | 41 | /* ********************************************************************* |
| 42 | * DMA Registers | 42 | * DMA Registers |
diff --git a/arch/mips/include/asm/sibyte/sb1250_genbus.h b/arch/mips/include/asm/sibyte/sb1250_genbus.h index 94e9c7c8e783..a96ded17bdc9 100644 --- a/arch/mips/include/asm/sibyte/sb1250_genbus.h +++ b/arch/mips/include/asm/sibyte/sb1250_genbus.h | |||
| @@ -34,7 +34,7 @@ | |||
| 34 | #ifndef _SB1250_GENBUS_H | 34 | #ifndef _SB1250_GENBUS_H |
| 35 | #define _SB1250_GENBUS_H | 35 | #define _SB1250_GENBUS_H |
| 36 | 36 | ||
| 37 | #include "sb1250_defs.h" | 37 | #include <asm/sibyte/sb1250_defs.h> |
| 38 | 38 | ||
| 39 | /* | 39 | /* |
| 40 | * Generic Bus Region Configuration Registers (Table 11-4) | 40 | * Generic Bus Region Configuration Registers (Table 11-4) |
diff --git a/arch/mips/include/asm/sibyte/sb1250_int.h b/arch/mips/include/asm/sibyte/sb1250_int.h index f2850b4bcfd4..dbea73ddd2fe 100644 --- a/arch/mips/include/asm/sibyte/sb1250_int.h +++ b/arch/mips/include/asm/sibyte/sb1250_int.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_INT_H | 33 | #ifndef _SB1250_INT_H |
| 34 | #define _SB1250_INT_H | 34 | #define _SB1250_INT_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* ********************************************************************* | 38 | /* ********************************************************************* |
| 39 | * Interrupt Mapper Constants | 39 | * Interrupt Mapper Constants |
diff --git a/arch/mips/include/asm/sibyte/sb1250_l2c.h b/arch/mips/include/asm/sibyte/sb1250_l2c.h index 6554dcf05cfe..b61a7491607d 100644 --- a/arch/mips/include/asm/sibyte/sb1250_l2c.h +++ b/arch/mips/include/asm/sibyte/sb1250_l2c.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_L2C_H | 33 | #ifndef _SB1250_L2C_H |
| 34 | #define _SB1250_L2C_H | 34 | #define _SB1250_L2C_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* | 38 | /* |
| 39 | * Level 2 Cache Tag register (Table 5-3) | 39 | * Level 2 Cache Tag register (Table 5-3) |
diff --git a/arch/mips/include/asm/sibyte/sb1250_ldt.h b/arch/mips/include/asm/sibyte/sb1250_ldt.h index 1e76cf137995..bf7f320d1a87 100644 --- a/arch/mips/include/asm/sibyte/sb1250_ldt.h +++ b/arch/mips/include/asm/sibyte/sb1250_ldt.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_LDT_H | 33 | #ifndef _SB1250_LDT_H |
| 34 | #define _SB1250_LDT_H | 34 | #define _SB1250_LDT_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | #define K_LDT_VENDOR_SIBYTE 0x166D | 38 | #define K_LDT_VENDOR_SIBYTE 0x166D |
| 39 | #define K_LDT_DEVICE_SB1250 0x0002 | 39 | #define K_LDT_DEVICE_SB1250 0x0002 |
diff --git a/arch/mips/include/asm/sibyte/sb1250_mac.h b/arch/mips/include/asm/sibyte/sb1250_mac.h index 77f787284235..cfc4d7870882 100644 --- a/arch/mips/include/asm/sibyte/sb1250_mac.h +++ b/arch/mips/include/asm/sibyte/sb1250_mac.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_MAC_H | 33 | #ifndef _SB1250_MAC_H |
| 34 | #define _SB1250_MAC_H | 34 | #define _SB1250_MAC_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* ********************************************************************* | 38 | /* ********************************************************************* |
| 39 | * Ethernet MAC Registers | 39 | * Ethernet MAC Registers |
diff --git a/arch/mips/include/asm/sibyte/sb1250_mc.h b/arch/mips/include/asm/sibyte/sb1250_mc.h index 1eb1b5a88736..15048dcaf22f 100644 --- a/arch/mips/include/asm/sibyte/sb1250_mc.h +++ b/arch/mips/include/asm/sibyte/sb1250_mc.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_MC_H | 33 | #ifndef _SB1250_MC_H |
| 34 | #define _SB1250_MC_H | 34 | #define _SB1250_MC_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* | 38 | /* |
| 39 | * Memory Channel Config Register (table 6-14) | 39 | * Memory Channel Config Register (table 6-14) |
diff --git a/arch/mips/include/asm/sibyte/sb1250_regs.h b/arch/mips/include/asm/sibyte/sb1250_regs.h index 8f53ec817a5e..29b9f0b26b3a 100644 --- a/arch/mips/include/asm/sibyte/sb1250_regs.h +++ b/arch/mips/include/asm/sibyte/sb1250_regs.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_REGS_H | 33 | #ifndef _SB1250_REGS_H |
| 34 | #define _SB1250_REGS_H | 34 | #define _SB1250_REGS_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | 38 | ||
| 39 | /* ********************************************************************* | 39 | /* ********************************************************************* |
diff --git a/arch/mips/include/asm/sibyte/sb1250_scd.h b/arch/mips/include/asm/sibyte/sb1250_scd.h index e49c3e89b5ee..615e165dbd21 100644 --- a/arch/mips/include/asm/sibyte/sb1250_scd.h +++ b/arch/mips/include/asm/sibyte/sb1250_scd.h | |||
| @@ -32,7 +32,7 @@ | |||
| 32 | #ifndef _SB1250_SCD_H | 32 | #ifndef _SB1250_SCD_H |
| 33 | #define _SB1250_SCD_H | 33 | #define _SB1250_SCD_H |
| 34 | 34 | ||
| 35 | #include "sb1250_defs.h" | 35 | #include <asm/sibyte/sb1250_defs.h> |
| 36 | 36 | ||
| 37 | /* ********************************************************************* | 37 | /* ********************************************************************* |
| 38 | * System control/debug registers | 38 | * System control/debug registers |
diff --git a/arch/mips/include/asm/sibyte/sb1250_smbus.h b/arch/mips/include/asm/sibyte/sb1250_smbus.h index 04769923cf1e..128d6b75b819 100644 --- a/arch/mips/include/asm/sibyte/sb1250_smbus.h +++ b/arch/mips/include/asm/sibyte/sb1250_smbus.h | |||
| @@ -34,7 +34,7 @@ | |||
| 34 | #ifndef _SB1250_SMBUS_H | 34 | #ifndef _SB1250_SMBUS_H |
| 35 | #define _SB1250_SMBUS_H | 35 | #define _SB1250_SMBUS_H |
| 36 | 36 | ||
| 37 | #include "sb1250_defs.h" | 37 | #include <asm/sibyte/sb1250_defs.h> |
| 38 | 38 | ||
| 39 | /* | 39 | /* |
| 40 | * SMBus Clock Frequency Register (Table 14-2) | 40 | * SMBus Clock Frequency Register (Table 14-2) |
diff --git a/arch/mips/include/asm/sibyte/sb1250_syncser.h b/arch/mips/include/asm/sibyte/sb1250_syncser.h index d4b8558e0bf1..274e9179d326 100644 --- a/arch/mips/include/asm/sibyte/sb1250_syncser.h +++ b/arch/mips/include/asm/sibyte/sb1250_syncser.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_SYNCSER_H | 33 | #ifndef _SB1250_SYNCSER_H |
| 34 | #define _SB1250_SYNCSER_H | 34 | #define _SB1250_SYNCSER_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* | 38 | /* |
| 39 | * Serial Mode Configuration Register | 39 | * Serial Mode Configuration Register |
diff --git a/arch/mips/include/asm/sibyte/sb1250_uart.h b/arch/mips/include/asm/sibyte/sb1250_uart.h index d835bf280140..bb99ecac5817 100644 --- a/arch/mips/include/asm/sibyte/sb1250_uart.h +++ b/arch/mips/include/asm/sibyte/sb1250_uart.h | |||
| @@ -33,7 +33,7 @@ | |||
| 33 | #ifndef _SB1250_UART_H | 33 | #ifndef _SB1250_UART_H |
| 34 | #define _SB1250_UART_H | 34 | #define _SB1250_UART_H |
| 35 | 35 | ||
| 36 | #include "sb1250_defs.h" | 36 | #include <asm/sibyte/sb1250_defs.h> |
| 37 | 37 | ||
| 38 | /* ********************************************************************** | 38 | /* ********************************************************************** |
| 39 | * DUART Registers | 39 | * DUART Registers |
diff --git a/arch/mips/include/uapi/asm/Kbuild b/arch/mips/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/mips/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/mips/pci/pci-octeon.c b/arch/mips/pci/pci-octeon.c index c5dfb2c87d44..4b0c347d7a82 100644 --- a/arch/mips/pci/pci-octeon.c +++ b/arch/mips/pci/pci-octeon.c | |||
| @@ -58,7 +58,7 @@ union octeon_pci_address { | |||
| 58 | } s; | 58 | } s; |
| 59 | }; | 59 | }; |
| 60 | 60 | ||
| 61 | int __initdata (*octeon_pcibios_map_irq)(const struct pci_dev *dev, | 61 | int __initconst (*octeon_pcibios_map_irq)(const struct pci_dev *dev, |
| 62 | u8 slot, u8 pin); | 62 | u8 slot, u8 pin); |
| 63 | enum octeon_dma_bar_type octeon_dma_bar_type = OCTEON_DMA_BAR_TYPE_INVALID; | 63 | enum octeon_dma_bar_type octeon_dma_bar_type = OCTEON_DMA_BAR_TYPE_INVALID; |
| 64 | 64 | ||
diff --git a/arch/mn10300/Makefile b/arch/mn10300/Makefile index 33188b6e81e4..a3d0fef3b126 100644 --- a/arch/mn10300/Makefile +++ b/arch/mn10300/Makefile | |||
| @@ -26,7 +26,7 @@ CHECKFLAGS += | |||
| 26 | PROCESSOR := unset | 26 | PROCESSOR := unset |
| 27 | UNIT := unset | 27 | UNIT := unset |
| 28 | 28 | ||
| 29 | KBUILD_CFLAGS += -mam33 -mmem-funcs -DCPU=AM33 | 29 | KBUILD_CFLAGS += -mam33 -DCPU=AM33 $(call cc-option,-mmem-funcs,) |
| 30 | KBUILD_AFLAGS += -mam33 -DCPU=AM33 | 30 | KBUILD_AFLAGS += -mam33 -DCPU=AM33 |
| 31 | 31 | ||
| 32 | ifeq ($(CONFIG_MN10300_CURRENT_IN_E2),y) | 32 | ifeq ($(CONFIG_MN10300_CURRENT_IN_E2),y) |
diff --git a/arch/mn10300/include/asm/elf.h b/arch/mn10300/include/asm/elf.h index 8157c9267f42..4ebd6b3a0a1e 100644 --- a/arch/mn10300/include/asm/elf.h +++ b/arch/mn10300/include/asm/elf.h | |||
| @@ -151,7 +151,8 @@ do { \ | |||
| 151 | #define ELF_PLATFORM (NULL) | 151 | #define ELF_PLATFORM (NULL) |
| 152 | 152 | ||
| 153 | #ifdef __KERNEL__ | 153 | #ifdef __KERNEL__ |
| 154 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 154 | #define SET_PERSONALITY(ex) \ |
| 155 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 155 | #endif | 156 | #endif |
| 156 | 157 | ||
| 157 | #endif /* _ASM_ELF_H */ | 158 | #endif /* _ASM_ELF_H */ |
diff --git a/arch/mn10300/include/uapi/asm/Kbuild b/arch/mn10300/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/mn10300/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/openrisc/include/asm/elf.h b/arch/openrisc/include/asm/elf.h index a8fe2c513070..225a7ff320ad 100644 --- a/arch/openrisc/include/asm/elf.h +++ b/arch/openrisc/include/asm/elf.h | |||
| @@ -110,7 +110,8 @@ extern void dump_elf_thread(elf_greg_t *dest, struct pt_regs *pt); | |||
| 110 | 110 | ||
| 111 | #define ELF_PLATFORM (NULL) | 111 | #define ELF_PLATFORM (NULL) |
| 112 | 112 | ||
| 113 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 113 | #define SET_PERSONALITY(ex) \ |
| 114 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 114 | 115 | ||
| 115 | #endif /* __KERNEL__ */ | 116 | #endif /* __KERNEL__ */ |
| 116 | #endif | 117 | #endif |
diff --git a/arch/openrisc/include/uapi/asm/Kbuild b/arch/openrisc/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/openrisc/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 3ff21b536f28..b87438bb3384 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig | |||
| @@ -13,6 +13,7 @@ config PARISC | |||
| 13 | select HAVE_PERF_EVENTS | 13 | select HAVE_PERF_EVENTS |
| 14 | select GENERIC_ATOMIC64 if !64BIT | 14 | select GENERIC_ATOMIC64 if !64BIT |
| 15 | select HAVE_GENERIC_HARDIRQS | 15 | select HAVE_GENERIC_HARDIRQS |
| 16 | select BROKEN_RODATA | ||
| 16 | select GENERIC_IRQ_PROBE | 17 | select GENERIC_IRQ_PROBE |
| 17 | select GENERIC_PCI_IOMAP | 18 | select GENERIC_PCI_IOMAP |
| 18 | select IRQ_PER_CPU | 19 | select IRQ_PER_CPU |
diff --git a/arch/parisc/hpux/fs.c b/arch/parisc/hpux/fs.c index c71eb6c79897..6785de7bd2a0 100644 --- a/arch/parisc/hpux/fs.c +++ b/arch/parisc/hpux/fs.c | |||
| @@ -109,33 +109,32 @@ Efault: | |||
| 109 | 109 | ||
| 110 | int hpux_getdents(unsigned int fd, struct hpux_dirent __user *dirent, unsigned int count) | 110 | int hpux_getdents(unsigned int fd, struct hpux_dirent __user *dirent, unsigned int count) |
| 111 | { | 111 | { |
| 112 | struct file * file; | 112 | struct fd arg; |
| 113 | struct hpux_dirent __user * lastdirent; | 113 | struct hpux_dirent __user * lastdirent; |
| 114 | struct getdents_callback buf; | 114 | struct getdents_callback buf; |
| 115 | int error = -EBADF; | 115 | int error; |
| 116 | 116 | ||
| 117 | file = fget(fd); | 117 | arg = fdget(fd); |
| 118 | if (!file) | 118 | if (!arg.file) |
| 119 | goto out; | 119 | return -EBADF; |
| 120 | 120 | ||
| 121 | buf.current_dir = dirent; | 121 | buf.current_dir = dirent; |
| 122 | buf.previous = NULL; | 122 | buf.previous = NULL; |
| 123 | buf.count = count; | 123 | buf.count = count; |
| 124 | buf.error = 0; | 124 | buf.error = 0; |
| 125 | 125 | ||
| 126 | error = vfs_readdir(file, filldir, &buf); | 126 | error = vfs_readdir(arg.file, filldir, &buf); |
| 127 | if (error >= 0) | 127 | if (error >= 0) |
| 128 | error = buf.error; | 128 | error = buf.error; |
| 129 | lastdirent = buf.previous; | 129 | lastdirent = buf.previous; |
| 130 | if (lastdirent) { | 130 | if (lastdirent) { |
| 131 | if (put_user(file->f_pos, &lastdirent->d_off)) | 131 | if (put_user(arg.file->f_pos, &lastdirent->d_off)) |
| 132 | error = -EFAULT; | 132 | error = -EFAULT; |
| 133 | else | 133 | else |
| 134 | error = count - buf.count; | 134 | error = count - buf.count; |
| 135 | } | 135 | } |
| 136 | 136 | ||
| 137 | fput(file); | 137 | fdput(arg); |
| 138 | out: | ||
| 139 | return error; | 138 | return error; |
| 140 | } | 139 | } |
| 141 | 140 | ||
diff --git a/arch/parisc/include/asm/compat.h b/arch/parisc/include/asm/compat.h index 760f331d4fa3..db7a662691a8 100644 --- a/arch/parisc/include/asm/compat.h +++ b/arch/parisc/include/asm/compat.h | |||
| @@ -36,6 +36,7 @@ typedef s64 compat_s64; | |||
| 36 | typedef u32 compat_uint_t; | 36 | typedef u32 compat_uint_t; |
| 37 | typedef u32 compat_ulong_t; | 37 | typedef u32 compat_ulong_t; |
| 38 | typedef u64 compat_u64; | 38 | typedef u64 compat_u64; |
| 39 | typedef u32 compat_uptr_t; | ||
| 39 | 40 | ||
| 40 | struct compat_timespec { | 41 | struct compat_timespec { |
| 41 | compat_time_t tv_sec; | 42 | compat_time_t tv_sec; |
| @@ -127,6 +128,63 @@ typedef u32 compat_old_sigset_t; /* at least 32 bits */ | |||
| 127 | 128 | ||
| 128 | typedef u32 compat_sigset_word; | 129 | typedef u32 compat_sigset_word; |
| 129 | 130 | ||
| 131 | typedef union compat_sigval { | ||
| 132 | compat_int_t sival_int; | ||
| 133 | compat_uptr_t sival_ptr; | ||
| 134 | } compat_sigval_t; | ||
| 135 | |||
| 136 | typedef struct compat_siginfo { | ||
| 137 | int si_signo; | ||
| 138 | int si_errno; | ||
| 139 | int si_code; | ||
| 140 | |||
| 141 | union { | ||
| 142 | int _pad[128/sizeof(int) - 3]; | ||
| 143 | |||
| 144 | /* kill() */ | ||
| 145 | struct { | ||
| 146 | unsigned int _pid; /* sender's pid */ | ||
| 147 | unsigned int _uid; /* sender's uid */ | ||
| 148 | } _kill; | ||
| 149 | |||
| 150 | /* POSIX.1b timers */ | ||
| 151 | struct { | ||
| 152 | compat_timer_t _tid; /* timer id */ | ||
| 153 | int _overrun; /* overrun count */ | ||
| 154 | char _pad[sizeof(unsigned int) - sizeof(int)]; | ||
| 155 | compat_sigval_t _sigval; /* same as below */ | ||
| 156 | int _sys_private; /* not to be passed to user */ | ||
| 157 | } _timer; | ||
| 158 | |||
| 159 | /* POSIX.1b signals */ | ||
| 160 | struct { | ||
| 161 | unsigned int _pid; /* sender's pid */ | ||
| 162 | unsigned int _uid; /* sender's uid */ | ||
| 163 | compat_sigval_t _sigval; | ||
| 164 | } _rt; | ||
| 165 | |||
| 166 | /* SIGCHLD */ | ||
| 167 | struct { | ||
| 168 | unsigned int _pid; /* which child */ | ||
| 169 | unsigned int _uid; /* sender's uid */ | ||
| 170 | int _status; /* exit code */ | ||
| 171 | compat_clock_t _utime; | ||
| 172 | compat_clock_t _stime; | ||
| 173 | } _sigchld; | ||
| 174 | |||
| 175 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 176 | struct { | ||
| 177 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 178 | } _sigfault; | ||
| 179 | |||
| 180 | /* SIGPOLL */ | ||
| 181 | struct { | ||
| 182 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 183 | int _fd; | ||
| 184 | } _sigpoll; | ||
| 185 | } _sifields; | ||
| 186 | } compat_siginfo_t; | ||
| 187 | |||
| 130 | #define COMPAT_OFF_T_MAX 0x7fffffff | 188 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 131 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 189 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 132 | 190 | ||
| @@ -136,7 +194,6 @@ typedef u32 compat_sigset_word; | |||
| 136 | * as pointers because the syscall entry code will have | 194 | * as pointers because the syscall entry code will have |
| 137 | * appropriately converted them already. | 195 | * appropriately converted them already. |
| 138 | */ | 196 | */ |
| 139 | typedef u32 compat_uptr_t; | ||
| 140 | 197 | ||
| 141 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 198 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 142 | { | 199 | { |
diff --git a/arch/parisc/include/uapi/asm/Kbuild b/arch/parisc/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/parisc/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/parisc/kernel/signal32.h b/arch/parisc/kernel/signal32.h index c7800846422c..08a88b5349a2 100644 --- a/arch/parisc/kernel/signal32.h +++ b/arch/parisc/kernel/signal32.h | |||
| @@ -55,58 +55,6 @@ struct k_sigaction32 { | |||
| 55 | struct compat_sigaction sa; | 55 | struct compat_sigaction sa; |
| 56 | }; | 56 | }; |
| 57 | 57 | ||
| 58 | typedef struct compat_siginfo { | ||
| 59 | int si_signo; | ||
| 60 | int si_errno; | ||
| 61 | int si_code; | ||
| 62 | |||
| 63 | union { | ||
| 64 | int _pad[((128/sizeof(int)) - 3)]; | ||
| 65 | |||
| 66 | /* kill() */ | ||
| 67 | struct { | ||
| 68 | unsigned int _pid; /* sender's pid */ | ||
| 69 | unsigned int _uid; /* sender's uid */ | ||
| 70 | } _kill; | ||
| 71 | |||
| 72 | /* POSIX.1b timers */ | ||
| 73 | struct { | ||
| 74 | compat_timer_t _tid; /* timer id */ | ||
| 75 | int _overrun; /* overrun count */ | ||
| 76 | char _pad[sizeof(unsigned int) - sizeof(int)]; | ||
| 77 | compat_sigval_t _sigval; /* same as below */ | ||
| 78 | int _sys_private; /* not to be passed to user */ | ||
| 79 | } _timer; | ||
| 80 | |||
| 81 | /* POSIX.1b signals */ | ||
| 82 | struct { | ||
| 83 | unsigned int _pid; /* sender's pid */ | ||
| 84 | unsigned int _uid; /* sender's uid */ | ||
| 85 | compat_sigval_t _sigval; | ||
| 86 | } _rt; | ||
| 87 | |||
| 88 | /* SIGCHLD */ | ||
| 89 | struct { | ||
| 90 | unsigned int _pid; /* which child */ | ||
| 91 | unsigned int _uid; /* sender's uid */ | ||
| 92 | int _status; /* exit code */ | ||
| 93 | compat_clock_t _utime; | ||
| 94 | compat_clock_t _stime; | ||
| 95 | } _sigchld; | ||
| 96 | |||
| 97 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 98 | struct { | ||
| 99 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 100 | } _sigfault; | ||
| 101 | |||
| 102 | /* SIGPOLL */ | ||
| 103 | struct { | ||
| 104 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 105 | int _fd; | ||
| 106 | } _sigpoll; | ||
| 107 | } _sifields; | ||
| 108 | } compat_siginfo_t; | ||
| 109 | |||
| 110 | int copy_siginfo_to_user32 (compat_siginfo_t __user *to, siginfo_t *from); | 58 | int copy_siginfo_to_user32 (compat_siginfo_t __user *to, siginfo_t *from); |
| 111 | int copy_siginfo_from_user32 (siginfo_t *to, compat_siginfo_t __user *from); | 59 | int copy_siginfo_from_user32 (siginfo_t *to, compat_siginfo_t __user *from); |
| 112 | 60 | ||
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 06b56245d78c..de7c4c53f5cf 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig | |||
| @@ -486,7 +486,8 @@ CONFIG_CRYPTO_TWOFISH=m | |||
| 486 | CONFIG_CRYPTO_LZO=m | 486 | CONFIG_CRYPTO_LZO=m |
| 487 | # CONFIG_CRYPTO_ANSI_CPRNG is not set | 487 | # CONFIG_CRYPTO_ANSI_CPRNG is not set |
| 488 | CONFIG_CRYPTO_HW=y | 488 | CONFIG_CRYPTO_HW=y |
| 489 | CONFIG_CRYPTO_DEV_NX=m | 489 | CONFIG_CRYPTO_DEV_NX=y |
| 490 | CONFIG_CRYPTO_DEV_NX_ENCRYPT=m | ||
| 490 | CONFIG_VIRTUALIZATION=y | 491 | CONFIG_VIRTUALIZATION=y |
| 491 | CONFIG_KVM_BOOK3S_64=m | 492 | CONFIG_KVM_BOOK3S_64=m |
| 492 | CONFIG_KVM_BOOK3S_64_HV=y | 493 | CONFIG_KVM_BOOK3S_64_HV=y |
diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig index 1f65b3c9b59a..9f4a9368f51b 100644 --- a/arch/powerpc/configs/pseries_defconfig +++ b/arch/powerpc/configs/pseries_defconfig | |||
| @@ -369,7 +369,8 @@ CONFIG_CRYPTO_TWOFISH=m | |||
| 369 | CONFIG_CRYPTO_LZO=m | 369 | CONFIG_CRYPTO_LZO=m |
| 370 | # CONFIG_CRYPTO_ANSI_CPRNG is not set | 370 | # CONFIG_CRYPTO_ANSI_CPRNG is not set |
| 371 | CONFIG_CRYPTO_HW=y | 371 | CONFIG_CRYPTO_HW=y |
| 372 | CONFIG_CRYPTO_DEV_NX=m | 372 | CONFIG_CRYPTO_DEV_NX=y |
| 373 | CONFIG_CRYPTO_DEV_NX_ENCRYPT=m | ||
| 373 | CONFIG_VIRTUALIZATION=y | 374 | CONFIG_VIRTUALIZATION=y |
| 374 | CONFIG_KVM_BOOK3S_64=m | 375 | CONFIG_KVM_BOOK3S_64=m |
| 375 | CONFIG_KVM_BOOK3S_64_HV=y | 376 | CONFIG_KVM_BOOK3S_64_HV=y |
diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index efdc92618b38..dc2cf9c6d9e6 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h | |||
| @@ -288,6 +288,16 @@ static __inline__ int test_bit_le(unsigned long nr, | |||
| 288 | return (tmp[nr >> 3] >> (nr & 7)) & 1; | 288 | return (tmp[nr >> 3] >> (nr & 7)) & 1; |
| 289 | } | 289 | } |
| 290 | 290 | ||
| 291 | static inline void set_bit_le(int nr, void *addr) | ||
| 292 | { | ||
| 293 | set_bit(nr ^ BITOP_LE_SWIZZLE, addr); | ||
| 294 | } | ||
| 295 | |||
| 296 | static inline void clear_bit_le(int nr, void *addr) | ||
| 297 | { | ||
| 298 | clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); | ||
| 299 | } | ||
| 300 | |||
| 291 | static inline void __set_bit_le(int nr, void *addr) | 301 | static inline void __set_bit_le(int nr, void *addr) |
| 292 | { | 302 | { |
| 293 | __set_bit(nr ^ BITOP_LE_SWIZZLE, addr); | 303 | __set_bit(nr ^ BITOP_LE_SWIZZLE, addr); |
diff --git a/arch/powerpc/include/asm/compat.h b/arch/powerpc/include/asm/compat.h index 88e602f6430d..84fdf6857c31 100644 --- a/arch/powerpc/include/asm/compat.h +++ b/arch/powerpc/include/asm/compat.h | |||
| @@ -38,6 +38,7 @@ typedef s64 compat_s64; | |||
| 38 | typedef u32 compat_uint_t; | 38 | typedef u32 compat_uint_t; |
| 39 | typedef u32 compat_ulong_t; | 39 | typedef u32 compat_ulong_t; |
| 40 | typedef u64 compat_u64; | 40 | typedef u64 compat_u64; |
| 41 | typedef u32 compat_uptr_t; | ||
| 41 | 42 | ||
| 42 | struct compat_timespec { | 43 | struct compat_timespec { |
| 43 | compat_time_t tv_sec; | 44 | compat_time_t tv_sec; |
| @@ -114,6 +115,64 @@ typedef u32 compat_old_sigset_t; | |||
| 114 | 115 | ||
| 115 | typedef u32 compat_sigset_word; | 116 | typedef u32 compat_sigset_word; |
| 116 | 117 | ||
| 118 | typedef union compat_sigval { | ||
| 119 | compat_int_t sival_int; | ||
| 120 | compat_uptr_t sival_ptr; | ||
| 121 | } compat_sigval_t; | ||
| 122 | |||
| 123 | #define SI_PAD_SIZE32 (128/sizeof(int) - 3) | ||
| 124 | |||
| 125 | typedef struct compat_siginfo { | ||
| 126 | int si_signo; | ||
| 127 | int si_errno; | ||
| 128 | int si_code; | ||
| 129 | |||
| 130 | union { | ||
| 131 | int _pad[SI_PAD_SIZE32]; | ||
| 132 | |||
| 133 | /* kill() */ | ||
| 134 | struct { | ||
| 135 | compat_pid_t _pid; /* sender's pid */ | ||
| 136 | __compat_uid_t _uid; /* sender's uid */ | ||
| 137 | } _kill; | ||
| 138 | |||
| 139 | /* POSIX.1b timers */ | ||
| 140 | struct { | ||
| 141 | compat_timer_t _tid; /* timer id */ | ||
| 142 | int _overrun; /* overrun count */ | ||
| 143 | compat_sigval_t _sigval; /* same as below */ | ||
| 144 | int _sys_private; /* not to be passed to user */ | ||
| 145 | } _timer; | ||
| 146 | |||
| 147 | /* POSIX.1b signals */ | ||
| 148 | struct { | ||
| 149 | compat_pid_t _pid; /* sender's pid */ | ||
| 150 | __compat_uid_t _uid; /* sender's uid */ | ||
| 151 | compat_sigval_t _sigval; | ||
| 152 | } _rt; | ||
| 153 | |||
| 154 | /* SIGCHLD */ | ||
| 155 | struct { | ||
| 156 | compat_pid_t _pid; /* which child */ | ||
| 157 | __compat_uid_t _uid; /* sender's uid */ | ||
| 158 | int _status; /* exit code */ | ||
| 159 | compat_clock_t _utime; | ||
| 160 | compat_clock_t _stime; | ||
| 161 | } _sigchld; | ||
| 162 | |||
| 163 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ | ||
| 164 | struct { | ||
| 165 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 166 | } _sigfault; | ||
| 167 | |||
| 168 | /* SIGPOLL */ | ||
| 169 | struct { | ||
| 170 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 171 | int _fd; | ||
| 172 | } _sigpoll; | ||
| 173 | } _sifields; | ||
| 174 | } compat_siginfo_t; | ||
| 175 | |||
| 117 | #define COMPAT_OFF_T_MAX 0x7fffffff | 176 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 118 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 177 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 119 | 178 | ||
| @@ -123,7 +182,6 @@ typedef u32 compat_sigset_word; | |||
| 123 | * as pointers because the syscall entry code will have | 182 | * as pointers because the syscall entry code will have |
| 124 | * appropriately converted them already. | 183 | * appropriately converted them already. |
| 125 | */ | 184 | */ |
| 126 | typedef u32 compat_uptr_t; | ||
| 127 | 185 | ||
| 128 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 186 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 129 | { | 187 | { |
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index a8bf5c673a3c..28e8f5e5c63e 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h | |||
| @@ -53,6 +53,8 @@ | |||
| 53 | 53 | ||
| 54 | struct kvm; | 54 | struct kvm; |
| 55 | extern int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); | 55 | extern int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); |
| 56 | extern int kvm_unmap_hva_range(struct kvm *kvm, | ||
| 57 | unsigned long start, unsigned long end); | ||
| 56 | extern int kvm_age_hva(struct kvm *kvm, unsigned long hva); | 58 | extern int kvm_age_hva(struct kvm *kvm, unsigned long hva); |
| 57 | extern int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); | 59 | extern int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); |
| 58 | extern void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); | 60 | extern void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); |
| @@ -220,6 +222,7 @@ struct revmap_entry { | |||
| 220 | #define KVMPPC_GOT_PAGE 0x80 | 222 | #define KVMPPC_GOT_PAGE 0x80 |
| 221 | 223 | ||
| 222 | struct kvm_arch_memory_slot { | 224 | struct kvm_arch_memory_slot { |
| 225 | unsigned long *rmap; | ||
| 223 | }; | 226 | }; |
| 224 | 227 | ||
| 225 | struct kvm_arch { | 228 | struct kvm_arch { |
diff --git a/arch/powerpc/include/asm/ps3.h b/arch/powerpc/include/asm/ps3.h index 7f065e178ec4..0e15db4d703b 100644 --- a/arch/powerpc/include/asm/ps3.h +++ b/arch/powerpc/include/asm/ps3.h | |||
| @@ -24,7 +24,7 @@ | |||
| 24 | #include <linux/init.h> | 24 | #include <linux/init.h> |
| 25 | #include <linux/types.h> | 25 | #include <linux/types.h> |
| 26 | #include <linux/device.h> | 26 | #include <linux/device.h> |
| 27 | #include "cell-pmu.h" | 27 | #include <asm/cell-pmu.h> |
| 28 | 28 | ||
| 29 | union ps3_firmware_version { | 29 | union ps3_firmware_version { |
| 30 | u64 raw; | 30 | u64 raw; |
diff --git a/arch/powerpc/include/asm/siginfo.h b/arch/powerpc/include/asm/siginfo.h index 49495b0534ed..ccce3ef5cd86 100644 --- a/arch/powerpc/include/asm/siginfo.h +++ b/arch/powerpc/include/asm/siginfo.h | |||
| @@ -10,7 +10,6 @@ | |||
| 10 | 10 | ||
| 11 | #ifdef __powerpc64__ | 11 | #ifdef __powerpc64__ |
| 12 | # define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) | 12 | # define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) |
| 13 | # define SI_PAD_SIZE32 ((SI_MAX_SIZE/sizeof(int)) - 3) | ||
| 14 | #endif | 13 | #endif |
| 15 | 14 | ||
| 16 | #include <asm-generic/siginfo.h> | 15 | #include <asm-generic/siginfo.h> |
diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h index 559ae1ee6706..840838769853 100644 --- a/arch/powerpc/include/asm/systbl.h +++ b/arch/powerpc/include/asm/systbl.h | |||
| @@ -189,7 +189,7 @@ SYSCALL_SPU(getcwd) | |||
| 189 | SYSCALL_SPU(capget) | 189 | SYSCALL_SPU(capget) |
| 190 | SYSCALL_SPU(capset) | 190 | SYSCALL_SPU(capset) |
| 191 | COMPAT_SYS(sigaltstack) | 191 | COMPAT_SYS(sigaltstack) |
| 192 | SYSX_SPU(sys_sendfile64,compat_sys_sendfile,sys_sendfile) | 192 | SYSX_SPU(sys_sendfile,compat_sys_sendfile_wrapper,sys_sendfile) |
| 193 | SYSCALL(ni_syscall) | 193 | SYSCALL(ni_syscall) |
| 194 | SYSCALL(ni_syscall) | 194 | SYSCALL(ni_syscall) |
| 195 | PPC_SYS(vfork) | 195 | PPC_SYS(vfork) |
| @@ -229,7 +229,7 @@ COMPAT_SYS_SPU(sched_setaffinity) | |||
| 229 | COMPAT_SYS_SPU(sched_getaffinity) | 229 | COMPAT_SYS_SPU(sched_getaffinity) |
| 230 | SYSCALL(ni_syscall) | 230 | SYSCALL(ni_syscall) |
| 231 | SYSCALL(ni_syscall) | 231 | SYSCALL(ni_syscall) |
| 232 | SYS32ONLY(sendfile64) | 232 | SYSX(sys_ni_syscall,compat_sys_sendfile64_wrapper,sys_sendfile64) |
| 233 | COMPAT_SYS_SPU(io_setup) | 233 | COMPAT_SYS_SPU(io_setup) |
| 234 | SYSCALL_SPU(io_destroy) | 234 | SYSCALL_SPU(io_destroy) |
| 235 | COMPAT_SYS_SPU(io_getevents) | 235 | COMPAT_SYS_SPU(io_getevents) |
diff --git a/arch/powerpc/include/asm/ucc_fast.h b/arch/powerpc/include/asm/ucc_fast.h index 839aab8bf37d..4644c840e2fa 100644 --- a/arch/powerpc/include/asm/ucc_fast.h +++ b/arch/powerpc/include/asm/ucc_fast.h | |||
| @@ -19,7 +19,7 @@ | |||
| 19 | #include <asm/immap_qe.h> | 19 | #include <asm/immap_qe.h> |
| 20 | #include <asm/qe.h> | 20 | #include <asm/qe.h> |
| 21 | 21 | ||
| 22 | #include "ucc.h" | 22 | #include <asm/ucc.h> |
| 23 | 23 | ||
| 24 | /* Receive BD's status */ | 24 | /* Receive BD's status */ |
| 25 | #define R_E 0x80000000 /* buffer empty */ | 25 | #define R_E 0x80000000 /* buffer empty */ |
diff --git a/arch/powerpc/include/asm/ucc_slow.h b/arch/powerpc/include/asm/ucc_slow.h index 0980e6ad335b..cf131ffdb8d1 100644 --- a/arch/powerpc/include/asm/ucc_slow.h +++ b/arch/powerpc/include/asm/ucc_slow.h | |||
| @@ -20,7 +20,7 @@ | |||
| 20 | #include <asm/immap_qe.h> | 20 | #include <asm/immap_qe.h> |
| 21 | #include <asm/qe.h> | 21 | #include <asm/qe.h> |
| 22 | 22 | ||
| 23 | #include "ucc.h" | 23 | #include <asm/ucc.h> |
| 24 | 24 | ||
| 25 | /* transmit BD's status */ | 25 | /* transmit BD's status */ |
| 26 | #define T_R 0x80000000 /* ready bit */ | 26 | #define T_R 0x80000000 /* ready bit */ |
diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h index bd377a368611..c683fa350add 100644 --- a/arch/powerpc/include/asm/unistd.h +++ b/arch/powerpc/include/asm/unistd.h | |||
| @@ -419,6 +419,7 @@ | |||
| 419 | #define __ARCH_WANT_COMPAT_SYS_TIME | 419 | #define __ARCH_WANT_COMPAT_SYS_TIME |
| 420 | #define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND | 420 | #define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND |
| 421 | #define __ARCH_WANT_SYS_NEWFSTATAT | 421 | #define __ARCH_WANT_SYS_NEWFSTATAT |
| 422 | #define __ARCH_WANT_COMPAT_SYS_SENDFILE | ||
| 422 | #endif | 423 | #endif |
| 423 | 424 | ||
| 424 | /* | 425 | /* |
diff --git a/arch/powerpc/include/uapi/asm/Kbuild b/arch/powerpc/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/powerpc/kernel/ppc32.h b/arch/powerpc/kernel/ppc32.h index dc16aefe1dd0..02fb0ee26093 100644 --- a/arch/powerpc/kernel/ppc32.h +++ b/arch/powerpc/kernel/ppc32.h | |||
| @@ -16,57 +16,6 @@ | |||
| 16 | 16 | ||
| 17 | /* These are here to support 32-bit syscalls on a 64-bit kernel. */ | 17 | /* These are here to support 32-bit syscalls on a 64-bit kernel. */ |
| 18 | 18 | ||
| 19 | typedef struct compat_siginfo { | ||
| 20 | int si_signo; | ||
| 21 | int si_errno; | ||
| 22 | int si_code; | ||
| 23 | |||
| 24 | union { | ||
| 25 | int _pad[SI_PAD_SIZE32]; | ||
| 26 | |||
| 27 | /* kill() */ | ||
| 28 | struct { | ||
| 29 | compat_pid_t _pid; /* sender's pid */ | ||
| 30 | compat_uid_t _uid; /* sender's uid */ | ||
| 31 | } _kill; | ||
| 32 | |||
| 33 | /* POSIX.1b timers */ | ||
| 34 | struct { | ||
| 35 | compat_timer_t _tid; /* timer id */ | ||
| 36 | int _overrun; /* overrun count */ | ||
| 37 | compat_sigval_t _sigval; /* same as below */ | ||
| 38 | int _sys_private; /* not to be passed to user */ | ||
| 39 | } _timer; | ||
| 40 | |||
| 41 | /* POSIX.1b signals */ | ||
| 42 | struct { | ||
| 43 | compat_pid_t _pid; /* sender's pid */ | ||
| 44 | compat_uid_t _uid; /* sender's uid */ | ||
| 45 | compat_sigval_t _sigval; | ||
| 46 | } _rt; | ||
| 47 | |||
| 48 | /* SIGCHLD */ | ||
| 49 | struct { | ||
| 50 | compat_pid_t _pid; /* which child */ | ||
| 51 | compat_uid_t _uid; /* sender's uid */ | ||
| 52 | int _status; /* exit code */ | ||
| 53 | compat_clock_t _utime; | ||
| 54 | compat_clock_t _stime; | ||
| 55 | } _sigchld; | ||
| 56 | |||
| 57 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ | ||
| 58 | struct { | ||
| 59 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 60 | } _sigfault; | ||
| 61 | |||
| 62 | /* SIGPOLL */ | ||
| 63 | struct { | ||
| 64 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 65 | int _fd; | ||
| 66 | } _sigpoll; | ||
| 67 | } _sifields; | ||
| 68 | } compat_siginfo_t; | ||
| 69 | |||
| 70 | #define __old_sigaction32 old_sigaction32 | 19 | #define __old_sigaction32 old_sigaction32 |
| 71 | 20 | ||
| 72 | struct __old_sigaction32 { | 21 | struct __old_sigaction32 { |
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 0794a3017b1b..47834a3f4938 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c | |||
| @@ -705,6 +705,7 @@ static void __init early_cmdline_parse(void) | |||
| 705 | #endif | 705 | #endif |
| 706 | #define OV5_TYPE1_AFFINITY 0x80 /* Type 1 NUMA affinity */ | 706 | #define OV5_TYPE1_AFFINITY 0x80 /* Type 1 NUMA affinity */ |
| 707 | #define OV5_PFO_HW_RNG 0x80 /* PFO Random Number Generator */ | 707 | #define OV5_PFO_HW_RNG 0x80 /* PFO Random Number Generator */ |
| 708 | #define OV5_PFO_HW_842 0x40 /* PFO Compression Accelerator */ | ||
| 708 | #define OV5_PFO_HW_ENCR 0x20 /* PFO Encryption Accelerator */ | 709 | #define OV5_PFO_HW_ENCR 0x20 /* PFO Encryption Accelerator */ |
| 709 | 710 | ||
| 710 | /* Option Vector 6: IBM PAPR hints */ | 711 | /* Option Vector 6: IBM PAPR hints */ |
| @@ -774,8 +775,7 @@ static unsigned char ibm_architecture_vec[] = { | |||
| 774 | 0, | 775 | 0, |
| 775 | 0, | 776 | 0, |
| 776 | 0, | 777 | 0, |
| 777 | OV5_PFO_HW_RNG | OV5_PFO_HW_ENCR, | 778 | OV5_PFO_HW_RNG | OV5_PFO_HW_ENCR | OV5_PFO_HW_842, |
| 778 | |||
| 779 | /* option vector 6: IBM PAPR hints */ | 779 | /* option vector 6: IBM PAPR hints */ |
| 780 | 4 - 2, /* length */ | 780 | 4 - 2, /* length */ |
| 781 | 0, | 781 | 0, |
| @@ -1624,6 +1624,63 @@ static void __init prom_instantiate_rtas(void) | |||
| 1624 | 1624 | ||
| 1625 | #ifdef CONFIG_PPC64 | 1625 | #ifdef CONFIG_PPC64 |
| 1626 | /* | 1626 | /* |
| 1627 | * Allocate room for and instantiate Stored Measurement Log (SML) | ||
| 1628 | */ | ||
| 1629 | static void __init prom_instantiate_sml(void) | ||
| 1630 | { | ||
| 1631 | phandle ibmvtpm_node; | ||
| 1632 | ihandle ibmvtpm_inst; | ||
| 1633 | u32 entry = 0, size = 0; | ||
| 1634 | u64 base; | ||
| 1635 | |||
| 1636 | prom_debug("prom_instantiate_sml: start...\n"); | ||
| 1637 | |||
| 1638 | ibmvtpm_node = call_prom("finddevice", 1, 1, ADDR("/ibm,vtpm")); | ||
| 1639 | prom_debug("ibmvtpm_node: %x\n", ibmvtpm_node); | ||
| 1640 | if (!PHANDLE_VALID(ibmvtpm_node)) | ||
| 1641 | return; | ||
| 1642 | |||
| 1643 | ibmvtpm_inst = call_prom("open", 1, 1, ADDR("/ibm,vtpm")); | ||
| 1644 | if (!IHANDLE_VALID(ibmvtpm_inst)) { | ||
| 1645 | prom_printf("opening vtpm package failed (%x)\n", ibmvtpm_inst); | ||
| 1646 | return; | ||
| 1647 | } | ||
| 1648 | |||
| 1649 | if (call_prom_ret("call-method", 2, 2, &size, | ||
| 1650 | ADDR("sml-get-handover-size"), | ||
| 1651 | ibmvtpm_inst) != 0 || size == 0) { | ||
| 1652 | prom_printf("SML get handover size failed\n"); | ||
| 1653 | return; | ||
| 1654 | } | ||
| 1655 | |||
| 1656 | base = alloc_down(size, PAGE_SIZE, 0); | ||
| 1657 | if (base == 0) | ||
| 1658 | prom_panic("Could not allocate memory for sml\n"); | ||
| 1659 | |||
| 1660 | prom_printf("instantiating sml at 0x%x...", base); | ||
| 1661 | |||
| 1662 | if (call_prom_ret("call-method", 4, 2, &entry, | ||
| 1663 | ADDR("sml-handover"), | ||
| 1664 | ibmvtpm_inst, size, base) != 0 || entry == 0) { | ||
| 1665 | prom_printf("SML handover failed\n"); | ||
| 1666 | return; | ||
| 1667 | } | ||
| 1668 | prom_printf(" done\n"); | ||
| 1669 | |||
| 1670 | reserve_mem(base, size); | ||
| 1671 | |||
| 1672 | prom_setprop(ibmvtpm_node, "/ibm,vtpm", "linux,sml-base", | ||
| 1673 | &base, sizeof(base)); | ||
| 1674 | prom_setprop(ibmvtpm_node, "/ibm,vtpm", "linux,sml-size", | ||
| 1675 | &size, sizeof(size)); | ||
| 1676 | |||
| 1677 | prom_debug("sml base = 0x%x\n", base); | ||
| 1678 | prom_debug("sml size = 0x%x\n", (long)size); | ||
| 1679 | |||
| 1680 | prom_debug("prom_instantiate_sml: end...\n"); | ||
| 1681 | } | ||
| 1682 | |||
| 1683 | /* | ||
| 1627 | * Allocate room for and initialize TCE tables | 1684 | * Allocate room for and initialize TCE tables |
| 1628 | */ | 1685 | */ |
| 1629 | static void __init prom_initialize_tce_table(void) | 1686 | static void __init prom_initialize_tce_table(void) |
| @@ -2916,6 +2973,11 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4, | |||
| 2916 | prom_instantiate_opal(); | 2973 | prom_instantiate_opal(); |
| 2917 | #endif | 2974 | #endif |
| 2918 | 2975 | ||
| 2976 | #ifdef CONFIG_PPC64 | ||
| 2977 | /* instantiate sml */ | ||
| 2978 | prom_instantiate_sml(); | ||
| 2979 | #endif | ||
| 2980 | |||
| 2919 | /* | 2981 | /* |
| 2920 | * On non-powermacs, put all CPUs in spin-loops. | 2982 | * On non-powermacs, put all CPUs in spin-loops. |
| 2921 | * | 2983 | * |
diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c index 81c570633ead..abd1112da54f 100644 --- a/arch/powerpc/kernel/sys_ppc32.c +++ b/arch/powerpc/kernel/sys_ppc32.c | |||
| @@ -143,48 +143,17 @@ long compat_sys_ipc(u32 call, u32 first, u32 second, u32 third, compat_uptr_t pt | |||
| 143 | * proper conversion (sign extension) between the register representation of a signed int (msr in 32-bit mode) | 143 | * proper conversion (sign extension) between the register representation of a signed int (msr in 32-bit mode) |
| 144 | * and the register representation of a signed int (msr in 64-bit mode) is performed. | 144 | * and the register representation of a signed int (msr in 64-bit mode) is performed. |
| 145 | */ | 145 | */ |
| 146 | asmlinkage long compat_sys_sendfile(u32 out_fd, u32 in_fd, compat_off_t __user * offset, u32 count) | 146 | asmlinkage long compat_sys_sendfile_wrapper(u32 out_fd, u32 in_fd, |
| 147 | compat_off_t __user *offset, u32 count) | ||
| 147 | { | 148 | { |
| 148 | mm_segment_t old_fs = get_fs(); | 149 | return compat_sys_sendfile((int)out_fd, (int)in_fd, offset, count); |
| 149 | int ret; | ||
| 150 | off_t of; | ||
| 151 | off_t __user *up; | ||
| 152 | |||
| 153 | if (offset && get_user(of, offset)) | ||
| 154 | return -EFAULT; | ||
| 155 | |||
| 156 | /* The __user pointer cast is valid because of the set_fs() */ | ||
| 157 | set_fs(KERNEL_DS); | ||
| 158 | up = offset ? (off_t __user *) &of : NULL; | ||
| 159 | ret = sys_sendfile((int)out_fd, (int)in_fd, up, count); | ||
| 160 | set_fs(old_fs); | ||
| 161 | |||
| 162 | if (offset && put_user(of, offset)) | ||
| 163 | return -EFAULT; | ||
| 164 | |||
| 165 | return ret; | ||
| 166 | } | 150 | } |
| 167 | 151 | ||
| 168 | asmlinkage int compat_sys_sendfile64(int out_fd, int in_fd, compat_loff_t __user *offset, s32 count) | 152 | asmlinkage long compat_sys_sendfile64_wrapper(u32 out_fd, u32 in_fd, |
| 153 | compat_loff_t __user *offset, u32 count) | ||
| 169 | { | 154 | { |
| 170 | mm_segment_t old_fs = get_fs(); | 155 | return sys_sendfile((int)out_fd, (int)in_fd, |
| 171 | int ret; | 156 | (off_t __user *)offset, count); |
| 172 | loff_t lof; | ||
| 173 | loff_t __user *up; | ||
| 174 | |||
| 175 | if (offset && get_user(lof, offset)) | ||
| 176 | return -EFAULT; | ||
| 177 | |||
| 178 | /* The __user pointer cast is valid because of the set_fs() */ | ||
| 179 | set_fs(KERNEL_DS); | ||
| 180 | up = offset ? (loff_t __user *) &lof : NULL; | ||
| 181 | ret = sys_sendfile64(out_fd, in_fd, up, count); | ||
| 182 | set_fs(old_fs); | ||
| 183 | |||
| 184 | if (offset && put_user(lof, offset)) | ||
| 185 | return -EFAULT; | ||
| 186 | |||
| 187 | return ret; | ||
| 188 | } | 157 | } |
| 189 | 158 | ||
| 190 | long compat_sys_execve(unsigned long a0, unsigned long a1, unsigned long a2, | 159 | long compat_sys_execve(unsigned long a0, unsigned long a1, unsigned long a2, |
diff --git a/arch/powerpc/kvm/44x_tlb.c b/arch/powerpc/kvm/44x_tlb.c index 33aa715dab28..5dd3ab469976 100644 --- a/arch/powerpc/kvm/44x_tlb.c +++ b/arch/powerpc/kvm/44x_tlb.c | |||
| @@ -319,7 +319,6 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, | |||
| 319 | if (is_error_page(new_page)) { | 319 | if (is_error_page(new_page)) { |
| 320 | printk(KERN_ERR "Couldn't get guest page for gfn %llx!\n", | 320 | printk(KERN_ERR "Couldn't get guest page for gfn %llx!\n", |
| 321 | (unsigned long long)gfn); | 321 | (unsigned long long)gfn); |
| 322 | kvm_release_page_clean(new_page); | ||
| 323 | return; | 322 | return; |
| 324 | } | 323 | } |
| 325 | hpaddr = page_to_phys(new_page); | 324 | hpaddr = page_to_phys(new_page); |
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index d03eb6f7b058..d95d11322a15 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c | |||
| @@ -705,7 +705,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, | |||
| 705 | goto out_unlock; | 705 | goto out_unlock; |
| 706 | hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID; | 706 | hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID; |
| 707 | 707 | ||
| 708 | rmap = &memslot->rmap[gfn - memslot->base_gfn]; | 708 | rmap = &memslot->arch.rmap[gfn - memslot->base_gfn]; |
| 709 | lock_rmap(rmap); | 709 | lock_rmap(rmap); |
| 710 | 710 | ||
| 711 | /* Check if we might have been invalidated; let the guest retry if so */ | 711 | /* Check if we might have been invalidated; let the guest retry if so */ |
| @@ -756,9 +756,12 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, | |||
| 756 | goto out_put; | 756 | goto out_put; |
| 757 | } | 757 | } |
| 758 | 758 | ||
| 759 | static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, | 759 | static int kvm_handle_hva_range(struct kvm *kvm, |
| 760 | int (*handler)(struct kvm *kvm, unsigned long *rmapp, | 760 | unsigned long start, |
| 761 | unsigned long gfn)) | 761 | unsigned long end, |
| 762 | int (*handler)(struct kvm *kvm, | ||
| 763 | unsigned long *rmapp, | ||
| 764 | unsigned long gfn)) | ||
| 762 | { | 765 | { |
| 763 | int ret; | 766 | int ret; |
| 764 | int retval = 0; | 767 | int retval = 0; |
| @@ -767,15 +770,25 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, | |||
| 767 | 770 | ||
| 768 | slots = kvm_memslots(kvm); | 771 | slots = kvm_memslots(kvm); |
| 769 | kvm_for_each_memslot(memslot, slots) { | 772 | kvm_for_each_memslot(memslot, slots) { |
| 770 | unsigned long start = memslot->userspace_addr; | 773 | unsigned long hva_start, hva_end; |
| 771 | unsigned long end; | 774 | gfn_t gfn, gfn_end; |
| 772 | 775 | ||
| 773 | end = start + (memslot->npages << PAGE_SHIFT); | 776 | hva_start = max(start, memslot->userspace_addr); |
| 774 | if (hva >= start && hva < end) { | 777 | hva_end = min(end, memslot->userspace_addr + |
| 775 | gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; | 778 | (memslot->npages << PAGE_SHIFT)); |
| 779 | if (hva_start >= hva_end) | ||
| 780 | continue; | ||
| 781 | /* | ||
| 782 | * {gfn(page) | page intersects with [hva_start, hva_end)} = | ||
| 783 | * {gfn, gfn+1, ..., gfn_end-1}. | ||
| 784 | */ | ||
| 785 | gfn = hva_to_gfn_memslot(hva_start, memslot); | ||
| 786 | gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); | ||
| 787 | |||
| 788 | for (; gfn < gfn_end; ++gfn) { | ||
| 789 | gfn_t gfn_offset = gfn - memslot->base_gfn; | ||
| 776 | 790 | ||
| 777 | ret = handler(kvm, &memslot->rmap[gfn_offset], | 791 | ret = handler(kvm, &memslot->arch.rmap[gfn_offset], gfn); |
| 778 | memslot->base_gfn + gfn_offset); | ||
| 779 | retval |= ret; | 792 | retval |= ret; |
| 780 | } | 793 | } |
| 781 | } | 794 | } |
| @@ -783,6 +796,13 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, | |||
| 783 | return retval; | 796 | return retval; |
| 784 | } | 797 | } |
| 785 | 798 | ||
| 799 | static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, | ||
| 800 | int (*handler)(struct kvm *kvm, unsigned long *rmapp, | ||
| 801 | unsigned long gfn)) | ||
| 802 | { | ||
| 803 | return kvm_handle_hva_range(kvm, hva, hva + 1, handler); | ||
| 804 | } | ||
| 805 | |||
| 786 | static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, | 806 | static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 787 | unsigned long gfn) | 807 | unsigned long gfn) |
| 788 | { | 808 | { |
| @@ -850,6 +870,13 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) | |||
| 850 | return 0; | 870 | return 0; |
| 851 | } | 871 | } |
| 852 | 872 | ||
| 873 | int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end) | ||
| 874 | { | ||
| 875 | if (kvm->arch.using_mmu_notifiers) | ||
| 876 | kvm_handle_hva_range(kvm, start, end, kvm_unmap_rmapp); | ||
| 877 | return 0; | ||
| 878 | } | ||
| 879 | |||
| 853 | static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, | 880 | static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 854 | unsigned long gfn) | 881 | unsigned long gfn) |
| 855 | { | 882 | { |
| @@ -1009,7 +1036,7 @@ long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) | |||
| 1009 | unsigned long *rmapp, *map; | 1036 | unsigned long *rmapp, *map; |
| 1010 | 1037 | ||
| 1011 | preempt_disable(); | 1038 | preempt_disable(); |
| 1012 | rmapp = memslot->rmap; | 1039 | rmapp = memslot->arch.rmap; |
| 1013 | map = memslot->dirty_bitmap; | 1040 | map = memslot->dirty_bitmap; |
| 1014 | for (i = 0; i < memslot->npages; ++i) { | 1041 | for (i = 0; i < memslot->npages; ++i) { |
| 1015 | if (kvm_test_clear_dirty(kvm, rmapp)) | 1042 | if (kvm_test_clear_dirty(kvm, rmapp)) |
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 5c70d19494f9..fb0e821622d4 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c | |||
| @@ -84,7 +84,7 @@ static void remove_revmap_chain(struct kvm *kvm, long pte_index, | |||
| 84 | if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) | 84 | if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) |
| 85 | return; | 85 | return; |
| 86 | 86 | ||
| 87 | rmap = real_vmalloc_addr(&memslot->rmap[gfn - memslot->base_gfn]); | 87 | rmap = real_vmalloc_addr(&memslot->arch.rmap[gfn - memslot->base_gfn]); |
| 88 | lock_rmap(rmap); | 88 | lock_rmap(rmap); |
| 89 | 89 | ||
| 90 | head = *rmap & KVMPPC_RMAP_INDEX; | 90 | head = *rmap & KVMPPC_RMAP_INDEX; |
| @@ -180,7 +180,7 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, | |||
| 180 | if (!slot_is_aligned(memslot, psize)) | 180 | if (!slot_is_aligned(memslot, psize)) |
| 181 | return H_PARAMETER; | 181 | return H_PARAMETER; |
| 182 | slot_fn = gfn - memslot->base_gfn; | 182 | slot_fn = gfn - memslot->base_gfn; |
| 183 | rmap = &memslot->rmap[slot_fn]; | 183 | rmap = &memslot->arch.rmap[slot_fn]; |
| 184 | 184 | ||
| 185 | if (!kvm->arch.using_mmu_notifiers) { | 185 | if (!kvm->arch.using_mmu_notifiers) { |
| 186 | physp = kvm->arch.slot_phys[memslot->id]; | 186 | physp = kvm->arch.slot_phys[memslot->id]; |
| @@ -197,7 +197,7 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, | |||
| 197 | pa &= PAGE_MASK; | 197 | pa &= PAGE_MASK; |
| 198 | } else { | 198 | } else { |
| 199 | /* Translate to host virtual address */ | 199 | /* Translate to host virtual address */ |
| 200 | hva = gfn_to_hva_memslot(memslot, gfn); | 200 | hva = __gfn_to_hva_memslot(memslot, gfn); |
| 201 | 201 | ||
| 202 | /* Look up the Linux PTE for the backing page */ | 202 | /* Look up the Linux PTE for the backing page */ |
| 203 | pte_size = psize; | 203 | pte_size = psize; |
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index a1baec340f7e..05c28f59f77f 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c | |||
| @@ -242,10 +242,8 @@ static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) | |||
| 242 | int i; | 242 | int i; |
| 243 | 243 | ||
| 244 | hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT); | 244 | hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT); |
| 245 | if (is_error_page(hpage)) { | 245 | if (is_error_page(hpage)) |
| 246 | kvm_release_page_clean(hpage); | ||
| 247 | return; | 246 | return; |
| 248 | } | ||
| 249 | 247 | ||
| 250 | hpage_offset = pte->raddr & ~PAGE_MASK; | 248 | hpage_offset = pte->raddr & ~PAGE_MASK; |
| 251 | hpage_offset &= ~0xFFFULL; | 249 | hpage_offset &= ~0xFFFULL; |
diff --git a/arch/powerpc/kvm/e500_tlb.c b/arch/powerpc/kvm/e500_tlb.c index a2b66717813d..ff38b664195d 100644 --- a/arch/powerpc/kvm/e500_tlb.c +++ b/arch/powerpc/kvm/e500_tlb.c | |||
| @@ -520,11 +520,10 @@ static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, | |||
| 520 | 520 | ||
| 521 | if (likely(!pfnmap)) { | 521 | if (likely(!pfnmap)) { |
| 522 | unsigned long tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT); | 522 | unsigned long tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT); |
| 523 | pfn = gfn_to_pfn_memslot(vcpu_e500->vcpu.kvm, slot, gfn); | 523 | pfn = gfn_to_pfn_memslot(slot, gfn); |
| 524 | if (is_error_pfn(pfn)) { | 524 | if (is_error_pfn(pfn)) { |
| 525 | printk(KERN_ERR "Couldn't get real page for gfn %lx!\n", | 525 | printk(KERN_ERR "Couldn't get real page for gfn %lx!\n", |
| 526 | (long)gfn); | 526 | (long)gfn); |
| 527 | kvm_release_pfn_clean(pfn); | ||
| 528 | return; | 527 | return; |
| 529 | } | 528 | } |
| 530 | 529 | ||
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 87f4dc886076..4d213b8b0fb5 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c | |||
| @@ -302,10 +302,18 @@ long kvm_arch_dev_ioctl(struct file *filp, | |||
| 302 | void kvm_arch_free_memslot(struct kvm_memory_slot *free, | 302 | void kvm_arch_free_memslot(struct kvm_memory_slot *free, |
| 303 | struct kvm_memory_slot *dont) | 303 | struct kvm_memory_slot *dont) |
| 304 | { | 304 | { |
| 305 | if (!dont || free->arch.rmap != dont->arch.rmap) { | ||
| 306 | vfree(free->arch.rmap); | ||
| 307 | free->arch.rmap = NULL; | ||
| 308 | } | ||
| 305 | } | 309 | } |
| 306 | 310 | ||
| 307 | int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages) | 311 | int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages) |
| 308 | { | 312 | { |
| 313 | slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap)); | ||
| 314 | if (!slot->arch.rmap) | ||
| 315 | return -ENOMEM; | ||
| 316 | |||
| 309 | return 0; | 317 | return 0; |
| 310 | } | 318 | } |
| 311 | 319 | ||
| @@ -326,8 +334,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, | |||
| 326 | kvmppc_core_commit_memory_region(kvm, mem); | 334 | kvmppc_core_commit_memory_region(kvm, mem); |
| 327 | } | 335 | } |
| 328 | 336 | ||
| 337 | void kvm_arch_flush_shadow_all(struct kvm *kvm) | ||
| 338 | { | ||
| 339 | } | ||
| 329 | 340 | ||
| 330 | void kvm_arch_flush_shadow(struct kvm *kvm) | 341 | void kvm_arch_flush_shadow_memslot(struct kvm *kvm, |
| 342 | struct kvm_memory_slot *slot) | ||
| 331 | { | 343 | { |
| 332 | } | 344 | } |
| 333 | 345 | ||
diff --git a/arch/powerpc/platforms/40x/ppc40x_simple.c b/arch/powerpc/platforms/40x/ppc40x_simple.c index 97612068fae3..969dddcf3320 100644 --- a/arch/powerpc/platforms/40x/ppc40x_simple.c +++ b/arch/powerpc/platforms/40x/ppc40x_simple.c | |||
| @@ -50,7 +50,7 @@ machine_device_initcall(ppc40x_simple, ppc40x_device_probe); | |||
| 50 | * Again, if your board needs to do things differently then create a | 50 | * Again, if your board needs to do things differently then create a |
| 51 | * board.c file for it rather than adding it to this list. | 51 | * board.c file for it rather than adding it to this list. |
| 52 | */ | 52 | */ |
| 53 | static const char *board[] __initdata = { | 53 | static const char * const board[] __initconst = { |
| 54 | "amcc,acadia", | 54 | "amcc,acadia", |
| 55 | "amcc,haleakala", | 55 | "amcc,haleakala", |
| 56 | "amcc,kilauea", | 56 | "amcc,kilauea", |
diff --git a/arch/powerpc/platforms/512x/mpc5121_generic.c b/arch/powerpc/platforms/512x/mpc5121_generic.c index 926731f1ff01..ca1ca6669990 100644 --- a/arch/powerpc/platforms/512x/mpc5121_generic.c +++ b/arch/powerpc/platforms/512x/mpc5121_generic.c | |||
| @@ -26,7 +26,7 @@ | |||
| 26 | /* | 26 | /* |
| 27 | * list of supported boards | 27 | * list of supported boards |
| 28 | */ | 28 | */ |
| 29 | static const char *board[] __initdata = { | 29 | static const char * const board[] __initconst = { |
| 30 | "prt,prtlvt", | 30 | "prt,prtlvt", |
| 31 | NULL | 31 | NULL |
| 32 | }; | 32 | }; |
diff --git a/arch/powerpc/platforms/52xx/lite5200.c b/arch/powerpc/platforms/52xx/lite5200.c index 01ffa64d2aa7..448d862bcf3d 100644 --- a/arch/powerpc/platforms/52xx/lite5200.c +++ b/arch/powerpc/platforms/52xx/lite5200.c | |||
| @@ -172,7 +172,7 @@ static void __init lite5200_setup_arch(void) | |||
| 172 | mpc52xx_setup_pci(); | 172 | mpc52xx_setup_pci(); |
| 173 | } | 173 | } |
| 174 | 174 | ||
| 175 | static const char *board[] __initdata = { | 175 | static const char * const board[] __initconst = { |
| 176 | "fsl,lite5200", | 176 | "fsl,lite5200", |
| 177 | "fsl,lite5200b", | 177 | "fsl,lite5200b", |
| 178 | NULL, | 178 | NULL, |
diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index 17d91b7da315..070d315dd6cd 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c | |||
| @@ -232,7 +232,7 @@ static void __init media5200_setup_arch(void) | |||
| 232 | } | 232 | } |
| 233 | 233 | ||
| 234 | /* list of the supported boards */ | 234 | /* list of the supported boards */ |
| 235 | static const char *board[] __initdata = { | 235 | static const char * const board[] __initconst = { |
| 236 | "fsl,media5200", | 236 | "fsl,media5200", |
| 237 | NULL | 237 | NULL |
| 238 | }; | 238 | }; |
diff --git a/arch/powerpc/platforms/83xx/mpc837x_rdb.c b/arch/powerpc/platforms/83xx/mpc837x_rdb.c index 16c9c9cbbb7f..eca1f0960fff 100644 --- a/arch/powerpc/platforms/83xx/mpc837x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc837x_rdb.c | |||
| @@ -60,7 +60,7 @@ static void __init mpc837x_rdb_setup_arch(void) | |||
| 60 | 60 | ||
| 61 | machine_device_initcall(mpc837x_rdb, mpc83xx_declare_of_platform_devices); | 61 | machine_device_initcall(mpc837x_rdb, mpc83xx_declare_of_platform_devices); |
| 62 | 62 | ||
| 63 | static const char *board[] __initdata = { | 63 | static const char * const board[] __initconst = { |
| 64 | "fsl,mpc8377rdb", | 64 | "fsl,mpc8377rdb", |
| 65 | "fsl,mpc8378rdb", | 65 | "fsl,mpc8378rdb", |
| 66 | "fsl,mpc8379rdb", | 66 | "fsl,mpc8379rdb", |
diff --git a/arch/powerpc/platforms/85xx/tqm85xx.c b/arch/powerpc/platforms/85xx/tqm85xx.c index 3e70a2035e53..b62fa87521a3 100644 --- a/arch/powerpc/platforms/85xx/tqm85xx.c +++ b/arch/powerpc/platforms/85xx/tqm85xx.c | |||
| @@ -125,7 +125,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_1520, | |||
| 125 | 125 | ||
| 126 | machine_device_initcall(tqm85xx, mpc85xx_common_publish_devices); | 126 | machine_device_initcall(tqm85xx, mpc85xx_common_publish_devices); |
| 127 | 127 | ||
| 128 | static const char *board[] __initdata = { | 128 | static const char * const board[] __initconst = { |
| 129 | "tqc,tqm8540", | 129 | "tqc,tqm8540", |
| 130 | "tqc,tqm8541", | 130 | "tqc,tqm8541", |
| 131 | "tqc,tqm8548", | 131 | "tqc,tqm8548", |
diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c index 714bbfc3162c..db4e638cf408 100644 --- a/arch/powerpc/platforms/cell/spu_syscalls.c +++ b/arch/powerpc/platforms/cell/spu_syscalls.c | |||
| @@ -69,8 +69,6 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags, | |||
| 69 | umode_t, mode, int, neighbor_fd) | 69 | umode_t, mode, int, neighbor_fd) |
| 70 | { | 70 | { |
| 71 | long ret; | 71 | long ret; |
| 72 | struct file *neighbor; | ||
| 73 | int fput_needed; | ||
| 74 | struct spufs_calls *calls; | 72 | struct spufs_calls *calls; |
| 75 | 73 | ||
| 76 | calls = spufs_calls_get(); | 74 | calls = spufs_calls_get(); |
| @@ -78,11 +76,11 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags, | |||
| 78 | return -ENOSYS; | 76 | return -ENOSYS; |
| 79 | 77 | ||
| 80 | if (flags & SPU_CREATE_AFFINITY_SPU) { | 78 | if (flags & SPU_CREATE_AFFINITY_SPU) { |
| 79 | struct fd neighbor = fdget(neighbor_fd); | ||
| 81 | ret = -EBADF; | 80 | ret = -EBADF; |
| 82 | neighbor = fget_light(neighbor_fd, &fput_needed); | 81 | if (neighbor.file) { |
| 83 | if (neighbor) { | 82 | ret = calls->create_thread(name, flags, mode, neighbor.file); |
| 84 | ret = calls->create_thread(name, flags, mode, neighbor); | 83 | fdput(neighbor); |
| 85 | fput_light(neighbor, fput_needed); | ||
| 86 | } | 84 | } |
| 87 | } else | 85 | } else |
| 88 | ret = calls->create_thread(name, flags, mode, NULL); | 86 | ret = calls->create_thread(name, flags, mode, NULL); |
| @@ -94,8 +92,7 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags, | |||
| 94 | asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) | 92 | asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) |
| 95 | { | 93 | { |
| 96 | long ret; | 94 | long ret; |
| 97 | struct file *filp; | 95 | struct fd arg; |
| 98 | int fput_needed; | ||
| 99 | struct spufs_calls *calls; | 96 | struct spufs_calls *calls; |
| 100 | 97 | ||
| 101 | calls = spufs_calls_get(); | 98 | calls = spufs_calls_get(); |
| @@ -103,10 +100,10 @@ asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) | |||
| 103 | return -ENOSYS; | 100 | return -ENOSYS; |
| 104 | 101 | ||
| 105 | ret = -EBADF; | 102 | ret = -EBADF; |
| 106 | filp = fget_light(fd, &fput_needed); | 103 | arg = fdget(fd); |
| 107 | if (filp) { | 104 | if (arg.file) { |
| 108 | ret = calls->spu_run(filp, unpc, ustatus); | 105 | ret = calls->spu_run(arg.file, unpc, ustatus); |
| 109 | fput_light(filp, fput_needed); | 106 | fdput(arg); |
| 110 | } | 107 | } |
| 111 | 108 | ||
| 112 | spufs_calls_put(calls); | 109 | spufs_calls_put(calls); |
diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index c2c5b078ba80..657e3f233a64 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c | |||
| @@ -106,6 +106,17 @@ static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) | |||
| 106 | return total; | 106 | return total; |
| 107 | } | 107 | } |
| 108 | 108 | ||
| 109 | static int match_context(const void *v, struct file *file, unsigned fd) | ||
| 110 | { | ||
| 111 | struct spu_context *ctx; | ||
| 112 | if (file->f_op != &spufs_context_fops) | ||
| 113 | return 0; | ||
| 114 | ctx = SPUFS_I(file->f_dentry->d_inode)->i_ctx; | ||
| 115 | if (ctx->flags & SPU_CREATE_NOSCHED) | ||
| 116 | return 0; | ||
| 117 | return fd + 1; | ||
| 118 | } | ||
| 119 | |||
| 109 | /* | 120 | /* |
| 110 | * The additional architecture-specific notes for Cell are various | 121 | * The additional architecture-specific notes for Cell are various |
| 111 | * context files in the spu context. | 122 | * context files in the spu context. |
| @@ -115,29 +126,18 @@ static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) | |||
| 115 | * internal functionality to dump them without needing to actually | 126 | * internal functionality to dump them without needing to actually |
| 116 | * open the files. | 127 | * open the files. |
| 117 | */ | 128 | */ |
| 129 | /* | ||
| 130 | * descriptor table is not shared, so files can't change or go away. | ||
| 131 | */ | ||
| 118 | static struct spu_context *coredump_next_context(int *fd) | 132 | static struct spu_context *coredump_next_context(int *fd) |
| 119 | { | 133 | { |
| 120 | struct fdtable *fdt = files_fdtable(current->files); | ||
| 121 | struct file *file; | 134 | struct file *file; |
| 122 | struct spu_context *ctx = NULL; | 135 | int n = iterate_fd(current->files, *fd, match_context, NULL); |
| 123 | 136 | if (!n) | |
| 124 | for (; *fd < fdt->max_fds; (*fd)++) { | 137 | return NULL; |
| 125 | if (!fd_is_open(*fd, fdt)) | 138 | *fd = n - 1; |
| 126 | continue; | 139 | file = fcheck(*fd); |
| 127 | 140 | return SPUFS_I(file->f_dentry->d_inode)->i_ctx; | |
| 128 | file = fcheck(*fd); | ||
| 129 | |||
| 130 | if (!file || file->f_op != &spufs_context_fops) | ||
| 131 | continue; | ||
| 132 | |||
| 133 | ctx = SPUFS_I(file->f_dentry->d_inode)->i_ctx; | ||
| 134 | if (ctx->flags & SPU_CREATE_NOSCHED) | ||
| 135 | continue; | ||
| 136 | |||
| 137 | break; | ||
| 138 | } | ||
| 139 | |||
| 140 | return ctx; | ||
| 141 | } | 141 | } |
| 142 | 142 | ||
| 143 | int spufs_coredump_extra_notes_size(void) | 143 | int spufs_coredump_extra_notes_size(void) |
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index e402a9dd4eda..da3c1a7dcd8e 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c | |||
| @@ -216,7 +216,6 @@ static struct crypto_alg aes_alg = { | |||
| 216 | .cra_blocksize = AES_BLOCK_SIZE, | 216 | .cra_blocksize = AES_BLOCK_SIZE, |
| 217 | .cra_ctxsize = sizeof(struct s390_aes_ctx), | 217 | .cra_ctxsize = sizeof(struct s390_aes_ctx), |
| 218 | .cra_module = THIS_MODULE, | 218 | .cra_module = THIS_MODULE, |
| 219 | .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), | ||
| 220 | .cra_init = fallback_init_cip, | 219 | .cra_init = fallback_init_cip, |
| 221 | .cra_exit = fallback_exit_cip, | 220 | .cra_exit = fallback_exit_cip, |
| 222 | .cra_u = { | 221 | .cra_u = { |
| @@ -398,7 +397,6 @@ static struct crypto_alg ecb_aes_alg = { | |||
| 398 | .cra_ctxsize = sizeof(struct s390_aes_ctx), | 397 | .cra_ctxsize = sizeof(struct s390_aes_ctx), |
| 399 | .cra_type = &crypto_blkcipher_type, | 398 | .cra_type = &crypto_blkcipher_type, |
| 400 | .cra_module = THIS_MODULE, | 399 | .cra_module = THIS_MODULE, |
| 401 | .cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list), | ||
| 402 | .cra_init = fallback_init_blk, | 400 | .cra_init = fallback_init_blk, |
| 403 | .cra_exit = fallback_exit_blk, | 401 | .cra_exit = fallback_exit_blk, |
| 404 | .cra_u = { | 402 | .cra_u = { |
| @@ -508,7 +506,6 @@ static struct crypto_alg cbc_aes_alg = { | |||
| 508 | .cra_ctxsize = sizeof(struct s390_aes_ctx), | 506 | .cra_ctxsize = sizeof(struct s390_aes_ctx), |
| 509 | .cra_type = &crypto_blkcipher_type, | 507 | .cra_type = &crypto_blkcipher_type, |
| 510 | .cra_module = THIS_MODULE, | 508 | .cra_module = THIS_MODULE, |
| 511 | .cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list), | ||
| 512 | .cra_init = fallback_init_blk, | 509 | .cra_init = fallback_init_blk, |
| 513 | .cra_exit = fallback_exit_blk, | 510 | .cra_exit = fallback_exit_blk, |
| 514 | .cra_u = { | 511 | .cra_u = { |
| @@ -710,7 +707,6 @@ static struct crypto_alg xts_aes_alg = { | |||
| 710 | .cra_ctxsize = sizeof(struct s390_xts_ctx), | 707 | .cra_ctxsize = sizeof(struct s390_xts_ctx), |
| 711 | .cra_type = &crypto_blkcipher_type, | 708 | .cra_type = &crypto_blkcipher_type, |
| 712 | .cra_module = THIS_MODULE, | 709 | .cra_module = THIS_MODULE, |
| 713 | .cra_list = LIST_HEAD_INIT(xts_aes_alg.cra_list), | ||
| 714 | .cra_init = xts_fallback_init, | 710 | .cra_init = xts_fallback_init, |
| 715 | .cra_exit = xts_fallback_exit, | 711 | .cra_exit = xts_fallback_exit, |
| 716 | .cra_u = { | 712 | .cra_u = { |
| @@ -832,7 +828,6 @@ static struct crypto_alg ctr_aes_alg = { | |||
| 832 | .cra_ctxsize = sizeof(struct s390_aes_ctx), | 828 | .cra_ctxsize = sizeof(struct s390_aes_ctx), |
| 833 | .cra_type = &crypto_blkcipher_type, | 829 | .cra_type = &crypto_blkcipher_type, |
| 834 | .cra_module = THIS_MODULE, | 830 | .cra_module = THIS_MODULE, |
| 835 | .cra_list = LIST_HEAD_INIT(ctr_aes_alg.cra_list), | ||
| 836 | .cra_u = { | 831 | .cra_u = { |
| 837 | .blkcipher = { | 832 | .blkcipher = { |
| 838 | .min_keysize = AES_MIN_KEY_SIZE, | 833 | .min_keysize = AES_MIN_KEY_SIZE, |
diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 1eaa371ca3ee..b49fb96f4207 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c | |||
| @@ -70,7 +70,6 @@ static struct crypto_alg des_alg = { | |||
| 70 | .cra_blocksize = DES_BLOCK_SIZE, | 70 | .cra_blocksize = DES_BLOCK_SIZE, |
| 71 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 71 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 72 | .cra_module = THIS_MODULE, | 72 | .cra_module = THIS_MODULE, |
| 73 | .cra_list = LIST_HEAD_INIT(des_alg.cra_list), | ||
| 74 | .cra_u = { | 73 | .cra_u = { |
| 75 | .cipher = { | 74 | .cipher = { |
| 76 | .cia_min_keysize = DES_KEY_SIZE, | 75 | .cia_min_keysize = DES_KEY_SIZE, |
| @@ -163,7 +162,6 @@ static struct crypto_alg ecb_des_alg = { | |||
| 163 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 162 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 164 | .cra_type = &crypto_blkcipher_type, | 163 | .cra_type = &crypto_blkcipher_type, |
| 165 | .cra_module = THIS_MODULE, | 164 | .cra_module = THIS_MODULE, |
| 166 | .cra_list = LIST_HEAD_INIT(ecb_des_alg.cra_list), | ||
| 167 | .cra_u = { | 165 | .cra_u = { |
| 168 | .blkcipher = { | 166 | .blkcipher = { |
| 169 | .min_keysize = DES_KEY_SIZE, | 167 | .min_keysize = DES_KEY_SIZE, |
| @@ -206,7 +204,6 @@ static struct crypto_alg cbc_des_alg = { | |||
| 206 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 204 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 207 | .cra_type = &crypto_blkcipher_type, | 205 | .cra_type = &crypto_blkcipher_type, |
| 208 | .cra_module = THIS_MODULE, | 206 | .cra_module = THIS_MODULE, |
| 209 | .cra_list = LIST_HEAD_INIT(cbc_des_alg.cra_list), | ||
| 210 | .cra_u = { | 207 | .cra_u = { |
| 211 | .blkcipher = { | 208 | .blkcipher = { |
| 212 | .min_keysize = DES_KEY_SIZE, | 209 | .min_keysize = DES_KEY_SIZE, |
| @@ -271,7 +268,6 @@ static struct crypto_alg des3_alg = { | |||
| 271 | .cra_blocksize = DES_BLOCK_SIZE, | 268 | .cra_blocksize = DES_BLOCK_SIZE, |
| 272 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 269 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 273 | .cra_module = THIS_MODULE, | 270 | .cra_module = THIS_MODULE, |
| 274 | .cra_list = LIST_HEAD_INIT(des3_alg.cra_list), | ||
| 275 | .cra_u = { | 271 | .cra_u = { |
| 276 | .cipher = { | 272 | .cipher = { |
| 277 | .cia_min_keysize = DES3_KEY_SIZE, | 273 | .cia_min_keysize = DES3_KEY_SIZE, |
| @@ -314,8 +310,6 @@ static struct crypto_alg ecb_des3_alg = { | |||
| 314 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 310 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 315 | .cra_type = &crypto_blkcipher_type, | 311 | .cra_type = &crypto_blkcipher_type, |
| 316 | .cra_module = THIS_MODULE, | 312 | .cra_module = THIS_MODULE, |
| 317 | .cra_list = LIST_HEAD_INIT( | ||
| 318 | ecb_des3_alg.cra_list), | ||
| 319 | .cra_u = { | 313 | .cra_u = { |
| 320 | .blkcipher = { | 314 | .blkcipher = { |
| 321 | .min_keysize = DES3_KEY_SIZE, | 315 | .min_keysize = DES3_KEY_SIZE, |
| @@ -358,8 +352,6 @@ static struct crypto_alg cbc_des3_alg = { | |||
| 358 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 352 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 359 | .cra_type = &crypto_blkcipher_type, | 353 | .cra_type = &crypto_blkcipher_type, |
| 360 | .cra_module = THIS_MODULE, | 354 | .cra_module = THIS_MODULE, |
| 361 | .cra_list = LIST_HEAD_INIT( | ||
| 362 | cbc_des3_alg.cra_list), | ||
| 363 | .cra_u = { | 355 | .cra_u = { |
| 364 | .blkcipher = { | 356 | .blkcipher = { |
| 365 | .min_keysize = DES3_KEY_SIZE, | 357 | .min_keysize = DES3_KEY_SIZE, |
| @@ -452,7 +444,6 @@ static struct crypto_alg ctr_des_alg = { | |||
| 452 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 444 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 453 | .cra_type = &crypto_blkcipher_type, | 445 | .cra_type = &crypto_blkcipher_type, |
| 454 | .cra_module = THIS_MODULE, | 446 | .cra_module = THIS_MODULE, |
| 455 | .cra_list = LIST_HEAD_INIT(ctr_des_alg.cra_list), | ||
| 456 | .cra_u = { | 447 | .cra_u = { |
| 457 | .blkcipher = { | 448 | .blkcipher = { |
| 458 | .min_keysize = DES_KEY_SIZE, | 449 | .min_keysize = DES_KEY_SIZE, |
| @@ -496,7 +487,6 @@ static struct crypto_alg ctr_des3_alg = { | |||
| 496 | .cra_ctxsize = sizeof(struct s390_des_ctx), | 487 | .cra_ctxsize = sizeof(struct s390_des_ctx), |
| 497 | .cra_type = &crypto_blkcipher_type, | 488 | .cra_type = &crypto_blkcipher_type, |
| 498 | .cra_module = THIS_MODULE, | 489 | .cra_module = THIS_MODULE, |
| 499 | .cra_list = LIST_HEAD_INIT(ctr_des3_alg.cra_list), | ||
| 500 | .cra_u = { | 490 | .cra_u = { |
| 501 | .blkcipher = { | 491 | .blkcipher = { |
| 502 | .min_keysize = DES3_KEY_SIZE, | 492 | .min_keysize = DES3_KEY_SIZE, |
diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c index b1bd170f24b1..1ebd3a15cca4 100644 --- a/arch/s390/crypto/ghash_s390.c +++ b/arch/s390/crypto/ghash_s390.c | |||
| @@ -135,7 +135,6 @@ static struct shash_alg ghash_alg = { | |||
| 135 | .cra_blocksize = GHASH_BLOCK_SIZE, | 135 | .cra_blocksize = GHASH_BLOCK_SIZE, |
| 136 | .cra_ctxsize = sizeof(struct ghash_ctx), | 136 | .cra_ctxsize = sizeof(struct ghash_ctx), |
| 137 | .cra_module = THIS_MODULE, | 137 | .cra_module = THIS_MODULE, |
| 138 | .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list), | ||
| 139 | }, | 138 | }, |
| 140 | }; | 139 | }; |
| 141 | 140 | ||
diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c index 124ec1a55cc9..06ea69bd387a 100644 --- a/arch/s390/hypfs/inode.c +++ b/arch/s390/hypfs/inode.c | |||
| @@ -72,8 +72,6 @@ static void hypfs_remove(struct dentry *dentry) | |||
| 72 | struct dentry *parent; | 72 | struct dentry *parent; |
| 73 | 73 | ||
| 74 | parent = dentry->d_parent; | 74 | parent = dentry->d_parent; |
| 75 | if (!parent || !parent->d_inode) | ||
| 76 | return; | ||
| 77 | mutex_lock(&parent->d_inode->i_mutex); | 75 | mutex_lock(&parent->d_inode->i_mutex); |
| 78 | if (hypfs_positive(dentry)) { | 76 | if (hypfs_positive(dentry)) { |
| 79 | if (S_ISDIR(dentry->d_inode->i_mode)) | 77 | if (S_ISDIR(dentry->d_inode->i_mode)) |
diff --git a/arch/s390/include/asm/compat.h b/arch/s390/include/asm/compat.h index 234f1d859cea..a34a9d612fc0 100644 --- a/arch/s390/include/asm/compat.h +++ b/arch/s390/include/asm/compat.h | |||
| @@ -65,6 +65,7 @@ typedef s64 compat_s64; | |||
| 65 | typedef u32 compat_uint_t; | 65 | typedef u32 compat_uint_t; |
| 66 | typedef u32 compat_ulong_t; | 66 | typedef u32 compat_ulong_t; |
| 67 | typedef u64 compat_u64; | 67 | typedef u64 compat_u64; |
| 68 | typedef u32 compat_uptr_t; | ||
| 68 | 69 | ||
| 69 | struct compat_timespec { | 70 | struct compat_timespec { |
| 70 | compat_time_t tv_sec; | 71 | compat_time_t tv_sec; |
| @@ -144,6 +145,79 @@ typedef u32 compat_old_sigset_t; /* at least 32 bits */ | |||
| 144 | 145 | ||
| 145 | typedef u32 compat_sigset_word; | 146 | typedef u32 compat_sigset_word; |
| 146 | 147 | ||
| 148 | typedef union compat_sigval { | ||
| 149 | compat_int_t sival_int; | ||
| 150 | compat_uptr_t sival_ptr; | ||
| 151 | } compat_sigval_t; | ||
| 152 | |||
| 153 | typedef struct compat_siginfo { | ||
| 154 | int si_signo; | ||
| 155 | int si_errno; | ||
| 156 | int si_code; | ||
| 157 | |||
| 158 | union { | ||
| 159 | int _pad[128/sizeof(int) - 3]; | ||
| 160 | |||
| 161 | /* kill() */ | ||
| 162 | struct { | ||
| 163 | pid_t _pid; /* sender's pid */ | ||
| 164 | uid_t _uid; /* sender's uid */ | ||
| 165 | } _kill; | ||
| 166 | |||
| 167 | /* POSIX.1b timers */ | ||
| 168 | struct { | ||
| 169 | compat_timer_t _tid; /* timer id */ | ||
| 170 | int _overrun; /* overrun count */ | ||
| 171 | compat_sigval_t _sigval; /* same as below */ | ||
| 172 | int _sys_private; /* not to be passed to user */ | ||
| 173 | } _timer; | ||
| 174 | |||
| 175 | /* POSIX.1b signals */ | ||
| 176 | struct { | ||
| 177 | pid_t _pid; /* sender's pid */ | ||
| 178 | uid_t _uid; /* sender's uid */ | ||
| 179 | compat_sigval_t _sigval; | ||
| 180 | } _rt; | ||
| 181 | |||
| 182 | /* SIGCHLD */ | ||
| 183 | struct { | ||
| 184 | pid_t _pid; /* which child */ | ||
| 185 | uid_t _uid; /* sender's uid */ | ||
| 186 | int _status;/* exit code */ | ||
| 187 | compat_clock_t _utime; | ||
| 188 | compat_clock_t _stime; | ||
| 189 | } _sigchld; | ||
| 190 | |||
| 191 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 192 | struct { | ||
| 193 | __u32 _addr; /* faulting insn/memory ref. - pointer */ | ||
| 194 | } _sigfault; | ||
| 195 | |||
| 196 | /* SIGPOLL */ | ||
| 197 | struct { | ||
| 198 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 199 | int _fd; | ||
| 200 | } _sigpoll; | ||
| 201 | } _sifields; | ||
| 202 | } compat_siginfo_t; | ||
| 203 | |||
| 204 | /* | ||
| 205 | * How these fields are to be accessed. | ||
| 206 | */ | ||
| 207 | #define si_pid _sifields._kill._pid | ||
| 208 | #define si_uid _sifields._kill._uid | ||
| 209 | #define si_status _sifields._sigchld._status | ||
| 210 | #define si_utime _sifields._sigchld._utime | ||
| 211 | #define si_stime _sifields._sigchld._stime | ||
| 212 | #define si_value _sifields._rt._sigval | ||
| 213 | #define si_int _sifields._rt._sigval.sival_int | ||
| 214 | #define si_ptr _sifields._rt._sigval.sival_ptr | ||
| 215 | #define si_addr _sifields._sigfault._addr | ||
| 216 | #define si_band _sifields._sigpoll._band | ||
| 217 | #define si_fd _sifields._sigpoll._fd | ||
| 218 | #define si_tid _sifields._timer._tid | ||
| 219 | #define si_overrun _sifields._timer._overrun | ||
| 220 | |||
| 147 | #define COMPAT_OFF_T_MAX 0x7fffffff | 221 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 148 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 222 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 149 | 223 | ||
| @@ -153,7 +227,6 @@ typedef u32 compat_sigset_word; | |||
| 153 | * as pointers because the syscall entry code will have | 227 | * as pointers because the syscall entry code will have |
| 154 | * appropriately converted them already. | 228 | * appropriately converted them already. |
| 155 | */ | 229 | */ |
| 156 | typedef u32 compat_uptr_t; | ||
| 157 | 230 | ||
| 158 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 231 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 159 | { | 232 | { |
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index f3e0aabfc6bc..56831dfa9198 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h | |||
| @@ -159,6 +159,7 @@ extern unsigned long thread_saved_pc(struct task_struct *t); | |||
| 159 | 159 | ||
| 160 | extern void show_code(struct pt_regs *regs); | 160 | extern void show_code(struct pt_regs *regs); |
| 161 | extern void print_fn_code(unsigned char *code, unsigned long len); | 161 | extern void print_fn_code(unsigned char *code, unsigned long len); |
| 162 | extern int insn_to_mnemonic(unsigned char *instruction, char buf[8]); | ||
| 162 | 163 | ||
| 163 | unsigned long get_wchan(struct task_struct *p); | 164 | unsigned long get_wchan(struct task_struct *p); |
| 164 | #define task_pt_regs(tsk) ((struct pt_regs *) \ | 165 | #define task_pt_regs(tsk) ((struct pt_regs *) \ |
diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/s390/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/s390/kernel/compat_linux.h b/arch/s390/kernel/compat_linux.h index 9635d759c2b9..90887bd98cf0 100644 --- a/arch/s390/kernel/compat_linux.h +++ b/arch/s390/kernel/compat_linux.h | |||
| @@ -23,74 +23,6 @@ struct old_sigaction32 { | |||
| 23 | __u32 sa_flags; | 23 | __u32 sa_flags; |
| 24 | __u32 sa_restorer; /* Another 32 bit pointer */ | 24 | __u32 sa_restorer; /* Another 32 bit pointer */ |
| 25 | }; | 25 | }; |
| 26 | |||
| 27 | typedef struct compat_siginfo { | ||
| 28 | int si_signo; | ||
| 29 | int si_errno; | ||
| 30 | int si_code; | ||
| 31 | |||
| 32 | union { | ||
| 33 | int _pad[((128/sizeof(int)) - 3)]; | ||
| 34 | |||
| 35 | /* kill() */ | ||
| 36 | struct { | ||
| 37 | pid_t _pid; /* sender's pid */ | ||
| 38 | uid_t _uid; /* sender's uid */ | ||
| 39 | } _kill; | ||
| 40 | |||
| 41 | /* POSIX.1b timers */ | ||
| 42 | struct { | ||
| 43 | compat_timer_t _tid; /* timer id */ | ||
| 44 | int _overrun; /* overrun count */ | ||
| 45 | compat_sigval_t _sigval; /* same as below */ | ||
| 46 | int _sys_private; /* not to be passed to user */ | ||
| 47 | } _timer; | ||
| 48 | |||
| 49 | /* POSIX.1b signals */ | ||
| 50 | struct { | ||
| 51 | pid_t _pid; /* sender's pid */ | ||
| 52 | uid_t _uid; /* sender's uid */ | ||
| 53 | compat_sigval_t _sigval; | ||
| 54 | } _rt; | ||
| 55 | |||
| 56 | /* SIGCHLD */ | ||
| 57 | struct { | ||
| 58 | pid_t _pid; /* which child */ | ||
| 59 | uid_t _uid; /* sender's uid */ | ||
| 60 | int _status;/* exit code */ | ||
| 61 | compat_clock_t _utime; | ||
| 62 | compat_clock_t _stime; | ||
| 63 | } _sigchld; | ||
| 64 | |||
| 65 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 66 | struct { | ||
| 67 | __u32 _addr; /* faulting insn/memory ref. - pointer */ | ||
| 68 | } _sigfault; | ||
| 69 | |||
| 70 | /* SIGPOLL */ | ||
| 71 | struct { | ||
| 72 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 73 | int _fd; | ||
| 74 | } _sigpoll; | ||
| 75 | } _sifields; | ||
| 76 | } compat_siginfo_t; | ||
| 77 | |||
| 78 | /* | ||
| 79 | * How these fields are to be accessed. | ||
| 80 | */ | ||
| 81 | #define si_pid _sifields._kill._pid | ||
| 82 | #define si_uid _sifields._kill._uid | ||
| 83 | #define si_status _sifields._sigchld._status | ||
| 84 | #define si_utime _sifields._sigchld._utime | ||
| 85 | #define si_stime _sifields._sigchld._stime | ||
| 86 | #define si_value _sifields._rt._sigval | ||
| 87 | #define si_int _sifields._rt._sigval.sival_int | ||
| 88 | #define si_ptr _sifields._rt._sigval.sival_ptr | ||
| 89 | #define si_addr _sifields._sigfault._addr | ||
| 90 | #define si_band _sifields._sigpoll._band | ||
| 91 | #define si_fd _sifields._sigpoll._fd | ||
| 92 | #define si_tid _sifields._timer._tid | ||
| 93 | #define si_overrun _sifields._timer._overrun | ||
| 94 | 26 | ||
| 95 | /* asm/sigcontext.h */ | 27 | /* asm/sigcontext.h */ |
| 96 | typedef union | 28 | typedef union |
diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index cc84a24c023f..f00286bd2ef9 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c | |||
| @@ -1501,6 +1501,33 @@ static struct insn *find_insn(unsigned char *code) | |||
| 1501 | return NULL; | 1501 | return NULL; |
| 1502 | } | 1502 | } |
| 1503 | 1503 | ||
| 1504 | /** | ||
| 1505 | * insn_to_mnemonic - decode an s390 instruction | ||
| 1506 | * @instruction: instruction to decode | ||
| 1507 | * @buf: buffer to fill with mnemonic | ||
| 1508 | * | ||
| 1509 | * Decode the instruction at @instruction and store the corresponding | ||
| 1510 | * mnemonic into @buf. | ||
| 1511 | * @buf is left unchanged if the instruction could not be decoded. | ||
| 1512 | * Returns: | ||
| 1513 | * %0 on success, %-ENOENT if the instruction was not found. | ||
| 1514 | */ | ||
| 1515 | int insn_to_mnemonic(unsigned char *instruction, char buf[8]) | ||
| 1516 | { | ||
| 1517 | struct insn *insn; | ||
| 1518 | |||
| 1519 | insn = find_insn(instruction); | ||
| 1520 | if (!insn) | ||
| 1521 | return -ENOENT; | ||
| 1522 | if (insn->name[0] == '\0') | ||
| 1523 | snprintf(buf, sizeof(buf), "%s", | ||
| 1524 | long_insn_name[(int) insn->name[1]]); | ||
| 1525 | else | ||
| 1526 | snprintf(buf, sizeof(buf), "%.5s", insn->name); | ||
| 1527 | return 0; | ||
| 1528 | } | ||
| 1529 | EXPORT_SYMBOL_GPL(insn_to_mnemonic); | ||
| 1530 | |||
| 1504 | static int print_insn(char *buffer, unsigned char *code, unsigned long addr) | 1531 | static int print_insn(char *buffer, unsigned char *code, unsigned long addr) |
| 1505 | { | 1532 | { |
| 1506 | struct insn *insn; | 1533 | struct insn *insn; |
diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 9b04a32e5695..b58dd869cb32 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig | |||
| @@ -21,6 +21,7 @@ config KVM | |||
| 21 | depends on HAVE_KVM && EXPERIMENTAL | 21 | depends on HAVE_KVM && EXPERIMENTAL |
| 22 | select PREEMPT_NOTIFIERS | 22 | select PREEMPT_NOTIFIERS |
| 23 | select ANON_INODES | 23 | select ANON_INODES |
| 24 | select HAVE_KVM_CPU_RELAX_INTERCEPT | ||
| 24 | ---help--- | 25 | ---help--- |
| 25 | Support hosting paravirtualized guest machines using the SIE | 26 | Support hosting paravirtualized guest machines using the SIE |
| 26 | virtualization capability on the mainframe. This should work | 27 | virtualization capability on the mainframe. This should work |
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index c88bb7793390..a390687feb13 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c | |||
| @@ -14,6 +14,8 @@ | |||
| 14 | #include <linux/kvm.h> | 14 | #include <linux/kvm.h> |
| 15 | #include <linux/kvm_host.h> | 15 | #include <linux/kvm_host.h> |
| 16 | #include "kvm-s390.h" | 16 | #include "kvm-s390.h" |
| 17 | #include "trace.h" | ||
| 18 | #include "trace-s390.h" | ||
| 17 | 19 | ||
| 18 | static int diag_release_pages(struct kvm_vcpu *vcpu) | 20 | static int diag_release_pages(struct kvm_vcpu *vcpu) |
| 19 | { | 21 | { |
| @@ -98,6 +100,7 @@ static int __diag_ipl_functions(struct kvm_vcpu *vcpu) | |||
| 98 | vcpu->run->exit_reason = KVM_EXIT_S390_RESET; | 100 | vcpu->run->exit_reason = KVM_EXIT_S390_RESET; |
| 99 | VCPU_EVENT(vcpu, 3, "requesting userspace resets %llx", | 101 | VCPU_EVENT(vcpu, 3, "requesting userspace resets %llx", |
| 100 | vcpu->run->s390_reset_flags); | 102 | vcpu->run->s390_reset_flags); |
| 103 | trace_kvm_s390_request_resets(vcpu->run->s390_reset_flags); | ||
| 101 | return -EREMOTE; | 104 | return -EREMOTE; |
| 102 | } | 105 | } |
| 103 | 106 | ||
| @@ -105,6 +108,7 @@ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu) | |||
| 105 | { | 108 | { |
| 106 | int code = (vcpu->arch.sie_block->ipb & 0xfff0000) >> 16; | 109 | int code = (vcpu->arch.sie_block->ipb & 0xfff0000) >> 16; |
| 107 | 110 | ||
| 111 | trace_kvm_s390_handle_diag(vcpu, code); | ||
| 108 | switch (code) { | 112 | switch (code) { |
| 109 | case 0x10: | 113 | case 0x10: |
| 110 | return diag_release_pages(vcpu); | 114 | return diag_release_pages(vcpu); |
diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index adae539f12e2..22798ec33fd1 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c | |||
| @@ -19,6 +19,8 @@ | |||
| 19 | 19 | ||
| 20 | #include "kvm-s390.h" | 20 | #include "kvm-s390.h" |
| 21 | #include "gaccess.h" | 21 | #include "gaccess.h" |
| 22 | #include "trace.h" | ||
| 23 | #include "trace-s390.h" | ||
| 22 | 24 | ||
| 23 | static int handle_lctlg(struct kvm_vcpu *vcpu) | 25 | static int handle_lctlg(struct kvm_vcpu *vcpu) |
| 24 | { | 26 | { |
| @@ -45,6 +47,7 @@ static int handle_lctlg(struct kvm_vcpu *vcpu) | |||
| 45 | 47 | ||
| 46 | VCPU_EVENT(vcpu, 5, "lctlg r1:%x, r3:%x,b2:%x,d2:%x", reg1, reg3, base2, | 48 | VCPU_EVENT(vcpu, 5, "lctlg r1:%x, r3:%x,b2:%x,d2:%x", reg1, reg3, base2, |
| 47 | disp2); | 49 | disp2); |
| 50 | trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, useraddr); | ||
| 48 | 51 | ||
| 49 | do { | 52 | do { |
| 50 | rc = get_guest_u64(vcpu, useraddr, | 53 | rc = get_guest_u64(vcpu, useraddr, |
| @@ -82,6 +85,7 @@ static int handle_lctl(struct kvm_vcpu *vcpu) | |||
| 82 | 85 | ||
| 83 | VCPU_EVENT(vcpu, 5, "lctl r1:%x, r3:%x,b2:%x,d2:%x", reg1, reg3, base2, | 86 | VCPU_EVENT(vcpu, 5, "lctl r1:%x, r3:%x,b2:%x,d2:%x", reg1, reg3, base2, |
| 84 | disp2); | 87 | disp2); |
| 88 | trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, useraddr); | ||
| 85 | 89 | ||
| 86 | reg = reg1; | 90 | reg = reg1; |
| 87 | do { | 91 | do { |
| @@ -135,6 +139,8 @@ static int handle_stop(struct kvm_vcpu *vcpu) | |||
| 135 | vcpu->stat.exit_stop_request++; | 139 | vcpu->stat.exit_stop_request++; |
| 136 | spin_lock_bh(&vcpu->arch.local_int.lock); | 140 | spin_lock_bh(&vcpu->arch.local_int.lock); |
| 137 | 141 | ||
| 142 | trace_kvm_s390_stop_request(vcpu->arch.local_int.action_bits); | ||
| 143 | |||
| 138 | if (vcpu->arch.local_int.action_bits & ACTION_RELOADVCPU_ON_STOP) { | 144 | if (vcpu->arch.local_int.action_bits & ACTION_RELOADVCPU_ON_STOP) { |
| 139 | vcpu->arch.local_int.action_bits &= ~ACTION_RELOADVCPU_ON_STOP; | 145 | vcpu->arch.local_int.action_bits &= ~ACTION_RELOADVCPU_ON_STOP; |
| 140 | rc = SIE_INTERCEPT_RERUNVCPU; | 146 | rc = SIE_INTERCEPT_RERUNVCPU; |
| @@ -171,6 +177,7 @@ static int handle_validity(struct kvm_vcpu *vcpu) | |||
| 171 | int rc; | 177 | int rc; |
| 172 | 178 | ||
| 173 | vcpu->stat.exit_validity++; | 179 | vcpu->stat.exit_validity++; |
| 180 | trace_kvm_s390_intercept_validity(vcpu, viwhy); | ||
| 174 | if (viwhy == 0x37) { | 181 | if (viwhy == 0x37) { |
| 175 | vmaddr = gmap_fault(vcpu->arch.sie_block->prefix, | 182 | vmaddr = gmap_fault(vcpu->arch.sie_block->prefix, |
| 176 | vcpu->arch.gmap); | 183 | vcpu->arch.gmap); |
| @@ -213,6 +220,9 @@ static int handle_instruction(struct kvm_vcpu *vcpu) | |||
| 213 | intercept_handler_t handler; | 220 | intercept_handler_t handler; |
| 214 | 221 | ||
| 215 | vcpu->stat.exit_instruction++; | 222 | vcpu->stat.exit_instruction++; |
| 223 | trace_kvm_s390_intercept_instruction(vcpu, | ||
| 224 | vcpu->arch.sie_block->ipa, | ||
| 225 | vcpu->arch.sie_block->ipb); | ||
| 216 | handler = instruction_handlers[vcpu->arch.sie_block->ipa >> 8]; | 226 | handler = instruction_handlers[vcpu->arch.sie_block->ipa >> 8]; |
| 217 | if (handler) | 227 | if (handler) |
| 218 | return handler(vcpu); | 228 | return handler(vcpu); |
| @@ -222,6 +232,7 @@ static int handle_instruction(struct kvm_vcpu *vcpu) | |||
| 222 | static int handle_prog(struct kvm_vcpu *vcpu) | 232 | static int handle_prog(struct kvm_vcpu *vcpu) |
| 223 | { | 233 | { |
| 224 | vcpu->stat.exit_program_interruption++; | 234 | vcpu->stat.exit_program_interruption++; |
| 235 | trace_kvm_s390_intercept_prog(vcpu, vcpu->arch.sie_block->iprcc); | ||
| 225 | return kvm_s390_inject_program_int(vcpu, vcpu->arch.sie_block->iprcc); | 236 | return kvm_s390_inject_program_int(vcpu, vcpu->arch.sie_block->iprcc); |
| 226 | } | 237 | } |
| 227 | 238 | ||
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index b7bc1aac8ed2..ff1e2f8ef94a 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c | |||
| @@ -19,6 +19,7 @@ | |||
| 19 | #include <asm/uaccess.h> | 19 | #include <asm/uaccess.h> |
| 20 | #include "kvm-s390.h" | 20 | #include "kvm-s390.h" |
| 21 | #include "gaccess.h" | 21 | #include "gaccess.h" |
| 22 | #include "trace-s390.h" | ||
| 22 | 23 | ||
| 23 | static int psw_extint_disabled(struct kvm_vcpu *vcpu) | 24 | static int psw_extint_disabled(struct kvm_vcpu *vcpu) |
| 24 | { | 25 | { |
| @@ -130,6 +131,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 130 | case KVM_S390_INT_EMERGENCY: | 131 | case KVM_S390_INT_EMERGENCY: |
| 131 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp emerg"); | 132 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp emerg"); |
| 132 | vcpu->stat.deliver_emergency_signal++; | 133 | vcpu->stat.deliver_emergency_signal++; |
| 134 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 135 | inti->emerg.code, 0); | ||
| 133 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1201); | 136 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1201); |
| 134 | if (rc == -EFAULT) | 137 | if (rc == -EFAULT) |
| 135 | exception = 1; | 138 | exception = 1; |
| @@ -152,6 +155,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 152 | case KVM_S390_INT_EXTERNAL_CALL: | 155 | case KVM_S390_INT_EXTERNAL_CALL: |
| 153 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp ext call"); | 156 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp ext call"); |
| 154 | vcpu->stat.deliver_external_call++; | 157 | vcpu->stat.deliver_external_call++; |
| 158 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 159 | inti->extcall.code, 0); | ||
| 155 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1202); | 160 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1202); |
| 156 | if (rc == -EFAULT) | 161 | if (rc == -EFAULT) |
| 157 | exception = 1; | 162 | exception = 1; |
| @@ -175,6 +180,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 175 | VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x", | 180 | VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x", |
| 176 | inti->ext.ext_params); | 181 | inti->ext.ext_params); |
| 177 | vcpu->stat.deliver_service_signal++; | 182 | vcpu->stat.deliver_service_signal++; |
| 183 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 184 | inti->ext.ext_params, 0); | ||
| 178 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2401); | 185 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2401); |
| 179 | if (rc == -EFAULT) | 186 | if (rc == -EFAULT) |
| 180 | exception = 1; | 187 | exception = 1; |
| @@ -198,6 +205,9 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 198 | VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx", | 205 | VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx", |
| 199 | inti->ext.ext_params, inti->ext.ext_params2); | 206 | inti->ext.ext_params, inti->ext.ext_params2); |
| 200 | vcpu->stat.deliver_virtio_interrupt++; | 207 | vcpu->stat.deliver_virtio_interrupt++; |
| 208 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 209 | inti->ext.ext_params, | ||
| 210 | inti->ext.ext_params2); | ||
| 201 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2603); | 211 | rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2603); |
| 202 | if (rc == -EFAULT) | 212 | if (rc == -EFAULT) |
| 203 | exception = 1; | 213 | exception = 1; |
| @@ -229,6 +239,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 229 | case KVM_S390_SIGP_STOP: | 239 | case KVM_S390_SIGP_STOP: |
| 230 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu stop"); | 240 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu stop"); |
| 231 | vcpu->stat.deliver_stop_signal++; | 241 | vcpu->stat.deliver_stop_signal++; |
| 242 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 243 | 0, 0); | ||
| 232 | __set_intercept_indicator(vcpu, inti); | 244 | __set_intercept_indicator(vcpu, inti); |
| 233 | break; | 245 | break; |
| 234 | 246 | ||
| @@ -236,12 +248,16 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 236 | VCPU_EVENT(vcpu, 4, "interrupt: set prefix to %x", | 248 | VCPU_EVENT(vcpu, 4, "interrupt: set prefix to %x", |
| 237 | inti->prefix.address); | 249 | inti->prefix.address); |
| 238 | vcpu->stat.deliver_prefix_signal++; | 250 | vcpu->stat.deliver_prefix_signal++; |
| 251 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 252 | inti->prefix.address, 0); | ||
| 239 | kvm_s390_set_prefix(vcpu, inti->prefix.address); | 253 | kvm_s390_set_prefix(vcpu, inti->prefix.address); |
| 240 | break; | 254 | break; |
| 241 | 255 | ||
| 242 | case KVM_S390_RESTART: | 256 | case KVM_S390_RESTART: |
| 243 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu restart"); | 257 | VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu restart"); |
| 244 | vcpu->stat.deliver_restart_signal++; | 258 | vcpu->stat.deliver_restart_signal++; |
| 259 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 260 | 0, 0); | ||
| 245 | rc = copy_to_guest(vcpu, offsetof(struct _lowcore, | 261 | rc = copy_to_guest(vcpu, offsetof(struct _lowcore, |
| 246 | restart_old_psw), &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); | 262 | restart_old_psw), &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); |
| 247 | if (rc == -EFAULT) | 263 | if (rc == -EFAULT) |
| @@ -259,6 +275,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu, | |||
| 259 | inti->pgm.code, | 275 | inti->pgm.code, |
| 260 | table[vcpu->arch.sie_block->ipa >> 14]); | 276 | table[vcpu->arch.sie_block->ipa >> 14]); |
| 261 | vcpu->stat.deliver_program_int++; | 277 | vcpu->stat.deliver_program_int++; |
| 278 | trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, | ||
| 279 | inti->pgm.code, 0); | ||
| 262 | rc = put_guest_u16(vcpu, __LC_PGM_INT_CODE, inti->pgm.code); | 280 | rc = put_guest_u16(vcpu, __LC_PGM_INT_CODE, inti->pgm.code); |
| 263 | if (rc == -EFAULT) | 281 | if (rc == -EFAULT) |
| 264 | exception = 1; | 282 | exception = 1; |
| @@ -405,9 +423,7 @@ no_timer: | |||
| 405 | set_current_state(TASK_INTERRUPTIBLE); | 423 | set_current_state(TASK_INTERRUPTIBLE); |
| 406 | spin_unlock_bh(&vcpu->arch.local_int.lock); | 424 | spin_unlock_bh(&vcpu->arch.local_int.lock); |
| 407 | spin_unlock(&vcpu->arch.local_int.float_int->lock); | 425 | spin_unlock(&vcpu->arch.local_int.float_int->lock); |
| 408 | vcpu_put(vcpu); | ||
| 409 | schedule(); | 426 | schedule(); |
| 410 | vcpu_load(vcpu); | ||
| 411 | spin_lock(&vcpu->arch.local_int.float_int->lock); | 427 | spin_lock(&vcpu->arch.local_int.float_int->lock); |
| 412 | spin_lock_bh(&vcpu->arch.local_int.lock); | 428 | spin_lock_bh(&vcpu->arch.local_int.lock); |
| 413 | } | 429 | } |
| @@ -515,6 +531,7 @@ int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code) | |||
| 515 | inti->pgm.code = code; | 531 | inti->pgm.code = code; |
| 516 | 532 | ||
| 517 | VCPU_EVENT(vcpu, 3, "inject: program check %d (from kernel)", code); | 533 | VCPU_EVENT(vcpu, 3, "inject: program check %d (from kernel)", code); |
| 534 | trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, inti->type, code, 0, 1); | ||
| 518 | spin_lock_bh(&li->lock); | 535 | spin_lock_bh(&li->lock); |
| 519 | list_add(&inti->list, &li->list); | 536 | list_add(&inti->list, &li->list); |
| 520 | atomic_set(&li->active, 1); | 537 | atomic_set(&li->active, 1); |
| @@ -556,6 +573,8 @@ int kvm_s390_inject_vm(struct kvm *kvm, | |||
| 556 | kfree(inti); | 573 | kfree(inti); |
| 557 | return -EINVAL; | 574 | return -EINVAL; |
| 558 | } | 575 | } |
| 576 | trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, | ||
| 577 | 2); | ||
| 559 | 578 | ||
| 560 | mutex_lock(&kvm->lock); | 579 | mutex_lock(&kvm->lock); |
| 561 | fi = &kvm->arch.float_int; | 580 | fi = &kvm->arch.float_int; |
| @@ -621,6 +640,8 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, | |||
| 621 | kfree(inti); | 640 | kfree(inti); |
| 622 | return -EINVAL; | 641 | return -EINVAL; |
| 623 | } | 642 | } |
| 643 | trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, s390int->type, s390int->parm, | ||
| 644 | s390int->parm64, 2); | ||
| 624 | 645 | ||
| 625 | mutex_lock(&vcpu->kvm->lock); | 646 | mutex_lock(&vcpu->kvm->lock); |
| 626 | li = &vcpu->arch.local_int; | 647 | li = &vcpu->arch.local_int; |
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index d470ccbfabae..ecced9d18986 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c | |||
| @@ -32,6 +32,10 @@ | |||
| 32 | #include "kvm-s390.h" | 32 | #include "kvm-s390.h" |
| 33 | #include "gaccess.h" | 33 | #include "gaccess.h" |
| 34 | 34 | ||
| 35 | #define CREATE_TRACE_POINTS | ||
| 36 | #include "trace.h" | ||
| 37 | #include "trace-s390.h" | ||
| 38 | |||
| 35 | #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU | 39 | #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU |
| 36 | 40 | ||
| 37 | struct kvm_stats_debugfs_item debugfs_entries[] = { | 41 | struct kvm_stats_debugfs_item debugfs_entries[] = { |
| @@ -242,6 +246,7 @@ out_err: | |||
| 242 | void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) | 246 | void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) |
| 243 | { | 247 | { |
| 244 | VCPU_EVENT(vcpu, 3, "%s", "free cpu"); | 248 | VCPU_EVENT(vcpu, 3, "%s", "free cpu"); |
| 249 | trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); | ||
| 245 | if (!kvm_is_ucontrol(vcpu->kvm)) { | 250 | if (!kvm_is_ucontrol(vcpu->kvm)) { |
| 246 | clear_bit(63 - vcpu->vcpu_id, | 251 | clear_bit(63 - vcpu->vcpu_id, |
| 247 | (unsigned long *) &vcpu->kvm->arch.sca->mcn); | 252 | (unsigned long *) &vcpu->kvm->arch.sca->mcn); |
| @@ -417,6 +422,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, | |||
| 417 | goto out_free_sie_block; | 422 | goto out_free_sie_block; |
| 418 | VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, | 423 | VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu, |
| 419 | vcpu->arch.sie_block); | 424 | vcpu->arch.sie_block); |
| 425 | trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); | ||
| 420 | 426 | ||
| 421 | return vcpu; | 427 | return vcpu; |
| 422 | out_free_sie_block: | 428 | out_free_sie_block: |
| @@ -607,18 +613,22 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) | |||
| 607 | local_irq_enable(); | 613 | local_irq_enable(); |
| 608 | VCPU_EVENT(vcpu, 6, "entering sie flags %x", | 614 | VCPU_EVENT(vcpu, 6, "entering sie flags %x", |
| 609 | atomic_read(&vcpu->arch.sie_block->cpuflags)); | 615 | atomic_read(&vcpu->arch.sie_block->cpuflags)); |
| 616 | trace_kvm_s390_sie_enter(vcpu, | ||
| 617 | atomic_read(&vcpu->arch.sie_block->cpuflags)); | ||
| 610 | rc = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); | 618 | rc = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); |
| 611 | if (rc) { | 619 | if (rc) { |
| 612 | if (kvm_is_ucontrol(vcpu->kvm)) { | 620 | if (kvm_is_ucontrol(vcpu->kvm)) { |
| 613 | rc = SIE_INTERCEPT_UCONTROL; | 621 | rc = SIE_INTERCEPT_UCONTROL; |
| 614 | } else { | 622 | } else { |
| 615 | VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); | 623 | VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); |
| 624 | trace_kvm_s390_sie_fault(vcpu); | ||
| 616 | kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); | 625 | kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); |
| 617 | rc = 0; | 626 | rc = 0; |
| 618 | } | 627 | } |
| 619 | } | 628 | } |
| 620 | VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", | 629 | VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", |
| 621 | vcpu->arch.sie_block->icptcode); | 630 | vcpu->arch.sie_block->icptcode); |
| 631 | trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); | ||
| 622 | local_irq_disable(); | 632 | local_irq_disable(); |
| 623 | kvm_guest_exit(); | 633 | kvm_guest_exit(); |
| 624 | local_irq_enable(); | 634 | local_irq_enable(); |
| @@ -959,7 +969,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, | |||
| 959 | return; | 969 | return; |
| 960 | } | 970 | } |
| 961 | 971 | ||
| 962 | void kvm_arch_flush_shadow(struct kvm *kvm) | 972 | void kvm_arch_flush_shadow_all(struct kvm *kvm) |
| 973 | { | ||
| 974 | } | ||
| 975 | |||
| 976 | void kvm_arch_flush_shadow_memslot(struct kvm *kvm, | ||
| 977 | struct kvm_memory_slot *slot) | ||
| 963 | { | 978 | { |
| 964 | } | 979 | } |
| 965 | 980 | ||
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index 310be61bead7..d768906f15c8 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c | |||
| @@ -20,6 +20,7 @@ | |||
| 20 | #include <asm/sysinfo.h> | 20 | #include <asm/sysinfo.h> |
| 21 | #include "gaccess.h" | 21 | #include "gaccess.h" |
| 22 | #include "kvm-s390.h" | 22 | #include "kvm-s390.h" |
| 23 | #include "trace.h" | ||
| 23 | 24 | ||
| 24 | static int handle_set_prefix(struct kvm_vcpu *vcpu) | 25 | static int handle_set_prefix(struct kvm_vcpu *vcpu) |
| 25 | { | 26 | { |
| @@ -59,6 +60,7 @@ static int handle_set_prefix(struct kvm_vcpu *vcpu) | |||
| 59 | kvm_s390_set_prefix(vcpu, address); | 60 | kvm_s390_set_prefix(vcpu, address); |
| 60 | 61 | ||
| 61 | VCPU_EVENT(vcpu, 5, "setting prefix to %x", address); | 62 | VCPU_EVENT(vcpu, 5, "setting prefix to %x", address); |
| 63 | trace_kvm_s390_handle_prefix(vcpu, 1, address); | ||
| 62 | out: | 64 | out: |
| 63 | return 0; | 65 | return 0; |
| 64 | } | 66 | } |
| @@ -91,6 +93,7 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu) | |||
| 91 | } | 93 | } |
| 92 | 94 | ||
| 93 | VCPU_EVENT(vcpu, 5, "storing prefix to %x", address); | 95 | VCPU_EVENT(vcpu, 5, "storing prefix to %x", address); |
| 96 | trace_kvm_s390_handle_prefix(vcpu, 0, address); | ||
| 94 | out: | 97 | out: |
| 95 | return 0; | 98 | return 0; |
| 96 | } | 99 | } |
| @@ -119,6 +122,7 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu) | |||
| 119 | } | 122 | } |
| 120 | 123 | ||
| 121 | VCPU_EVENT(vcpu, 5, "storing cpu address to %llx", useraddr); | 124 | VCPU_EVENT(vcpu, 5, "storing cpu address to %llx", useraddr); |
| 125 | trace_kvm_s390_handle_stap(vcpu, useraddr); | ||
| 122 | out: | 126 | out: |
| 123 | return 0; | 127 | return 0; |
| 124 | } | 128 | } |
| @@ -164,9 +168,11 @@ static int handle_stfl(struct kvm_vcpu *vcpu) | |||
| 164 | &facility_list, sizeof(facility_list)); | 168 | &facility_list, sizeof(facility_list)); |
| 165 | if (rc == -EFAULT) | 169 | if (rc == -EFAULT) |
| 166 | kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); | 170 | kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); |
| 167 | else | 171 | else { |
| 168 | VCPU_EVENT(vcpu, 5, "store facility list value %x", | 172 | VCPU_EVENT(vcpu, 5, "store facility list value %x", |
| 169 | facility_list); | 173 | facility_list); |
| 174 | trace_kvm_s390_handle_stfl(vcpu, facility_list); | ||
| 175 | } | ||
| 170 | return 0; | 176 | return 0; |
| 171 | } | 177 | } |
| 172 | 178 | ||
| @@ -278,6 +284,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu) | |||
| 278 | kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); | 284 | kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); |
| 279 | goto out_mem; | 285 | goto out_mem; |
| 280 | } | 286 | } |
| 287 | trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); | ||
| 281 | free_page(mem); | 288 | free_page(mem); |
| 282 | vcpu->arch.sie_block->gpsw.mask &= ~(3ul << 44); | 289 | vcpu->arch.sie_block->gpsw.mask &= ~(3ul << 44); |
| 283 | vcpu->run->s.regs.gprs[0] = 0; | 290 | vcpu->run->s.regs.gprs[0] = 0; |
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c index 56f80e1f98f7..566ddf6e8dfb 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/sigp.c | |||
| @@ -18,6 +18,7 @@ | |||
| 18 | #include <asm/sigp.h> | 18 | #include <asm/sigp.h> |
| 19 | #include "gaccess.h" | 19 | #include "gaccess.h" |
| 20 | #include "kvm-s390.h" | 20 | #include "kvm-s390.h" |
| 21 | #include "trace.h" | ||
| 21 | 22 | ||
| 22 | static int __sigp_sense(struct kvm_vcpu *vcpu, u16 cpu_addr, | 23 | static int __sigp_sense(struct kvm_vcpu *vcpu, u16 cpu_addr, |
| 23 | u64 *reg) | 24 | u64 *reg) |
| @@ -344,6 +345,7 @@ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu) | |||
| 344 | else | 345 | else |
| 345 | parameter = vcpu->run->s.regs.gprs[r1 + 1]; | 346 | parameter = vcpu->run->s.regs.gprs[r1 + 1]; |
| 346 | 347 | ||
| 348 | trace_kvm_s390_handle_sigp(vcpu, order_code, cpu_addr, parameter); | ||
| 347 | switch (order_code) { | 349 | switch (order_code) { |
| 348 | case SIGP_SENSE: | 350 | case SIGP_SENSE: |
| 349 | vcpu->stat.instruction_sigp_sense++; | 351 | vcpu->stat.instruction_sigp_sense++; |
diff --git a/arch/s390/kvm/trace-s390.h b/arch/s390/kvm/trace-s390.h new file mode 100644 index 000000000000..90fdf85b5ff7 --- /dev/null +++ b/arch/s390/kvm/trace-s390.h | |||
| @@ -0,0 +1,210 @@ | |||
| 1 | #if !defined(_TRACE_KVMS390_H) || defined(TRACE_HEADER_MULTI_READ) | ||
| 2 | #define _TRACE_KVMS390_H | ||
| 3 | |||
| 4 | #include <linux/tracepoint.h> | ||
| 5 | |||
| 6 | #undef TRACE_SYSTEM | ||
| 7 | #define TRACE_SYSTEM kvm-s390 | ||
| 8 | #define TRACE_INCLUDE_PATH . | ||
| 9 | #undef TRACE_INCLUDE_FILE | ||
| 10 | #define TRACE_INCLUDE_FILE trace-s390 | ||
| 11 | |||
| 12 | /* | ||
| 13 | * Trace point for the creation of the kvm instance. | ||
| 14 | */ | ||
| 15 | TRACE_EVENT(kvm_s390_create_vm, | ||
| 16 | TP_PROTO(unsigned long type), | ||
| 17 | TP_ARGS(type), | ||
| 18 | |||
| 19 | TP_STRUCT__entry( | ||
| 20 | __field(unsigned long, type) | ||
| 21 | ), | ||
| 22 | |||
| 23 | TP_fast_assign( | ||
| 24 | __entry->type = type; | ||
| 25 | ), | ||
| 26 | |||
| 27 | TP_printk("create vm%s", | ||
| 28 | __entry->type & KVM_VM_S390_UCONTROL ? " (UCONTROL)" : "") | ||
| 29 | ); | ||
| 30 | |||
| 31 | /* | ||
| 32 | * Trace points for creation and destruction of vpcus. | ||
| 33 | */ | ||
| 34 | TRACE_EVENT(kvm_s390_create_vcpu, | ||
| 35 | TP_PROTO(unsigned int id, struct kvm_vcpu *vcpu, | ||
| 36 | struct kvm_s390_sie_block *sie_block), | ||
| 37 | TP_ARGS(id, vcpu, sie_block), | ||
| 38 | |||
| 39 | TP_STRUCT__entry( | ||
| 40 | __field(unsigned int, id) | ||
| 41 | __field(struct kvm_vcpu *, vcpu) | ||
| 42 | __field(struct kvm_s390_sie_block *, sie_block) | ||
| 43 | ), | ||
| 44 | |||
| 45 | TP_fast_assign( | ||
| 46 | __entry->id = id; | ||
| 47 | __entry->vcpu = vcpu; | ||
| 48 | __entry->sie_block = sie_block; | ||
| 49 | ), | ||
| 50 | |||
| 51 | TP_printk("create cpu %d at %p, sie block at %p", __entry->id, | ||
| 52 | __entry->vcpu, __entry->sie_block) | ||
| 53 | ); | ||
| 54 | |||
| 55 | TRACE_EVENT(kvm_s390_destroy_vcpu, | ||
| 56 | TP_PROTO(unsigned int id), | ||
| 57 | TP_ARGS(id), | ||
| 58 | |||
| 59 | TP_STRUCT__entry( | ||
| 60 | __field(unsigned int, id) | ||
| 61 | ), | ||
| 62 | |||
| 63 | TP_fast_assign( | ||
| 64 | __entry->id = id; | ||
| 65 | ), | ||
| 66 | |||
| 67 | TP_printk("destroy cpu %d", __entry->id) | ||
| 68 | ); | ||
| 69 | |||
| 70 | /* | ||
| 71 | * Trace points for injection of interrupts, either per machine or | ||
| 72 | * per vcpu. | ||
| 73 | */ | ||
| 74 | |||
| 75 | #define kvm_s390_int_type \ | ||
| 76 | {KVM_S390_SIGP_STOP, "sigp stop"}, \ | ||
| 77 | {KVM_S390_PROGRAM_INT, "program interrupt"}, \ | ||
| 78 | {KVM_S390_SIGP_SET_PREFIX, "sigp set prefix"}, \ | ||
| 79 | {KVM_S390_RESTART, "sigp restart"}, \ | ||
| 80 | {KVM_S390_INT_VIRTIO, "virtio interrupt"}, \ | ||
| 81 | {KVM_S390_INT_SERVICE, "sclp interrupt"}, \ | ||
| 82 | {KVM_S390_INT_EMERGENCY, "sigp emergency"}, \ | ||
| 83 | {KVM_S390_INT_EXTERNAL_CALL, "sigp ext call"} | ||
| 84 | |||
| 85 | TRACE_EVENT(kvm_s390_inject_vm, | ||
| 86 | TP_PROTO(__u64 type, __u32 parm, __u64 parm64, int who), | ||
| 87 | TP_ARGS(type, parm, parm64, who), | ||
| 88 | |||
| 89 | TP_STRUCT__entry( | ||
| 90 | __field(__u32, inttype) | ||
| 91 | __field(__u32, parm) | ||
| 92 | __field(__u64, parm64) | ||
| 93 | __field(int, who) | ||
| 94 | ), | ||
| 95 | |||
| 96 | TP_fast_assign( | ||
| 97 | __entry->inttype = type & 0x00000000ffffffff; | ||
| 98 | __entry->parm = parm; | ||
| 99 | __entry->parm64 = parm64; | ||
| 100 | __entry->who = who; | ||
| 101 | ), | ||
| 102 | |||
| 103 | TP_printk("inject%s: type:%x (%s) parm:%x parm64:%llx", | ||
| 104 | (__entry->who == 1) ? " (from kernel)" : | ||
| 105 | (__entry->who == 2) ? " (from user)" : "", | ||
| 106 | __entry->inttype, | ||
| 107 | __print_symbolic(__entry->inttype, kvm_s390_int_type), | ||
| 108 | __entry->parm, __entry->parm64) | ||
| 109 | ); | ||
| 110 | |||
| 111 | TRACE_EVENT(kvm_s390_inject_vcpu, | ||
| 112 | TP_PROTO(unsigned int id, __u64 type, __u32 parm, __u64 parm64, \ | ||
| 113 | int who), | ||
| 114 | TP_ARGS(id, type, parm, parm64, who), | ||
| 115 | |||
| 116 | TP_STRUCT__entry( | ||
| 117 | __field(int, id) | ||
| 118 | __field(__u32, inttype) | ||
| 119 | __field(__u32, parm) | ||
| 120 | __field(__u64, parm64) | ||
| 121 | __field(int, who) | ||
| 122 | ), | ||
| 123 | |||
| 124 | TP_fast_assign( | ||
| 125 | __entry->id = id; | ||
| 126 | __entry->inttype = type & 0x00000000ffffffff; | ||
| 127 | __entry->parm = parm; | ||
| 128 | __entry->parm64 = parm64; | ||
| 129 | __entry->who = who; | ||
| 130 | ), | ||
| 131 | |||
| 132 | TP_printk("inject%s (vcpu %d): type:%x (%s) parm:%x parm64:%llx", | ||
| 133 | (__entry->who == 1) ? " (from kernel)" : | ||
| 134 | (__entry->who == 2) ? " (from user)" : "", | ||
| 135 | __entry->id, __entry->inttype, | ||
| 136 | __print_symbolic(__entry->inttype, kvm_s390_int_type), | ||
| 137 | __entry->parm, __entry->parm64) | ||
| 138 | ); | ||
| 139 | |||
| 140 | /* | ||
| 141 | * Trace point for the actual delivery of interrupts. | ||
| 142 | */ | ||
| 143 | TRACE_EVENT(kvm_s390_deliver_interrupt, | ||
| 144 | TP_PROTO(unsigned int id, __u64 type, __u32 data0, __u64 data1), | ||
| 145 | TP_ARGS(id, type, data0, data1), | ||
| 146 | |||
| 147 | TP_STRUCT__entry( | ||
| 148 | __field(int, id) | ||
| 149 | __field(__u32, inttype) | ||
| 150 | __field(__u32, data0) | ||
| 151 | __field(__u64, data1) | ||
| 152 | ), | ||
| 153 | |||
| 154 | TP_fast_assign( | ||
| 155 | __entry->id = id; | ||
| 156 | __entry->inttype = type & 0x00000000ffffffff; | ||
| 157 | __entry->data0 = data0; | ||
| 158 | __entry->data1 = data1; | ||
| 159 | ), | ||
| 160 | |||
| 161 | TP_printk("deliver interrupt (vcpu %d): type:%x (%s) " \ | ||
| 162 | "data:%08x %016llx", | ||
| 163 | __entry->id, __entry->inttype, | ||
| 164 | __print_symbolic(__entry->inttype, kvm_s390_int_type), | ||
| 165 | __entry->data0, __entry->data1) | ||
| 166 | ); | ||
| 167 | |||
| 168 | /* | ||
| 169 | * Trace point for resets that may be requested from userspace. | ||
| 170 | */ | ||
| 171 | TRACE_EVENT(kvm_s390_request_resets, | ||
| 172 | TP_PROTO(__u64 resets), | ||
| 173 | TP_ARGS(resets), | ||
| 174 | |||
| 175 | TP_STRUCT__entry( | ||
| 176 | __field(__u64, resets) | ||
| 177 | ), | ||
| 178 | |||
| 179 | TP_fast_assign( | ||
| 180 | __entry->resets = resets; | ||
| 181 | ), | ||
| 182 | |||
| 183 | TP_printk("requesting userspace resets %llx", | ||
| 184 | __entry->resets) | ||
| 185 | ); | ||
| 186 | |||
| 187 | /* | ||
| 188 | * Trace point for a vcpu's stop requests. | ||
| 189 | */ | ||
| 190 | TRACE_EVENT(kvm_s390_stop_request, | ||
| 191 | TP_PROTO(unsigned int action_bits), | ||
| 192 | TP_ARGS(action_bits), | ||
| 193 | |||
| 194 | TP_STRUCT__entry( | ||
| 195 | __field(unsigned int, action_bits) | ||
| 196 | ), | ||
| 197 | |||
| 198 | TP_fast_assign( | ||
| 199 | __entry->action_bits = action_bits; | ||
| 200 | ), | ||
| 201 | |||
| 202 | TP_printk("stop request, action_bits = %08x", | ||
| 203 | __entry->action_bits) | ||
| 204 | ); | ||
| 205 | |||
| 206 | |||
| 207 | #endif /* _TRACE_KVMS390_H */ | ||
| 208 | |||
| 209 | /* This part must be outside protection */ | ||
| 210 | #include <trace/define_trace.h> | ||
diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/trace.h new file mode 100644 index 000000000000..2b29e62351d3 --- /dev/null +++ b/arch/s390/kvm/trace.h | |||
| @@ -0,0 +1,341 @@ | |||
| 1 | #if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) | ||
| 2 | #define _TRACE_KVM_H | ||
| 3 | |||
| 4 | #include <linux/tracepoint.h> | ||
| 5 | #include <asm/sigp.h> | ||
| 6 | #include <asm/debug.h> | ||
| 7 | |||
| 8 | #undef TRACE_SYSTEM | ||
| 9 | #define TRACE_SYSTEM kvm | ||
| 10 | #define TRACE_INCLUDE_PATH . | ||
| 11 | #undef TRACE_INCLUDE_FILE | ||
| 12 | #define TRACE_INCLUDE_FILE trace | ||
| 13 | |||
| 14 | /* | ||
| 15 | * Helpers for vcpu-specific tracepoints containing the same information | ||
| 16 | * as s390dbf VCPU_EVENTs. | ||
| 17 | */ | ||
| 18 | #define VCPU_PROTO_COMMON struct kvm_vcpu *vcpu | ||
| 19 | #define VCPU_ARGS_COMMON vcpu | ||
| 20 | #define VCPU_FIELD_COMMON __field(int, id) \ | ||
| 21 | __field(unsigned long, pswmask) \ | ||
| 22 | __field(unsigned long, pswaddr) | ||
| 23 | #define VCPU_ASSIGN_COMMON do { \ | ||
| 24 | __entry->id = vcpu->vcpu_id; \ | ||
| 25 | __entry->pswmask = vcpu->arch.sie_block->gpsw.mask; \ | ||
| 26 | __entry->pswaddr = vcpu->arch.sie_block->gpsw.addr; \ | ||
| 27 | } while (0); | ||
| 28 | #define VCPU_TP_PRINTK(p_str, p_args...) \ | ||
| 29 | TP_printk("%02d[%016lx-%016lx]: " p_str, __entry->id, \ | ||
| 30 | __entry->pswmask, __entry->pswaddr, p_args) | ||
| 31 | |||
| 32 | /* | ||
| 33 | * Tracepoints for SIE entry and exit. | ||
| 34 | */ | ||
| 35 | TRACE_EVENT(kvm_s390_sie_enter, | ||
| 36 | TP_PROTO(VCPU_PROTO_COMMON, int cpuflags), | ||
| 37 | TP_ARGS(VCPU_ARGS_COMMON, cpuflags), | ||
| 38 | |||
| 39 | TP_STRUCT__entry( | ||
| 40 | VCPU_FIELD_COMMON | ||
| 41 | __field(int, cpuflags) | ||
| 42 | ), | ||
| 43 | |||
| 44 | TP_fast_assign( | ||
| 45 | VCPU_ASSIGN_COMMON | ||
| 46 | __entry->cpuflags = cpuflags; | ||
| 47 | ), | ||
| 48 | |||
| 49 | VCPU_TP_PRINTK("entering sie flags %x", __entry->cpuflags) | ||
| 50 | ); | ||
| 51 | |||
| 52 | TRACE_EVENT(kvm_s390_sie_fault, | ||
| 53 | TP_PROTO(VCPU_PROTO_COMMON), | ||
| 54 | TP_ARGS(VCPU_ARGS_COMMON), | ||
| 55 | |||
| 56 | TP_STRUCT__entry( | ||
| 57 | VCPU_FIELD_COMMON | ||
| 58 | ), | ||
| 59 | |||
| 60 | TP_fast_assign( | ||
| 61 | VCPU_ASSIGN_COMMON | ||
| 62 | ), | ||
| 63 | |||
| 64 | VCPU_TP_PRINTK("%s", "fault in sie instruction") | ||
| 65 | ); | ||
| 66 | |||
| 67 | #define sie_intercept_code \ | ||
| 68 | {0x04, "Instruction"}, \ | ||
| 69 | {0x08, "Program interruption"}, \ | ||
| 70 | {0x0C, "Instruction and program interuption"}, \ | ||
| 71 | {0x10, "External request"}, \ | ||
| 72 | {0x14, "External interruption"}, \ | ||
| 73 | {0x18, "I/O request"}, \ | ||
| 74 | {0x1C, "Wait state"}, \ | ||
| 75 | {0x20, "Validity"}, \ | ||
| 76 | {0x28, "Stop request"} | ||
| 77 | |||
| 78 | TRACE_EVENT(kvm_s390_sie_exit, | ||
| 79 | TP_PROTO(VCPU_PROTO_COMMON, u8 icptcode), | ||
| 80 | TP_ARGS(VCPU_ARGS_COMMON, icptcode), | ||
| 81 | |||
| 82 | TP_STRUCT__entry( | ||
| 83 | VCPU_FIELD_COMMON | ||
| 84 | __field(u8, icptcode) | ||
| 85 | ), | ||
| 86 | |||
| 87 | TP_fast_assign( | ||
| 88 | VCPU_ASSIGN_COMMON | ||
| 89 | __entry->icptcode = icptcode; | ||
| 90 | ), | ||
| 91 | |||
| 92 | VCPU_TP_PRINTK("exit sie icptcode %d (%s)", __entry->icptcode, | ||
| 93 | __print_symbolic(__entry->icptcode, | ||
| 94 | sie_intercept_code)) | ||
| 95 | ); | ||
| 96 | |||
| 97 | /* | ||
| 98 | * Trace point for intercepted instructions. | ||
| 99 | */ | ||
| 100 | TRACE_EVENT(kvm_s390_intercept_instruction, | ||
| 101 | TP_PROTO(VCPU_PROTO_COMMON, __u16 ipa, __u32 ipb), | ||
| 102 | TP_ARGS(VCPU_ARGS_COMMON, ipa, ipb), | ||
| 103 | |||
| 104 | TP_STRUCT__entry( | ||
| 105 | VCPU_FIELD_COMMON | ||
| 106 | __field(__u64, instruction) | ||
| 107 | __field(char, insn[8]) | ||
| 108 | ), | ||
| 109 | |||
| 110 | TP_fast_assign( | ||
| 111 | VCPU_ASSIGN_COMMON | ||
| 112 | __entry->instruction = ((__u64)ipa << 48) | | ||
| 113 | ((__u64)ipb << 16); | ||
| 114 | ), | ||
| 115 | |||
| 116 | VCPU_TP_PRINTK("intercepted instruction %016llx (%s)", | ||
| 117 | __entry->instruction, | ||
| 118 | insn_to_mnemonic((unsigned char *) | ||
| 119 | &__entry->instruction, | ||
| 120 | __entry->insn) ? | ||
| 121 | "unknown" : __entry->insn) | ||
| 122 | ); | ||
| 123 | |||
| 124 | /* | ||
| 125 | * Trace point for intercepted program interruptions. | ||
| 126 | */ | ||
| 127 | TRACE_EVENT(kvm_s390_intercept_prog, | ||
| 128 | TP_PROTO(VCPU_PROTO_COMMON, __u16 code), | ||
| 129 | TP_ARGS(VCPU_ARGS_COMMON, code), | ||
| 130 | |||
| 131 | TP_STRUCT__entry( | ||
| 132 | VCPU_FIELD_COMMON | ||
| 133 | __field(__u16, code) | ||
| 134 | ), | ||
| 135 | |||
| 136 | TP_fast_assign( | ||
| 137 | VCPU_ASSIGN_COMMON | ||
| 138 | __entry->code = code; | ||
| 139 | ), | ||
| 140 | |||
| 141 | VCPU_TP_PRINTK("intercepted program interruption %04x", | ||
| 142 | __entry->code) | ||
| 143 | ); | ||
| 144 | |||
| 145 | /* | ||
| 146 | * Trace point for validity intercepts. | ||
| 147 | */ | ||
| 148 | TRACE_EVENT(kvm_s390_intercept_validity, | ||
| 149 | TP_PROTO(VCPU_PROTO_COMMON, __u16 viwhy), | ||
| 150 | TP_ARGS(VCPU_ARGS_COMMON, viwhy), | ||
| 151 | |||
| 152 | TP_STRUCT__entry( | ||
| 153 | VCPU_FIELD_COMMON | ||
| 154 | __field(__u16, viwhy) | ||
| 155 | ), | ||
| 156 | |||
| 157 | TP_fast_assign( | ||
| 158 | VCPU_ASSIGN_COMMON | ||
| 159 | __entry->viwhy = viwhy; | ||
| 160 | ), | ||
| 161 | |||
| 162 | VCPU_TP_PRINTK("got validity intercept %04x", __entry->viwhy) | ||
| 163 | ); | ||
| 164 | |||
| 165 | /* | ||
| 166 | * Trace points for instructions that are of special interest. | ||
| 167 | */ | ||
| 168 | |||
| 169 | #define sigp_order_codes \ | ||
| 170 | {SIGP_SENSE, "sense"}, \ | ||
| 171 | {SIGP_EXTERNAL_CALL, "external call"}, \ | ||
| 172 | {SIGP_EMERGENCY_SIGNAL, "emergency signal"}, \ | ||
| 173 | {SIGP_STOP, "stop"}, \ | ||
| 174 | {SIGP_STOP_AND_STORE_STATUS, "stop and store status"}, \ | ||
| 175 | {SIGP_SET_ARCHITECTURE, "set architecture"}, \ | ||
| 176 | {SIGP_SET_PREFIX, "set prefix"}, \ | ||
| 177 | {SIGP_SENSE_RUNNING, "sense running"}, \ | ||
| 178 | {SIGP_RESTART, "restart"} | ||
| 179 | |||
| 180 | TRACE_EVENT(kvm_s390_handle_sigp, | ||
| 181 | TP_PROTO(VCPU_PROTO_COMMON, __u8 order_code, __u16 cpu_addr, \ | ||
| 182 | __u32 parameter), | ||
| 183 | TP_ARGS(VCPU_ARGS_COMMON, order_code, cpu_addr, parameter), | ||
| 184 | |||
| 185 | TP_STRUCT__entry( | ||
| 186 | VCPU_FIELD_COMMON | ||
| 187 | __field(__u8, order_code) | ||
| 188 | __field(__u16, cpu_addr) | ||
| 189 | __field(__u32, parameter) | ||
| 190 | ), | ||
| 191 | |||
| 192 | TP_fast_assign( | ||
| 193 | VCPU_ASSIGN_COMMON | ||
| 194 | __entry->order_code = order_code; | ||
| 195 | __entry->cpu_addr = cpu_addr; | ||
| 196 | __entry->parameter = parameter; | ||
| 197 | ), | ||
| 198 | |||
| 199 | VCPU_TP_PRINTK("handle sigp order %02x (%s), cpu address %04x, " \ | ||
| 200 | "parameter %08x", __entry->order_code, | ||
| 201 | __print_symbolic(__entry->order_code, | ||
| 202 | sigp_order_codes), | ||
| 203 | __entry->cpu_addr, __entry->parameter) | ||
| 204 | ); | ||
| 205 | |||
| 206 | #define diagnose_codes \ | ||
| 207 | {0x10, "release pages"}, \ | ||
| 208 | {0x44, "time slice end"}, \ | ||
| 209 | {0x308, "ipl functions"}, \ | ||
| 210 | {0x500, "kvm hypercall"}, \ | ||
| 211 | {0x501, "kvm breakpoint"} | ||
| 212 | |||
| 213 | TRACE_EVENT(kvm_s390_handle_diag, | ||
| 214 | TP_PROTO(VCPU_PROTO_COMMON, __u16 code), | ||
| 215 | TP_ARGS(VCPU_ARGS_COMMON, code), | ||
| 216 | |||
| 217 | TP_STRUCT__entry( | ||
| 218 | VCPU_FIELD_COMMON | ||
| 219 | __field(__u16, code) | ||
| 220 | ), | ||
| 221 | |||
| 222 | TP_fast_assign( | ||
| 223 | VCPU_ASSIGN_COMMON | ||
| 224 | __entry->code = code; | ||
| 225 | ), | ||
| 226 | |||
| 227 | VCPU_TP_PRINTK("handle diagnose call %04x (%s)", __entry->code, | ||
| 228 | __print_symbolic(__entry->code, diagnose_codes)) | ||
| 229 | ); | ||
| 230 | |||
| 231 | TRACE_EVENT(kvm_s390_handle_lctl, | ||
| 232 | TP_PROTO(VCPU_PROTO_COMMON, int g, int reg1, int reg3, u64 addr), | ||
| 233 | TP_ARGS(VCPU_ARGS_COMMON, g, reg1, reg3, addr), | ||
| 234 | |||
| 235 | TP_STRUCT__entry( | ||
| 236 | VCPU_FIELD_COMMON | ||
| 237 | __field(int, g) | ||
| 238 | __field(int, reg1) | ||
| 239 | __field(int, reg3) | ||
| 240 | __field(u64, addr) | ||
| 241 | ), | ||
| 242 | |||
| 243 | TP_fast_assign( | ||
| 244 | VCPU_ASSIGN_COMMON | ||
| 245 | __entry->g = g; | ||
| 246 | __entry->reg1 = reg1; | ||
| 247 | __entry->reg3 = reg3; | ||
| 248 | __entry->addr = addr; | ||
| 249 | ), | ||
| 250 | |||
| 251 | VCPU_TP_PRINTK("%s: loading cr %x-%x from %016llx", | ||
| 252 | __entry->g ? "lctlg" : "lctl", | ||
| 253 | __entry->reg1, __entry->reg3, __entry->addr) | ||
| 254 | ); | ||
| 255 | |||
| 256 | TRACE_EVENT(kvm_s390_handle_prefix, | ||
| 257 | TP_PROTO(VCPU_PROTO_COMMON, int set, u32 address), | ||
| 258 | TP_ARGS(VCPU_ARGS_COMMON, set, address), | ||
| 259 | |||
| 260 | TP_STRUCT__entry( | ||
| 261 | VCPU_FIELD_COMMON | ||
| 262 | __field(int, set) | ||
| 263 | __field(u32, address) | ||
| 264 | ), | ||
| 265 | |||
| 266 | TP_fast_assign( | ||
| 267 | VCPU_ASSIGN_COMMON | ||
| 268 | __entry->set = set; | ||
| 269 | __entry->address = address; | ||
| 270 | ), | ||
| 271 | |||
| 272 | VCPU_TP_PRINTK("%s prefix to %08x", | ||
| 273 | __entry->set ? "setting" : "storing", | ||
| 274 | __entry->address) | ||
| 275 | ); | ||
| 276 | |||
| 277 | TRACE_EVENT(kvm_s390_handle_stap, | ||
| 278 | TP_PROTO(VCPU_PROTO_COMMON, u64 address), | ||
| 279 | TP_ARGS(VCPU_ARGS_COMMON, address), | ||
| 280 | |||
| 281 | TP_STRUCT__entry( | ||
| 282 | VCPU_FIELD_COMMON | ||
| 283 | __field(u64, address) | ||
| 284 | ), | ||
| 285 | |||
| 286 | TP_fast_assign( | ||
| 287 | VCPU_ASSIGN_COMMON | ||
| 288 | __entry->address = address; | ||
| 289 | ), | ||
| 290 | |||
| 291 | VCPU_TP_PRINTK("storing cpu address to %016llx", | ||
| 292 | __entry->address) | ||
| 293 | ); | ||
| 294 | |||
| 295 | TRACE_EVENT(kvm_s390_handle_stfl, | ||
| 296 | TP_PROTO(VCPU_PROTO_COMMON, unsigned int facility_list), | ||
| 297 | TP_ARGS(VCPU_ARGS_COMMON, facility_list), | ||
| 298 | |||
| 299 | TP_STRUCT__entry( | ||
| 300 | VCPU_FIELD_COMMON | ||
| 301 | __field(unsigned int, facility_list) | ||
| 302 | ), | ||
| 303 | |||
| 304 | TP_fast_assign( | ||
| 305 | VCPU_ASSIGN_COMMON | ||
| 306 | __entry->facility_list = facility_list; | ||
| 307 | ), | ||
| 308 | |||
| 309 | VCPU_TP_PRINTK("store facility list value %08x", | ||
| 310 | __entry->facility_list) | ||
| 311 | ); | ||
| 312 | |||
| 313 | TRACE_EVENT(kvm_s390_handle_stsi, | ||
| 314 | TP_PROTO(VCPU_PROTO_COMMON, int fc, int sel1, int sel2, u64 addr), | ||
| 315 | TP_ARGS(VCPU_ARGS_COMMON, fc, sel1, sel2, addr), | ||
| 316 | |||
| 317 | TP_STRUCT__entry( | ||
| 318 | VCPU_FIELD_COMMON | ||
| 319 | __field(int, fc) | ||
| 320 | __field(int, sel1) | ||
| 321 | __field(int, sel2) | ||
| 322 | __field(u64, addr) | ||
| 323 | ), | ||
| 324 | |||
| 325 | TP_fast_assign( | ||
| 326 | VCPU_ASSIGN_COMMON | ||
| 327 | __entry->fc = fc; | ||
| 328 | __entry->sel1 = sel1; | ||
| 329 | __entry->sel2 = sel2; | ||
| 330 | __entry->addr = addr; | ||
| 331 | ), | ||
| 332 | |||
| 333 | VCPU_TP_PRINTK("STSI %d.%d.%d information stored to %016llx", | ||
| 334 | __entry->fc, __entry->sel1, __entry->sel2, | ||
| 335 | __entry->addr) | ||
| 336 | ); | ||
| 337 | |||
| 338 | #endif /* _TRACE_KVM_H */ | ||
| 339 | |||
| 340 | /* This part must be outside protection */ | ||
| 341 | #include <trace/define_trace.h> | ||
diff --git a/arch/score/Kconfig b/arch/score/Kconfig index ba0f412920be..461c23747491 100644 --- a/arch/score/Kconfig +++ b/arch/score/Kconfig | |||
| @@ -5,6 +5,7 @@ config SCORE | |||
| 5 | select HAVE_GENERIC_HARDIRQS | 5 | select HAVE_GENERIC_HARDIRQS |
| 6 | select GENERIC_IRQ_SHOW | 6 | select GENERIC_IRQ_SHOW |
| 7 | select GENERIC_IOMAP | 7 | select GENERIC_IOMAP |
| 8 | select GENERIC_ATOMIC64 | ||
| 8 | select HAVE_MEMBLOCK | 9 | select HAVE_MEMBLOCK |
| 9 | select HAVE_MEMBLOCK_NODE_MAP | 10 | select HAVE_MEMBLOCK_NODE_MAP |
| 10 | select ARCH_DISCARD_MEMBLOCK | 11 | select ARCH_DISCARD_MEMBLOCK |
diff --git a/arch/score/include/asm/elf.h b/arch/score/include/asm/elf.h index f478ce94181f..5d566c7a0af2 100644 --- a/arch/score/include/asm/elf.h +++ b/arch/score/include/asm/elf.h | |||
| @@ -54,7 +54,7 @@ typedef elf_fpreg_t elf_fpregset_t; | |||
| 54 | 54 | ||
| 55 | #define SET_PERSONALITY(ex) \ | 55 | #define SET_PERSONALITY(ex) \ |
| 56 | do { \ | 56 | do { \ |
| 57 | set_personality(PER_LINUX); \ | 57 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))); \ |
| 58 | } while (0) | 58 | } while (0) |
| 59 | 59 | ||
| 60 | struct task_struct; | 60 | struct task_struct; |
diff --git a/arch/score/include/uapi/asm/Kbuild b/arch/score/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/score/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/score/kernel/sys_score.c b/arch/score/kernel/sys_score.c index e478bf9a7e91..21e867974066 100644 --- a/arch/score/kernel/sys_score.c +++ b/arch/score/kernel/sys_score.c | |||
| @@ -112,6 +112,7 @@ score_execve(struct pt_regs *regs) | |||
| 112 | * Do a system call from kernel instead of calling sys_execve so we | 112 | * Do a system call from kernel instead of calling sys_execve so we |
| 113 | * end up with proper pt_regs. | 113 | * end up with proper pt_regs. |
| 114 | */ | 114 | */ |
| 115 | asmlinkage | ||
| 115 | int kernel_execve(const char *filename, | 116 | int kernel_execve(const char *filename, |
| 116 | const char *const argv[], | 117 | const char *const argv[], |
| 117 | const char *const envp[]) | 118 | const char *const envp[]) |
diff --git a/arch/sh/include/asm/bl_bit.h b/arch/sh/include/asm/bl_bit.h index 45e6b9fc37a0..06e4163c6746 100644 --- a/arch/sh/include/asm/bl_bit.h +++ b/arch/sh/include/asm/bl_bit.h | |||
| @@ -2,9 +2,9 @@ | |||
| 2 | #define __ASM_SH_BL_BIT_H | 2 | #define __ASM_SH_BL_BIT_H |
| 3 | 3 | ||
| 4 | #ifdef CONFIG_SUPERH32 | 4 | #ifdef CONFIG_SUPERH32 |
| 5 | # include "bl_bit_32.h" | 5 | # include <asm/bl_bit_32.h> |
| 6 | #else | 6 | #else |
| 7 | # include "bl_bit_64.h" | 7 | # include <asm/bl_bit_64.h> |
| 8 | #endif | 8 | #endif |
| 9 | 9 | ||
| 10 | #endif /* __ASM_SH_BL_BIT_H */ | 10 | #endif /* __ASM_SH_BL_BIT_H */ |
diff --git a/arch/sh/include/asm/cache_insns.h b/arch/sh/include/asm/cache_insns.h index d25fbe53090d..355cb06b7a30 100644 --- a/arch/sh/include/asm/cache_insns.h +++ b/arch/sh/include/asm/cache_insns.h | |||
| @@ -3,9 +3,9 @@ | |||
| 3 | 3 | ||
| 4 | 4 | ||
| 5 | #ifdef CONFIG_SUPERH32 | 5 | #ifdef CONFIG_SUPERH32 |
| 6 | # include "cache_insns_32.h" | 6 | # include <asm/cache_insns_32.h> |
| 7 | #else | 7 | #else |
| 8 | # include "cache_insns_64.h" | 8 | # include <asm/cache_insns_64.h> |
| 9 | #endif | 9 | #endif |
| 10 | 10 | ||
| 11 | #endif /* __ASM_SH_CACHE_INSNS_H */ | 11 | #endif /* __ASM_SH_CACHE_INSNS_H */ |
diff --git a/arch/sh/include/asm/checksum.h b/arch/sh/include/asm/checksum.h index fc26d1f4b590..34ae26204524 100644 --- a/arch/sh/include/asm/checksum.h +++ b/arch/sh/include/asm/checksum.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_SUPERH32 | 1 | #ifdef CONFIG_SUPERH32 |
| 2 | # include "checksum_32.h" | 2 | # include <asm/checksum_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include <asm-generic/checksum.h> | 4 | # include <asm-generic/checksum.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/sh/include/asm/elf.h b/arch/sh/include/asm/elf.h index f38112be67d2..37924afa8d8a 100644 --- a/arch/sh/include/asm/elf.h +++ b/arch/sh/include/asm/elf.h | |||
| @@ -183,7 +183,8 @@ do { \ | |||
| 183 | } while (0) | 183 | } while (0) |
| 184 | #endif | 184 | #endif |
| 185 | 185 | ||
| 186 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT) | 186 | #define SET_PERSONALITY(ex) \ |
| 187 | set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK))) | ||
| 187 | 188 | ||
| 188 | #ifdef CONFIG_VSYSCALL | 189 | #ifdef CONFIG_VSYSCALL |
| 189 | /* vDSO has arch_setup_additional_pages */ | 190 | /* vDSO has arch_setup_additional_pages */ |
diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h index 0cf60a628814..73a23f4617a3 100644 --- a/arch/sh/include/asm/io.h +++ b/arch/sh/include/asm/io.h | |||
| @@ -134,7 +134,7 @@ __BUILD_MEMORY_STRING(__raw_, q, u64) | |||
| 134 | * load/store instructions. sh_io_port_base is the virtual address to | 134 | * load/store instructions. sh_io_port_base is the virtual address to |
| 135 | * which all ports are being mapped. | 135 | * which all ports are being mapped. |
| 136 | */ | 136 | */ |
| 137 | extern const unsigned long sh_io_port_base; | 137 | extern unsigned long sh_io_port_base; |
| 138 | 138 | ||
| 139 | static inline void __set_io_port_base(unsigned long pbase) | 139 | static inline void __set_io_port_base(unsigned long pbase) |
| 140 | { | 140 | { |
diff --git a/arch/sh/include/asm/mmu_context.h b/arch/sh/include/asm/mmu_context.h index 384c7471a374..21c5088788da 100644 --- a/arch/sh/include/asm/mmu_context.h +++ b/arch/sh/include/asm/mmu_context.h | |||
| @@ -46,9 +46,9 @@ | |||
| 46 | #define MMU_VPN_MASK 0xfffff000 | 46 | #define MMU_VPN_MASK 0xfffff000 |
| 47 | 47 | ||
| 48 | #if defined(CONFIG_SUPERH32) | 48 | #if defined(CONFIG_SUPERH32) |
| 49 | #include "mmu_context_32.h" | 49 | #include <asm/mmu_context_32.h> |
| 50 | #else | 50 | #else |
| 51 | #include "mmu_context_64.h" | 51 | #include <asm/mmu_context_64.h> |
| 52 | #endif | 52 | #endif |
| 53 | 53 | ||
| 54 | /* | 54 | /* |
diff --git a/arch/sh/include/asm/posix_types.h b/arch/sh/include/asm/posix_types.h index 4eeb723aee7e..f08449bcbde7 100644 --- a/arch/sh/include/asm/posix_types.h +++ b/arch/sh/include/asm/posix_types.h | |||
| @@ -1,13 +1,13 @@ | |||
| 1 | #ifdef __KERNEL__ | 1 | #ifdef __KERNEL__ |
| 2 | # ifdef CONFIG_SUPERH32 | 2 | # ifdef CONFIG_SUPERH32 |
| 3 | # include "posix_types_32.h" | 3 | # include <asm/posix_types_32.h> |
| 4 | # else | 4 | # else |
| 5 | # include "posix_types_64.h" | 5 | # include <asm/posix_types_64.h> |
| 6 | # endif | 6 | # endif |
| 7 | #else | 7 | #else |
| 8 | # ifdef __SH5__ | 8 | # ifdef __SH5__ |
| 9 | # include "posix_types_64.h" | 9 | # include <asm/posix_types_64.h> |
| 10 | # else | 10 | # else |
| 11 | # include "posix_types_32.h" | 11 | # include <asm/posix_types_32.h> |
| 12 | # endif | 12 | # endif |
| 13 | #endif /* __KERNEL__ */ | 13 | #endif /* __KERNEL__ */ |
diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h index 3d14aeaef57c..5448f9bbf4ab 100644 --- a/arch/sh/include/asm/processor.h +++ b/arch/sh/include/asm/processor.h | |||
| @@ -175,9 +175,9 @@ extern unsigned int instruction_size(unsigned int insn); | |||
| 175 | #endif /* __ASSEMBLY__ */ | 175 | #endif /* __ASSEMBLY__ */ |
| 176 | 176 | ||
| 177 | #ifdef CONFIG_SUPERH32 | 177 | #ifdef CONFIG_SUPERH32 |
| 178 | # include "processor_32.h" | 178 | # include <asm/processor_32.h> |
| 179 | #else | 179 | #else |
| 180 | # include "processor_64.h" | 180 | # include <asm/processor_64.h> |
| 181 | #endif | 181 | #endif |
| 182 | 182 | ||
| 183 | #endif /* __ASM_SH_PROCESSOR_H */ | 183 | #endif /* __ASM_SH_PROCESSOR_H */ |
diff --git a/arch/sh/include/asm/ptrace.h b/arch/sh/include/asm/ptrace.h index c7b7e1ed194a..a4a38dff997a 100644 --- a/arch/sh/include/asm/ptrace.h +++ b/arch/sh/include/asm/ptrace.h | |||
| @@ -25,9 +25,9 @@ | |||
| 25 | #define PT_TEXT_LEN 252 | 25 | #define PT_TEXT_LEN 252 |
| 26 | 26 | ||
| 27 | #if defined(__SH5__) || defined(CONFIG_CPU_SH5) | 27 | #if defined(__SH5__) || defined(CONFIG_CPU_SH5) |
| 28 | #include "ptrace_64.h" | 28 | #include <asm/ptrace_64.h> |
| 29 | #else | 29 | #else |
| 30 | #include "ptrace_32.h" | 30 | #include <asm/ptrace_32.h> |
| 31 | #endif | 31 | #endif |
| 32 | 32 | ||
| 33 | #ifdef __KERNEL__ | 33 | #ifdef __KERNEL__ |
diff --git a/arch/sh/include/asm/string.h b/arch/sh/include/asm/string.h index 8c1ea21dc0ae..114011fa08af 100644 --- a/arch/sh/include/asm/string.h +++ b/arch/sh/include/asm/string.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_SUPERH32 | 1 | #ifdef CONFIG_SUPERH32 |
| 2 | # include "string_32.h" | 2 | # include <asm/string_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "string_64.h" | 4 | # include <asm/string_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/sh/include/asm/switch_to.h b/arch/sh/include/asm/switch_to.h index 62b1941813e3..bcd722fc8347 100644 --- a/arch/sh/include/asm/switch_to.h +++ b/arch/sh/include/asm/switch_to.h | |||
| @@ -11,9 +11,9 @@ | |||
| 11 | #define __ASM_SH_SWITCH_TO_H | 11 | #define __ASM_SH_SWITCH_TO_H |
| 12 | 12 | ||
| 13 | #ifdef CONFIG_SUPERH32 | 13 | #ifdef CONFIG_SUPERH32 |
| 14 | # include "switch_to_32.h" | 14 | # include <asm/switch_to_32.h> |
| 15 | #else | 15 | #else |
| 16 | # include "switch_to_64.h" | 16 | # include <asm/switch_to_64.h> |
| 17 | #endif | 17 | #endif |
| 18 | 18 | ||
| 19 | #endif /* __ASM_SH_SWITCH_TO_H */ | 19 | #endif /* __ASM_SH_SWITCH_TO_H */ |
diff --git a/arch/sh/include/asm/syscall.h b/arch/sh/include/asm/syscall.h index aa7777bdc370..847128da6eac 100644 --- a/arch/sh/include/asm/syscall.h +++ b/arch/sh/include/asm/syscall.h | |||
| @@ -4,9 +4,9 @@ | |||
| 4 | extern const unsigned long sys_call_table[]; | 4 | extern const unsigned long sys_call_table[]; |
| 5 | 5 | ||
| 6 | #ifdef CONFIG_SUPERH32 | 6 | #ifdef CONFIG_SUPERH32 |
| 7 | # include "syscall_32.h" | 7 | # include <asm/syscall_32.h> |
| 8 | #else | 8 | #else |
| 9 | # include "syscall_64.h" | 9 | # include <asm/syscall_64.h> |
| 10 | #endif | 10 | #endif |
| 11 | 11 | ||
| 12 | #endif /* __ASM_SH_SYSCALL_H */ | 12 | #endif /* __ASM_SH_SYSCALL_H */ |
diff --git a/arch/sh/include/asm/syscalls.h b/arch/sh/include/asm/syscalls.h index 507725af2e54..3dbfef06f6b2 100644 --- a/arch/sh/include/asm/syscalls.h +++ b/arch/sh/include/asm/syscalls.h | |||
| @@ -11,9 +11,9 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, | |||
| 11 | unsigned long fd, unsigned long pgoff); | 11 | unsigned long fd, unsigned long pgoff); |
| 12 | 12 | ||
| 13 | #ifdef CONFIG_SUPERH32 | 13 | #ifdef CONFIG_SUPERH32 |
| 14 | # include "syscalls_32.h" | 14 | # include <asm/syscalls_32.h> |
| 15 | #else | 15 | #else |
| 16 | # include "syscalls_64.h" | 16 | # include <asm/syscalls_64.h> |
| 17 | #endif | 17 | #endif |
| 18 | 18 | ||
| 19 | #endif /* __KERNEL__ */ | 19 | #endif /* __KERNEL__ */ |
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h index ec88bfcdf7ce..e61d43d9f689 100644 --- a/arch/sh/include/asm/tlb.h +++ b/arch/sh/include/asm/tlb.h | |||
| @@ -2,7 +2,7 @@ | |||
| 2 | #define __ASM_SH_TLB_H | 2 | #define __ASM_SH_TLB_H |
| 3 | 3 | ||
| 4 | #ifdef CONFIG_SUPERH64 | 4 | #ifdef CONFIG_SUPERH64 |
| 5 | # include "tlb_64.h" | 5 | # include <asm/tlb_64.h> |
| 6 | #endif | 6 | #endif |
| 7 | 7 | ||
| 8 | #ifndef __ASSEMBLY__ | 8 | #ifndef __ASSEMBLY__ |
diff --git a/arch/sh/include/asm/traps.h b/arch/sh/include/asm/traps.h index afd9df8d0641..9cc149a0dbd1 100644 --- a/arch/sh/include/asm/traps.h +++ b/arch/sh/include/asm/traps.h | |||
| @@ -4,9 +4,9 @@ | |||
| 4 | #include <linux/compiler.h> | 4 | #include <linux/compiler.h> |
| 5 | 5 | ||
| 6 | #ifdef CONFIG_SUPERH32 | 6 | #ifdef CONFIG_SUPERH32 |
| 7 | # include "traps_32.h" | 7 | # include <asm/traps_32.h> |
| 8 | #else | 8 | #else |
| 9 | # include "traps_64.h" | 9 | # include <asm/traps_64.h> |
| 10 | #endif | 10 | #endif |
| 11 | 11 | ||
| 12 | BUILD_TRAP_HANDLER(address_error); | 12 | BUILD_TRAP_HANDLER(address_error); |
diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 8698a80ed00c..9486376605f4 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h | |||
| @@ -97,9 +97,9 @@ struct __large_struct { unsigned long buf[100]; }; | |||
| 97 | }) | 97 | }) |
| 98 | 98 | ||
| 99 | #ifdef CONFIG_SUPERH32 | 99 | #ifdef CONFIG_SUPERH32 |
| 100 | # include "uaccess_32.h" | 100 | # include <asm/uaccess_32.h> |
| 101 | #else | 101 | #else |
| 102 | # include "uaccess_64.h" | 102 | # include <asm/uaccess_64.h> |
| 103 | #endif | 103 | #endif |
| 104 | 104 | ||
| 105 | extern long strncpy_from_user(char *dest, const char __user *src, long count); | 105 | extern long strncpy_from_user(char *dest, const char __user *src, long count); |
diff --git a/arch/sh/include/asm/unistd.h b/arch/sh/include/asm/unistd.h index 7bc67076baac..307201a854f3 100644 --- a/arch/sh/include/asm/unistd.h +++ b/arch/sh/include/asm/unistd.h | |||
| @@ -1,8 +1,8 @@ | |||
| 1 | #ifdef __KERNEL__ | 1 | #ifdef __KERNEL__ |
| 2 | # ifdef CONFIG_SUPERH32 | 2 | # ifdef CONFIG_SUPERH32 |
| 3 | # include "unistd_32.h" | 3 | # include <asm/unistd_32.h> |
| 4 | # else | 4 | # else |
| 5 | # include "unistd_64.h" | 5 | # include <asm/unistd_64.h> |
| 6 | # endif | 6 | # endif |
| 7 | 7 | ||
| 8 | # define __ARCH_WANT_SYS_RT_SIGSUSPEND | 8 | # define __ARCH_WANT_SYS_RT_SIGSUSPEND |
| @@ -40,8 +40,8 @@ | |||
| 40 | 40 | ||
| 41 | #else | 41 | #else |
| 42 | # ifdef __SH5__ | 42 | # ifdef __SH5__ |
| 43 | # include "unistd_64.h" | 43 | # include <asm/unistd_64.h> |
| 44 | # else | 44 | # else |
| 45 | # include "unistd_32.h" | 45 | # include <asm/unistd_32.h> |
| 46 | # endif | 46 | # endif |
| 47 | #endif | 47 | #endif |
diff --git a/arch/sh/include/mach-ecovec24/mach/romimage.h b/arch/sh/include/mach-ecovec24/mach/romimage.h index d63ef51ec186..60f3e8af05fa 100644 --- a/arch/sh/include/mach-ecovec24/mach/romimage.h +++ b/arch/sh/include/mach-ecovec24/mach/romimage.h | |||
| @@ -6,7 +6,7 @@ | |||
| 6 | */ | 6 | */ |
| 7 | 7 | ||
| 8 | #include <asm/romimage-macros.h> | 8 | #include <asm/romimage-macros.h> |
| 9 | #include "partner-jet-setup.txt" | 9 | #include <mach/partner-jet-setup.txt> |
| 10 | 10 | ||
| 11 | /* execute icbi after enabling cache */ | 11 | /* execute icbi after enabling cache */ |
| 12 | mov.l 1f, r0 | 12 | mov.l 1f, r0 |
diff --git a/arch/sh/include/mach-kfr2r09/mach/romimage.h b/arch/sh/include/mach-kfr2r09/mach/romimage.h index 7a883167c846..1afae21ced5f 100644 --- a/arch/sh/include/mach-kfr2r09/mach/romimage.h +++ b/arch/sh/include/mach-kfr2r09/mach/romimage.h | |||
| @@ -6,7 +6,7 @@ | |||
| 6 | */ | 6 | */ |
| 7 | 7 | ||
| 8 | #include <asm/romimage-macros.h> | 8 | #include <asm/romimage-macros.h> |
| 9 | #include "partner-jet-setup.txt" | 9 | #include <mach/partner-jet-setup.txt> |
| 10 | 10 | ||
| 11 | /* execute icbi after enabling cache */ | 11 | /* execute icbi after enabling cache */ |
| 12 | mov.l 1f, r0 | 12 | mov.l 1f, r0 |
diff --git a/arch/sh/include/uapi/asm/Kbuild b/arch/sh/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/sh/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/sh/kernel/ioport.c b/arch/sh/kernel/ioport.c index e3ad6103e7c1..cca14ba84a37 100644 --- a/arch/sh/kernel/ioport.c +++ b/arch/sh/kernel/ioport.c | |||
| @@ -11,7 +11,7 @@ | |||
| 11 | #include <linux/module.h> | 11 | #include <linux/module.h> |
| 12 | #include <linux/io.h> | 12 | #include <linux/io.h> |
| 13 | 13 | ||
| 14 | const unsigned long sh_io_port_base __read_mostly = -1; | 14 | unsigned long sh_io_port_base __read_mostly = -1; |
| 15 | EXPORT_SYMBOL(sh_io_port_base); | 15 | EXPORT_SYMBOL(sh_io_port_base); |
| 16 | 16 | ||
| 17 | void __iomem *__ioport_map(unsigned long addr, unsigned int size) | 17 | void __iomem *__ioport_map(unsigned long addr, unsigned int size) |
diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h index b8be20d42a0a..cef99fbc0a21 100644 --- a/arch/sparc/include/asm/compat.h +++ b/arch/sparc/include/asm/compat.h | |||
| @@ -36,6 +36,7 @@ typedef s64 compat_s64; | |||
| 36 | typedef u32 compat_uint_t; | 36 | typedef u32 compat_uint_t; |
| 37 | typedef u32 compat_ulong_t; | 37 | typedef u32 compat_ulong_t; |
| 38 | typedef u64 compat_u64; | 38 | typedef u64 compat_u64; |
| 39 | typedef u32 compat_uptr_t; | ||
| 39 | 40 | ||
| 40 | struct compat_timespec { | 41 | struct compat_timespec { |
| 41 | compat_time_t tv_sec; | 42 | compat_time_t tv_sec; |
| @@ -147,6 +148,65 @@ typedef u32 compat_old_sigset_t; | |||
| 147 | 148 | ||
| 148 | typedef u32 compat_sigset_word; | 149 | typedef u32 compat_sigset_word; |
| 149 | 150 | ||
| 151 | typedef union compat_sigval { | ||
| 152 | compat_int_t sival_int; | ||
| 153 | compat_uptr_t sival_ptr; | ||
| 154 | } compat_sigval_t; | ||
| 155 | |||
| 156 | #define SI_PAD_SIZE32 (128/sizeof(int) - 3) | ||
| 157 | |||
| 158 | typedef struct compat_siginfo { | ||
| 159 | int si_signo; | ||
| 160 | int si_errno; | ||
| 161 | int si_code; | ||
| 162 | |||
| 163 | union { | ||
| 164 | int _pad[SI_PAD_SIZE32]; | ||
| 165 | |||
| 166 | /* kill() */ | ||
| 167 | struct { | ||
| 168 | compat_pid_t _pid; /* sender's pid */ | ||
| 169 | unsigned int _uid; /* sender's uid */ | ||
| 170 | } _kill; | ||
| 171 | |||
| 172 | /* POSIX.1b timers */ | ||
| 173 | struct { | ||
| 174 | compat_timer_t _tid; /* timer id */ | ||
| 175 | int _overrun; /* overrun count */ | ||
| 176 | compat_sigval_t _sigval; /* same as below */ | ||
| 177 | int _sys_private; /* not to be passed to user */ | ||
| 178 | } _timer; | ||
| 179 | |||
| 180 | /* POSIX.1b signals */ | ||
| 181 | struct { | ||
| 182 | compat_pid_t _pid; /* sender's pid */ | ||
| 183 | unsigned int _uid; /* sender's uid */ | ||
| 184 | compat_sigval_t _sigval; | ||
| 185 | } _rt; | ||
| 186 | |||
| 187 | /* SIGCHLD */ | ||
| 188 | struct { | ||
| 189 | compat_pid_t _pid; /* which child */ | ||
| 190 | unsigned int _uid; /* sender's uid */ | ||
| 191 | int _status; /* exit code */ | ||
| 192 | compat_clock_t _utime; | ||
| 193 | compat_clock_t _stime; | ||
| 194 | } _sigchld; | ||
| 195 | |||
| 196 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ | ||
| 197 | struct { | ||
| 198 | u32 _addr; /* faulting insn/memory ref. */ | ||
| 199 | int _trapno; | ||
| 200 | } _sigfault; | ||
| 201 | |||
| 202 | /* SIGPOLL */ | ||
| 203 | struct { | ||
| 204 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 205 | int _fd; | ||
| 206 | } _sigpoll; | ||
| 207 | } _sifields; | ||
| 208 | } compat_siginfo_t; | ||
| 209 | |||
| 150 | #define COMPAT_OFF_T_MAX 0x7fffffff | 210 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 151 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 211 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 152 | 212 | ||
| @@ -156,7 +216,6 @@ typedef u32 compat_sigset_word; | |||
| 156 | * as pointers because the syscall entry code will have | 216 | * as pointers because the syscall entry code will have |
| 157 | * appropriately converted them already. | 217 | * appropriately converted them already. |
| 158 | */ | 218 | */ |
| 159 | typedef u32 compat_uptr_t; | ||
| 160 | 219 | ||
| 161 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 220 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 162 | { | 221 | { |
diff --git a/arch/sparc/include/asm/elf_32.h b/arch/sparc/include/asm/elf_32.h index 2d4d755cba9e..ac74a2c98e6d 100644 --- a/arch/sparc/include/asm/elf_32.h +++ b/arch/sparc/include/asm/elf_32.h | |||
| @@ -128,6 +128,7 @@ typedef struct { | |||
| 128 | 128 | ||
| 129 | #define ELF_PLATFORM (NULL) | 129 | #define ELF_PLATFORM (NULL) |
| 130 | 130 | ||
| 131 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX) | 131 | #define SET_PERSONALITY(ex) \ |
| 132 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))) | ||
| 132 | 133 | ||
| 133 | #endif /* !(__ASMSPARC_ELF_H) */ | 134 | #endif /* !(__ASMSPARC_ELF_H) */ |
diff --git a/arch/sparc/include/asm/oplib_32.h b/arch/sparc/include/asm/oplib_32.h index 27517879a6c2..c72f3045820c 100644 --- a/arch/sparc/include/asm/oplib_32.h +++ b/arch/sparc/include/asm/oplib_32.h | |||
| @@ -94,7 +94,7 @@ extern int prom_getprev(void); | |||
| 94 | extern void prom_console_write_buf(const char *buf, int len); | 94 | extern void prom_console_write_buf(const char *buf, int len); |
| 95 | 95 | ||
| 96 | /* Prom's internal routines, don't use in kernel/boot code. */ | 96 | /* Prom's internal routines, don't use in kernel/boot code. */ |
| 97 | extern void prom_printf(const char *fmt, ...); | 97 | extern __printf(1, 2) void prom_printf(const char *fmt, ...); |
| 98 | extern void prom_write(const char *buf, unsigned int len); | 98 | extern void prom_write(const char *buf, unsigned int len); |
| 99 | 99 | ||
| 100 | /* Multiprocessor operations... */ | 100 | /* Multiprocessor operations... */ |
diff --git a/arch/sparc/include/asm/oplib_64.h b/arch/sparc/include/asm/oplib_64.h index 97a90475c314..a12dbe3b7762 100644 --- a/arch/sparc/include/asm/oplib_64.h +++ b/arch/sparc/include/asm/oplib_64.h | |||
| @@ -98,7 +98,7 @@ extern unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); | |||
| 98 | extern void prom_console_write_buf(const char *buf, int len); | 98 | extern void prom_console_write_buf(const char *buf, int len); |
| 99 | 99 | ||
| 100 | /* Prom's internal routines, don't use in kernel/boot code. */ | 100 | /* Prom's internal routines, don't use in kernel/boot code. */ |
| 101 | extern void prom_printf(const char *fmt, ...); | 101 | extern __printf(1, 2) void prom_printf(const char *fmt, ...); |
| 102 | extern void prom_write(const char *buf, unsigned int len); | 102 | extern void prom_write(const char *buf, unsigned int len); |
| 103 | 103 | ||
| 104 | /* Multiprocessor operations... */ | 104 | /* Multiprocessor operations... */ |
diff --git a/arch/sparc/include/asm/siginfo.h b/arch/sparc/include/asm/siginfo.h index 215900fce21b..dbc182c438b4 100644 --- a/arch/sparc/include/asm/siginfo.h +++ b/arch/sparc/include/asm/siginfo.h | |||
| @@ -3,7 +3,6 @@ | |||
| 3 | 3 | ||
| 4 | #if defined(__sparc__) && defined(__arch64__) | 4 | #if defined(__sparc__) && defined(__arch64__) |
| 5 | 5 | ||
| 6 | #define SI_PAD_SIZE32 ((SI_MAX_SIZE/sizeof(int)) - 3) | ||
| 7 | #define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) | 6 | #define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) |
| 8 | #define __ARCH_SI_BAND_T int | 7 | #define __ARCH_SI_BAND_T int |
| 9 | 8 | ||
diff --git a/arch/sparc/include/asm/unistd.h b/arch/sparc/include/asm/unistd.h index fb2693464807..d9a677c51926 100644 --- a/arch/sparc/include/asm/unistd.h +++ b/arch/sparc/include/asm/unistd.h | |||
| @@ -447,6 +447,7 @@ | |||
| 447 | #else | 447 | #else |
| 448 | #define __ARCH_WANT_COMPAT_SYS_TIME | 448 | #define __ARCH_WANT_COMPAT_SYS_TIME |
| 449 | #define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND | 449 | #define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND |
| 450 | #define __ARCH_WANT_COMPAT_SYS_SENDFILE | ||
| 450 | #endif | 451 | #endif |
| 451 | 452 | ||
| 452 | /* | 453 | /* |
diff --git a/arch/sparc/include/uapi/asm/Kbuild b/arch/sparc/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..7518ad286963 --- /dev/null +++ b/arch/sparc/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,5 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | # User exported sparc header files | ||
| 3 | |||
| 4 | include include/uapi/asm-generic/Kbuild.asm | ||
| 5 | |||
diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c index 1032df43ec95..c0a2de0fd624 100644 --- a/arch/sparc/kernel/hvapi.c +++ b/arch/sparc/kernel/hvapi.c | |||
| @@ -194,7 +194,7 @@ void __init sun4v_hvapi_init(void) | |||
| 194 | 194 | ||
| 195 | bad: | 195 | bad: |
| 196 | prom_printf("HVAPI: Cannot register API group " | 196 | prom_printf("HVAPI: Cannot register API group " |
| 197 | "%lx with major(%u) minor(%u)\n", | 197 | "%lx with major(%lu) minor(%lu)\n", |
| 198 | group, major, minor); | 198 | group, major, minor); |
| 199 | prom_halt(); | 199 | prom_halt(); |
| 200 | } | 200 | } |
diff --git a/arch/sparc/kernel/prom_64.c b/arch/sparc/kernel/prom_64.c index 340c5b976d28..d397d7fc5c28 100644 --- a/arch/sparc/kernel/prom_64.c +++ b/arch/sparc/kernel/prom_64.c | |||
| @@ -37,7 +37,7 @@ void * __init prom_early_alloc(unsigned long size) | |||
| 37 | void *ret; | 37 | void *ret; |
| 38 | 38 | ||
| 39 | if (!paddr) { | 39 | if (!paddr) { |
| 40 | prom_printf("prom_early_alloc(%lu) failed\n"); | 40 | prom_printf("prom_early_alloc(%lu) failed\n", size); |
| 41 | prom_halt(); | 41 | prom_halt(); |
| 42 | } | 42 | } |
| 43 | 43 | ||
diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c index a53e0a5fd3a3..53e48f721ce3 100644 --- a/arch/sparc/kernel/signal32.c +++ b/arch/sparc/kernel/signal32.c | |||
| @@ -54,58 +54,6 @@ struct signal_frame32 { | |||
| 54 | /* __siginfo_rwin_t * */u32 rwin_save; | 54 | /* __siginfo_rwin_t * */u32 rwin_save; |
| 55 | } __attribute__((aligned(8))); | 55 | } __attribute__((aligned(8))); |
| 56 | 56 | ||
| 57 | typedef struct compat_siginfo{ | ||
| 58 | int si_signo; | ||
| 59 | int si_errno; | ||
| 60 | int si_code; | ||
| 61 | |||
| 62 | union { | ||
| 63 | int _pad[SI_PAD_SIZE32]; | ||
| 64 | |||
| 65 | /* kill() */ | ||
| 66 | struct { | ||
| 67 | compat_pid_t _pid; /* sender's pid */ | ||
| 68 | unsigned int _uid; /* sender's uid */ | ||
| 69 | } _kill; | ||
| 70 | |||
| 71 | /* POSIX.1b timers */ | ||
| 72 | struct { | ||
| 73 | compat_timer_t _tid; /* timer id */ | ||
| 74 | int _overrun; /* overrun count */ | ||
| 75 | compat_sigval_t _sigval; /* same as below */ | ||
| 76 | int _sys_private; /* not to be passed to user */ | ||
| 77 | } _timer; | ||
| 78 | |||
| 79 | /* POSIX.1b signals */ | ||
| 80 | struct { | ||
| 81 | compat_pid_t _pid; /* sender's pid */ | ||
| 82 | unsigned int _uid; /* sender's uid */ | ||
| 83 | compat_sigval_t _sigval; | ||
| 84 | } _rt; | ||
| 85 | |||
| 86 | /* SIGCHLD */ | ||
| 87 | struct { | ||
| 88 | compat_pid_t _pid; /* which child */ | ||
| 89 | unsigned int _uid; /* sender's uid */ | ||
| 90 | int _status; /* exit code */ | ||
| 91 | compat_clock_t _utime; | ||
| 92 | compat_clock_t _stime; | ||
| 93 | } _sigchld; | ||
| 94 | |||
| 95 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ | ||
| 96 | struct { | ||
| 97 | u32 _addr; /* faulting insn/memory ref. */ | ||
| 98 | int _trapno; | ||
| 99 | } _sigfault; | ||
| 100 | |||
| 101 | /* SIGPOLL */ | ||
| 102 | struct { | ||
| 103 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 104 | int _fd; | ||
| 105 | } _sigpoll; | ||
| 106 | } _sifields; | ||
| 107 | }compat_siginfo_t; | ||
| 108 | |||
| 109 | struct rt_signal_frame32 { | 57 | struct rt_signal_frame32 { |
| 110 | struct sparc_stackf32 ss; | 58 | struct sparc_stackf32 ss; |
| 111 | compat_siginfo_t info; | 59 | compat_siginfo_t info; |
diff --git a/arch/sparc/kernel/sys32.S b/arch/sparc/kernel/sys32.S index d97f3eb72e06..44025f4ba41f 100644 --- a/arch/sparc/kernel/sys32.S +++ b/arch/sparc/kernel/sys32.S | |||
| @@ -90,7 +90,7 @@ SIGN1(sys32_mkdir, sys_mkdir, %o1) | |||
| 90 | SIGN3(sys32_futex, compat_sys_futex, %o1, %o2, %o5) | 90 | SIGN3(sys32_futex, compat_sys_futex, %o1, %o2, %o5) |
| 91 | SIGN1(sys32_sysfs, compat_sys_sysfs, %o0) | 91 | SIGN1(sys32_sysfs, compat_sys_sysfs, %o0) |
| 92 | SIGN2(sys32_sendfile, compat_sys_sendfile, %o0, %o1) | 92 | SIGN2(sys32_sendfile, compat_sys_sendfile, %o0, %o1) |
| 93 | SIGN2(sys32_sendfile64, compat_sys_sendfile64, %o0, %o1) | 93 | SIGN2(sys32_sendfile64, sys_sendfile, %o0, %o1) |
| 94 | SIGN1(sys32_prctl, sys_prctl, %o0) | 94 | SIGN1(sys32_prctl, sys_prctl, %o0) |
| 95 | SIGN1(sys32_sched_rr_get_interval, compat_sys_sched_rr_get_interval, %o0) | 95 | SIGN1(sys32_sched_rr_get_interval, compat_sys_sched_rr_get_interval, %o0) |
| 96 | SIGN2(sys32_waitpid, sys_waitpid, %o0, %o2) | 96 | SIGN2(sys32_waitpid, sys_waitpid, %o0, %o2) |
diff --git a/arch/sparc/kernel/sys_sparc32.c b/arch/sparc/kernel/sys_sparc32.c index f7392336961f..d862499eb01c 100644 --- a/arch/sparc/kernel/sys_sparc32.c +++ b/arch/sparc/kernel/sys_sparc32.c | |||
| @@ -506,52 +506,6 @@ long compat_sys_fadvise64_64(int fd, | |||
| 506 | advice); | 506 | advice); |
| 507 | } | 507 | } |
| 508 | 508 | ||
| 509 | asmlinkage long compat_sys_sendfile(int out_fd, int in_fd, | ||
| 510 | compat_off_t __user *offset, | ||
| 511 | compat_size_t count) | ||
| 512 | { | ||
| 513 | mm_segment_t old_fs = get_fs(); | ||
| 514 | int ret; | ||
| 515 | off_t of; | ||
| 516 | |||
| 517 | if (offset && get_user(of, offset)) | ||
| 518 | return -EFAULT; | ||
| 519 | |||
| 520 | set_fs(KERNEL_DS); | ||
| 521 | ret = sys_sendfile(out_fd, in_fd, | ||
| 522 | offset ? (off_t __user *) &of : NULL, | ||
| 523 | count); | ||
| 524 | set_fs(old_fs); | ||
| 525 | |||
| 526 | if (offset && put_user(of, offset)) | ||
| 527 | return -EFAULT; | ||
| 528 | |||
| 529 | return ret; | ||
| 530 | } | ||
| 531 | |||
| 532 | asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd, | ||
| 533 | compat_loff_t __user *offset, | ||
| 534 | compat_size_t count) | ||
| 535 | { | ||
| 536 | mm_segment_t old_fs = get_fs(); | ||
| 537 | int ret; | ||
| 538 | loff_t lof; | ||
| 539 | |||
| 540 | if (offset && get_user(lof, offset)) | ||
| 541 | return -EFAULT; | ||
| 542 | |||
| 543 | set_fs(KERNEL_DS); | ||
| 544 | ret = sys_sendfile64(out_fd, in_fd, | ||
| 545 | offset ? (loff_t __user *) &lof : NULL, | ||
| 546 | count); | ||
| 547 | set_fs(old_fs); | ||
| 548 | |||
| 549 | if (offset && put_user(lof, offset)) | ||
| 550 | return -EFAULT; | ||
| 551 | |||
| 552 | return ret; | ||
| 553 | } | ||
| 554 | |||
| 555 | /* This is just a version for 32-bit applications which does | 509 | /* This is just a version for 32-bit applications which does |
| 556 | * not force O_LARGEFILE on. | 510 | * not force O_LARGEFILE on. |
| 557 | */ | 511 | */ |
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 3b05e6697710..fa1f1d375ffc 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c | |||
| @@ -850,7 +850,7 @@ void __init cheetah_ecache_flush_init(void) | |||
| 850 | ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size); | 850 | ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size); |
| 851 | 851 | ||
| 852 | if (ecache_flush_physbase == ~0UL) { | 852 | if (ecache_flush_physbase == ~0UL) { |
| 853 | prom_printf("cheetah_ecache_flush_init: Cannot find %d byte " | 853 | prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte " |
| 854 | "contiguous physical memory.\n", | 854 | "contiguous physical memory.\n", |
| 855 | ecache_flush_size); | 855 | ecache_flush_size); |
| 856 | prom_halt(); | 856 | prom_halt(); |
diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S index 03eadf66b0d3..2c20ad63ddbf 100644 --- a/arch/sparc/lib/NG2memcpy.S +++ b/arch/sparc/lib/NG2memcpy.S | |||
| @@ -14,7 +14,7 @@ | |||
| 14 | #define FPRS_FEF 0x04 | 14 | #define FPRS_FEF 0x04 |
| 15 | #ifdef MEMCPY_DEBUG | 15 | #ifdef MEMCPY_DEBUG |
| 16 | #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ | 16 | #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ |
| 17 | clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; | 17 | clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0; |
| 18 | #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs | 18 | #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs |
| 19 | #else | 19 | #else |
| 20 | #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs | 20 | #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs |
| @@ -182,13 +182,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 182 | cmp %g2, 0 | 182 | cmp %g2, 0 |
| 183 | tne %xcc, 5 | 183 | tne %xcc, 5 |
| 184 | PREAMBLE | 184 | PREAMBLE |
| 185 | mov %o0, GLOBAL_SPARE | 185 | mov %o0, %o3 |
| 186 | cmp %o2, 0 | 186 | cmp %o2, 0 |
| 187 | be,pn %XCC, 85f | 187 | be,pn %XCC, 85f |
| 188 | or %o0, %o1, %o3 | 188 | or %o0, %o1, GLOBAL_SPARE |
| 189 | cmp %o2, 16 | 189 | cmp %o2, 16 |
| 190 | blu,a,pn %XCC, 80f | 190 | blu,a,pn %XCC, 80f |
| 191 | or %o3, %o2, %o3 | 191 | or GLOBAL_SPARE, %o2, GLOBAL_SPARE |
| 192 | 192 | ||
| 193 | /* 2 blocks (128 bytes) is the minimum we can do the block | 193 | /* 2 blocks (128 bytes) is the minimum we can do the block |
| 194 | * copy with. We need to ensure that we'll iterate at least | 194 | * copy with. We need to ensure that we'll iterate at least |
| @@ -202,7 +202,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 202 | */ | 202 | */ |
| 203 | cmp %o2, (4 * 64) | 203 | cmp %o2, (4 * 64) |
| 204 | blu,pt %XCC, 75f | 204 | blu,pt %XCC, 75f |
| 205 | andcc %o3, 0x7, %g0 | 205 | andcc GLOBAL_SPARE, 0x7, %g0 |
| 206 | 206 | ||
| 207 | /* %o0: dst | 207 | /* %o0: dst |
| 208 | * %o1: src | 208 | * %o1: src |
| @@ -404,13 +404,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 404 | * over. If anything is left, we copy it one byte at a time. | 404 | * over. If anything is left, we copy it one byte at a time. |
| 405 | */ | 405 | */ |
| 406 | brz,pt %o2, 85f | 406 | brz,pt %o2, 85f |
| 407 | sub %o0, %o1, %o3 | 407 | sub %o0, %o1, GLOBAL_SPARE |
| 408 | ba,a,pt %XCC, 90f | 408 | ba,a,pt %XCC, 90f |
| 409 | 409 | ||
| 410 | .align 64 | 410 | .align 64 |
| 411 | 75: /* 16 < len <= 64 */ | 411 | 75: /* 16 < len <= 64 */ |
| 412 | bne,pn %XCC, 75f | 412 | bne,pn %XCC, 75f |
| 413 | sub %o0, %o1, %o3 | 413 | sub %o0, %o1, GLOBAL_SPARE |
| 414 | 414 | ||
| 415 | 72: | 415 | 72: |
| 416 | andn %o2, 0xf, %o4 | 416 | andn %o2, 0xf, %o4 |
| @@ -420,9 +420,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 420 | add %o1, 0x08, %o1 | 420 | add %o1, 0x08, %o1 |
| 421 | EX_LD(LOAD(ldx, %o1, %g1)) | 421 | EX_LD(LOAD(ldx, %o1, %g1)) |
| 422 | sub %o1, 0x08, %o1 | 422 | sub %o1, 0x08, %o1 |
| 423 | EX_ST(STORE(stx, %o5, %o1 + %o3)) | 423 | EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) |
| 424 | add %o1, 0x8, %o1 | 424 | add %o1, 0x8, %o1 |
| 425 | EX_ST(STORE(stx, %g1, %o1 + %o3)) | 425 | EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) |
| 426 | bgu,pt %XCC, 1b | 426 | bgu,pt %XCC, 1b |
| 427 | add %o1, 0x8, %o1 | 427 | add %o1, 0x8, %o1 |
| 428 | 73: andcc %o2, 0x8, %g0 | 428 | 73: andcc %o2, 0x8, %g0 |
| @@ -430,14 +430,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 430 | nop | 430 | nop |
| 431 | sub %o2, 0x8, %o2 | 431 | sub %o2, 0x8, %o2 |
| 432 | EX_LD(LOAD(ldx, %o1, %o5)) | 432 | EX_LD(LOAD(ldx, %o1, %o5)) |
| 433 | EX_ST(STORE(stx, %o5, %o1 + %o3)) | 433 | EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) |
| 434 | add %o1, 0x8, %o1 | 434 | add %o1, 0x8, %o1 |
| 435 | 1: andcc %o2, 0x4, %g0 | 435 | 1: andcc %o2, 0x4, %g0 |
| 436 | be,pt %XCC, 1f | 436 | be,pt %XCC, 1f |
| 437 | nop | 437 | nop |
| 438 | sub %o2, 0x4, %o2 | 438 | sub %o2, 0x4, %o2 |
| 439 | EX_LD(LOAD(lduw, %o1, %o5)) | 439 | EX_LD(LOAD(lduw, %o1, %o5)) |
| 440 | EX_ST(STORE(stw, %o5, %o1 + %o3)) | 440 | EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) |
| 441 | add %o1, 0x4, %o1 | 441 | add %o1, 0x4, %o1 |
| 442 | 1: cmp %o2, 0 | 442 | 1: cmp %o2, 0 |
| 443 | be,pt %XCC, 85f | 443 | be,pt %XCC, 85f |
| @@ -454,11 +454,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 454 | 454 | ||
| 455 | 1: subcc %g1, 1, %g1 | 455 | 1: subcc %g1, 1, %g1 |
| 456 | EX_LD(LOAD(ldub, %o1, %o5)) | 456 | EX_LD(LOAD(ldub, %o1, %o5)) |
| 457 | EX_ST(STORE(stb, %o5, %o1 + %o3)) | 457 | EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) |
| 458 | bgu,pt %icc, 1b | 458 | bgu,pt %icc, 1b |
| 459 | add %o1, 1, %o1 | 459 | add %o1, 1, %o1 |
| 460 | 460 | ||
| 461 | 2: add %o1, %o3, %o0 | 461 | 2: add %o1, GLOBAL_SPARE, %o0 |
| 462 | andcc %o1, 0x7, %g1 | 462 | andcc %o1, 0x7, %g1 |
| 463 | bne,pt %icc, 8f | 463 | bne,pt %icc, 8f |
| 464 | sll %g1, 3, %g1 | 464 | sll %g1, 3, %g1 |
| @@ -468,16 +468,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 468 | nop | 468 | nop |
| 469 | ba,a,pt %xcc, 73b | 469 | ba,a,pt %xcc, 73b |
| 470 | 470 | ||
| 471 | 8: mov 64, %o3 | 471 | 8: mov 64, GLOBAL_SPARE |
| 472 | andn %o1, 0x7, %o1 | 472 | andn %o1, 0x7, %o1 |
| 473 | EX_LD(LOAD(ldx, %o1, %g2)) | 473 | EX_LD(LOAD(ldx, %o1, %g2)) |
| 474 | sub %o3, %g1, %o3 | 474 | sub GLOBAL_SPARE, %g1, GLOBAL_SPARE |
| 475 | andn %o2, 0x7, %o4 | 475 | andn %o2, 0x7, %o4 |
| 476 | sllx %g2, %g1, %g2 | 476 | sllx %g2, %g1, %g2 |
| 477 | 1: add %o1, 0x8, %o1 | 477 | 1: add %o1, 0x8, %o1 |
| 478 | EX_LD(LOAD(ldx, %o1, %g3)) | 478 | EX_LD(LOAD(ldx, %o1, %g3)) |
| 479 | subcc %o4, 0x8, %o4 | 479 | subcc %o4, 0x8, %o4 |
| 480 | srlx %g3, %o3, %o5 | 480 | srlx %g3, GLOBAL_SPARE, %o5 |
| 481 | or %o5, %g2, %o5 | 481 | or %o5, %g2, %o5 |
| 482 | EX_ST(STORE(stx, %o5, %o0)) | 482 | EX_ST(STORE(stx, %o5, %o0)) |
| 483 | add %o0, 0x8, %o0 | 483 | add %o0, 0x8, %o0 |
| @@ -489,32 +489,32 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |||
| 489 | be,pn %icc, 85f | 489 | be,pn %icc, 85f |
| 490 | add %o1, %g1, %o1 | 490 | add %o1, %g1, %o1 |
| 491 | ba,pt %xcc, 90f | 491 | ba,pt %xcc, 90f |
| 492 | sub %o0, %o1, %o3 | 492 | sub %o0, %o1, GLOBAL_SPARE |
| 493 | 493 | ||
| 494 | .align 64 | 494 | .align 64 |
| 495 | 80: /* 0 < len <= 16 */ | 495 | 80: /* 0 < len <= 16 */ |
| 496 | andcc %o3, 0x3, %g0 | 496 | andcc GLOBAL_SPARE, 0x3, %g0 |
| 497 | bne,pn %XCC, 90f | 497 | bne,pn %XCC, 90f |
| 498 | sub %o0, %o1, %o3 | 498 | sub %o0, %o1, GLOBAL_SPARE |
| 499 | 499 | ||
| 500 | 1: | 500 | 1: |
| 501 | subcc %o2, 4, %o2 | 501 | subcc %o2, 4, %o2 |
| 502 | EX_LD(LOAD(lduw, %o1, %g1)) | 502 | EX_LD(LOAD(lduw, %o1, %g1)) |
| 503 | EX_ST(STORE(stw, %g1, %o1 + %o3)) | 503 | EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) |
| 504 | bgu,pt %XCC, 1b | 504 | bgu,pt %XCC, 1b |
| 505 | add %o1, 4, %o1 | 505 | add %o1, 4, %o1 |
| 506 | 506 | ||
| 507 | 85: retl | 507 | 85: retl |
| 508 | mov EX_RETVAL(GLOBAL_SPARE), %o0 | 508 | mov EX_RETVAL(%o3), %o0 |
| 509 | 509 | ||
| 510 | .align 32 | 510 | .align 32 |
| 511 | 90: | 511 | 90: |
| 512 | subcc %o2, 1, %o2 | 512 | subcc %o2, 1, %o2 |
| 513 | EX_LD(LOAD(ldub, %o1, %g1)) | 513 | EX_LD(LOAD(ldub, %o1, %g1)) |
| 514 | EX_ST(STORE(stb, %g1, %o1 + %o3)) | 514 | EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) |
| 515 | bgu,pt %XCC, 90b | 515 | bgu,pt %XCC, 90b |
| 516 | add %o1, 1, %o1 | 516 | add %o1, 1, %o1 |
| 517 | retl | 517 | retl |
| 518 | mov EX_RETVAL(GLOBAL_SPARE), %o0 | 518 | mov EX_RETVAL(%o3), %o0 |
| 519 | 519 | ||
| 520 | .size FUNC_NAME, .-FUNC_NAME | 520 | .size FUNC_NAME, .-FUNC_NAME |
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 696bb095e0fc..7a9b788c6ced 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c | |||
| @@ -119,7 +119,8 @@ static void __init read_obp_memory(const char *property, | |||
| 119 | 119 | ||
| 120 | ret = prom_getproperty(node, property, (char *) regs, prop_size); | 120 | ret = prom_getproperty(node, property, (char *) regs, prop_size); |
| 121 | if (ret == -1) { | 121 | if (ret == -1) { |
| 122 | prom_printf("Couldn't get %s property from /memory.\n"); | 122 | prom_printf("Couldn't get %s property from /memory.\n", |
| 123 | property); | ||
| 123 | prom_halt(); | 124 | prom_halt(); |
| 124 | } | 125 | } |
| 125 | 126 | ||
| @@ -497,7 +498,7 @@ static void __init read_obp_translations(void) | |||
| 497 | prom_halt(); | 498 | prom_halt(); |
| 498 | } | 499 | } |
| 499 | if (unlikely(n > sizeof(prom_trans))) { | 500 | if (unlikely(n > sizeof(prom_trans))) { |
| 500 | prom_printf("prom_mappings: Size %Zd is too big.\n", n); | 501 | prom_printf("prom_mappings: Size %d is too big.\n", n); |
| 501 | prom_halt(); | 502 | prom_halt(); |
| 502 | } | 503 | } |
| 503 | 504 | ||
| @@ -559,7 +560,7 @@ static void __init hypervisor_tlb_lock(unsigned long vaddr, | |||
| 559 | unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu); | 560 | unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu); |
| 560 | 561 | ||
| 561 | if (ret != 0) { | 562 | if (ret != 0) { |
| 562 | prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: " | 563 | prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: " |
| 563 | "errors with %lx\n", vaddr, 0, pte, mmu, ret); | 564 | "errors with %lx\n", vaddr, 0, pte, mmu, ret); |
| 564 | prom_halt(); | 565 | prom_halt(); |
| 565 | } | 566 | } |
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index a8a58cad9d2b..0f4f7191fbba 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c | |||
| @@ -90,8 +90,8 @@ static void __init sbus_iommu_init(struct platform_device *op) | |||
| 90 | it to us. */ | 90 | it to us. */ |
| 91 | tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); | 91 | tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); |
| 92 | if (!tmp) { | 92 | if (!tmp) { |
| 93 | prom_printf("Unable to allocate iommu table [0x%08x]\n", | 93 | prom_printf("Unable to allocate iommu table [0x%lx]\n", |
| 94 | IOMMU_NPTES*sizeof(iopte_t)); | 94 | IOMMU_NPTES * sizeof(iopte_t)); |
| 95 | prom_halt(); | 95 | prom_halt(); |
| 96 | } | 96 | } |
| 97 | iommu->page_table = (iopte_t *)tmp; | 97 | iommu->page_table = (iopte_t *)tmp; |
diff --git a/arch/tile/include/asm/compat.h b/arch/tile/include/asm/compat.h index 6e74450ff0a1..3063e6fc8daa 100644 --- a/arch/tile/include/asm/compat.h +++ b/arch/tile/include/asm/compat.h | |||
| @@ -110,6 +110,68 @@ struct compat_flock64 { | |||
| 110 | 110 | ||
| 111 | typedef u32 compat_sigset_word; | 111 | typedef u32 compat_sigset_word; |
| 112 | 112 | ||
| 113 | typedef union compat_sigval { | ||
| 114 | compat_int_t sival_int; | ||
| 115 | compat_uptr_t sival_ptr; | ||
| 116 | } compat_sigval_t; | ||
| 117 | |||
| 118 | #define COMPAT_SI_PAD_SIZE (128/sizeof(int) - 3) | ||
| 119 | |||
| 120 | typedef struct compat_siginfo { | ||
| 121 | int si_signo; | ||
| 122 | int si_errno; | ||
| 123 | int si_code; | ||
| 124 | |||
| 125 | union { | ||
| 126 | int _pad[COMPAT_SI_PAD_SIZE]; | ||
| 127 | |||
| 128 | /* kill() */ | ||
| 129 | struct { | ||
| 130 | unsigned int _pid; /* sender's pid */ | ||
| 131 | unsigned int _uid; /* sender's uid */ | ||
| 132 | } _kill; | ||
| 133 | |||
| 134 | /* POSIX.1b timers */ | ||
| 135 | struct { | ||
| 136 | compat_timer_t _tid; /* timer id */ | ||
| 137 | int _overrun; /* overrun count */ | ||
| 138 | compat_sigval_t _sigval; /* same as below */ | ||
| 139 | int _sys_private; /* not to be passed to user */ | ||
| 140 | int _overrun_incr; /* amount to add to overrun */ | ||
| 141 | } _timer; | ||
| 142 | |||
| 143 | /* POSIX.1b signals */ | ||
| 144 | struct { | ||
| 145 | unsigned int _pid; /* sender's pid */ | ||
| 146 | unsigned int _uid; /* sender's uid */ | ||
| 147 | compat_sigval_t _sigval; | ||
| 148 | } _rt; | ||
| 149 | |||
| 150 | /* SIGCHLD */ | ||
| 151 | struct { | ||
| 152 | unsigned int _pid; /* which child */ | ||
| 153 | unsigned int _uid; /* sender's uid */ | ||
| 154 | int _status; /* exit code */ | ||
| 155 | compat_clock_t _utime; | ||
| 156 | compat_clock_t _stime; | ||
| 157 | } _sigchld; | ||
| 158 | |||
| 159 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 160 | struct { | ||
| 161 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 162 | #ifdef __ARCH_SI_TRAPNO | ||
| 163 | int _trapno; /* TRAP # which caused the signal */ | ||
| 164 | #endif | ||
| 165 | } _sigfault; | ||
| 166 | |||
| 167 | /* SIGPOLL */ | ||
| 168 | struct { | ||
| 169 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 170 | int _fd; | ||
| 171 | } _sigpoll; | ||
| 172 | } _sifields; | ||
| 173 | } compat_siginfo_t; | ||
| 174 | |||
| 113 | #define COMPAT_OFF_T_MAX 0x7fffffff | 175 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 114 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 176 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 115 | 177 | ||
diff --git a/arch/tile/include/asm/elf.h b/arch/tile/include/asm/elf.h index d16d006d660e..f8ccf08f6934 100644 --- a/arch/tile/include/asm/elf.h +++ b/arch/tile/include/asm/elf.h | |||
| @@ -156,12 +156,12 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm, | |||
| 156 | #undef SET_PERSONALITY | 156 | #undef SET_PERSONALITY |
| 157 | #define SET_PERSONALITY(ex) \ | 157 | #define SET_PERSONALITY(ex) \ |
| 158 | do { \ | 158 | do { \ |
| 159 | current->personality = PER_LINUX; \ | 159 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))); \ |
| 160 | current_thread_info()->status &= ~TS_COMPAT; \ | 160 | current_thread_info()->status &= ~TS_COMPAT; \ |
| 161 | } while (0) | 161 | } while (0) |
| 162 | #define COMPAT_SET_PERSONALITY(ex) \ | 162 | #define COMPAT_SET_PERSONALITY(ex) \ |
| 163 | do { \ | 163 | do { \ |
| 164 | current->personality = PER_LINUX_32BIT; \ | 164 | set_personality(PER_LINUX | (current->personality & (~PER_MASK))); \ |
| 165 | current_thread_info()->status |= TS_COMPAT; \ | 165 | current_thread_info()->status |= TS_COMPAT; \ |
| 166 | } while (0) | 166 | } while (0) |
| 167 | 167 | ||
diff --git a/arch/tile/include/gxio/dma_queue.h b/arch/tile/include/gxio/dma_queue.h index 00654feb7db0..b9e45e37649e 100644 --- a/arch/tile/include/gxio/dma_queue.h +++ b/arch/tile/include/gxio/dma_queue.h | |||
| @@ -19,7 +19,7 @@ | |||
| 19 | * DMA queue management APIs shared between TRIO and mPIPE. | 19 | * DMA queue management APIs shared between TRIO and mPIPE. |
| 20 | */ | 20 | */ |
| 21 | 21 | ||
| 22 | #include "common.h" | 22 | #include <gxio/common.h> |
| 23 | 23 | ||
| 24 | /* The credit counter lives in the high 32 bits. */ | 24 | /* The credit counter lives in the high 32 bits. */ |
| 25 | #define DMA_QUEUE_CREDIT_SHIFT 32 | 25 | #define DMA_QUEUE_CREDIT_SHIFT 32 |
diff --git a/arch/tile/include/gxio/mpipe.h b/arch/tile/include/gxio/mpipe.h index 78c598618c97..b74f470ed11e 100644 --- a/arch/tile/include/gxio/mpipe.h +++ b/arch/tile/include/gxio/mpipe.h | |||
| @@ -21,8 +21,8 @@ | |||
| 21 | * resources. | 21 | * resources. |
| 22 | */ | 22 | */ |
| 23 | 23 | ||
| 24 | #include "common.h" | 24 | #include <gxio/common.h> |
| 25 | #include "dma_queue.h" | 25 | #include <gxio/dma_queue.h> |
| 26 | 26 | ||
| 27 | #include <linux/time.h> | 27 | #include <linux/time.h> |
| 28 | 28 | ||
diff --git a/arch/tile/include/gxio/trio.h b/arch/tile/include/gxio/trio.h index 77b80cdd46d8..df10a662cc25 100644 --- a/arch/tile/include/gxio/trio.h +++ b/arch/tile/include/gxio/trio.h | |||
| @@ -140,8 +140,8 @@ | |||
| 140 | 140 | ||
| 141 | #include <linux/types.h> | 141 | #include <linux/types.h> |
| 142 | 142 | ||
| 143 | #include "common.h" | 143 | #include <gxio/common.h> |
| 144 | #include "dma_queue.h" | 144 | #include <gxio/dma_queue.h> |
| 145 | 145 | ||
| 146 | #include <arch/trio_constants.h> | 146 | #include <arch/trio_constants.h> |
| 147 | #include <arch/trio.h> | 147 | #include <arch/trio.h> |
diff --git a/arch/tile/include/gxio/usb_host.h b/arch/tile/include/gxio/usb_host.h index a60a126e4565..5eedec0e988e 100644 --- a/arch/tile/include/gxio/usb_host.h +++ b/arch/tile/include/gxio/usb_host.h | |||
| @@ -14,7 +14,7 @@ | |||
| 14 | #ifndef _GXIO_USB_H_ | 14 | #ifndef _GXIO_USB_H_ |
| 15 | #define _GXIO_USB_H_ | 15 | #define _GXIO_USB_H_ |
| 16 | 16 | ||
| 17 | #include "common.h" | 17 | #include <gxio/common.h> |
| 18 | 18 | ||
| 19 | #include <hv/drv_usb_host_intf.h> | 19 | #include <hv/drv_usb_host_intf.h> |
| 20 | #include <hv/iorpc.h> | 20 | #include <hv/iorpc.h> |
diff --git a/arch/tile/include/hv/iorpc.h b/arch/tile/include/hv/iorpc.h index 89c72a5d9341..ddf1604482b3 100644 --- a/arch/tile/include/hv/iorpc.h +++ b/arch/tile/include/hv/iorpc.h | |||
| @@ -248,7 +248,7 @@ | |||
| 248 | #if defined(__HV__) | 248 | #if defined(__HV__) |
| 249 | #include <hv/hypervisor.h> | 249 | #include <hv/hypervisor.h> |
| 250 | #elif defined(__KERNEL__) | 250 | #elif defined(__KERNEL__) |
| 251 | #include "hypervisor.h" | 251 | #include <hv/hypervisor.h> |
| 252 | #include <linux/types.h> | 252 | #include <linux/types.h> |
| 253 | #else | 253 | #else |
| 254 | #include <stdint.h> | 254 | #include <stdint.h> |
diff --git a/arch/tile/include/uapi/arch/Kbuild b/arch/tile/include/uapi/arch/Kbuild new file mode 100644 index 000000000000..aafaa5aa54d4 --- /dev/null +++ b/arch/tile/include/uapi/arch/Kbuild | |||
| @@ -0,0 +1 @@ | |||
| # UAPI Header export list | |||
diff --git a/arch/tile/include/uapi/asm/Kbuild b/arch/tile/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/tile/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/tile/kernel/compat_signal.c b/arch/tile/kernel/compat_signal.c index 474571b84085..7bc0859a9f5e 100644 --- a/arch/tile/kernel/compat_signal.c +++ b/arch/tile/kernel/compat_signal.c | |||
| @@ -55,63 +55,6 @@ struct compat_ucontext { | |||
| 55 | sigset_t uc_sigmask; /* mask last for extensibility */ | 55 | sigset_t uc_sigmask; /* mask last for extensibility */ |
| 56 | }; | 56 | }; |
| 57 | 57 | ||
| 58 | #define COMPAT_SI_PAD_SIZE ((SI_MAX_SIZE - 3 * sizeof(int)) / sizeof(int)) | ||
| 59 | |||
| 60 | struct compat_siginfo { | ||
| 61 | int si_signo; | ||
| 62 | int si_errno; | ||
| 63 | int si_code; | ||
| 64 | |||
| 65 | union { | ||
| 66 | int _pad[COMPAT_SI_PAD_SIZE]; | ||
| 67 | |||
| 68 | /* kill() */ | ||
| 69 | struct { | ||
| 70 | unsigned int _pid; /* sender's pid */ | ||
| 71 | unsigned int _uid; /* sender's uid */ | ||
| 72 | } _kill; | ||
| 73 | |||
| 74 | /* POSIX.1b timers */ | ||
| 75 | struct { | ||
| 76 | compat_timer_t _tid; /* timer id */ | ||
| 77 | int _overrun; /* overrun count */ | ||
| 78 | compat_sigval_t _sigval; /* same as below */ | ||
| 79 | int _sys_private; /* not to be passed to user */ | ||
| 80 | int _overrun_incr; /* amount to add to overrun */ | ||
| 81 | } _timer; | ||
| 82 | |||
| 83 | /* POSIX.1b signals */ | ||
| 84 | struct { | ||
| 85 | unsigned int _pid; /* sender's pid */ | ||
| 86 | unsigned int _uid; /* sender's uid */ | ||
| 87 | compat_sigval_t _sigval; | ||
| 88 | } _rt; | ||
| 89 | |||
| 90 | /* SIGCHLD */ | ||
| 91 | struct { | ||
| 92 | unsigned int _pid; /* which child */ | ||
| 93 | unsigned int _uid; /* sender's uid */ | ||
| 94 | int _status; /* exit code */ | ||
| 95 | compat_clock_t _utime; | ||
| 96 | compat_clock_t _stime; | ||
| 97 | } _sigchld; | ||
| 98 | |||
| 99 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 100 | struct { | ||
| 101 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 102 | #ifdef __ARCH_SI_TRAPNO | ||
| 103 | int _trapno; /* TRAP # which caused the signal */ | ||
| 104 | #endif | ||
| 105 | } _sigfault; | ||
| 106 | |||
| 107 | /* SIGPOLL */ | ||
| 108 | struct { | ||
| 109 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 110 | int _fd; | ||
| 111 | } _sigpoll; | ||
| 112 | } _sifields; | ||
| 113 | }; | ||
| 114 | |||
| 115 | struct compat_rt_sigframe { | 58 | struct compat_rt_sigframe { |
| 116 | unsigned char save_area[C_ABI_SAVE_AREA_SIZE]; /* caller save area */ | 59 | unsigned char save_area[C_ABI_SAVE_AREA_SIZE]; /* caller save area */ |
| 117 | struct compat_siginfo info; | 60 | struct compat_siginfo info; |
diff --git a/arch/um/Makefile b/arch/um/Makefile index 097091059aaa..133f7de2a13d 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile | |||
| @@ -66,7 +66,9 @@ USER_CFLAGS = $(patsubst $(KERNEL_DEFINES),,$(patsubst -D__KERNEL__,,\ | |||
| 66 | include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS) | 66 | include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS) |
| 67 | 67 | ||
| 68 | KBUILD_CPPFLAGS += -I$(srctree)/$(HOST_DIR)/include \ | 68 | KBUILD_CPPFLAGS += -I$(srctree)/$(HOST_DIR)/include \ |
| 69 | -I$(HOST_DIR)/include/generated | 69 | -I$(srctree)/$(HOST_DIR)/include/uapi \ |
| 70 | -I$(HOST_DIR)/include/generated \ | ||
| 71 | -I$(HOST_DIR)/include/generated/uapi | ||
| 70 | 72 | ||
| 71 | # -Derrno=kernel_errno - This turns all kernel references to errno into | 73 | # -Derrno=kernel_errno - This turns all kernel references to errno into |
| 72 | # kernel_errno to separate them from the libc errno. This allows -fno-common | 74 | # kernel_errno to separate them from the libc errno. This allows -fno-common |
diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c index c17de0db6736..9efeb6da48bc 100644 --- a/arch/um/drivers/mconsole_kern.c +++ b/arch/um/drivers/mconsole_kern.c | |||
| @@ -21,6 +21,9 @@ | |||
| 21 | #include <linux/un.h> | 21 | #include <linux/un.h> |
| 22 | #include <linux/workqueue.h> | 22 | #include <linux/workqueue.h> |
| 23 | #include <linux/mutex.h> | 23 | #include <linux/mutex.h> |
| 24 | #include <linux/fs.h> | ||
| 25 | #include <linux/mount.h> | ||
| 26 | #include <linux/file.h> | ||
| 24 | #include <asm/uaccess.h> | 27 | #include <asm/uaccess.h> |
| 25 | #include <asm/switch_to.h> | 28 | #include <asm/switch_to.h> |
| 26 | 29 | ||
| @@ -118,90 +121,38 @@ void mconsole_log(struct mc_request *req) | |||
| 118 | mconsole_reply(req, "", 0, 0); | 121 | mconsole_reply(req, "", 0, 0); |
| 119 | } | 122 | } |
| 120 | 123 | ||
| 121 | /* This is a more convoluted version of mconsole_proc, which has some stability | ||
| 122 | * problems; however, we need it fixed, because it is expected that UML users | ||
| 123 | * mount HPPFS instead of procfs on /proc. And we want mconsole_proc to still | ||
| 124 | * show the real procfs content, not the ones from hppfs.*/ | ||
| 125 | #if 0 | ||
| 126 | void mconsole_proc(struct mc_request *req) | 124 | void mconsole_proc(struct mc_request *req) |
| 127 | { | 125 | { |
| 128 | struct vfsmount *mnt = current->nsproxy->pid_ns->proc_mnt; | 126 | struct vfsmount *mnt = current->nsproxy->pid_ns->proc_mnt; |
| 129 | struct file *file; | ||
| 130 | int n; | ||
| 131 | char *ptr = req->request.data, *buf; | ||
| 132 | mm_segment_t old_fs = get_fs(); | ||
| 133 | |||
| 134 | ptr += strlen("proc"); | ||
| 135 | ptr = skip_spaces(ptr); | ||
| 136 | |||
| 137 | file = file_open_root(mnt->mnt_root, mnt, ptr, O_RDONLY); | ||
| 138 | if (IS_ERR(file)) { | ||
| 139 | mconsole_reply(req, "Failed to open file", 1, 0); | ||
| 140 | goto out; | ||
| 141 | } | ||
| 142 | |||
| 143 | buf = kmalloc(PAGE_SIZE, GFP_KERNEL); | ||
| 144 | if (buf == NULL) { | ||
| 145 | mconsole_reply(req, "Failed to allocate buffer", 1, 0); | ||
| 146 | goto out_fput; | ||
| 147 | } | ||
| 148 | |||
| 149 | if (file->f_op->read) { | ||
| 150 | do { | ||
| 151 | loff_t pos; | ||
| 152 | set_fs(KERNEL_DS); | ||
| 153 | n = vfs_read(file, buf, PAGE_SIZE - 1, &pos); | ||
| 154 | file_pos_write(file, pos); | ||
| 155 | set_fs(old_fs); | ||
| 156 | if (n >= 0) { | ||
| 157 | buf[n] = '\0'; | ||
| 158 | mconsole_reply(req, buf, 0, (n > 0)); | ||
| 159 | } | ||
| 160 | else { | ||
| 161 | mconsole_reply(req, "Read of file failed", | ||
| 162 | 1, 0); | ||
| 163 | goto out_free; | ||
| 164 | } | ||
| 165 | } while (n > 0); | ||
| 166 | } | ||
| 167 | else mconsole_reply(req, "", 0, 0); | ||
| 168 | |||
| 169 | out_free: | ||
| 170 | kfree(buf); | ||
| 171 | out_fput: | ||
| 172 | fput(file); | ||
| 173 | out: ; | ||
| 174 | } | ||
| 175 | #endif | ||
| 176 | |||
| 177 | void mconsole_proc(struct mc_request *req) | ||
| 178 | { | ||
| 179 | char path[64]; | ||
| 180 | char *buf; | 127 | char *buf; |
| 181 | int len; | 128 | int len; |
| 182 | int fd; | 129 | struct file *file; |
| 183 | int first_chunk = 1; | 130 | int first_chunk = 1; |
| 184 | char *ptr = req->request.data; | 131 | char *ptr = req->request.data; |
| 185 | 132 | ||
| 186 | ptr += strlen("proc"); | 133 | ptr += strlen("proc"); |
| 187 | ptr = skip_spaces(ptr); | 134 | ptr = skip_spaces(ptr); |
| 188 | snprintf(path, sizeof(path), "/proc/%s", ptr); | ||
| 189 | 135 | ||
| 190 | fd = sys_open(path, 0, 0); | 136 | file = file_open_root(mnt->mnt_root, mnt, ptr, O_RDONLY); |
| 191 | if (fd < 0) { | 137 | if (IS_ERR(file)) { |
| 192 | mconsole_reply(req, "Failed to open file", 1, 0); | 138 | mconsole_reply(req, "Failed to open file", 1, 0); |
| 193 | printk(KERN_ERR "open %s: %d\n",path,fd); | 139 | printk(KERN_ERR "open /proc/%s: %ld\n", ptr, PTR_ERR(file)); |
| 194 | goto out; | 140 | goto out; |
| 195 | } | 141 | } |
| 196 | 142 | ||
| 197 | buf = kmalloc(PAGE_SIZE, GFP_KERNEL); | 143 | buf = kmalloc(PAGE_SIZE, GFP_KERNEL); |
| 198 | if (buf == NULL) { | 144 | if (buf == NULL) { |
| 199 | mconsole_reply(req, "Failed to allocate buffer", 1, 0); | 145 | mconsole_reply(req, "Failed to allocate buffer", 1, 0); |
| 200 | goto out_close; | 146 | goto out_fput; |
| 201 | } | 147 | } |
| 202 | 148 | ||
| 203 | for (;;) { | 149 | do { |
| 204 | len = sys_read(fd, buf, PAGE_SIZE-1); | 150 | loff_t pos; |
| 151 | mm_segment_t old_fs = get_fs(); | ||
| 152 | set_fs(KERNEL_DS); | ||
| 153 | len = vfs_read(file, buf, PAGE_SIZE - 1, &pos); | ||
| 154 | set_fs(old_fs); | ||
| 155 | file->f_pos = pos; | ||
| 205 | if (len < 0) { | 156 | if (len < 0) { |
| 206 | mconsole_reply(req, "Read of file failed", 1, 0); | 157 | mconsole_reply(req, "Read of file failed", 1, 0); |
| 207 | goto out_free; | 158 | goto out_free; |
| @@ -211,22 +162,14 @@ void mconsole_proc(struct mc_request *req) | |||
| 211 | mconsole_reply(req, "\n", 0, 1); | 162 | mconsole_reply(req, "\n", 0, 1); |
| 212 | first_chunk = 0; | 163 | first_chunk = 0; |
| 213 | } | 164 | } |
| 214 | if (len == PAGE_SIZE-1) { | 165 | buf[len] = '\0'; |
| 215 | buf[len] = '\0'; | 166 | mconsole_reply(req, buf, 0, (len > 0)); |
| 216 | mconsole_reply(req, buf, 0, 1); | 167 | } while (len > 0); |
| 217 | } else { | ||
| 218 | buf[len] = '\0'; | ||
| 219 | mconsole_reply(req, buf, 0, 0); | ||
| 220 | break; | ||
| 221 | } | ||
| 222 | } | ||
| 223 | |||
| 224 | out_free: | 168 | out_free: |
| 225 | kfree(buf); | 169 | kfree(buf); |
| 226 | out_close: | 170 | out_fput: |
| 227 | sys_close(fd); | 171 | fput(file); |
| 228 | out: | 172 | out: ; |
| 229 | /* nothing */; | ||
| 230 | } | 173 | } |
| 231 | 174 | ||
| 232 | #define UML_MCONSOLE_HELPTEXT \ | 175 | #define UML_MCONSOLE_HELPTEXT \ |
diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig index b0a47433341e..1e638e75a6b7 100644 --- a/arch/unicore32/Kconfig +++ b/arch/unicore32/Kconfig | |||
| @@ -6,6 +6,7 @@ config UNICORE32 | |||
| 6 | select HAVE_DMA_ATTRS | 6 | select HAVE_DMA_ATTRS |
| 7 | select HAVE_KERNEL_GZIP | 7 | select HAVE_KERNEL_GZIP |
| 8 | select HAVE_KERNEL_BZIP2 | 8 | select HAVE_KERNEL_BZIP2 |
| 9 | select GENERIC_ATOMIC64 | ||
| 9 | select HAVE_KERNEL_LZO | 10 | select HAVE_KERNEL_LZO |
| 10 | select HAVE_KERNEL_LZMA | 11 | select HAVE_KERNEL_LZMA |
| 11 | select ARCH_HAVE_CUSTOM_GPIO_H | 12 | select ARCH_HAVE_CUSTOM_GPIO_H |
diff --git a/arch/unicore32/include/mach/PKUnity.h b/arch/unicore32/include/mach/PKUnity.h index 8040d575dddb..46705afcbf5a 100644 --- a/arch/unicore32/include/mach/PKUnity.h +++ b/arch/unicore32/include/mach/PKUnity.h | |||
| @@ -15,7 +15,7 @@ | |||
| 15 | #error You must include hardware.h not PKUnity.h | 15 | #error You must include hardware.h not PKUnity.h |
| 16 | #endif | 16 | #endif |
| 17 | 17 | ||
| 18 | #include "bitfield.h" | 18 | #include <mach/bitfield.h> |
| 19 | 19 | ||
| 20 | /* | 20 | /* |
| 21 | * Memory Definitions | 21 | * Memory Definitions |
| @@ -32,7 +32,7 @@ | |||
| 32 | * 0x98000000 - 0x9FFFFFFF 128MB PCI PCI-AHB MEM-mapping | 32 | * 0x98000000 - 0x9FFFFFFF 128MB PCI PCI-AHB MEM-mapping |
| 33 | */ | 33 | */ |
| 34 | #define PKUNITY_PCI_BASE io_p2v(0x80000000) /* 0x80000000 - 0xBFFFFFFF 1GB */ | 34 | #define PKUNITY_PCI_BASE io_p2v(0x80000000) /* 0x80000000 - 0xBFFFFFFF 1GB */ |
| 35 | #include "regs-pci.h" | 35 | #include <mach/regs-pci.h> |
| 36 | 36 | ||
| 37 | #define PKUNITY_PCICFG_BASE (PKUNITY_PCI_BASE + 0x0) | 37 | #define PKUNITY_PCICFG_BASE (PKUNITY_PCI_BASE + 0x0) |
| 38 | #define PKUNITY_PCIBRI_BASE (PKUNITY_PCI_BASE + 0x00010000) | 38 | #define PKUNITY_PCIBRI_BASE (PKUNITY_PCI_BASE + 0x00010000) |
| @@ -50,18 +50,18 @@ | |||
| 50 | #define PKUNITY_ARBITER_BASE (PKUNITY_AHB_BASE + 0x000000) /* AHB-2 */ | 50 | #define PKUNITY_ARBITER_BASE (PKUNITY_AHB_BASE + 0x000000) /* AHB-2 */ |
| 51 | #define PKUNITY_DDR2CTRL_BASE (PKUNITY_AHB_BASE + 0x100000) /* AHB-3 */ | 51 | #define PKUNITY_DDR2CTRL_BASE (PKUNITY_AHB_BASE + 0x100000) /* AHB-3 */ |
| 52 | #define PKUNITY_DMAC_BASE (PKUNITY_AHB_BASE + 0x200000) /* AHB-4 */ | 52 | #define PKUNITY_DMAC_BASE (PKUNITY_AHB_BASE + 0x200000) /* AHB-4 */ |
| 53 | #include "regs-dmac.h" | 53 | #include <mach/regs-dmac.h> |
| 54 | #define PKUNITY_UMAL_BASE (PKUNITY_AHB_BASE + 0x300000) /* AHB-5 */ | 54 | #define PKUNITY_UMAL_BASE (PKUNITY_AHB_BASE + 0x300000) /* AHB-5 */ |
| 55 | #include "regs-umal.h" | 55 | #include <mach/regs-umal.h> |
| 56 | #define PKUNITY_USB_BASE (PKUNITY_AHB_BASE + 0x400000) /* AHB-6 */ | 56 | #define PKUNITY_USB_BASE (PKUNITY_AHB_BASE + 0x400000) /* AHB-6 */ |
| 57 | #define PKUNITY_SATA_BASE (PKUNITY_AHB_BASE + 0x500000) /* AHB-7 */ | 57 | #define PKUNITY_SATA_BASE (PKUNITY_AHB_BASE + 0x500000) /* AHB-7 */ |
| 58 | #define PKUNITY_SMC_BASE (PKUNITY_AHB_BASE + 0x600000) /* AHB-8 */ | 58 | #define PKUNITY_SMC_BASE (PKUNITY_AHB_BASE + 0x600000) /* AHB-8 */ |
| 59 | /* AHB-9 is for APB bridge */ | 59 | /* AHB-9 is for APB bridge */ |
| 60 | #define PKUNITY_MME_BASE (PKUNITY_AHB_BASE + 0x700000) /* AHB-10 */ | 60 | #define PKUNITY_MME_BASE (PKUNITY_AHB_BASE + 0x700000) /* AHB-10 */ |
| 61 | #define PKUNITY_UNIGFX_BASE (PKUNITY_AHB_BASE + 0x800000) /* AHB-11 */ | 61 | #define PKUNITY_UNIGFX_BASE (PKUNITY_AHB_BASE + 0x800000) /* AHB-11 */ |
| 62 | #include "regs-unigfx.h" | 62 | #include <mach/regs-unigfx.h> |
| 63 | #define PKUNITY_NAND_BASE (PKUNITY_AHB_BASE + 0x900000) /* AHB-12 */ | 63 | #define PKUNITY_NAND_BASE (PKUNITY_AHB_BASE + 0x900000) /* AHB-12 */ |
| 64 | #include "regs-nand.h" | 64 | #include <mach/regs-nand.h> |
| 65 | #define PKUNITY_H264D_BASE (PKUNITY_AHB_BASE + 0xA00000) /* AHB-13 */ | 65 | #define PKUNITY_H264D_BASE (PKUNITY_AHB_BASE + 0xA00000) /* AHB-13 */ |
| 66 | #define PKUNITY_H264E_BASE (PKUNITY_AHB_BASE + 0xB00000) /* AHB-14 */ | 66 | #define PKUNITY_H264E_BASE (PKUNITY_AHB_BASE + 0xB00000) /* AHB-14 */ |
| 67 | 67 | ||
| @@ -72,27 +72,27 @@ | |||
| 72 | 72 | ||
| 73 | #define PKUNITY_UART0_BASE (PKUNITY_APB_BASE + 0x000000) /* APB-0 */ | 73 | #define PKUNITY_UART0_BASE (PKUNITY_APB_BASE + 0x000000) /* APB-0 */ |
| 74 | #define PKUNITY_UART1_BASE (PKUNITY_APB_BASE + 0x100000) /* APB-1 */ | 74 | #define PKUNITY_UART1_BASE (PKUNITY_APB_BASE + 0x100000) /* APB-1 */ |
| 75 | #include "regs-uart.h" | 75 | #include <mach/regs-uart.h> |
| 76 | #define PKUNITY_I2C_BASE (PKUNITY_APB_BASE + 0x200000) /* APB-2 */ | 76 | #define PKUNITY_I2C_BASE (PKUNITY_APB_BASE + 0x200000) /* APB-2 */ |
| 77 | #include "regs-i2c.h" | 77 | #include <mach/regs-i2c.h> |
| 78 | #define PKUNITY_SPI_BASE (PKUNITY_APB_BASE + 0x300000) /* APB-3 */ | 78 | #define PKUNITY_SPI_BASE (PKUNITY_APB_BASE + 0x300000) /* APB-3 */ |
| 79 | #include "regs-spi.h" | 79 | #include <mach/regs-spi.h> |
| 80 | #define PKUNITY_AC97_BASE (PKUNITY_APB_BASE + 0x400000) /* APB-4 */ | 80 | #define PKUNITY_AC97_BASE (PKUNITY_APB_BASE + 0x400000) /* APB-4 */ |
| 81 | #include "regs-ac97.h" | 81 | #include <mach/regs-ac97.h> |
| 82 | #define PKUNITY_GPIO_BASE (PKUNITY_APB_BASE + 0x500000) /* APB-5 */ | 82 | #define PKUNITY_GPIO_BASE (PKUNITY_APB_BASE + 0x500000) /* APB-5 */ |
| 83 | #include "regs-gpio.h" | 83 | #include <mach/regs-gpio.h> |
| 84 | #define PKUNITY_INTC_BASE (PKUNITY_APB_BASE + 0x600000) /* APB-6 */ | 84 | #define PKUNITY_INTC_BASE (PKUNITY_APB_BASE + 0x600000) /* APB-6 */ |
| 85 | #include "regs-intc.h" | 85 | #include <mach/regs-intc.h> |
| 86 | #define PKUNITY_RTC_BASE (PKUNITY_APB_BASE + 0x700000) /* APB-7 */ | 86 | #define PKUNITY_RTC_BASE (PKUNITY_APB_BASE + 0x700000) /* APB-7 */ |
| 87 | #include "regs-rtc.h" | 87 | #include <mach/regs-rtc.h> |
| 88 | #define PKUNITY_OST_BASE (PKUNITY_APB_BASE + 0x800000) /* APB-8 */ | 88 | #define PKUNITY_OST_BASE (PKUNITY_APB_BASE + 0x800000) /* APB-8 */ |
| 89 | #include "regs-ost.h" | 89 | #include <mach/regs-ost.h> |
| 90 | #define PKUNITY_RESETC_BASE (PKUNITY_APB_BASE + 0x900000) /* APB-9 */ | 90 | #define PKUNITY_RESETC_BASE (PKUNITY_APB_BASE + 0x900000) /* APB-9 */ |
| 91 | #include "regs-resetc.h" | 91 | #include <mach/regs-resetc.h> |
| 92 | #define PKUNITY_PM_BASE (PKUNITY_APB_BASE + 0xA00000) /* APB-10 */ | 92 | #define PKUNITY_PM_BASE (PKUNITY_APB_BASE + 0xA00000) /* APB-10 */ |
| 93 | #include "regs-pm.h" | 93 | #include <mach/regs-pm.h> |
| 94 | #define PKUNITY_PS2_BASE (PKUNITY_APB_BASE + 0xB00000) /* APB-11 */ | 94 | #define PKUNITY_PS2_BASE (PKUNITY_APB_BASE + 0xB00000) /* APB-11 */ |
| 95 | #include "regs-ps2.h" | 95 | #include <mach/regs-ps2.h> |
| 96 | #define PKUNITY_SDC_BASE (PKUNITY_APB_BASE + 0xC00000) /* APB-12 */ | 96 | #define PKUNITY_SDC_BASE (PKUNITY_APB_BASE + 0xC00000) /* APB-12 */ |
| 97 | #include "regs-sdc.h" | 97 | #include <mach/regs-sdc.h> |
| 98 | 98 | ||
diff --git a/arch/unicore32/include/mach/hardware.h b/arch/unicore32/include/mach/hardware.h index 930bea6e129a..9e20b5d9ed50 100644 --- a/arch/unicore32/include/mach/hardware.h +++ b/arch/unicore32/include/mach/hardware.h | |||
| @@ -15,7 +15,7 @@ | |||
| 15 | #ifndef __MACH_PUV3_HARDWARE_H__ | 15 | #ifndef __MACH_PUV3_HARDWARE_H__ |
| 16 | #define __MACH_PUV3_HARDWARE_H__ | 16 | #define __MACH_PUV3_HARDWARE_H__ |
| 17 | 17 | ||
| 18 | #include "PKUnity.h" | 18 | #include <mach/PKUnity.h> |
| 19 | 19 | ||
| 20 | #ifndef __ASSEMBLY__ | 20 | #ifndef __ASSEMBLY__ |
| 21 | #define io_p2v(x) (void __iomem *)((x) - PKUNITY_MMIO_BASE) | 21 | #define io_p2v(x) (void __iomem *)((x) - PKUNITY_MMIO_BASE) |
diff --git a/arch/unicore32/include/mach/uncompress.h b/arch/unicore32/include/mach/uncompress.h index 142d3e7958a9..9be67c9d3b53 100644 --- a/arch/unicore32/include/mach/uncompress.h +++ b/arch/unicore32/include/mach/uncompress.h | |||
| @@ -13,8 +13,8 @@ | |||
| 13 | #ifndef __MACH_PUV3_UNCOMPRESS_H__ | 13 | #ifndef __MACH_PUV3_UNCOMPRESS_H__ |
| 14 | #define __MACH_PUV3_UNCOMPRESS_H__ | 14 | #define __MACH_PUV3_UNCOMPRESS_H__ |
| 15 | 15 | ||
| 16 | #include "hardware.h" | 16 | #include <mach/hardware.h> |
| 17 | #include "ocd.h" | 17 | #include <mach/ocd.h> |
| 18 | 18 | ||
| 19 | extern char input_data[]; | 19 | extern char input_data[]; |
| 20 | extern char input_data_end[]; | 20 | extern char input_data_end[]; |
diff --git a/arch/unicore32/include/uapi/asm/Kbuild b/arch/unicore32/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/unicore32/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7f9a395c5254..b72777ff32a9 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
| @@ -586,23 +586,18 @@ config PARAVIRT_TIME_ACCOUNTING | |||
| 586 | 586 | ||
| 587 | source "arch/x86/xen/Kconfig" | 587 | source "arch/x86/xen/Kconfig" |
| 588 | 588 | ||
| 589 | config KVM_CLOCK | ||
| 590 | bool "KVM paravirtualized clock" | ||
| 591 | select PARAVIRT | ||
| 592 | select PARAVIRT_CLOCK | ||
| 593 | ---help--- | ||
| 594 | Turning on this option will allow you to run a paravirtualized clock | ||
| 595 | when running over the KVM hypervisor. Instead of relying on a PIT | ||
| 596 | (or probably other) emulation by the underlying device model, the host | ||
| 597 | provides the guest with timing infrastructure such as time of day, and | ||
| 598 | system time | ||
| 599 | |||
| 600 | config KVM_GUEST | 589 | config KVM_GUEST |
| 601 | bool "KVM Guest support" | 590 | bool "KVM Guest support (including kvmclock)" |
| 591 | select PARAVIRT | ||
| 602 | select PARAVIRT | 592 | select PARAVIRT |
| 593 | select PARAVIRT_CLOCK | ||
| 594 | default y if PARAVIRT_GUEST | ||
| 603 | ---help--- | 595 | ---help--- |
| 604 | This option enables various optimizations for running under the KVM | 596 | This option enables various optimizations for running under the KVM |
| 605 | hypervisor. | 597 | hypervisor. It includes a paravirtualized clock, so that instead |
| 598 | of relying on a PIT (or probably other) emulation by the | ||
| 599 | underlying device model, the host provides the guest with | ||
| 600 | timing infrastructure such as time of day, and system time | ||
| 606 | 601 | ||
| 607 | source "arch/x86/lguest/Kconfig" | 602 | source "arch/x86/lguest/Kconfig" |
| 608 | 603 | ||
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index f7535bedc33f..ce03476d8c8f 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile | |||
| @@ -37,7 +37,7 @@ setup-y += video-bios.o | |||
| 37 | targets += $(setup-y) | 37 | targets += $(setup-y) |
| 38 | hostprogs-y := mkcpustr tools/build | 38 | hostprogs-y := mkcpustr tools/build |
| 39 | 39 | ||
| 40 | HOST_EXTRACFLAGS += -I$(srctree)/tools/include $(LINUXINCLUDE) \ | 40 | HOST_EXTRACFLAGS += -I$(srctree)/tools/include $(USERINCLUDE) \ |
| 41 | -D__EXPORTED_HEADERS__ | 41 | -D__EXPORTED_HEADERS__ |
| 42 | 42 | ||
| 43 | $(obj)/cpu.o: $(obj)/cpustr.h | 43 | $(obj)/cpu.o: $(obj)/cpustr.h |
| @@ -52,7 +52,7 @@ $(obj)/cpustr.h: $(obj)/mkcpustr FORCE | |||
| 52 | 52 | ||
| 53 | # How to compile the 16-bit code. Note we always compile for -march=i386, | 53 | # How to compile the 16-bit code. Note we always compile for -march=i386, |
| 54 | # that way we can complain to the user if the CPU is insufficient. | 54 | # that way we can complain to the user if the CPU is insufficient. |
| 55 | KBUILD_CFLAGS := $(LINUXINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \ | 55 | KBUILD_CFLAGS := $(USERINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \ |
| 56 | -DDISABLE_BRANCH_PROFILING \ | 56 | -DDISABLE_BRANCH_PROFILING \ |
| 57 | -Wall -Wstrict-prototypes \ | 57 | -Wall -Wstrict-prototypes \ |
| 58 | -march=i386 -mregparm=3 \ | 58 | -march=i386 -mregparm=3 \ |
diff --git a/arch/x86/boot/mkcpustr.c b/arch/x86/boot/mkcpustr.c index 919257f526f2..4579eff0ef4d 100644 --- a/arch/x86/boot/mkcpustr.c +++ b/arch/x86/boot/mkcpustr.c | |||
| @@ -15,6 +15,8 @@ | |||
| 15 | 15 | ||
| 16 | #include <stdio.h> | 16 | #include <stdio.h> |
| 17 | 17 | ||
| 18 | #include "../include/asm/required-features.h" | ||
| 19 | #include "../include/asm/cpufeature.h" | ||
| 18 | #include "../kernel/cpu/capflags.c" | 20 | #include "../kernel/cpu/capflags.c" |
| 19 | 21 | ||
| 20 | int main(void) | 22 | int main(void) |
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index e908e5de82d3..5bacb4a226ac 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile | |||
| @@ -12,6 +12,8 @@ obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o | |||
| 12 | 12 | ||
| 13 | obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o | 13 | obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o |
| 14 | obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o | 14 | obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o |
| 15 | obj-$(CONFIG_CRYPTO_CAST5_AVX_X86_64) += cast5-avx-x86_64.o | ||
| 16 | obj-$(CONFIG_CRYPTO_CAST6_AVX_X86_64) += cast6-avx-x86_64.o | ||
| 15 | obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o | 17 | obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o |
| 16 | obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o | 18 | obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o |
| 17 | obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o | 19 | obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o |
| @@ -32,6 +34,8 @@ serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o | |||
| 32 | 34 | ||
| 33 | aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o | 35 | aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o |
| 34 | camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o | 36 | camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o |
| 37 | cast5-avx-x86_64-y := cast5-avx-x86_64-asm_64.o cast5_avx_glue.o | ||
| 38 | cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o | ||
| 35 | blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o | 39 | blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o |
| 36 | twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o | 40 | twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o |
| 37 | twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o | 41 | twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o |
diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c index 59b37deb8c8d..aafe8ce0d65d 100644 --- a/arch/x86/crypto/aes_glue.c +++ b/arch/x86/crypto/aes_glue.c | |||
| @@ -40,7 +40,6 @@ static struct crypto_alg aes_alg = { | |||
| 40 | .cra_blocksize = AES_BLOCK_SIZE, | 40 | .cra_blocksize = AES_BLOCK_SIZE, |
| 41 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | 41 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), |
| 42 | .cra_module = THIS_MODULE, | 42 | .cra_module = THIS_MODULE, |
| 43 | .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), | ||
| 44 | .cra_u = { | 43 | .cra_u = { |
| 45 | .cipher = { | 44 | .cipher = { |
| 46 | .cia_min_keysize = AES_MIN_KEY_SIZE, | 45 | .cia_min_keysize = AES_MIN_KEY_SIZE, |
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 34fdcff4d2c8..7c04d0da709b 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c | |||
| @@ -28,6 +28,9 @@ | |||
| 28 | #include <crypto/aes.h> | 28 | #include <crypto/aes.h> |
| 29 | #include <crypto/cryptd.h> | 29 | #include <crypto/cryptd.h> |
| 30 | #include <crypto/ctr.h> | 30 | #include <crypto/ctr.h> |
| 31 | #include <crypto/b128ops.h> | ||
| 32 | #include <crypto/lrw.h> | ||
| 33 | #include <crypto/xts.h> | ||
| 31 | #include <asm/cpu_device_id.h> | 34 | #include <asm/cpu_device_id.h> |
| 32 | #include <asm/i387.h> | 35 | #include <asm/i387.h> |
| 33 | #include <asm/crypto/aes.h> | 36 | #include <asm/crypto/aes.h> |
| @@ -41,18 +44,10 @@ | |||
| 41 | #define HAS_CTR | 44 | #define HAS_CTR |
| 42 | #endif | 45 | #endif |
| 43 | 46 | ||
| 44 | #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) | ||
| 45 | #define HAS_LRW | ||
| 46 | #endif | ||
| 47 | |||
| 48 | #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) | 47 | #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) |
| 49 | #define HAS_PCBC | 48 | #define HAS_PCBC |
| 50 | #endif | 49 | #endif |
| 51 | 50 | ||
| 52 | #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE) | ||
| 53 | #define HAS_XTS | ||
| 54 | #endif | ||
| 55 | |||
| 56 | /* This data is stored at the end of the crypto_tfm struct. | 51 | /* This data is stored at the end of the crypto_tfm struct. |
| 57 | * It's a type of per "session" data storage location. | 52 | * It's a type of per "session" data storage location. |
| 58 | * This needs to be 16 byte aligned. | 53 | * This needs to be 16 byte aligned. |
| @@ -79,6 +74,16 @@ struct aesni_hash_subkey_req_data { | |||
| 79 | #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) | 74 | #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) |
| 80 | #define RFC4106_HASH_SUBKEY_SIZE 16 | 75 | #define RFC4106_HASH_SUBKEY_SIZE 16 |
| 81 | 76 | ||
| 77 | struct aesni_lrw_ctx { | ||
| 78 | struct lrw_table_ctx lrw_table; | ||
| 79 | u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; | ||
| 80 | }; | ||
| 81 | |||
| 82 | struct aesni_xts_ctx { | ||
| 83 | u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; | ||
| 84 | u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; | ||
| 85 | }; | ||
| 86 | |||
| 82 | asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, | 87 | asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, |
| 83 | unsigned int key_len); | 88 | unsigned int key_len); |
| 84 | asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, | 89 | asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, |
| @@ -398,13 +403,6 @@ static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) | |||
| 398 | #endif | 403 | #endif |
| 399 | #endif | 404 | #endif |
| 400 | 405 | ||
| 401 | #ifdef HAS_LRW | ||
| 402 | static int ablk_lrw_init(struct crypto_tfm *tfm) | ||
| 403 | { | ||
| 404 | return ablk_init_common(tfm, "fpu(lrw(__driver-aes-aesni))"); | ||
| 405 | } | ||
| 406 | #endif | ||
| 407 | |||
| 408 | #ifdef HAS_PCBC | 406 | #ifdef HAS_PCBC |
| 409 | static int ablk_pcbc_init(struct crypto_tfm *tfm) | 407 | static int ablk_pcbc_init(struct crypto_tfm *tfm) |
| 410 | { | 408 | { |
| @@ -412,12 +410,160 @@ static int ablk_pcbc_init(struct crypto_tfm *tfm) | |||
| 412 | } | 410 | } |
| 413 | #endif | 411 | #endif |
| 414 | 412 | ||
| 415 | #ifdef HAS_XTS | 413 | static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) |
| 416 | static int ablk_xts_init(struct crypto_tfm *tfm) | ||
| 417 | { | 414 | { |
| 418 | return ablk_init_common(tfm, "fpu(xts(__driver-aes-aesni))"); | 415 | aesni_ecb_enc(ctx, blks, blks, nbytes); |
| 416 | } | ||
| 417 | |||
| 418 | static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) | ||
| 419 | { | ||
| 420 | aesni_ecb_dec(ctx, blks, blks, nbytes); | ||
| 421 | } | ||
| 422 | |||
| 423 | static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, | ||
| 424 | unsigned int keylen) | ||
| 425 | { | ||
| 426 | struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); | ||
| 427 | int err; | ||
| 428 | |||
| 429 | err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, | ||
| 430 | keylen - AES_BLOCK_SIZE); | ||
| 431 | if (err) | ||
| 432 | return err; | ||
| 433 | |||
| 434 | return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); | ||
| 435 | } | ||
| 436 | |||
| 437 | static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) | ||
| 438 | { | ||
| 439 | struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); | ||
| 440 | |||
| 441 | lrw_free_table(&ctx->lrw_table); | ||
| 442 | } | ||
| 443 | |||
| 444 | static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 445 | struct scatterlist *src, unsigned int nbytes) | ||
| 446 | { | ||
| 447 | struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 448 | be128 buf[8]; | ||
| 449 | struct lrw_crypt_req req = { | ||
| 450 | .tbuf = buf, | ||
| 451 | .tbuflen = sizeof(buf), | ||
| 452 | |||
| 453 | .table_ctx = &ctx->lrw_table, | ||
| 454 | .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), | ||
| 455 | .crypt_fn = lrw_xts_encrypt_callback, | ||
| 456 | }; | ||
| 457 | int ret; | ||
| 458 | |||
| 459 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 460 | |||
| 461 | kernel_fpu_begin(); | ||
| 462 | ret = lrw_crypt(desc, dst, src, nbytes, &req); | ||
| 463 | kernel_fpu_end(); | ||
| 464 | |||
| 465 | return ret; | ||
| 466 | } | ||
| 467 | |||
| 468 | static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 469 | struct scatterlist *src, unsigned int nbytes) | ||
| 470 | { | ||
| 471 | struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 472 | be128 buf[8]; | ||
| 473 | struct lrw_crypt_req req = { | ||
| 474 | .tbuf = buf, | ||
| 475 | .tbuflen = sizeof(buf), | ||
| 476 | |||
| 477 | .table_ctx = &ctx->lrw_table, | ||
| 478 | .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), | ||
| 479 | .crypt_fn = lrw_xts_decrypt_callback, | ||
| 480 | }; | ||
| 481 | int ret; | ||
| 482 | |||
| 483 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 484 | |||
| 485 | kernel_fpu_begin(); | ||
| 486 | ret = lrw_crypt(desc, dst, src, nbytes, &req); | ||
| 487 | kernel_fpu_end(); | ||
| 488 | |||
| 489 | return ret; | ||
| 490 | } | ||
| 491 | |||
| 492 | static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, | ||
| 493 | unsigned int keylen) | ||
| 494 | { | ||
| 495 | struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); | ||
| 496 | u32 *flags = &tfm->crt_flags; | ||
| 497 | int err; | ||
| 498 | |||
| 499 | /* key consists of keys of equal size concatenated, therefore | ||
| 500 | * the length must be even | ||
| 501 | */ | ||
| 502 | if (keylen % 2) { | ||
| 503 | *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | ||
| 504 | return -EINVAL; | ||
| 505 | } | ||
| 506 | |||
| 507 | /* first half of xts-key is for crypt */ | ||
| 508 | err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); | ||
| 509 | if (err) | ||
| 510 | return err; | ||
| 511 | |||
| 512 | /* second half of xts-key is for tweak */ | ||
| 513 | return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, | ||
| 514 | keylen / 2); | ||
| 515 | } | ||
| 516 | |||
| 517 | |||
| 518 | static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 519 | struct scatterlist *src, unsigned int nbytes) | ||
| 520 | { | ||
| 521 | struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 522 | be128 buf[8]; | ||
| 523 | struct xts_crypt_req req = { | ||
| 524 | .tbuf = buf, | ||
| 525 | .tbuflen = sizeof(buf), | ||
| 526 | |||
| 527 | .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), | ||
| 528 | .tweak_fn = XTS_TWEAK_CAST(aesni_enc), | ||
| 529 | .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), | ||
| 530 | .crypt_fn = lrw_xts_encrypt_callback, | ||
| 531 | }; | ||
| 532 | int ret; | ||
| 533 | |||
| 534 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 535 | |||
| 536 | kernel_fpu_begin(); | ||
| 537 | ret = xts_crypt(desc, dst, src, nbytes, &req); | ||
| 538 | kernel_fpu_end(); | ||
| 539 | |||
| 540 | return ret; | ||
| 541 | } | ||
| 542 | |||
| 543 | static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 544 | struct scatterlist *src, unsigned int nbytes) | ||
| 545 | { | ||
| 546 | struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 547 | be128 buf[8]; | ||
| 548 | struct xts_crypt_req req = { | ||
| 549 | .tbuf = buf, | ||
| 550 | .tbuflen = sizeof(buf), | ||
| 551 | |||
| 552 | .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), | ||
| 553 | .tweak_fn = XTS_TWEAK_CAST(aesni_enc), | ||
| 554 | .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), | ||
| 555 | .crypt_fn = lrw_xts_decrypt_callback, | ||
| 556 | }; | ||
| 557 | int ret; | ||
| 558 | |||
| 559 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 560 | |||
| 561 | kernel_fpu_begin(); | ||
| 562 | ret = xts_crypt(desc, dst, src, nbytes, &req); | ||
| 563 | kernel_fpu_end(); | ||
| 564 | |||
| 565 | return ret; | ||
| 419 | } | 566 | } |
| 420 | #endif | ||
| 421 | 567 | ||
| 422 | #ifdef CONFIG_X86_64 | 568 | #ifdef CONFIG_X86_64 |
| 423 | static int rfc4106_init(struct crypto_tfm *tfm) | 569 | static int rfc4106_init(struct crypto_tfm *tfm) |
| @@ -1035,10 +1181,10 @@ static struct crypto_alg aesni_algs[] = { { | |||
| 1035 | }, | 1181 | }, |
| 1036 | #endif | 1182 | #endif |
| 1037 | #endif | 1183 | #endif |
| 1038 | #ifdef HAS_LRW | 1184 | #ifdef HAS_PCBC |
| 1039 | }, { | 1185 | }, { |
| 1040 | .cra_name = "lrw(aes)", | 1186 | .cra_name = "pcbc(aes)", |
| 1041 | .cra_driver_name = "lrw-aes-aesni", | 1187 | .cra_driver_name = "pcbc-aes-aesni", |
| 1042 | .cra_priority = 400, | 1188 | .cra_priority = 400, |
| 1043 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | 1189 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, |
| 1044 | .cra_blocksize = AES_BLOCK_SIZE, | 1190 | .cra_blocksize = AES_BLOCK_SIZE, |
| @@ -1046,12 +1192,12 @@ static struct crypto_alg aesni_algs[] = { { | |||
| 1046 | .cra_alignmask = 0, | 1192 | .cra_alignmask = 0, |
| 1047 | .cra_type = &crypto_ablkcipher_type, | 1193 | .cra_type = &crypto_ablkcipher_type, |
| 1048 | .cra_module = THIS_MODULE, | 1194 | .cra_module = THIS_MODULE, |
| 1049 | .cra_init = ablk_lrw_init, | 1195 | .cra_init = ablk_pcbc_init, |
| 1050 | .cra_exit = ablk_exit, | 1196 | .cra_exit = ablk_exit, |
| 1051 | .cra_u = { | 1197 | .cra_u = { |
| 1052 | .ablkcipher = { | 1198 | .ablkcipher = { |
| 1053 | .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, | 1199 | .min_keysize = AES_MIN_KEY_SIZE, |
| 1054 | .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, | 1200 | .max_keysize = AES_MAX_KEY_SIZE, |
| 1055 | .ivsize = AES_BLOCK_SIZE, | 1201 | .ivsize = AES_BLOCK_SIZE, |
| 1056 | .setkey = ablk_set_key, | 1202 | .setkey = ablk_set_key, |
| 1057 | .encrypt = ablk_encrypt, | 1203 | .encrypt = ablk_encrypt, |
| @@ -1059,10 +1205,50 @@ static struct crypto_alg aesni_algs[] = { { | |||
| 1059 | }, | 1205 | }, |
| 1060 | }, | 1206 | }, |
| 1061 | #endif | 1207 | #endif |
| 1062 | #ifdef HAS_PCBC | ||
| 1063 | }, { | 1208 | }, { |
| 1064 | .cra_name = "pcbc(aes)", | 1209 | .cra_name = "__lrw-aes-aesni", |
| 1065 | .cra_driver_name = "pcbc-aes-aesni", | 1210 | .cra_driver_name = "__driver-lrw-aes-aesni", |
| 1211 | .cra_priority = 0, | ||
| 1212 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 1213 | .cra_blocksize = AES_BLOCK_SIZE, | ||
| 1214 | .cra_ctxsize = sizeof(struct aesni_lrw_ctx), | ||
| 1215 | .cra_alignmask = 0, | ||
| 1216 | .cra_type = &crypto_blkcipher_type, | ||
| 1217 | .cra_module = THIS_MODULE, | ||
| 1218 | .cra_exit = lrw_aesni_exit_tfm, | ||
| 1219 | .cra_u = { | ||
| 1220 | .blkcipher = { | ||
| 1221 | .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, | ||
| 1222 | .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, | ||
| 1223 | .ivsize = AES_BLOCK_SIZE, | ||
| 1224 | .setkey = lrw_aesni_setkey, | ||
| 1225 | .encrypt = lrw_encrypt, | ||
| 1226 | .decrypt = lrw_decrypt, | ||
| 1227 | }, | ||
| 1228 | }, | ||
| 1229 | }, { | ||
| 1230 | .cra_name = "__xts-aes-aesni", | ||
| 1231 | .cra_driver_name = "__driver-xts-aes-aesni", | ||
| 1232 | .cra_priority = 0, | ||
| 1233 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 1234 | .cra_blocksize = AES_BLOCK_SIZE, | ||
| 1235 | .cra_ctxsize = sizeof(struct aesni_xts_ctx), | ||
| 1236 | .cra_alignmask = 0, | ||
| 1237 | .cra_type = &crypto_blkcipher_type, | ||
| 1238 | .cra_module = THIS_MODULE, | ||
| 1239 | .cra_u = { | ||
| 1240 | .blkcipher = { | ||
| 1241 | .min_keysize = 2 * AES_MIN_KEY_SIZE, | ||
| 1242 | .max_keysize = 2 * AES_MAX_KEY_SIZE, | ||
| 1243 | .ivsize = AES_BLOCK_SIZE, | ||
| 1244 | .setkey = xts_aesni_setkey, | ||
| 1245 | .encrypt = xts_encrypt, | ||
| 1246 | .decrypt = xts_decrypt, | ||
| 1247 | }, | ||
| 1248 | }, | ||
| 1249 | }, { | ||
| 1250 | .cra_name = "lrw(aes)", | ||
| 1251 | .cra_driver_name = "lrw-aes-aesni", | ||
| 1066 | .cra_priority = 400, | 1252 | .cra_priority = 400, |
| 1067 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | 1253 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, |
| 1068 | .cra_blocksize = AES_BLOCK_SIZE, | 1254 | .cra_blocksize = AES_BLOCK_SIZE, |
| @@ -1070,20 +1256,18 @@ static struct crypto_alg aesni_algs[] = { { | |||
| 1070 | .cra_alignmask = 0, | 1256 | .cra_alignmask = 0, |
| 1071 | .cra_type = &crypto_ablkcipher_type, | 1257 | .cra_type = &crypto_ablkcipher_type, |
| 1072 | .cra_module = THIS_MODULE, | 1258 | .cra_module = THIS_MODULE, |
| 1073 | .cra_init = ablk_pcbc_init, | 1259 | .cra_init = ablk_init, |
| 1074 | .cra_exit = ablk_exit, | 1260 | .cra_exit = ablk_exit, |
| 1075 | .cra_u = { | 1261 | .cra_u = { |
| 1076 | .ablkcipher = { | 1262 | .ablkcipher = { |
| 1077 | .min_keysize = AES_MIN_KEY_SIZE, | 1263 | .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, |
| 1078 | .max_keysize = AES_MAX_KEY_SIZE, | 1264 | .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, |
| 1079 | .ivsize = AES_BLOCK_SIZE, | 1265 | .ivsize = AES_BLOCK_SIZE, |
| 1080 | .setkey = ablk_set_key, | 1266 | .setkey = ablk_set_key, |
| 1081 | .encrypt = ablk_encrypt, | 1267 | .encrypt = ablk_encrypt, |
| 1082 | .decrypt = ablk_decrypt, | 1268 | .decrypt = ablk_decrypt, |
| 1083 | }, | 1269 | }, |
| 1084 | }, | 1270 | }, |
| 1085 | #endif | ||
| 1086 | #ifdef HAS_XTS | ||
| 1087 | }, { | 1271 | }, { |
| 1088 | .cra_name = "xts(aes)", | 1272 | .cra_name = "xts(aes)", |
| 1089 | .cra_driver_name = "xts-aes-aesni", | 1273 | .cra_driver_name = "xts-aes-aesni", |
| @@ -1094,7 +1278,7 @@ static struct crypto_alg aesni_algs[] = { { | |||
| 1094 | .cra_alignmask = 0, | 1278 | .cra_alignmask = 0, |
| 1095 | .cra_type = &crypto_ablkcipher_type, | 1279 | .cra_type = &crypto_ablkcipher_type, |
| 1096 | .cra_module = THIS_MODULE, | 1280 | .cra_module = THIS_MODULE, |
| 1097 | .cra_init = ablk_xts_init, | 1281 | .cra_init = ablk_init, |
| 1098 | .cra_exit = ablk_exit, | 1282 | .cra_exit = ablk_exit, |
| 1099 | .cra_u = { | 1283 | .cra_u = { |
| 1100 | .ablkcipher = { | 1284 | .ablkcipher = { |
| @@ -1106,7 +1290,6 @@ static struct crypto_alg aesni_algs[] = { { | |||
| 1106 | .decrypt = ablk_decrypt, | 1290 | .decrypt = ablk_decrypt, |
| 1107 | }, | 1291 | }, |
| 1108 | }, | 1292 | }, |
| 1109 | #endif | ||
| 1110 | } }; | 1293 | } }; |
| 1111 | 1294 | ||
| 1112 | 1295 | ||
| @@ -1118,7 +1301,7 @@ MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); | |||
| 1118 | 1301 | ||
| 1119 | static int __init aesni_init(void) | 1302 | static int __init aesni_init(void) |
| 1120 | { | 1303 | { |
| 1121 | int err, i; | 1304 | int err; |
| 1122 | 1305 | ||
| 1123 | if (!x86_match_cpu(aesni_cpu_id)) | 1306 | if (!x86_match_cpu(aesni_cpu_id)) |
| 1124 | return -ENODEV; | 1307 | return -ENODEV; |
| @@ -1127,9 +1310,6 @@ static int __init aesni_init(void) | |||
| 1127 | if (err) | 1310 | if (err) |
| 1128 | return err; | 1311 | return err; |
| 1129 | 1312 | ||
| 1130 | for (i = 0; i < ARRAY_SIZE(aesni_algs); i++) | ||
| 1131 | INIT_LIST_HEAD(&aesni_algs[i].cra_list); | ||
| 1132 | |||
| 1133 | return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); | 1313 | return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); |
| 1134 | } | 1314 | } |
| 1135 | 1315 | ||
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index 7967474de8f7..50ec333b70e6 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c | |||
| @@ -367,7 +367,6 @@ static struct crypto_alg bf_algs[4] = { { | |||
| 367 | .cra_ctxsize = sizeof(struct bf_ctx), | 367 | .cra_ctxsize = sizeof(struct bf_ctx), |
| 368 | .cra_alignmask = 0, | 368 | .cra_alignmask = 0, |
| 369 | .cra_module = THIS_MODULE, | 369 | .cra_module = THIS_MODULE, |
| 370 | .cra_list = LIST_HEAD_INIT(bf_algs[0].cra_list), | ||
| 371 | .cra_u = { | 370 | .cra_u = { |
| 372 | .cipher = { | 371 | .cipher = { |
| 373 | .cia_min_keysize = BF_MIN_KEY_SIZE, | 372 | .cia_min_keysize = BF_MIN_KEY_SIZE, |
| @@ -387,7 +386,6 @@ static struct crypto_alg bf_algs[4] = { { | |||
| 387 | .cra_alignmask = 0, | 386 | .cra_alignmask = 0, |
| 388 | .cra_type = &crypto_blkcipher_type, | 387 | .cra_type = &crypto_blkcipher_type, |
| 389 | .cra_module = THIS_MODULE, | 388 | .cra_module = THIS_MODULE, |
| 390 | .cra_list = LIST_HEAD_INIT(bf_algs[1].cra_list), | ||
| 391 | .cra_u = { | 389 | .cra_u = { |
| 392 | .blkcipher = { | 390 | .blkcipher = { |
| 393 | .min_keysize = BF_MIN_KEY_SIZE, | 391 | .min_keysize = BF_MIN_KEY_SIZE, |
| @@ -407,7 +405,6 @@ static struct crypto_alg bf_algs[4] = { { | |||
| 407 | .cra_alignmask = 0, | 405 | .cra_alignmask = 0, |
| 408 | .cra_type = &crypto_blkcipher_type, | 406 | .cra_type = &crypto_blkcipher_type, |
| 409 | .cra_module = THIS_MODULE, | 407 | .cra_module = THIS_MODULE, |
| 410 | .cra_list = LIST_HEAD_INIT(bf_algs[2].cra_list), | ||
| 411 | .cra_u = { | 408 | .cra_u = { |
| 412 | .blkcipher = { | 409 | .blkcipher = { |
| 413 | .min_keysize = BF_MIN_KEY_SIZE, | 410 | .min_keysize = BF_MIN_KEY_SIZE, |
| @@ -428,7 +425,6 @@ static struct crypto_alg bf_algs[4] = { { | |||
| 428 | .cra_alignmask = 0, | 425 | .cra_alignmask = 0, |
| 429 | .cra_type = &crypto_blkcipher_type, | 426 | .cra_type = &crypto_blkcipher_type, |
| 430 | .cra_module = THIS_MODULE, | 427 | .cra_module = THIS_MODULE, |
| 431 | .cra_list = LIST_HEAD_INIT(bf_algs[3].cra_list), | ||
| 432 | .cra_u = { | 428 | .cra_u = { |
| 433 | .blkcipher = { | 429 | .blkcipher = { |
| 434 | .min_keysize = BF_MIN_KEY_SIZE, | 430 | .min_keysize = BF_MIN_KEY_SIZE, |
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index eeb2b3b743e9..42ffd2bbab5b 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c | |||
| @@ -92,715 +92,715 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | |||
| 92 | 92 | ||
| 93 | /* camellia sboxes */ | 93 | /* camellia sboxes */ |
| 94 | const u64 camellia_sp10011110[256] = { | 94 | const u64 camellia_sp10011110[256] = { |
| 95 | 0x7000007070707000, 0x8200008282828200, 0x2c00002c2c2c2c00, | 95 | 0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL, |
| 96 | 0xec0000ecececec00, 0xb30000b3b3b3b300, 0x2700002727272700, | 96 | 0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL, |
| 97 | 0xc00000c0c0c0c000, 0xe50000e5e5e5e500, 0xe40000e4e4e4e400, | 97 | 0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL, |
| 98 | 0x8500008585858500, 0x5700005757575700, 0x3500003535353500, | 98 | 0x8500008585858500ULL, 0x5700005757575700ULL, 0x3500003535353500ULL, |
| 99 | 0xea0000eaeaeaea00, 0x0c00000c0c0c0c00, 0xae0000aeaeaeae00, | 99 | 0xea0000eaeaeaea00ULL, 0x0c00000c0c0c0c00ULL, 0xae0000aeaeaeae00ULL, |
| 100 | 0x4100004141414100, 0x2300002323232300, 0xef0000efefefef00, | 100 | 0x4100004141414100ULL, 0x2300002323232300ULL, 0xef0000efefefef00ULL, |
| 101 | 0x6b00006b6b6b6b00, 0x9300009393939300, 0x4500004545454500, | 101 | 0x6b00006b6b6b6b00ULL, 0x9300009393939300ULL, 0x4500004545454500ULL, |
| 102 | 0x1900001919191900, 0xa50000a5a5a5a500, 0x2100002121212100, | 102 | 0x1900001919191900ULL, 0xa50000a5a5a5a500ULL, 0x2100002121212100ULL, |
| 103 | 0xed0000edededed00, 0x0e00000e0e0e0e00, 0x4f00004f4f4f4f00, | 103 | 0xed0000edededed00ULL, 0x0e00000e0e0e0e00ULL, 0x4f00004f4f4f4f00ULL, |
| 104 | 0x4e00004e4e4e4e00, 0x1d00001d1d1d1d00, 0x6500006565656500, | 104 | 0x4e00004e4e4e4e00ULL, 0x1d00001d1d1d1d00ULL, 0x6500006565656500ULL, |
| 105 | 0x9200009292929200, 0xbd0000bdbdbdbd00, 0x8600008686868600, | 105 | 0x9200009292929200ULL, 0xbd0000bdbdbdbd00ULL, 0x8600008686868600ULL, |
| 106 | 0xb80000b8b8b8b800, 0xaf0000afafafaf00, 0x8f00008f8f8f8f00, | 106 | 0xb80000b8b8b8b800ULL, 0xaf0000afafafaf00ULL, 0x8f00008f8f8f8f00ULL, |
| 107 | 0x7c00007c7c7c7c00, 0xeb0000ebebebeb00, 0x1f00001f1f1f1f00, | 107 | 0x7c00007c7c7c7c00ULL, 0xeb0000ebebebeb00ULL, 0x1f00001f1f1f1f00ULL, |
| 108 | 0xce0000cececece00, 0x3e00003e3e3e3e00, 0x3000003030303000, | 108 | 0xce0000cececece00ULL, 0x3e00003e3e3e3e00ULL, 0x3000003030303000ULL, |
| 109 | 0xdc0000dcdcdcdc00, 0x5f00005f5f5f5f00, 0x5e00005e5e5e5e00, | 109 | 0xdc0000dcdcdcdc00ULL, 0x5f00005f5f5f5f00ULL, 0x5e00005e5e5e5e00ULL, |
| 110 | 0xc50000c5c5c5c500, 0x0b00000b0b0b0b00, 0x1a00001a1a1a1a00, | 110 | 0xc50000c5c5c5c500ULL, 0x0b00000b0b0b0b00ULL, 0x1a00001a1a1a1a00ULL, |
| 111 | 0xa60000a6a6a6a600, 0xe10000e1e1e1e100, 0x3900003939393900, | 111 | 0xa60000a6a6a6a600ULL, 0xe10000e1e1e1e100ULL, 0x3900003939393900ULL, |
| 112 | 0xca0000cacacaca00, 0xd50000d5d5d5d500, 0x4700004747474700, | 112 | 0xca0000cacacaca00ULL, 0xd50000d5d5d5d500ULL, 0x4700004747474700ULL, |
| 113 | 0x5d00005d5d5d5d00, 0x3d00003d3d3d3d00, 0xd90000d9d9d9d900, | 113 | 0x5d00005d5d5d5d00ULL, 0x3d00003d3d3d3d00ULL, 0xd90000d9d9d9d900ULL, |
| 114 | 0x0100000101010100, 0x5a00005a5a5a5a00, 0xd60000d6d6d6d600, | 114 | 0x0100000101010100ULL, 0x5a00005a5a5a5a00ULL, 0xd60000d6d6d6d600ULL, |
| 115 | 0x5100005151515100, 0x5600005656565600, 0x6c00006c6c6c6c00, | 115 | 0x5100005151515100ULL, 0x5600005656565600ULL, 0x6c00006c6c6c6c00ULL, |
| 116 | 0x4d00004d4d4d4d00, 0x8b00008b8b8b8b00, 0x0d00000d0d0d0d00, | 116 | 0x4d00004d4d4d4d00ULL, 0x8b00008b8b8b8b00ULL, 0x0d00000d0d0d0d00ULL, |
| 117 | 0x9a00009a9a9a9a00, 0x6600006666666600, 0xfb0000fbfbfbfb00, | 117 | 0x9a00009a9a9a9a00ULL, 0x6600006666666600ULL, 0xfb0000fbfbfbfb00ULL, |
| 118 | 0xcc0000cccccccc00, 0xb00000b0b0b0b000, 0x2d00002d2d2d2d00, | 118 | 0xcc0000cccccccc00ULL, 0xb00000b0b0b0b000ULL, 0x2d00002d2d2d2d00ULL, |
| 119 | 0x7400007474747400, 0x1200001212121200, 0x2b00002b2b2b2b00, | 119 | 0x7400007474747400ULL, 0x1200001212121200ULL, 0x2b00002b2b2b2b00ULL, |
| 120 | 0x2000002020202000, 0xf00000f0f0f0f000, 0xb10000b1b1b1b100, | 120 | 0x2000002020202000ULL, 0xf00000f0f0f0f000ULL, 0xb10000b1b1b1b100ULL, |
| 121 | 0x8400008484848400, 0x9900009999999900, 0xdf0000dfdfdfdf00, | 121 | 0x8400008484848400ULL, 0x9900009999999900ULL, 0xdf0000dfdfdfdf00ULL, |
| 122 | 0x4c00004c4c4c4c00, 0xcb0000cbcbcbcb00, 0xc20000c2c2c2c200, | 122 | 0x4c00004c4c4c4c00ULL, 0xcb0000cbcbcbcb00ULL, 0xc20000c2c2c2c200ULL, |
| 123 | 0x3400003434343400, 0x7e00007e7e7e7e00, 0x7600007676767600, | 123 | 0x3400003434343400ULL, 0x7e00007e7e7e7e00ULL, 0x7600007676767600ULL, |
| 124 | 0x0500000505050500, 0x6d00006d6d6d6d00, 0xb70000b7b7b7b700, | 124 | 0x0500000505050500ULL, 0x6d00006d6d6d6d00ULL, 0xb70000b7b7b7b700ULL, |
| 125 | 0xa90000a9a9a9a900, 0x3100003131313100, 0xd10000d1d1d1d100, | 125 | 0xa90000a9a9a9a900ULL, 0x3100003131313100ULL, 0xd10000d1d1d1d100ULL, |
| 126 | 0x1700001717171700, 0x0400000404040400, 0xd70000d7d7d7d700, | 126 | 0x1700001717171700ULL, 0x0400000404040400ULL, 0xd70000d7d7d7d700ULL, |
| 127 | 0x1400001414141400, 0x5800005858585800, 0x3a00003a3a3a3a00, | 127 | 0x1400001414141400ULL, 0x5800005858585800ULL, 0x3a00003a3a3a3a00ULL, |
| 128 | 0x6100006161616100, 0xde0000dededede00, 0x1b00001b1b1b1b00, | 128 | 0x6100006161616100ULL, 0xde0000dededede00ULL, 0x1b00001b1b1b1b00ULL, |
| 129 | 0x1100001111111100, 0x1c00001c1c1c1c00, 0x3200003232323200, | 129 | 0x1100001111111100ULL, 0x1c00001c1c1c1c00ULL, 0x3200003232323200ULL, |
| 130 | 0x0f00000f0f0f0f00, 0x9c00009c9c9c9c00, 0x1600001616161600, | 130 | 0x0f00000f0f0f0f00ULL, 0x9c00009c9c9c9c00ULL, 0x1600001616161600ULL, |
| 131 | 0x5300005353535300, 0x1800001818181800, 0xf20000f2f2f2f200, | 131 | 0x5300005353535300ULL, 0x1800001818181800ULL, 0xf20000f2f2f2f200ULL, |
| 132 | 0x2200002222222200, 0xfe0000fefefefe00, 0x4400004444444400, | 132 | 0x2200002222222200ULL, 0xfe0000fefefefe00ULL, 0x4400004444444400ULL, |
| 133 | 0xcf0000cfcfcfcf00, 0xb20000b2b2b2b200, 0xc30000c3c3c3c300, | 133 | 0xcf0000cfcfcfcf00ULL, 0xb20000b2b2b2b200ULL, 0xc30000c3c3c3c300ULL, |
| 134 | 0xb50000b5b5b5b500, 0x7a00007a7a7a7a00, 0x9100009191919100, | 134 | 0xb50000b5b5b5b500ULL, 0x7a00007a7a7a7a00ULL, 0x9100009191919100ULL, |
| 135 | 0x2400002424242400, 0x0800000808080800, 0xe80000e8e8e8e800, | 135 | 0x2400002424242400ULL, 0x0800000808080800ULL, 0xe80000e8e8e8e800ULL, |
| 136 | 0xa80000a8a8a8a800, 0x6000006060606000, 0xfc0000fcfcfcfc00, | 136 | 0xa80000a8a8a8a800ULL, 0x6000006060606000ULL, 0xfc0000fcfcfcfc00ULL, |
| 137 | 0x6900006969696900, 0x5000005050505000, 0xaa0000aaaaaaaa00, | 137 | 0x6900006969696900ULL, 0x5000005050505000ULL, 0xaa0000aaaaaaaa00ULL, |
| 138 | 0xd00000d0d0d0d000, 0xa00000a0a0a0a000, 0x7d00007d7d7d7d00, | 138 | 0xd00000d0d0d0d000ULL, 0xa00000a0a0a0a000ULL, 0x7d00007d7d7d7d00ULL, |
| 139 | 0xa10000a1a1a1a100, 0x8900008989898900, 0x6200006262626200, | 139 | 0xa10000a1a1a1a100ULL, 0x8900008989898900ULL, 0x6200006262626200ULL, |
| 140 | 0x9700009797979700, 0x5400005454545400, 0x5b00005b5b5b5b00, | 140 | 0x9700009797979700ULL, 0x5400005454545400ULL, 0x5b00005b5b5b5b00ULL, |
| 141 | 0x1e00001e1e1e1e00, 0x9500009595959500, 0xe00000e0e0e0e000, | 141 | 0x1e00001e1e1e1e00ULL, 0x9500009595959500ULL, 0xe00000e0e0e0e000ULL, |
| 142 | 0xff0000ffffffff00, 0x6400006464646400, 0xd20000d2d2d2d200, | 142 | 0xff0000ffffffff00ULL, 0x6400006464646400ULL, 0xd20000d2d2d2d200ULL, |
| 143 | 0x1000001010101000, 0xc40000c4c4c4c400, 0x0000000000000000, | 143 | 0x1000001010101000ULL, 0xc40000c4c4c4c400ULL, 0x0000000000000000ULL, |
| 144 | 0x4800004848484800, 0xa30000a3a3a3a300, 0xf70000f7f7f7f700, | 144 | 0x4800004848484800ULL, 0xa30000a3a3a3a300ULL, 0xf70000f7f7f7f700ULL, |
| 145 | 0x7500007575757500, 0xdb0000dbdbdbdb00, 0x8a00008a8a8a8a00, | 145 | 0x7500007575757500ULL, 0xdb0000dbdbdbdb00ULL, 0x8a00008a8a8a8a00ULL, |
| 146 | 0x0300000303030300, 0xe60000e6e6e6e600, 0xda0000dadadada00, | 146 | 0x0300000303030300ULL, 0xe60000e6e6e6e600ULL, 0xda0000dadadada00ULL, |
| 147 | 0x0900000909090900, 0x3f00003f3f3f3f00, 0xdd0000dddddddd00, | 147 | 0x0900000909090900ULL, 0x3f00003f3f3f3f00ULL, 0xdd0000dddddddd00ULL, |
| 148 | 0x9400009494949400, 0x8700008787878700, 0x5c00005c5c5c5c00, | 148 | 0x9400009494949400ULL, 0x8700008787878700ULL, 0x5c00005c5c5c5c00ULL, |
| 149 | 0x8300008383838300, 0x0200000202020200, 0xcd0000cdcdcdcd00, | 149 | 0x8300008383838300ULL, 0x0200000202020200ULL, 0xcd0000cdcdcdcd00ULL, |
| 150 | 0x4a00004a4a4a4a00, 0x9000009090909000, 0x3300003333333300, | 150 | 0x4a00004a4a4a4a00ULL, 0x9000009090909000ULL, 0x3300003333333300ULL, |
| 151 | 0x7300007373737300, 0x6700006767676700, 0xf60000f6f6f6f600, | 151 | 0x7300007373737300ULL, 0x6700006767676700ULL, 0xf60000f6f6f6f600ULL, |
| 152 | 0xf30000f3f3f3f300, 0x9d00009d9d9d9d00, 0x7f00007f7f7f7f00, | 152 | 0xf30000f3f3f3f300ULL, 0x9d00009d9d9d9d00ULL, 0x7f00007f7f7f7f00ULL, |
| 153 | 0xbf0000bfbfbfbf00, 0xe20000e2e2e2e200, 0x5200005252525200, | 153 | 0xbf0000bfbfbfbf00ULL, 0xe20000e2e2e2e200ULL, 0x5200005252525200ULL, |
| 154 | 0x9b00009b9b9b9b00, 0xd80000d8d8d8d800, 0x2600002626262600, | 154 | 0x9b00009b9b9b9b00ULL, 0xd80000d8d8d8d800ULL, 0x2600002626262600ULL, |
| 155 | 0xc80000c8c8c8c800, 0x3700003737373700, 0xc60000c6c6c6c600, | 155 | 0xc80000c8c8c8c800ULL, 0x3700003737373700ULL, 0xc60000c6c6c6c600ULL, |
| 156 | 0x3b00003b3b3b3b00, 0x8100008181818100, 0x9600009696969600, | 156 | 0x3b00003b3b3b3b00ULL, 0x8100008181818100ULL, 0x9600009696969600ULL, |
| 157 | 0x6f00006f6f6f6f00, 0x4b00004b4b4b4b00, 0x1300001313131300, | 157 | 0x6f00006f6f6f6f00ULL, 0x4b00004b4b4b4b00ULL, 0x1300001313131300ULL, |
| 158 | 0xbe0000bebebebe00, 0x6300006363636300, 0x2e00002e2e2e2e00, | 158 | 0xbe0000bebebebe00ULL, 0x6300006363636300ULL, 0x2e00002e2e2e2e00ULL, |
| 159 | 0xe90000e9e9e9e900, 0x7900007979797900, 0xa70000a7a7a7a700, | 159 | 0xe90000e9e9e9e900ULL, 0x7900007979797900ULL, 0xa70000a7a7a7a700ULL, |
| 160 | 0x8c00008c8c8c8c00, 0x9f00009f9f9f9f00, 0x6e00006e6e6e6e00, | 160 | 0x8c00008c8c8c8c00ULL, 0x9f00009f9f9f9f00ULL, 0x6e00006e6e6e6e00ULL, |
| 161 | 0xbc0000bcbcbcbc00, 0x8e00008e8e8e8e00, 0x2900002929292900, | 161 | 0xbc0000bcbcbcbc00ULL, 0x8e00008e8e8e8e00ULL, 0x2900002929292900ULL, |
| 162 | 0xf50000f5f5f5f500, 0xf90000f9f9f9f900, 0xb60000b6b6b6b600, | 162 | 0xf50000f5f5f5f500ULL, 0xf90000f9f9f9f900ULL, 0xb60000b6b6b6b600ULL, |
| 163 | 0x2f00002f2f2f2f00, 0xfd0000fdfdfdfd00, 0xb40000b4b4b4b400, | 163 | 0x2f00002f2f2f2f00ULL, 0xfd0000fdfdfdfd00ULL, 0xb40000b4b4b4b400ULL, |
| 164 | 0x5900005959595900, 0x7800007878787800, 0x9800009898989800, | 164 | 0x5900005959595900ULL, 0x7800007878787800ULL, 0x9800009898989800ULL, |
| 165 | 0x0600000606060600, 0x6a00006a6a6a6a00, 0xe70000e7e7e7e700, | 165 | 0x0600000606060600ULL, 0x6a00006a6a6a6a00ULL, 0xe70000e7e7e7e700ULL, |
| 166 | 0x4600004646464600, 0x7100007171717100, 0xba0000babababa00, | 166 | 0x4600004646464600ULL, 0x7100007171717100ULL, 0xba0000babababa00ULL, |
| 167 | 0xd40000d4d4d4d400, 0x2500002525252500, 0xab0000abababab00, | 167 | 0xd40000d4d4d4d400ULL, 0x2500002525252500ULL, 0xab0000abababab00ULL, |
| 168 | 0x4200004242424200, 0x8800008888888800, 0xa20000a2a2a2a200, | 168 | 0x4200004242424200ULL, 0x8800008888888800ULL, 0xa20000a2a2a2a200ULL, |
| 169 | 0x8d00008d8d8d8d00, 0xfa0000fafafafa00, 0x7200007272727200, | 169 | 0x8d00008d8d8d8d00ULL, 0xfa0000fafafafa00ULL, 0x7200007272727200ULL, |
| 170 | 0x0700000707070700, 0xb90000b9b9b9b900, 0x5500005555555500, | 170 | 0x0700000707070700ULL, 0xb90000b9b9b9b900ULL, 0x5500005555555500ULL, |
| 171 | 0xf80000f8f8f8f800, 0xee0000eeeeeeee00, 0xac0000acacacac00, | 171 | 0xf80000f8f8f8f800ULL, 0xee0000eeeeeeee00ULL, 0xac0000acacacac00ULL, |
| 172 | 0x0a00000a0a0a0a00, 0x3600003636363600, 0x4900004949494900, | 172 | 0x0a00000a0a0a0a00ULL, 0x3600003636363600ULL, 0x4900004949494900ULL, |
| 173 | 0x2a00002a2a2a2a00, 0x6800006868686800, 0x3c00003c3c3c3c00, | 173 | 0x2a00002a2a2a2a00ULL, 0x6800006868686800ULL, 0x3c00003c3c3c3c00ULL, |
| 174 | 0x3800003838383800, 0xf10000f1f1f1f100, 0xa40000a4a4a4a400, | 174 | 0x3800003838383800ULL, 0xf10000f1f1f1f100ULL, 0xa40000a4a4a4a400ULL, |
| 175 | 0x4000004040404000, 0x2800002828282800, 0xd30000d3d3d3d300, | 175 | 0x4000004040404000ULL, 0x2800002828282800ULL, 0xd30000d3d3d3d300ULL, |
| 176 | 0x7b00007b7b7b7b00, 0xbb0000bbbbbbbb00, 0xc90000c9c9c9c900, | 176 | 0x7b00007b7b7b7b00ULL, 0xbb0000bbbbbbbb00ULL, 0xc90000c9c9c9c900ULL, |
| 177 | 0x4300004343434300, 0xc10000c1c1c1c100, 0x1500001515151500, | 177 | 0x4300004343434300ULL, 0xc10000c1c1c1c100ULL, 0x1500001515151500ULL, |
| 178 | 0xe30000e3e3e3e300, 0xad0000adadadad00, 0xf40000f4f4f4f400, | 178 | 0xe30000e3e3e3e300ULL, 0xad0000adadadad00ULL, 0xf40000f4f4f4f400ULL, |
| 179 | 0x7700007777777700, 0xc70000c7c7c7c700, 0x8000008080808000, | 179 | 0x7700007777777700ULL, 0xc70000c7c7c7c700ULL, 0x8000008080808000ULL, |
| 180 | 0x9e00009e9e9e9e00, | 180 | 0x9e00009e9e9e9e00ULL, |
| 181 | }; | 181 | }; |
| 182 | 182 | ||
| 183 | const u64 camellia_sp22000222[256] = { | 183 | const u64 camellia_sp22000222[256] = { |
| 184 | 0xe0e0000000e0e0e0, 0x0505000000050505, 0x5858000000585858, | 184 | 0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL, |
| 185 | 0xd9d9000000d9d9d9, 0x6767000000676767, 0x4e4e0000004e4e4e, | 185 | 0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL, |
| 186 | 0x8181000000818181, 0xcbcb000000cbcbcb, 0xc9c9000000c9c9c9, | 186 | 0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL, |
| 187 | 0x0b0b0000000b0b0b, 0xaeae000000aeaeae, 0x6a6a0000006a6a6a, | 187 | 0x0b0b0000000b0b0bULL, 0xaeae000000aeaeaeULL, 0x6a6a0000006a6a6aULL, |
| 188 | 0xd5d5000000d5d5d5, 0x1818000000181818, 0x5d5d0000005d5d5d, | 188 | 0xd5d5000000d5d5d5ULL, 0x1818000000181818ULL, 0x5d5d0000005d5d5dULL, |
| 189 | 0x8282000000828282, 0x4646000000464646, 0xdfdf000000dfdfdf, | 189 | 0x8282000000828282ULL, 0x4646000000464646ULL, 0xdfdf000000dfdfdfULL, |
| 190 | 0xd6d6000000d6d6d6, 0x2727000000272727, 0x8a8a0000008a8a8a, | 190 | 0xd6d6000000d6d6d6ULL, 0x2727000000272727ULL, 0x8a8a0000008a8a8aULL, |
| 191 | 0x3232000000323232, 0x4b4b0000004b4b4b, 0x4242000000424242, | 191 | 0x3232000000323232ULL, 0x4b4b0000004b4b4bULL, 0x4242000000424242ULL, |
| 192 | 0xdbdb000000dbdbdb, 0x1c1c0000001c1c1c, 0x9e9e0000009e9e9e, | 192 | 0xdbdb000000dbdbdbULL, 0x1c1c0000001c1c1cULL, 0x9e9e0000009e9e9eULL, |
| 193 | 0x9c9c0000009c9c9c, 0x3a3a0000003a3a3a, 0xcaca000000cacaca, | 193 | 0x9c9c0000009c9c9cULL, 0x3a3a0000003a3a3aULL, 0xcaca000000cacacaULL, |
| 194 | 0x2525000000252525, 0x7b7b0000007b7b7b, 0x0d0d0000000d0d0d, | 194 | 0x2525000000252525ULL, 0x7b7b0000007b7b7bULL, 0x0d0d0000000d0d0dULL, |
| 195 | 0x7171000000717171, 0x5f5f0000005f5f5f, 0x1f1f0000001f1f1f, | 195 | 0x7171000000717171ULL, 0x5f5f0000005f5f5fULL, 0x1f1f0000001f1f1fULL, |
| 196 | 0xf8f8000000f8f8f8, 0xd7d7000000d7d7d7, 0x3e3e0000003e3e3e, | 196 | 0xf8f8000000f8f8f8ULL, 0xd7d7000000d7d7d7ULL, 0x3e3e0000003e3e3eULL, |
| 197 | 0x9d9d0000009d9d9d, 0x7c7c0000007c7c7c, 0x6060000000606060, | 197 | 0x9d9d0000009d9d9dULL, 0x7c7c0000007c7c7cULL, 0x6060000000606060ULL, |
| 198 | 0xb9b9000000b9b9b9, 0xbebe000000bebebe, 0xbcbc000000bcbcbc, | 198 | 0xb9b9000000b9b9b9ULL, 0xbebe000000bebebeULL, 0xbcbc000000bcbcbcULL, |
| 199 | 0x8b8b0000008b8b8b, 0x1616000000161616, 0x3434000000343434, | 199 | 0x8b8b0000008b8b8bULL, 0x1616000000161616ULL, 0x3434000000343434ULL, |
| 200 | 0x4d4d0000004d4d4d, 0xc3c3000000c3c3c3, 0x7272000000727272, | 200 | 0x4d4d0000004d4d4dULL, 0xc3c3000000c3c3c3ULL, 0x7272000000727272ULL, |
| 201 | 0x9595000000959595, 0xabab000000ababab, 0x8e8e0000008e8e8e, | 201 | 0x9595000000959595ULL, 0xabab000000abababULL, 0x8e8e0000008e8e8eULL, |
| 202 | 0xbaba000000bababa, 0x7a7a0000007a7a7a, 0xb3b3000000b3b3b3, | 202 | 0xbaba000000bababaULL, 0x7a7a0000007a7a7aULL, 0xb3b3000000b3b3b3ULL, |
| 203 | 0x0202000000020202, 0xb4b4000000b4b4b4, 0xadad000000adadad, | 203 | 0x0202000000020202ULL, 0xb4b4000000b4b4b4ULL, 0xadad000000adadadULL, |
| 204 | 0xa2a2000000a2a2a2, 0xacac000000acacac, 0xd8d8000000d8d8d8, | 204 | 0xa2a2000000a2a2a2ULL, 0xacac000000acacacULL, 0xd8d8000000d8d8d8ULL, |
| 205 | 0x9a9a0000009a9a9a, 0x1717000000171717, 0x1a1a0000001a1a1a, | 205 | 0x9a9a0000009a9a9aULL, 0x1717000000171717ULL, 0x1a1a0000001a1a1aULL, |
| 206 | 0x3535000000353535, 0xcccc000000cccccc, 0xf7f7000000f7f7f7, | 206 | 0x3535000000353535ULL, 0xcccc000000ccccccULL, 0xf7f7000000f7f7f7ULL, |
| 207 | 0x9999000000999999, 0x6161000000616161, 0x5a5a0000005a5a5a, | 207 | 0x9999000000999999ULL, 0x6161000000616161ULL, 0x5a5a0000005a5a5aULL, |
| 208 | 0xe8e8000000e8e8e8, 0x2424000000242424, 0x5656000000565656, | 208 | 0xe8e8000000e8e8e8ULL, 0x2424000000242424ULL, 0x5656000000565656ULL, |
| 209 | 0x4040000000404040, 0xe1e1000000e1e1e1, 0x6363000000636363, | 209 | 0x4040000000404040ULL, 0xe1e1000000e1e1e1ULL, 0x6363000000636363ULL, |
| 210 | 0x0909000000090909, 0x3333000000333333, 0xbfbf000000bfbfbf, | 210 | 0x0909000000090909ULL, 0x3333000000333333ULL, 0xbfbf000000bfbfbfULL, |
| 211 | 0x9898000000989898, 0x9797000000979797, 0x8585000000858585, | 211 | 0x9898000000989898ULL, 0x9797000000979797ULL, 0x8585000000858585ULL, |
| 212 | 0x6868000000686868, 0xfcfc000000fcfcfc, 0xecec000000ececec, | 212 | 0x6868000000686868ULL, 0xfcfc000000fcfcfcULL, 0xecec000000ecececULL, |
| 213 | 0x0a0a0000000a0a0a, 0xdada000000dadada, 0x6f6f0000006f6f6f, | 213 | 0x0a0a0000000a0a0aULL, 0xdada000000dadadaULL, 0x6f6f0000006f6f6fULL, |
| 214 | 0x5353000000535353, 0x6262000000626262, 0xa3a3000000a3a3a3, | 214 | 0x5353000000535353ULL, 0x6262000000626262ULL, 0xa3a3000000a3a3a3ULL, |
| 215 | 0x2e2e0000002e2e2e, 0x0808000000080808, 0xafaf000000afafaf, | 215 | 0x2e2e0000002e2e2eULL, 0x0808000000080808ULL, 0xafaf000000afafafULL, |
| 216 | 0x2828000000282828, 0xb0b0000000b0b0b0, 0x7474000000747474, | 216 | 0x2828000000282828ULL, 0xb0b0000000b0b0b0ULL, 0x7474000000747474ULL, |
| 217 | 0xc2c2000000c2c2c2, 0xbdbd000000bdbdbd, 0x3636000000363636, | 217 | 0xc2c2000000c2c2c2ULL, 0xbdbd000000bdbdbdULL, 0x3636000000363636ULL, |
| 218 | 0x2222000000222222, 0x3838000000383838, 0x6464000000646464, | 218 | 0x2222000000222222ULL, 0x3838000000383838ULL, 0x6464000000646464ULL, |
| 219 | 0x1e1e0000001e1e1e, 0x3939000000393939, 0x2c2c0000002c2c2c, | 219 | 0x1e1e0000001e1e1eULL, 0x3939000000393939ULL, 0x2c2c0000002c2c2cULL, |
| 220 | 0xa6a6000000a6a6a6, 0x3030000000303030, 0xe5e5000000e5e5e5, | 220 | 0xa6a6000000a6a6a6ULL, 0x3030000000303030ULL, 0xe5e5000000e5e5e5ULL, |
| 221 | 0x4444000000444444, 0xfdfd000000fdfdfd, 0x8888000000888888, | 221 | 0x4444000000444444ULL, 0xfdfd000000fdfdfdULL, 0x8888000000888888ULL, |
| 222 | 0x9f9f0000009f9f9f, 0x6565000000656565, 0x8787000000878787, | 222 | 0x9f9f0000009f9f9fULL, 0x6565000000656565ULL, 0x8787000000878787ULL, |
| 223 | 0x6b6b0000006b6b6b, 0xf4f4000000f4f4f4, 0x2323000000232323, | 223 | 0x6b6b0000006b6b6bULL, 0xf4f4000000f4f4f4ULL, 0x2323000000232323ULL, |
| 224 | 0x4848000000484848, 0x1010000000101010, 0xd1d1000000d1d1d1, | 224 | 0x4848000000484848ULL, 0x1010000000101010ULL, 0xd1d1000000d1d1d1ULL, |
| 225 | 0x5151000000515151, 0xc0c0000000c0c0c0, 0xf9f9000000f9f9f9, | 225 | 0x5151000000515151ULL, 0xc0c0000000c0c0c0ULL, 0xf9f9000000f9f9f9ULL, |
| 226 | 0xd2d2000000d2d2d2, 0xa0a0000000a0a0a0, 0x5555000000555555, | 226 | 0xd2d2000000d2d2d2ULL, 0xa0a0000000a0a0a0ULL, 0x5555000000555555ULL, |
| 227 | 0xa1a1000000a1a1a1, 0x4141000000414141, 0xfafa000000fafafa, | 227 | 0xa1a1000000a1a1a1ULL, 0x4141000000414141ULL, 0xfafa000000fafafaULL, |
| 228 | 0x4343000000434343, 0x1313000000131313, 0xc4c4000000c4c4c4, | 228 | 0x4343000000434343ULL, 0x1313000000131313ULL, 0xc4c4000000c4c4c4ULL, |
| 229 | 0x2f2f0000002f2f2f, 0xa8a8000000a8a8a8, 0xb6b6000000b6b6b6, | 229 | 0x2f2f0000002f2f2fULL, 0xa8a8000000a8a8a8ULL, 0xb6b6000000b6b6b6ULL, |
| 230 | 0x3c3c0000003c3c3c, 0x2b2b0000002b2b2b, 0xc1c1000000c1c1c1, | 230 | 0x3c3c0000003c3c3cULL, 0x2b2b0000002b2b2bULL, 0xc1c1000000c1c1c1ULL, |
| 231 | 0xffff000000ffffff, 0xc8c8000000c8c8c8, 0xa5a5000000a5a5a5, | 231 | 0xffff000000ffffffULL, 0xc8c8000000c8c8c8ULL, 0xa5a5000000a5a5a5ULL, |
| 232 | 0x2020000000202020, 0x8989000000898989, 0x0000000000000000, | 232 | 0x2020000000202020ULL, 0x8989000000898989ULL, 0x0000000000000000ULL, |
| 233 | 0x9090000000909090, 0x4747000000474747, 0xefef000000efefef, | 233 | 0x9090000000909090ULL, 0x4747000000474747ULL, 0xefef000000efefefULL, |
| 234 | 0xeaea000000eaeaea, 0xb7b7000000b7b7b7, 0x1515000000151515, | 234 | 0xeaea000000eaeaeaULL, 0xb7b7000000b7b7b7ULL, 0x1515000000151515ULL, |
| 235 | 0x0606000000060606, 0xcdcd000000cdcdcd, 0xb5b5000000b5b5b5, | 235 | 0x0606000000060606ULL, 0xcdcd000000cdcdcdULL, 0xb5b5000000b5b5b5ULL, |
| 236 | 0x1212000000121212, 0x7e7e0000007e7e7e, 0xbbbb000000bbbbbb, | 236 | 0x1212000000121212ULL, 0x7e7e0000007e7e7eULL, 0xbbbb000000bbbbbbULL, |
| 237 | 0x2929000000292929, 0x0f0f0000000f0f0f, 0xb8b8000000b8b8b8, | 237 | 0x2929000000292929ULL, 0x0f0f0000000f0f0fULL, 0xb8b8000000b8b8b8ULL, |
| 238 | 0x0707000000070707, 0x0404000000040404, 0x9b9b0000009b9b9b, | 238 | 0x0707000000070707ULL, 0x0404000000040404ULL, 0x9b9b0000009b9b9bULL, |
| 239 | 0x9494000000949494, 0x2121000000212121, 0x6666000000666666, | 239 | 0x9494000000949494ULL, 0x2121000000212121ULL, 0x6666000000666666ULL, |
| 240 | 0xe6e6000000e6e6e6, 0xcece000000cecece, 0xeded000000ededed, | 240 | 0xe6e6000000e6e6e6ULL, 0xcece000000cececeULL, 0xeded000000edededULL, |
| 241 | 0xe7e7000000e7e7e7, 0x3b3b0000003b3b3b, 0xfefe000000fefefe, | 241 | 0xe7e7000000e7e7e7ULL, 0x3b3b0000003b3b3bULL, 0xfefe000000fefefeULL, |
| 242 | 0x7f7f0000007f7f7f, 0xc5c5000000c5c5c5, 0xa4a4000000a4a4a4, | 242 | 0x7f7f0000007f7f7fULL, 0xc5c5000000c5c5c5ULL, 0xa4a4000000a4a4a4ULL, |
| 243 | 0x3737000000373737, 0xb1b1000000b1b1b1, 0x4c4c0000004c4c4c, | 243 | 0x3737000000373737ULL, 0xb1b1000000b1b1b1ULL, 0x4c4c0000004c4c4cULL, |
| 244 | 0x9191000000919191, 0x6e6e0000006e6e6e, 0x8d8d0000008d8d8d, | 244 | 0x9191000000919191ULL, 0x6e6e0000006e6e6eULL, 0x8d8d0000008d8d8dULL, |
| 245 | 0x7676000000767676, 0x0303000000030303, 0x2d2d0000002d2d2d, | 245 | 0x7676000000767676ULL, 0x0303000000030303ULL, 0x2d2d0000002d2d2dULL, |
| 246 | 0xdede000000dedede, 0x9696000000969696, 0x2626000000262626, | 246 | 0xdede000000dededeULL, 0x9696000000969696ULL, 0x2626000000262626ULL, |
| 247 | 0x7d7d0000007d7d7d, 0xc6c6000000c6c6c6, 0x5c5c0000005c5c5c, | 247 | 0x7d7d0000007d7d7dULL, 0xc6c6000000c6c6c6ULL, 0x5c5c0000005c5c5cULL, |
| 248 | 0xd3d3000000d3d3d3, 0xf2f2000000f2f2f2, 0x4f4f0000004f4f4f, | 248 | 0xd3d3000000d3d3d3ULL, 0xf2f2000000f2f2f2ULL, 0x4f4f0000004f4f4fULL, |
| 249 | 0x1919000000191919, 0x3f3f0000003f3f3f, 0xdcdc000000dcdcdc, | 249 | 0x1919000000191919ULL, 0x3f3f0000003f3f3fULL, 0xdcdc000000dcdcdcULL, |
| 250 | 0x7979000000797979, 0x1d1d0000001d1d1d, 0x5252000000525252, | 250 | 0x7979000000797979ULL, 0x1d1d0000001d1d1dULL, 0x5252000000525252ULL, |
| 251 | 0xebeb000000ebebeb, 0xf3f3000000f3f3f3, 0x6d6d0000006d6d6d, | 251 | 0xebeb000000ebebebULL, 0xf3f3000000f3f3f3ULL, 0x6d6d0000006d6d6dULL, |
| 252 | 0x5e5e0000005e5e5e, 0xfbfb000000fbfbfb, 0x6969000000696969, | 252 | 0x5e5e0000005e5e5eULL, 0xfbfb000000fbfbfbULL, 0x6969000000696969ULL, |
| 253 | 0xb2b2000000b2b2b2, 0xf0f0000000f0f0f0, 0x3131000000313131, | 253 | 0xb2b2000000b2b2b2ULL, 0xf0f0000000f0f0f0ULL, 0x3131000000313131ULL, |
| 254 | 0x0c0c0000000c0c0c, 0xd4d4000000d4d4d4, 0xcfcf000000cfcfcf, | 254 | 0x0c0c0000000c0c0cULL, 0xd4d4000000d4d4d4ULL, 0xcfcf000000cfcfcfULL, |
| 255 | 0x8c8c0000008c8c8c, 0xe2e2000000e2e2e2, 0x7575000000757575, | 255 | 0x8c8c0000008c8c8cULL, 0xe2e2000000e2e2e2ULL, 0x7575000000757575ULL, |
| 256 | 0xa9a9000000a9a9a9, 0x4a4a0000004a4a4a, 0x5757000000575757, | 256 | 0xa9a9000000a9a9a9ULL, 0x4a4a0000004a4a4aULL, 0x5757000000575757ULL, |
| 257 | 0x8484000000848484, 0x1111000000111111, 0x4545000000454545, | 257 | 0x8484000000848484ULL, 0x1111000000111111ULL, 0x4545000000454545ULL, |
| 258 | 0x1b1b0000001b1b1b, 0xf5f5000000f5f5f5, 0xe4e4000000e4e4e4, | 258 | 0x1b1b0000001b1b1bULL, 0xf5f5000000f5f5f5ULL, 0xe4e4000000e4e4e4ULL, |
| 259 | 0x0e0e0000000e0e0e, 0x7373000000737373, 0xaaaa000000aaaaaa, | 259 | 0x0e0e0000000e0e0eULL, 0x7373000000737373ULL, 0xaaaa000000aaaaaaULL, |
| 260 | 0xf1f1000000f1f1f1, 0xdddd000000dddddd, 0x5959000000595959, | 260 | 0xf1f1000000f1f1f1ULL, 0xdddd000000ddddddULL, 0x5959000000595959ULL, |
| 261 | 0x1414000000141414, 0x6c6c0000006c6c6c, 0x9292000000929292, | 261 | 0x1414000000141414ULL, 0x6c6c0000006c6c6cULL, 0x9292000000929292ULL, |
| 262 | 0x5454000000545454, 0xd0d0000000d0d0d0, 0x7878000000787878, | 262 | 0x5454000000545454ULL, 0xd0d0000000d0d0d0ULL, 0x7878000000787878ULL, |
| 263 | 0x7070000000707070, 0xe3e3000000e3e3e3, 0x4949000000494949, | 263 | 0x7070000000707070ULL, 0xe3e3000000e3e3e3ULL, 0x4949000000494949ULL, |
| 264 | 0x8080000000808080, 0x5050000000505050, 0xa7a7000000a7a7a7, | 264 | 0x8080000000808080ULL, 0x5050000000505050ULL, 0xa7a7000000a7a7a7ULL, |
| 265 | 0xf6f6000000f6f6f6, 0x7777000000777777, 0x9393000000939393, | 265 | 0xf6f6000000f6f6f6ULL, 0x7777000000777777ULL, 0x9393000000939393ULL, |
| 266 | 0x8686000000868686, 0x8383000000838383, 0x2a2a0000002a2a2a, | 266 | 0x8686000000868686ULL, 0x8383000000838383ULL, 0x2a2a0000002a2a2aULL, |
| 267 | 0xc7c7000000c7c7c7, 0x5b5b0000005b5b5b, 0xe9e9000000e9e9e9, | 267 | 0xc7c7000000c7c7c7ULL, 0x5b5b0000005b5b5bULL, 0xe9e9000000e9e9e9ULL, |
| 268 | 0xeeee000000eeeeee, 0x8f8f0000008f8f8f, 0x0101000000010101, | 268 | 0xeeee000000eeeeeeULL, 0x8f8f0000008f8f8fULL, 0x0101000000010101ULL, |
| 269 | 0x3d3d0000003d3d3d, | 269 | 0x3d3d0000003d3d3dULL, |
| 270 | }; | 270 | }; |
| 271 | 271 | ||
| 272 | const u64 camellia_sp03303033[256] = { | 272 | const u64 camellia_sp03303033[256] = { |
| 273 | 0x0038380038003838, 0x0041410041004141, 0x0016160016001616, | 273 | 0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL, |
| 274 | 0x0076760076007676, 0x00d9d900d900d9d9, 0x0093930093009393, | 274 | 0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL, |
| 275 | 0x0060600060006060, 0x00f2f200f200f2f2, 0x0072720072007272, | 275 | 0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL, |
| 276 | 0x00c2c200c200c2c2, 0x00abab00ab00abab, 0x009a9a009a009a9a, | 276 | 0x00c2c200c200c2c2ULL, 0x00abab00ab00ababULL, 0x009a9a009a009a9aULL, |
| 277 | 0x0075750075007575, 0x0006060006000606, 0x0057570057005757, | 277 | 0x0075750075007575ULL, 0x0006060006000606ULL, 0x0057570057005757ULL, |
| 278 | 0x00a0a000a000a0a0, 0x0091910091009191, 0x00f7f700f700f7f7, | 278 | 0x00a0a000a000a0a0ULL, 0x0091910091009191ULL, 0x00f7f700f700f7f7ULL, |
| 279 | 0x00b5b500b500b5b5, 0x00c9c900c900c9c9, 0x00a2a200a200a2a2, | 279 | 0x00b5b500b500b5b5ULL, 0x00c9c900c900c9c9ULL, 0x00a2a200a200a2a2ULL, |
| 280 | 0x008c8c008c008c8c, 0x00d2d200d200d2d2, 0x0090900090009090, | 280 | 0x008c8c008c008c8cULL, 0x00d2d200d200d2d2ULL, 0x0090900090009090ULL, |
| 281 | 0x00f6f600f600f6f6, 0x0007070007000707, 0x00a7a700a700a7a7, | 281 | 0x00f6f600f600f6f6ULL, 0x0007070007000707ULL, 0x00a7a700a700a7a7ULL, |
| 282 | 0x0027270027002727, 0x008e8e008e008e8e, 0x00b2b200b200b2b2, | 282 | 0x0027270027002727ULL, 0x008e8e008e008e8eULL, 0x00b2b200b200b2b2ULL, |
| 283 | 0x0049490049004949, 0x00dede00de00dede, 0x0043430043004343, | 283 | 0x0049490049004949ULL, 0x00dede00de00dedeULL, 0x0043430043004343ULL, |
| 284 | 0x005c5c005c005c5c, 0x00d7d700d700d7d7, 0x00c7c700c700c7c7, | 284 | 0x005c5c005c005c5cULL, 0x00d7d700d700d7d7ULL, 0x00c7c700c700c7c7ULL, |
| 285 | 0x003e3e003e003e3e, 0x00f5f500f500f5f5, 0x008f8f008f008f8f, | 285 | 0x003e3e003e003e3eULL, 0x00f5f500f500f5f5ULL, 0x008f8f008f008f8fULL, |
| 286 | 0x0067670067006767, 0x001f1f001f001f1f, 0x0018180018001818, | 286 | 0x0067670067006767ULL, 0x001f1f001f001f1fULL, 0x0018180018001818ULL, |
| 287 | 0x006e6e006e006e6e, 0x00afaf00af00afaf, 0x002f2f002f002f2f, | 287 | 0x006e6e006e006e6eULL, 0x00afaf00af00afafULL, 0x002f2f002f002f2fULL, |
| 288 | 0x00e2e200e200e2e2, 0x0085850085008585, 0x000d0d000d000d0d, | 288 | 0x00e2e200e200e2e2ULL, 0x0085850085008585ULL, 0x000d0d000d000d0dULL, |
| 289 | 0x0053530053005353, 0x00f0f000f000f0f0, 0x009c9c009c009c9c, | 289 | 0x0053530053005353ULL, 0x00f0f000f000f0f0ULL, 0x009c9c009c009c9cULL, |
| 290 | 0x0065650065006565, 0x00eaea00ea00eaea, 0x00a3a300a300a3a3, | 290 | 0x0065650065006565ULL, 0x00eaea00ea00eaeaULL, 0x00a3a300a300a3a3ULL, |
| 291 | 0x00aeae00ae00aeae, 0x009e9e009e009e9e, 0x00ecec00ec00ecec, | 291 | 0x00aeae00ae00aeaeULL, 0x009e9e009e009e9eULL, 0x00ecec00ec00ececULL, |
| 292 | 0x0080800080008080, 0x002d2d002d002d2d, 0x006b6b006b006b6b, | 292 | 0x0080800080008080ULL, 0x002d2d002d002d2dULL, 0x006b6b006b006b6bULL, |
| 293 | 0x00a8a800a800a8a8, 0x002b2b002b002b2b, 0x0036360036003636, | 293 | 0x00a8a800a800a8a8ULL, 0x002b2b002b002b2bULL, 0x0036360036003636ULL, |
| 294 | 0x00a6a600a600a6a6, 0x00c5c500c500c5c5, 0x0086860086008686, | 294 | 0x00a6a600a600a6a6ULL, 0x00c5c500c500c5c5ULL, 0x0086860086008686ULL, |
| 295 | 0x004d4d004d004d4d, 0x0033330033003333, 0x00fdfd00fd00fdfd, | 295 | 0x004d4d004d004d4dULL, 0x0033330033003333ULL, 0x00fdfd00fd00fdfdULL, |
| 296 | 0x0066660066006666, 0x0058580058005858, 0x0096960096009696, | 296 | 0x0066660066006666ULL, 0x0058580058005858ULL, 0x0096960096009696ULL, |
| 297 | 0x003a3a003a003a3a, 0x0009090009000909, 0x0095950095009595, | 297 | 0x003a3a003a003a3aULL, 0x0009090009000909ULL, 0x0095950095009595ULL, |
| 298 | 0x0010100010001010, 0x0078780078007878, 0x00d8d800d800d8d8, | 298 | 0x0010100010001010ULL, 0x0078780078007878ULL, 0x00d8d800d800d8d8ULL, |
| 299 | 0x0042420042004242, 0x00cccc00cc00cccc, 0x00efef00ef00efef, | 299 | 0x0042420042004242ULL, 0x00cccc00cc00ccccULL, 0x00efef00ef00efefULL, |
| 300 | 0x0026260026002626, 0x00e5e500e500e5e5, 0x0061610061006161, | 300 | 0x0026260026002626ULL, 0x00e5e500e500e5e5ULL, 0x0061610061006161ULL, |
| 301 | 0x001a1a001a001a1a, 0x003f3f003f003f3f, 0x003b3b003b003b3b, | 301 | 0x001a1a001a001a1aULL, 0x003f3f003f003f3fULL, 0x003b3b003b003b3bULL, |
| 302 | 0x0082820082008282, 0x00b6b600b600b6b6, 0x00dbdb00db00dbdb, | 302 | 0x0082820082008282ULL, 0x00b6b600b600b6b6ULL, 0x00dbdb00db00dbdbULL, |
| 303 | 0x00d4d400d400d4d4, 0x0098980098009898, 0x00e8e800e800e8e8, | 303 | 0x00d4d400d400d4d4ULL, 0x0098980098009898ULL, 0x00e8e800e800e8e8ULL, |
| 304 | 0x008b8b008b008b8b, 0x0002020002000202, 0x00ebeb00eb00ebeb, | 304 | 0x008b8b008b008b8bULL, 0x0002020002000202ULL, 0x00ebeb00eb00ebebULL, |
| 305 | 0x000a0a000a000a0a, 0x002c2c002c002c2c, 0x001d1d001d001d1d, | 305 | 0x000a0a000a000a0aULL, 0x002c2c002c002c2cULL, 0x001d1d001d001d1dULL, |
| 306 | 0x00b0b000b000b0b0, 0x006f6f006f006f6f, 0x008d8d008d008d8d, | 306 | 0x00b0b000b000b0b0ULL, 0x006f6f006f006f6fULL, 0x008d8d008d008d8dULL, |
| 307 | 0x0088880088008888, 0x000e0e000e000e0e, 0x0019190019001919, | 307 | 0x0088880088008888ULL, 0x000e0e000e000e0eULL, 0x0019190019001919ULL, |
| 308 | 0x0087870087008787, 0x004e4e004e004e4e, 0x000b0b000b000b0b, | 308 | 0x0087870087008787ULL, 0x004e4e004e004e4eULL, 0x000b0b000b000b0bULL, |
| 309 | 0x00a9a900a900a9a9, 0x000c0c000c000c0c, 0x0079790079007979, | 309 | 0x00a9a900a900a9a9ULL, 0x000c0c000c000c0cULL, 0x0079790079007979ULL, |
| 310 | 0x0011110011001111, 0x007f7f007f007f7f, 0x0022220022002222, | 310 | 0x0011110011001111ULL, 0x007f7f007f007f7fULL, 0x0022220022002222ULL, |
| 311 | 0x00e7e700e700e7e7, 0x0059590059005959, 0x00e1e100e100e1e1, | 311 | 0x00e7e700e700e7e7ULL, 0x0059590059005959ULL, 0x00e1e100e100e1e1ULL, |
| 312 | 0x00dada00da00dada, 0x003d3d003d003d3d, 0x00c8c800c800c8c8, | 312 | 0x00dada00da00dadaULL, 0x003d3d003d003d3dULL, 0x00c8c800c800c8c8ULL, |
| 313 | 0x0012120012001212, 0x0004040004000404, 0x0074740074007474, | 313 | 0x0012120012001212ULL, 0x0004040004000404ULL, 0x0074740074007474ULL, |
| 314 | 0x0054540054005454, 0x0030300030003030, 0x007e7e007e007e7e, | 314 | 0x0054540054005454ULL, 0x0030300030003030ULL, 0x007e7e007e007e7eULL, |
| 315 | 0x00b4b400b400b4b4, 0x0028280028002828, 0x0055550055005555, | 315 | 0x00b4b400b400b4b4ULL, 0x0028280028002828ULL, 0x0055550055005555ULL, |
| 316 | 0x0068680068006868, 0x0050500050005050, 0x00bebe00be00bebe, | 316 | 0x0068680068006868ULL, 0x0050500050005050ULL, 0x00bebe00be00bebeULL, |
| 317 | 0x00d0d000d000d0d0, 0x00c4c400c400c4c4, 0x0031310031003131, | 317 | 0x00d0d000d000d0d0ULL, 0x00c4c400c400c4c4ULL, 0x0031310031003131ULL, |
| 318 | 0x00cbcb00cb00cbcb, 0x002a2a002a002a2a, 0x00adad00ad00adad, | 318 | 0x00cbcb00cb00cbcbULL, 0x002a2a002a002a2aULL, 0x00adad00ad00adadULL, |
| 319 | 0x000f0f000f000f0f, 0x00caca00ca00caca, 0x0070700070007070, | 319 | 0x000f0f000f000f0fULL, 0x00caca00ca00cacaULL, 0x0070700070007070ULL, |
| 320 | 0x00ffff00ff00ffff, 0x0032320032003232, 0x0069690069006969, | 320 | 0x00ffff00ff00ffffULL, 0x0032320032003232ULL, 0x0069690069006969ULL, |
| 321 | 0x0008080008000808, 0x0062620062006262, 0x0000000000000000, | 321 | 0x0008080008000808ULL, 0x0062620062006262ULL, 0x0000000000000000ULL, |
| 322 | 0x0024240024002424, 0x00d1d100d100d1d1, 0x00fbfb00fb00fbfb, | 322 | 0x0024240024002424ULL, 0x00d1d100d100d1d1ULL, 0x00fbfb00fb00fbfbULL, |
| 323 | 0x00baba00ba00baba, 0x00eded00ed00eded, 0x0045450045004545, | 323 | 0x00baba00ba00babaULL, 0x00eded00ed00ededULL, 0x0045450045004545ULL, |
| 324 | 0x0081810081008181, 0x0073730073007373, 0x006d6d006d006d6d, | 324 | 0x0081810081008181ULL, 0x0073730073007373ULL, 0x006d6d006d006d6dULL, |
| 325 | 0x0084840084008484, 0x009f9f009f009f9f, 0x00eeee00ee00eeee, | 325 | 0x0084840084008484ULL, 0x009f9f009f009f9fULL, 0x00eeee00ee00eeeeULL, |
| 326 | 0x004a4a004a004a4a, 0x00c3c300c300c3c3, 0x002e2e002e002e2e, | 326 | 0x004a4a004a004a4aULL, 0x00c3c300c300c3c3ULL, 0x002e2e002e002e2eULL, |
| 327 | 0x00c1c100c100c1c1, 0x0001010001000101, 0x00e6e600e600e6e6, | 327 | 0x00c1c100c100c1c1ULL, 0x0001010001000101ULL, 0x00e6e600e600e6e6ULL, |
| 328 | 0x0025250025002525, 0x0048480048004848, 0x0099990099009999, | 328 | 0x0025250025002525ULL, 0x0048480048004848ULL, 0x0099990099009999ULL, |
| 329 | 0x00b9b900b900b9b9, 0x00b3b300b300b3b3, 0x007b7b007b007b7b, | 329 | 0x00b9b900b900b9b9ULL, 0x00b3b300b300b3b3ULL, 0x007b7b007b007b7bULL, |
| 330 | 0x00f9f900f900f9f9, 0x00cece00ce00cece, 0x00bfbf00bf00bfbf, | 330 | 0x00f9f900f900f9f9ULL, 0x00cece00ce00ceceULL, 0x00bfbf00bf00bfbfULL, |
| 331 | 0x00dfdf00df00dfdf, 0x0071710071007171, 0x0029290029002929, | 331 | 0x00dfdf00df00dfdfULL, 0x0071710071007171ULL, 0x0029290029002929ULL, |
| 332 | 0x00cdcd00cd00cdcd, 0x006c6c006c006c6c, 0x0013130013001313, | 332 | 0x00cdcd00cd00cdcdULL, 0x006c6c006c006c6cULL, 0x0013130013001313ULL, |
| 333 | 0x0064640064006464, 0x009b9b009b009b9b, 0x0063630063006363, | 333 | 0x0064640064006464ULL, 0x009b9b009b009b9bULL, 0x0063630063006363ULL, |
| 334 | 0x009d9d009d009d9d, 0x00c0c000c000c0c0, 0x004b4b004b004b4b, | 334 | 0x009d9d009d009d9dULL, 0x00c0c000c000c0c0ULL, 0x004b4b004b004b4bULL, |
| 335 | 0x00b7b700b700b7b7, 0x00a5a500a500a5a5, 0x0089890089008989, | 335 | 0x00b7b700b700b7b7ULL, 0x00a5a500a500a5a5ULL, 0x0089890089008989ULL, |
| 336 | 0x005f5f005f005f5f, 0x00b1b100b100b1b1, 0x0017170017001717, | 336 | 0x005f5f005f005f5fULL, 0x00b1b100b100b1b1ULL, 0x0017170017001717ULL, |
| 337 | 0x00f4f400f400f4f4, 0x00bcbc00bc00bcbc, 0x00d3d300d300d3d3, | 337 | 0x00f4f400f400f4f4ULL, 0x00bcbc00bc00bcbcULL, 0x00d3d300d300d3d3ULL, |
| 338 | 0x0046460046004646, 0x00cfcf00cf00cfcf, 0x0037370037003737, | 338 | 0x0046460046004646ULL, 0x00cfcf00cf00cfcfULL, 0x0037370037003737ULL, |
| 339 | 0x005e5e005e005e5e, 0x0047470047004747, 0x0094940094009494, | 339 | 0x005e5e005e005e5eULL, 0x0047470047004747ULL, 0x0094940094009494ULL, |
| 340 | 0x00fafa00fa00fafa, 0x00fcfc00fc00fcfc, 0x005b5b005b005b5b, | 340 | 0x00fafa00fa00fafaULL, 0x00fcfc00fc00fcfcULL, 0x005b5b005b005b5bULL, |
| 341 | 0x0097970097009797, 0x00fefe00fe00fefe, 0x005a5a005a005a5a, | 341 | 0x0097970097009797ULL, 0x00fefe00fe00fefeULL, 0x005a5a005a005a5aULL, |
| 342 | 0x00acac00ac00acac, 0x003c3c003c003c3c, 0x004c4c004c004c4c, | 342 | 0x00acac00ac00acacULL, 0x003c3c003c003c3cULL, 0x004c4c004c004c4cULL, |
| 343 | 0x0003030003000303, 0x0035350035003535, 0x00f3f300f300f3f3, | 343 | 0x0003030003000303ULL, 0x0035350035003535ULL, 0x00f3f300f300f3f3ULL, |
| 344 | 0x0023230023002323, 0x00b8b800b800b8b8, 0x005d5d005d005d5d, | 344 | 0x0023230023002323ULL, 0x00b8b800b800b8b8ULL, 0x005d5d005d005d5dULL, |
| 345 | 0x006a6a006a006a6a, 0x0092920092009292, 0x00d5d500d500d5d5, | 345 | 0x006a6a006a006a6aULL, 0x0092920092009292ULL, 0x00d5d500d500d5d5ULL, |
| 346 | 0x0021210021002121, 0x0044440044004444, 0x0051510051005151, | 346 | 0x0021210021002121ULL, 0x0044440044004444ULL, 0x0051510051005151ULL, |
| 347 | 0x00c6c600c600c6c6, 0x007d7d007d007d7d, 0x0039390039003939, | 347 | 0x00c6c600c600c6c6ULL, 0x007d7d007d007d7dULL, 0x0039390039003939ULL, |
| 348 | 0x0083830083008383, 0x00dcdc00dc00dcdc, 0x00aaaa00aa00aaaa, | 348 | 0x0083830083008383ULL, 0x00dcdc00dc00dcdcULL, 0x00aaaa00aa00aaaaULL, |
| 349 | 0x007c7c007c007c7c, 0x0077770077007777, 0x0056560056005656, | 349 | 0x007c7c007c007c7cULL, 0x0077770077007777ULL, 0x0056560056005656ULL, |
| 350 | 0x0005050005000505, 0x001b1b001b001b1b, 0x00a4a400a400a4a4, | 350 | 0x0005050005000505ULL, 0x001b1b001b001b1bULL, 0x00a4a400a400a4a4ULL, |
| 351 | 0x0015150015001515, 0x0034340034003434, 0x001e1e001e001e1e, | 351 | 0x0015150015001515ULL, 0x0034340034003434ULL, 0x001e1e001e001e1eULL, |
| 352 | 0x001c1c001c001c1c, 0x00f8f800f800f8f8, 0x0052520052005252, | 352 | 0x001c1c001c001c1cULL, 0x00f8f800f800f8f8ULL, 0x0052520052005252ULL, |
| 353 | 0x0020200020002020, 0x0014140014001414, 0x00e9e900e900e9e9, | 353 | 0x0020200020002020ULL, 0x0014140014001414ULL, 0x00e9e900e900e9e9ULL, |
| 354 | 0x00bdbd00bd00bdbd, 0x00dddd00dd00dddd, 0x00e4e400e400e4e4, | 354 | 0x00bdbd00bd00bdbdULL, 0x00dddd00dd00ddddULL, 0x00e4e400e400e4e4ULL, |
| 355 | 0x00a1a100a100a1a1, 0x00e0e000e000e0e0, 0x008a8a008a008a8a, | 355 | 0x00a1a100a100a1a1ULL, 0x00e0e000e000e0e0ULL, 0x008a8a008a008a8aULL, |
| 356 | 0x00f1f100f100f1f1, 0x00d6d600d600d6d6, 0x007a7a007a007a7a, | 356 | 0x00f1f100f100f1f1ULL, 0x00d6d600d600d6d6ULL, 0x007a7a007a007a7aULL, |
| 357 | 0x00bbbb00bb00bbbb, 0x00e3e300e300e3e3, 0x0040400040004040, | 357 | 0x00bbbb00bb00bbbbULL, 0x00e3e300e300e3e3ULL, 0x0040400040004040ULL, |
| 358 | 0x004f4f004f004f4f, | 358 | 0x004f4f004f004f4fULL, |
| 359 | }; | 359 | }; |
| 360 | 360 | ||
| 361 | const u64 camellia_sp00444404[256] = { | 361 | const u64 camellia_sp00444404[256] = { |
| 362 | 0x0000707070700070, 0x00002c2c2c2c002c, 0x0000b3b3b3b300b3, | 362 | 0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL, |
| 363 | 0x0000c0c0c0c000c0, 0x0000e4e4e4e400e4, 0x0000575757570057, | 363 | 0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL, |
| 364 | 0x0000eaeaeaea00ea, 0x0000aeaeaeae00ae, 0x0000232323230023, | 364 | 0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL, |
| 365 | 0x00006b6b6b6b006b, 0x0000454545450045, 0x0000a5a5a5a500a5, | 365 | 0x00006b6b6b6b006bULL, 0x0000454545450045ULL, 0x0000a5a5a5a500a5ULL, |
| 366 | 0x0000edededed00ed, 0x00004f4f4f4f004f, 0x00001d1d1d1d001d, | 366 | 0x0000edededed00edULL, 0x00004f4f4f4f004fULL, 0x00001d1d1d1d001dULL, |
| 367 | 0x0000929292920092, 0x0000868686860086, 0x0000afafafaf00af, | 367 | 0x0000929292920092ULL, 0x0000868686860086ULL, 0x0000afafafaf00afULL, |
| 368 | 0x00007c7c7c7c007c, 0x00001f1f1f1f001f, 0x00003e3e3e3e003e, | 368 | 0x00007c7c7c7c007cULL, 0x00001f1f1f1f001fULL, 0x00003e3e3e3e003eULL, |
| 369 | 0x0000dcdcdcdc00dc, 0x00005e5e5e5e005e, 0x00000b0b0b0b000b, | 369 | 0x0000dcdcdcdc00dcULL, 0x00005e5e5e5e005eULL, 0x00000b0b0b0b000bULL, |
| 370 | 0x0000a6a6a6a600a6, 0x0000393939390039, 0x0000d5d5d5d500d5, | 370 | 0x0000a6a6a6a600a6ULL, 0x0000393939390039ULL, 0x0000d5d5d5d500d5ULL, |
| 371 | 0x00005d5d5d5d005d, 0x0000d9d9d9d900d9, 0x00005a5a5a5a005a, | 371 | 0x00005d5d5d5d005dULL, 0x0000d9d9d9d900d9ULL, 0x00005a5a5a5a005aULL, |
| 372 | 0x0000515151510051, 0x00006c6c6c6c006c, 0x00008b8b8b8b008b, | 372 | 0x0000515151510051ULL, 0x00006c6c6c6c006cULL, 0x00008b8b8b8b008bULL, |
| 373 | 0x00009a9a9a9a009a, 0x0000fbfbfbfb00fb, 0x0000b0b0b0b000b0, | 373 | 0x00009a9a9a9a009aULL, 0x0000fbfbfbfb00fbULL, 0x0000b0b0b0b000b0ULL, |
| 374 | 0x0000747474740074, 0x00002b2b2b2b002b, 0x0000f0f0f0f000f0, | 374 | 0x0000747474740074ULL, 0x00002b2b2b2b002bULL, 0x0000f0f0f0f000f0ULL, |
| 375 | 0x0000848484840084, 0x0000dfdfdfdf00df, 0x0000cbcbcbcb00cb, | 375 | 0x0000848484840084ULL, 0x0000dfdfdfdf00dfULL, 0x0000cbcbcbcb00cbULL, |
| 376 | 0x0000343434340034, 0x0000767676760076, 0x00006d6d6d6d006d, | 376 | 0x0000343434340034ULL, 0x0000767676760076ULL, 0x00006d6d6d6d006dULL, |
| 377 | 0x0000a9a9a9a900a9, 0x0000d1d1d1d100d1, 0x0000040404040004, | 377 | 0x0000a9a9a9a900a9ULL, 0x0000d1d1d1d100d1ULL, 0x0000040404040004ULL, |
| 378 | 0x0000141414140014, 0x00003a3a3a3a003a, 0x0000dededede00de, | 378 | 0x0000141414140014ULL, 0x00003a3a3a3a003aULL, 0x0000dededede00deULL, |
| 379 | 0x0000111111110011, 0x0000323232320032, 0x00009c9c9c9c009c, | 379 | 0x0000111111110011ULL, 0x0000323232320032ULL, 0x00009c9c9c9c009cULL, |
| 380 | 0x0000535353530053, 0x0000f2f2f2f200f2, 0x0000fefefefe00fe, | 380 | 0x0000535353530053ULL, 0x0000f2f2f2f200f2ULL, 0x0000fefefefe00feULL, |
| 381 | 0x0000cfcfcfcf00cf, 0x0000c3c3c3c300c3, 0x00007a7a7a7a007a, | 381 | 0x0000cfcfcfcf00cfULL, 0x0000c3c3c3c300c3ULL, 0x00007a7a7a7a007aULL, |
| 382 | 0x0000242424240024, 0x0000e8e8e8e800e8, 0x0000606060600060, | 382 | 0x0000242424240024ULL, 0x0000e8e8e8e800e8ULL, 0x0000606060600060ULL, |
| 383 | 0x0000696969690069, 0x0000aaaaaaaa00aa, 0x0000a0a0a0a000a0, | 383 | 0x0000696969690069ULL, 0x0000aaaaaaaa00aaULL, 0x0000a0a0a0a000a0ULL, |
| 384 | 0x0000a1a1a1a100a1, 0x0000626262620062, 0x0000545454540054, | 384 | 0x0000a1a1a1a100a1ULL, 0x0000626262620062ULL, 0x0000545454540054ULL, |
| 385 | 0x00001e1e1e1e001e, 0x0000e0e0e0e000e0, 0x0000646464640064, | 385 | 0x00001e1e1e1e001eULL, 0x0000e0e0e0e000e0ULL, 0x0000646464640064ULL, |
| 386 | 0x0000101010100010, 0x0000000000000000, 0x0000a3a3a3a300a3, | 386 | 0x0000101010100010ULL, 0x0000000000000000ULL, 0x0000a3a3a3a300a3ULL, |
| 387 | 0x0000757575750075, 0x00008a8a8a8a008a, 0x0000e6e6e6e600e6, | 387 | 0x0000757575750075ULL, 0x00008a8a8a8a008aULL, 0x0000e6e6e6e600e6ULL, |
| 388 | 0x0000090909090009, 0x0000dddddddd00dd, 0x0000878787870087, | 388 | 0x0000090909090009ULL, 0x0000dddddddd00ddULL, 0x0000878787870087ULL, |
| 389 | 0x0000838383830083, 0x0000cdcdcdcd00cd, 0x0000909090900090, | 389 | 0x0000838383830083ULL, 0x0000cdcdcdcd00cdULL, 0x0000909090900090ULL, |
| 390 | 0x0000737373730073, 0x0000f6f6f6f600f6, 0x00009d9d9d9d009d, | 390 | 0x0000737373730073ULL, 0x0000f6f6f6f600f6ULL, 0x00009d9d9d9d009dULL, |
| 391 | 0x0000bfbfbfbf00bf, 0x0000525252520052, 0x0000d8d8d8d800d8, | 391 | 0x0000bfbfbfbf00bfULL, 0x0000525252520052ULL, 0x0000d8d8d8d800d8ULL, |
| 392 | 0x0000c8c8c8c800c8, 0x0000c6c6c6c600c6, 0x0000818181810081, | 392 | 0x0000c8c8c8c800c8ULL, 0x0000c6c6c6c600c6ULL, 0x0000818181810081ULL, |
| 393 | 0x00006f6f6f6f006f, 0x0000131313130013, 0x0000636363630063, | 393 | 0x00006f6f6f6f006fULL, 0x0000131313130013ULL, 0x0000636363630063ULL, |
| 394 | 0x0000e9e9e9e900e9, 0x0000a7a7a7a700a7, 0x00009f9f9f9f009f, | 394 | 0x0000e9e9e9e900e9ULL, 0x0000a7a7a7a700a7ULL, 0x00009f9f9f9f009fULL, |
| 395 | 0x0000bcbcbcbc00bc, 0x0000292929290029, 0x0000f9f9f9f900f9, | 395 | 0x0000bcbcbcbc00bcULL, 0x0000292929290029ULL, 0x0000f9f9f9f900f9ULL, |
| 396 | 0x00002f2f2f2f002f, 0x0000b4b4b4b400b4, 0x0000787878780078, | 396 | 0x00002f2f2f2f002fULL, 0x0000b4b4b4b400b4ULL, 0x0000787878780078ULL, |
| 397 | 0x0000060606060006, 0x0000e7e7e7e700e7, 0x0000717171710071, | 397 | 0x0000060606060006ULL, 0x0000e7e7e7e700e7ULL, 0x0000717171710071ULL, |
| 398 | 0x0000d4d4d4d400d4, 0x0000abababab00ab, 0x0000888888880088, | 398 | 0x0000d4d4d4d400d4ULL, 0x0000abababab00abULL, 0x0000888888880088ULL, |
| 399 | 0x00008d8d8d8d008d, 0x0000727272720072, 0x0000b9b9b9b900b9, | 399 | 0x00008d8d8d8d008dULL, 0x0000727272720072ULL, 0x0000b9b9b9b900b9ULL, |
| 400 | 0x0000f8f8f8f800f8, 0x0000acacacac00ac, 0x0000363636360036, | 400 | 0x0000f8f8f8f800f8ULL, 0x0000acacacac00acULL, 0x0000363636360036ULL, |
| 401 | 0x00002a2a2a2a002a, 0x00003c3c3c3c003c, 0x0000f1f1f1f100f1, | 401 | 0x00002a2a2a2a002aULL, 0x00003c3c3c3c003cULL, 0x0000f1f1f1f100f1ULL, |
| 402 | 0x0000404040400040, 0x0000d3d3d3d300d3, 0x0000bbbbbbbb00bb, | 402 | 0x0000404040400040ULL, 0x0000d3d3d3d300d3ULL, 0x0000bbbbbbbb00bbULL, |
| 403 | 0x0000434343430043, 0x0000151515150015, 0x0000adadadad00ad, | 403 | 0x0000434343430043ULL, 0x0000151515150015ULL, 0x0000adadadad00adULL, |
| 404 | 0x0000777777770077, 0x0000808080800080, 0x0000828282820082, | 404 | 0x0000777777770077ULL, 0x0000808080800080ULL, 0x0000828282820082ULL, |
| 405 | 0x0000ecececec00ec, 0x0000272727270027, 0x0000e5e5e5e500e5, | 405 | 0x0000ecececec00ecULL, 0x0000272727270027ULL, 0x0000e5e5e5e500e5ULL, |
| 406 | 0x0000858585850085, 0x0000353535350035, 0x00000c0c0c0c000c, | 406 | 0x0000858585850085ULL, 0x0000353535350035ULL, 0x00000c0c0c0c000cULL, |
| 407 | 0x0000414141410041, 0x0000efefefef00ef, 0x0000939393930093, | 407 | 0x0000414141410041ULL, 0x0000efefefef00efULL, 0x0000939393930093ULL, |
| 408 | 0x0000191919190019, 0x0000212121210021, 0x00000e0e0e0e000e, | 408 | 0x0000191919190019ULL, 0x0000212121210021ULL, 0x00000e0e0e0e000eULL, |
| 409 | 0x00004e4e4e4e004e, 0x0000656565650065, 0x0000bdbdbdbd00bd, | 409 | 0x00004e4e4e4e004eULL, 0x0000656565650065ULL, 0x0000bdbdbdbd00bdULL, |
| 410 | 0x0000b8b8b8b800b8, 0x00008f8f8f8f008f, 0x0000ebebebeb00eb, | 410 | 0x0000b8b8b8b800b8ULL, 0x00008f8f8f8f008fULL, 0x0000ebebebeb00ebULL, |
| 411 | 0x0000cececece00ce, 0x0000303030300030, 0x00005f5f5f5f005f, | 411 | 0x0000cececece00ceULL, 0x0000303030300030ULL, 0x00005f5f5f5f005fULL, |
| 412 | 0x0000c5c5c5c500c5, 0x00001a1a1a1a001a, 0x0000e1e1e1e100e1, | 412 | 0x0000c5c5c5c500c5ULL, 0x00001a1a1a1a001aULL, 0x0000e1e1e1e100e1ULL, |
| 413 | 0x0000cacacaca00ca, 0x0000474747470047, 0x00003d3d3d3d003d, | 413 | 0x0000cacacaca00caULL, 0x0000474747470047ULL, 0x00003d3d3d3d003dULL, |
| 414 | 0x0000010101010001, 0x0000d6d6d6d600d6, 0x0000565656560056, | 414 | 0x0000010101010001ULL, 0x0000d6d6d6d600d6ULL, 0x0000565656560056ULL, |
| 415 | 0x00004d4d4d4d004d, 0x00000d0d0d0d000d, 0x0000666666660066, | 415 | 0x00004d4d4d4d004dULL, 0x00000d0d0d0d000dULL, 0x0000666666660066ULL, |
| 416 | 0x0000cccccccc00cc, 0x00002d2d2d2d002d, 0x0000121212120012, | 416 | 0x0000cccccccc00ccULL, 0x00002d2d2d2d002dULL, 0x0000121212120012ULL, |
| 417 | 0x0000202020200020, 0x0000b1b1b1b100b1, 0x0000999999990099, | 417 | 0x0000202020200020ULL, 0x0000b1b1b1b100b1ULL, 0x0000999999990099ULL, |
| 418 | 0x00004c4c4c4c004c, 0x0000c2c2c2c200c2, 0x00007e7e7e7e007e, | 418 | 0x00004c4c4c4c004cULL, 0x0000c2c2c2c200c2ULL, 0x00007e7e7e7e007eULL, |
| 419 | 0x0000050505050005, 0x0000b7b7b7b700b7, 0x0000313131310031, | 419 | 0x0000050505050005ULL, 0x0000b7b7b7b700b7ULL, 0x0000313131310031ULL, |
| 420 | 0x0000171717170017, 0x0000d7d7d7d700d7, 0x0000585858580058, | 420 | 0x0000171717170017ULL, 0x0000d7d7d7d700d7ULL, 0x0000585858580058ULL, |
| 421 | 0x0000616161610061, 0x00001b1b1b1b001b, 0x00001c1c1c1c001c, | 421 | 0x0000616161610061ULL, 0x00001b1b1b1b001bULL, 0x00001c1c1c1c001cULL, |
| 422 | 0x00000f0f0f0f000f, 0x0000161616160016, 0x0000181818180018, | 422 | 0x00000f0f0f0f000fULL, 0x0000161616160016ULL, 0x0000181818180018ULL, |
| 423 | 0x0000222222220022, 0x0000444444440044, 0x0000b2b2b2b200b2, | 423 | 0x0000222222220022ULL, 0x0000444444440044ULL, 0x0000b2b2b2b200b2ULL, |
| 424 | 0x0000b5b5b5b500b5, 0x0000919191910091, 0x0000080808080008, | 424 | 0x0000b5b5b5b500b5ULL, 0x0000919191910091ULL, 0x0000080808080008ULL, |
| 425 | 0x0000a8a8a8a800a8, 0x0000fcfcfcfc00fc, 0x0000505050500050, | 425 | 0x0000a8a8a8a800a8ULL, 0x0000fcfcfcfc00fcULL, 0x0000505050500050ULL, |
| 426 | 0x0000d0d0d0d000d0, 0x00007d7d7d7d007d, 0x0000898989890089, | 426 | 0x0000d0d0d0d000d0ULL, 0x00007d7d7d7d007dULL, 0x0000898989890089ULL, |
| 427 | 0x0000979797970097, 0x00005b5b5b5b005b, 0x0000959595950095, | 427 | 0x0000979797970097ULL, 0x00005b5b5b5b005bULL, 0x0000959595950095ULL, |
| 428 | 0x0000ffffffff00ff, 0x0000d2d2d2d200d2, 0x0000c4c4c4c400c4, | 428 | 0x0000ffffffff00ffULL, 0x0000d2d2d2d200d2ULL, 0x0000c4c4c4c400c4ULL, |
| 429 | 0x0000484848480048, 0x0000f7f7f7f700f7, 0x0000dbdbdbdb00db, | 429 | 0x0000484848480048ULL, 0x0000f7f7f7f700f7ULL, 0x0000dbdbdbdb00dbULL, |
| 430 | 0x0000030303030003, 0x0000dadadada00da, 0x00003f3f3f3f003f, | 430 | 0x0000030303030003ULL, 0x0000dadadada00daULL, 0x00003f3f3f3f003fULL, |
| 431 | 0x0000949494940094, 0x00005c5c5c5c005c, 0x0000020202020002, | 431 | 0x0000949494940094ULL, 0x00005c5c5c5c005cULL, 0x0000020202020002ULL, |
| 432 | 0x00004a4a4a4a004a, 0x0000333333330033, 0x0000676767670067, | 432 | 0x00004a4a4a4a004aULL, 0x0000333333330033ULL, 0x0000676767670067ULL, |
| 433 | 0x0000f3f3f3f300f3, 0x00007f7f7f7f007f, 0x0000e2e2e2e200e2, | 433 | 0x0000f3f3f3f300f3ULL, 0x00007f7f7f7f007fULL, 0x0000e2e2e2e200e2ULL, |
| 434 | 0x00009b9b9b9b009b, 0x0000262626260026, 0x0000373737370037, | 434 | 0x00009b9b9b9b009bULL, 0x0000262626260026ULL, 0x0000373737370037ULL, |
| 435 | 0x00003b3b3b3b003b, 0x0000969696960096, 0x00004b4b4b4b004b, | 435 | 0x00003b3b3b3b003bULL, 0x0000969696960096ULL, 0x00004b4b4b4b004bULL, |
| 436 | 0x0000bebebebe00be, 0x00002e2e2e2e002e, 0x0000797979790079, | 436 | 0x0000bebebebe00beULL, 0x00002e2e2e2e002eULL, 0x0000797979790079ULL, |
| 437 | 0x00008c8c8c8c008c, 0x00006e6e6e6e006e, 0x00008e8e8e8e008e, | 437 | 0x00008c8c8c8c008cULL, 0x00006e6e6e6e006eULL, 0x00008e8e8e8e008eULL, |
| 438 | 0x0000f5f5f5f500f5, 0x0000b6b6b6b600b6, 0x0000fdfdfdfd00fd, | 438 | 0x0000f5f5f5f500f5ULL, 0x0000b6b6b6b600b6ULL, 0x0000fdfdfdfd00fdULL, |
| 439 | 0x0000595959590059, 0x0000989898980098, 0x00006a6a6a6a006a, | 439 | 0x0000595959590059ULL, 0x0000989898980098ULL, 0x00006a6a6a6a006aULL, |
| 440 | 0x0000464646460046, 0x0000babababa00ba, 0x0000252525250025, | 440 | 0x0000464646460046ULL, 0x0000babababa00baULL, 0x0000252525250025ULL, |
| 441 | 0x0000424242420042, 0x0000a2a2a2a200a2, 0x0000fafafafa00fa, | 441 | 0x0000424242420042ULL, 0x0000a2a2a2a200a2ULL, 0x0000fafafafa00faULL, |
| 442 | 0x0000070707070007, 0x0000555555550055, 0x0000eeeeeeee00ee, | 442 | 0x0000070707070007ULL, 0x0000555555550055ULL, 0x0000eeeeeeee00eeULL, |
| 443 | 0x00000a0a0a0a000a, 0x0000494949490049, 0x0000686868680068, | 443 | 0x00000a0a0a0a000aULL, 0x0000494949490049ULL, 0x0000686868680068ULL, |
| 444 | 0x0000383838380038, 0x0000a4a4a4a400a4, 0x0000282828280028, | 444 | 0x0000383838380038ULL, 0x0000a4a4a4a400a4ULL, 0x0000282828280028ULL, |
| 445 | 0x00007b7b7b7b007b, 0x0000c9c9c9c900c9, 0x0000c1c1c1c100c1, | 445 | 0x00007b7b7b7b007bULL, 0x0000c9c9c9c900c9ULL, 0x0000c1c1c1c100c1ULL, |
| 446 | 0x0000e3e3e3e300e3, 0x0000f4f4f4f400f4, 0x0000c7c7c7c700c7, | 446 | 0x0000e3e3e3e300e3ULL, 0x0000f4f4f4f400f4ULL, 0x0000c7c7c7c700c7ULL, |
| 447 | 0x00009e9e9e9e009e, | 447 | 0x00009e9e9e9e009eULL, |
| 448 | }; | 448 | }; |
| 449 | 449 | ||
| 450 | const u64 camellia_sp02220222[256] = { | 450 | const u64 camellia_sp02220222[256] = { |
| 451 | 0x00e0e0e000e0e0e0, 0x0005050500050505, 0x0058585800585858, | 451 | 0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL, |
| 452 | 0x00d9d9d900d9d9d9, 0x0067676700676767, 0x004e4e4e004e4e4e, | 452 | 0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL, |
| 453 | 0x0081818100818181, 0x00cbcbcb00cbcbcb, 0x00c9c9c900c9c9c9, | 453 | 0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL, |
| 454 | 0x000b0b0b000b0b0b, 0x00aeaeae00aeaeae, 0x006a6a6a006a6a6a, | 454 | 0x000b0b0b000b0b0bULL, 0x00aeaeae00aeaeaeULL, 0x006a6a6a006a6a6aULL, |
| 455 | 0x00d5d5d500d5d5d5, 0x0018181800181818, 0x005d5d5d005d5d5d, | 455 | 0x00d5d5d500d5d5d5ULL, 0x0018181800181818ULL, 0x005d5d5d005d5d5dULL, |
| 456 | 0x0082828200828282, 0x0046464600464646, 0x00dfdfdf00dfdfdf, | 456 | 0x0082828200828282ULL, 0x0046464600464646ULL, 0x00dfdfdf00dfdfdfULL, |
| 457 | 0x00d6d6d600d6d6d6, 0x0027272700272727, 0x008a8a8a008a8a8a, | 457 | 0x00d6d6d600d6d6d6ULL, 0x0027272700272727ULL, 0x008a8a8a008a8a8aULL, |
| 458 | 0x0032323200323232, 0x004b4b4b004b4b4b, 0x0042424200424242, | 458 | 0x0032323200323232ULL, 0x004b4b4b004b4b4bULL, 0x0042424200424242ULL, |
| 459 | 0x00dbdbdb00dbdbdb, 0x001c1c1c001c1c1c, 0x009e9e9e009e9e9e, | 459 | 0x00dbdbdb00dbdbdbULL, 0x001c1c1c001c1c1cULL, 0x009e9e9e009e9e9eULL, |
| 460 | 0x009c9c9c009c9c9c, 0x003a3a3a003a3a3a, 0x00cacaca00cacaca, | 460 | 0x009c9c9c009c9c9cULL, 0x003a3a3a003a3a3aULL, 0x00cacaca00cacacaULL, |
| 461 | 0x0025252500252525, 0x007b7b7b007b7b7b, 0x000d0d0d000d0d0d, | 461 | 0x0025252500252525ULL, 0x007b7b7b007b7b7bULL, 0x000d0d0d000d0d0dULL, |
| 462 | 0x0071717100717171, 0x005f5f5f005f5f5f, 0x001f1f1f001f1f1f, | 462 | 0x0071717100717171ULL, 0x005f5f5f005f5f5fULL, 0x001f1f1f001f1f1fULL, |
| 463 | 0x00f8f8f800f8f8f8, 0x00d7d7d700d7d7d7, 0x003e3e3e003e3e3e, | 463 | 0x00f8f8f800f8f8f8ULL, 0x00d7d7d700d7d7d7ULL, 0x003e3e3e003e3e3eULL, |
| 464 | 0x009d9d9d009d9d9d, 0x007c7c7c007c7c7c, 0x0060606000606060, | 464 | 0x009d9d9d009d9d9dULL, 0x007c7c7c007c7c7cULL, 0x0060606000606060ULL, |
| 465 | 0x00b9b9b900b9b9b9, 0x00bebebe00bebebe, 0x00bcbcbc00bcbcbc, | 465 | 0x00b9b9b900b9b9b9ULL, 0x00bebebe00bebebeULL, 0x00bcbcbc00bcbcbcULL, |
| 466 | 0x008b8b8b008b8b8b, 0x0016161600161616, 0x0034343400343434, | 466 | 0x008b8b8b008b8b8bULL, 0x0016161600161616ULL, 0x0034343400343434ULL, |
| 467 | 0x004d4d4d004d4d4d, 0x00c3c3c300c3c3c3, 0x0072727200727272, | 467 | 0x004d4d4d004d4d4dULL, 0x00c3c3c300c3c3c3ULL, 0x0072727200727272ULL, |
| 468 | 0x0095959500959595, 0x00ababab00ababab, 0x008e8e8e008e8e8e, | 468 | 0x0095959500959595ULL, 0x00ababab00abababULL, 0x008e8e8e008e8e8eULL, |
| 469 | 0x00bababa00bababa, 0x007a7a7a007a7a7a, 0x00b3b3b300b3b3b3, | 469 | 0x00bababa00bababaULL, 0x007a7a7a007a7a7aULL, 0x00b3b3b300b3b3b3ULL, |
| 470 | 0x0002020200020202, 0x00b4b4b400b4b4b4, 0x00adadad00adadad, | 470 | 0x0002020200020202ULL, 0x00b4b4b400b4b4b4ULL, 0x00adadad00adadadULL, |
| 471 | 0x00a2a2a200a2a2a2, 0x00acacac00acacac, 0x00d8d8d800d8d8d8, | 471 | 0x00a2a2a200a2a2a2ULL, 0x00acacac00acacacULL, 0x00d8d8d800d8d8d8ULL, |
| 472 | 0x009a9a9a009a9a9a, 0x0017171700171717, 0x001a1a1a001a1a1a, | 472 | 0x009a9a9a009a9a9aULL, 0x0017171700171717ULL, 0x001a1a1a001a1a1aULL, |
| 473 | 0x0035353500353535, 0x00cccccc00cccccc, 0x00f7f7f700f7f7f7, | 473 | 0x0035353500353535ULL, 0x00cccccc00ccccccULL, 0x00f7f7f700f7f7f7ULL, |
| 474 | 0x0099999900999999, 0x0061616100616161, 0x005a5a5a005a5a5a, | 474 | 0x0099999900999999ULL, 0x0061616100616161ULL, 0x005a5a5a005a5a5aULL, |
| 475 | 0x00e8e8e800e8e8e8, 0x0024242400242424, 0x0056565600565656, | 475 | 0x00e8e8e800e8e8e8ULL, 0x0024242400242424ULL, 0x0056565600565656ULL, |
| 476 | 0x0040404000404040, 0x00e1e1e100e1e1e1, 0x0063636300636363, | 476 | 0x0040404000404040ULL, 0x00e1e1e100e1e1e1ULL, 0x0063636300636363ULL, |
| 477 | 0x0009090900090909, 0x0033333300333333, 0x00bfbfbf00bfbfbf, | 477 | 0x0009090900090909ULL, 0x0033333300333333ULL, 0x00bfbfbf00bfbfbfULL, |
| 478 | 0x0098989800989898, 0x0097979700979797, 0x0085858500858585, | 478 | 0x0098989800989898ULL, 0x0097979700979797ULL, 0x0085858500858585ULL, |
| 479 | 0x0068686800686868, 0x00fcfcfc00fcfcfc, 0x00ececec00ececec, | 479 | 0x0068686800686868ULL, 0x00fcfcfc00fcfcfcULL, 0x00ececec00ecececULL, |
| 480 | 0x000a0a0a000a0a0a, 0x00dadada00dadada, 0x006f6f6f006f6f6f, | 480 | 0x000a0a0a000a0a0aULL, 0x00dadada00dadadaULL, 0x006f6f6f006f6f6fULL, |
| 481 | 0x0053535300535353, 0x0062626200626262, 0x00a3a3a300a3a3a3, | 481 | 0x0053535300535353ULL, 0x0062626200626262ULL, 0x00a3a3a300a3a3a3ULL, |
| 482 | 0x002e2e2e002e2e2e, 0x0008080800080808, 0x00afafaf00afafaf, | 482 | 0x002e2e2e002e2e2eULL, 0x0008080800080808ULL, 0x00afafaf00afafafULL, |
| 483 | 0x0028282800282828, 0x00b0b0b000b0b0b0, 0x0074747400747474, | 483 | 0x0028282800282828ULL, 0x00b0b0b000b0b0b0ULL, 0x0074747400747474ULL, |
| 484 | 0x00c2c2c200c2c2c2, 0x00bdbdbd00bdbdbd, 0x0036363600363636, | 484 | 0x00c2c2c200c2c2c2ULL, 0x00bdbdbd00bdbdbdULL, 0x0036363600363636ULL, |
| 485 | 0x0022222200222222, 0x0038383800383838, 0x0064646400646464, | 485 | 0x0022222200222222ULL, 0x0038383800383838ULL, 0x0064646400646464ULL, |
| 486 | 0x001e1e1e001e1e1e, 0x0039393900393939, 0x002c2c2c002c2c2c, | 486 | 0x001e1e1e001e1e1eULL, 0x0039393900393939ULL, 0x002c2c2c002c2c2cULL, |
| 487 | 0x00a6a6a600a6a6a6, 0x0030303000303030, 0x00e5e5e500e5e5e5, | 487 | 0x00a6a6a600a6a6a6ULL, 0x0030303000303030ULL, 0x00e5e5e500e5e5e5ULL, |
| 488 | 0x0044444400444444, 0x00fdfdfd00fdfdfd, 0x0088888800888888, | 488 | 0x0044444400444444ULL, 0x00fdfdfd00fdfdfdULL, 0x0088888800888888ULL, |
| 489 | 0x009f9f9f009f9f9f, 0x0065656500656565, 0x0087878700878787, | 489 | 0x009f9f9f009f9f9fULL, 0x0065656500656565ULL, 0x0087878700878787ULL, |
| 490 | 0x006b6b6b006b6b6b, 0x00f4f4f400f4f4f4, 0x0023232300232323, | 490 | 0x006b6b6b006b6b6bULL, 0x00f4f4f400f4f4f4ULL, 0x0023232300232323ULL, |
| 491 | 0x0048484800484848, 0x0010101000101010, 0x00d1d1d100d1d1d1, | 491 | 0x0048484800484848ULL, 0x0010101000101010ULL, 0x00d1d1d100d1d1d1ULL, |
| 492 | 0x0051515100515151, 0x00c0c0c000c0c0c0, 0x00f9f9f900f9f9f9, | 492 | 0x0051515100515151ULL, 0x00c0c0c000c0c0c0ULL, 0x00f9f9f900f9f9f9ULL, |
| 493 | 0x00d2d2d200d2d2d2, 0x00a0a0a000a0a0a0, 0x0055555500555555, | 493 | 0x00d2d2d200d2d2d2ULL, 0x00a0a0a000a0a0a0ULL, 0x0055555500555555ULL, |
| 494 | 0x00a1a1a100a1a1a1, 0x0041414100414141, 0x00fafafa00fafafa, | 494 | 0x00a1a1a100a1a1a1ULL, 0x0041414100414141ULL, 0x00fafafa00fafafaULL, |
| 495 | 0x0043434300434343, 0x0013131300131313, 0x00c4c4c400c4c4c4, | 495 | 0x0043434300434343ULL, 0x0013131300131313ULL, 0x00c4c4c400c4c4c4ULL, |
| 496 | 0x002f2f2f002f2f2f, 0x00a8a8a800a8a8a8, 0x00b6b6b600b6b6b6, | 496 | 0x002f2f2f002f2f2fULL, 0x00a8a8a800a8a8a8ULL, 0x00b6b6b600b6b6b6ULL, |
| 497 | 0x003c3c3c003c3c3c, 0x002b2b2b002b2b2b, 0x00c1c1c100c1c1c1, | 497 | 0x003c3c3c003c3c3cULL, 0x002b2b2b002b2b2bULL, 0x00c1c1c100c1c1c1ULL, |
| 498 | 0x00ffffff00ffffff, 0x00c8c8c800c8c8c8, 0x00a5a5a500a5a5a5, | 498 | 0x00ffffff00ffffffULL, 0x00c8c8c800c8c8c8ULL, 0x00a5a5a500a5a5a5ULL, |
| 499 | 0x0020202000202020, 0x0089898900898989, 0x0000000000000000, | 499 | 0x0020202000202020ULL, 0x0089898900898989ULL, 0x0000000000000000ULL, |
| 500 | 0x0090909000909090, 0x0047474700474747, 0x00efefef00efefef, | 500 | 0x0090909000909090ULL, 0x0047474700474747ULL, 0x00efefef00efefefULL, |
| 501 | 0x00eaeaea00eaeaea, 0x00b7b7b700b7b7b7, 0x0015151500151515, | 501 | 0x00eaeaea00eaeaeaULL, 0x00b7b7b700b7b7b7ULL, 0x0015151500151515ULL, |
| 502 | 0x0006060600060606, 0x00cdcdcd00cdcdcd, 0x00b5b5b500b5b5b5, | 502 | 0x0006060600060606ULL, 0x00cdcdcd00cdcdcdULL, 0x00b5b5b500b5b5b5ULL, |
| 503 | 0x0012121200121212, 0x007e7e7e007e7e7e, 0x00bbbbbb00bbbbbb, | 503 | 0x0012121200121212ULL, 0x007e7e7e007e7e7eULL, 0x00bbbbbb00bbbbbbULL, |
| 504 | 0x0029292900292929, 0x000f0f0f000f0f0f, 0x00b8b8b800b8b8b8, | 504 | 0x0029292900292929ULL, 0x000f0f0f000f0f0fULL, 0x00b8b8b800b8b8b8ULL, |
| 505 | 0x0007070700070707, 0x0004040400040404, 0x009b9b9b009b9b9b, | 505 | 0x0007070700070707ULL, 0x0004040400040404ULL, 0x009b9b9b009b9b9bULL, |
| 506 | 0x0094949400949494, 0x0021212100212121, 0x0066666600666666, | 506 | 0x0094949400949494ULL, 0x0021212100212121ULL, 0x0066666600666666ULL, |
| 507 | 0x00e6e6e600e6e6e6, 0x00cecece00cecece, 0x00ededed00ededed, | 507 | 0x00e6e6e600e6e6e6ULL, 0x00cecece00cececeULL, 0x00ededed00edededULL, |
| 508 | 0x00e7e7e700e7e7e7, 0x003b3b3b003b3b3b, 0x00fefefe00fefefe, | 508 | 0x00e7e7e700e7e7e7ULL, 0x003b3b3b003b3b3bULL, 0x00fefefe00fefefeULL, |
| 509 | 0x007f7f7f007f7f7f, 0x00c5c5c500c5c5c5, 0x00a4a4a400a4a4a4, | 509 | 0x007f7f7f007f7f7fULL, 0x00c5c5c500c5c5c5ULL, 0x00a4a4a400a4a4a4ULL, |
| 510 | 0x0037373700373737, 0x00b1b1b100b1b1b1, 0x004c4c4c004c4c4c, | 510 | 0x0037373700373737ULL, 0x00b1b1b100b1b1b1ULL, 0x004c4c4c004c4c4cULL, |
| 511 | 0x0091919100919191, 0x006e6e6e006e6e6e, 0x008d8d8d008d8d8d, | 511 | 0x0091919100919191ULL, 0x006e6e6e006e6e6eULL, 0x008d8d8d008d8d8dULL, |
| 512 | 0x0076767600767676, 0x0003030300030303, 0x002d2d2d002d2d2d, | 512 | 0x0076767600767676ULL, 0x0003030300030303ULL, 0x002d2d2d002d2d2dULL, |
| 513 | 0x00dedede00dedede, 0x0096969600969696, 0x0026262600262626, | 513 | 0x00dedede00dededeULL, 0x0096969600969696ULL, 0x0026262600262626ULL, |
| 514 | 0x007d7d7d007d7d7d, 0x00c6c6c600c6c6c6, 0x005c5c5c005c5c5c, | 514 | 0x007d7d7d007d7d7dULL, 0x00c6c6c600c6c6c6ULL, 0x005c5c5c005c5c5cULL, |
| 515 | 0x00d3d3d300d3d3d3, 0x00f2f2f200f2f2f2, 0x004f4f4f004f4f4f, | 515 | 0x00d3d3d300d3d3d3ULL, 0x00f2f2f200f2f2f2ULL, 0x004f4f4f004f4f4fULL, |
| 516 | 0x0019191900191919, 0x003f3f3f003f3f3f, 0x00dcdcdc00dcdcdc, | 516 | 0x0019191900191919ULL, 0x003f3f3f003f3f3fULL, 0x00dcdcdc00dcdcdcULL, |
| 517 | 0x0079797900797979, 0x001d1d1d001d1d1d, 0x0052525200525252, | 517 | 0x0079797900797979ULL, 0x001d1d1d001d1d1dULL, 0x0052525200525252ULL, |
| 518 | 0x00ebebeb00ebebeb, 0x00f3f3f300f3f3f3, 0x006d6d6d006d6d6d, | 518 | 0x00ebebeb00ebebebULL, 0x00f3f3f300f3f3f3ULL, 0x006d6d6d006d6d6dULL, |
| 519 | 0x005e5e5e005e5e5e, 0x00fbfbfb00fbfbfb, 0x0069696900696969, | 519 | 0x005e5e5e005e5e5eULL, 0x00fbfbfb00fbfbfbULL, 0x0069696900696969ULL, |
| 520 | 0x00b2b2b200b2b2b2, 0x00f0f0f000f0f0f0, 0x0031313100313131, | 520 | 0x00b2b2b200b2b2b2ULL, 0x00f0f0f000f0f0f0ULL, 0x0031313100313131ULL, |
| 521 | 0x000c0c0c000c0c0c, 0x00d4d4d400d4d4d4, 0x00cfcfcf00cfcfcf, | 521 | 0x000c0c0c000c0c0cULL, 0x00d4d4d400d4d4d4ULL, 0x00cfcfcf00cfcfcfULL, |
| 522 | 0x008c8c8c008c8c8c, 0x00e2e2e200e2e2e2, 0x0075757500757575, | 522 | 0x008c8c8c008c8c8cULL, 0x00e2e2e200e2e2e2ULL, 0x0075757500757575ULL, |
| 523 | 0x00a9a9a900a9a9a9, 0x004a4a4a004a4a4a, 0x0057575700575757, | 523 | 0x00a9a9a900a9a9a9ULL, 0x004a4a4a004a4a4aULL, 0x0057575700575757ULL, |
| 524 | 0x0084848400848484, 0x0011111100111111, 0x0045454500454545, | 524 | 0x0084848400848484ULL, 0x0011111100111111ULL, 0x0045454500454545ULL, |
| 525 | 0x001b1b1b001b1b1b, 0x00f5f5f500f5f5f5, 0x00e4e4e400e4e4e4, | 525 | 0x001b1b1b001b1b1bULL, 0x00f5f5f500f5f5f5ULL, 0x00e4e4e400e4e4e4ULL, |
| 526 | 0x000e0e0e000e0e0e, 0x0073737300737373, 0x00aaaaaa00aaaaaa, | 526 | 0x000e0e0e000e0e0eULL, 0x0073737300737373ULL, 0x00aaaaaa00aaaaaaULL, |
| 527 | 0x00f1f1f100f1f1f1, 0x00dddddd00dddddd, 0x0059595900595959, | 527 | 0x00f1f1f100f1f1f1ULL, 0x00dddddd00ddddddULL, 0x0059595900595959ULL, |
| 528 | 0x0014141400141414, 0x006c6c6c006c6c6c, 0x0092929200929292, | 528 | 0x0014141400141414ULL, 0x006c6c6c006c6c6cULL, 0x0092929200929292ULL, |
| 529 | 0x0054545400545454, 0x00d0d0d000d0d0d0, 0x0078787800787878, | 529 | 0x0054545400545454ULL, 0x00d0d0d000d0d0d0ULL, 0x0078787800787878ULL, |
| 530 | 0x0070707000707070, 0x00e3e3e300e3e3e3, 0x0049494900494949, | 530 | 0x0070707000707070ULL, 0x00e3e3e300e3e3e3ULL, 0x0049494900494949ULL, |
| 531 | 0x0080808000808080, 0x0050505000505050, 0x00a7a7a700a7a7a7, | 531 | 0x0080808000808080ULL, 0x0050505000505050ULL, 0x00a7a7a700a7a7a7ULL, |
| 532 | 0x00f6f6f600f6f6f6, 0x0077777700777777, 0x0093939300939393, | 532 | 0x00f6f6f600f6f6f6ULL, 0x0077777700777777ULL, 0x0093939300939393ULL, |
| 533 | 0x0086868600868686, 0x0083838300838383, 0x002a2a2a002a2a2a, | 533 | 0x0086868600868686ULL, 0x0083838300838383ULL, 0x002a2a2a002a2a2aULL, |
| 534 | 0x00c7c7c700c7c7c7, 0x005b5b5b005b5b5b, 0x00e9e9e900e9e9e9, | 534 | 0x00c7c7c700c7c7c7ULL, 0x005b5b5b005b5b5bULL, 0x00e9e9e900e9e9e9ULL, |
| 535 | 0x00eeeeee00eeeeee, 0x008f8f8f008f8f8f, 0x0001010100010101, | 535 | 0x00eeeeee00eeeeeeULL, 0x008f8f8f008f8f8fULL, 0x0001010100010101ULL, |
| 536 | 0x003d3d3d003d3d3d, | 536 | 0x003d3d3d003d3d3dULL, |
| 537 | }; | 537 | }; |
| 538 | 538 | ||
| 539 | const u64 camellia_sp30333033[256] = { | 539 | const u64 camellia_sp30333033[256] = { |
| 540 | 0x3800383838003838, 0x4100414141004141, 0x1600161616001616, | 540 | 0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL, |
| 541 | 0x7600767676007676, 0xd900d9d9d900d9d9, 0x9300939393009393, | 541 | 0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL, |
| 542 | 0x6000606060006060, 0xf200f2f2f200f2f2, 0x7200727272007272, | 542 | 0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL, |
| 543 | 0xc200c2c2c200c2c2, 0xab00ababab00abab, 0x9a009a9a9a009a9a, | 543 | 0xc200c2c2c200c2c2ULL, 0xab00ababab00ababULL, 0x9a009a9a9a009a9aULL, |
| 544 | 0x7500757575007575, 0x0600060606000606, 0x5700575757005757, | 544 | 0x7500757575007575ULL, 0x0600060606000606ULL, 0x5700575757005757ULL, |
| 545 | 0xa000a0a0a000a0a0, 0x9100919191009191, 0xf700f7f7f700f7f7, | 545 | 0xa000a0a0a000a0a0ULL, 0x9100919191009191ULL, 0xf700f7f7f700f7f7ULL, |
| 546 | 0xb500b5b5b500b5b5, 0xc900c9c9c900c9c9, 0xa200a2a2a200a2a2, | 546 | 0xb500b5b5b500b5b5ULL, 0xc900c9c9c900c9c9ULL, 0xa200a2a2a200a2a2ULL, |
| 547 | 0x8c008c8c8c008c8c, 0xd200d2d2d200d2d2, 0x9000909090009090, | 547 | 0x8c008c8c8c008c8cULL, 0xd200d2d2d200d2d2ULL, 0x9000909090009090ULL, |
| 548 | 0xf600f6f6f600f6f6, 0x0700070707000707, 0xa700a7a7a700a7a7, | 548 | 0xf600f6f6f600f6f6ULL, 0x0700070707000707ULL, 0xa700a7a7a700a7a7ULL, |
| 549 | 0x2700272727002727, 0x8e008e8e8e008e8e, 0xb200b2b2b200b2b2, | 549 | 0x2700272727002727ULL, 0x8e008e8e8e008e8eULL, 0xb200b2b2b200b2b2ULL, |
| 550 | 0x4900494949004949, 0xde00dedede00dede, 0x4300434343004343, | 550 | 0x4900494949004949ULL, 0xde00dedede00dedeULL, 0x4300434343004343ULL, |
| 551 | 0x5c005c5c5c005c5c, 0xd700d7d7d700d7d7, 0xc700c7c7c700c7c7, | 551 | 0x5c005c5c5c005c5cULL, 0xd700d7d7d700d7d7ULL, 0xc700c7c7c700c7c7ULL, |
| 552 | 0x3e003e3e3e003e3e, 0xf500f5f5f500f5f5, 0x8f008f8f8f008f8f, | 552 | 0x3e003e3e3e003e3eULL, 0xf500f5f5f500f5f5ULL, 0x8f008f8f8f008f8fULL, |
| 553 | 0x6700676767006767, 0x1f001f1f1f001f1f, 0x1800181818001818, | 553 | 0x6700676767006767ULL, 0x1f001f1f1f001f1fULL, 0x1800181818001818ULL, |
| 554 | 0x6e006e6e6e006e6e, 0xaf00afafaf00afaf, 0x2f002f2f2f002f2f, | 554 | 0x6e006e6e6e006e6eULL, 0xaf00afafaf00afafULL, 0x2f002f2f2f002f2fULL, |
| 555 | 0xe200e2e2e200e2e2, 0x8500858585008585, 0x0d000d0d0d000d0d, | 555 | 0xe200e2e2e200e2e2ULL, 0x8500858585008585ULL, 0x0d000d0d0d000d0dULL, |
| 556 | 0x5300535353005353, 0xf000f0f0f000f0f0, 0x9c009c9c9c009c9c, | 556 | 0x5300535353005353ULL, 0xf000f0f0f000f0f0ULL, 0x9c009c9c9c009c9cULL, |
| 557 | 0x6500656565006565, 0xea00eaeaea00eaea, 0xa300a3a3a300a3a3, | 557 | 0x6500656565006565ULL, 0xea00eaeaea00eaeaULL, 0xa300a3a3a300a3a3ULL, |
| 558 | 0xae00aeaeae00aeae, 0x9e009e9e9e009e9e, 0xec00ececec00ecec, | 558 | 0xae00aeaeae00aeaeULL, 0x9e009e9e9e009e9eULL, 0xec00ececec00ececULL, |
| 559 | 0x8000808080008080, 0x2d002d2d2d002d2d, 0x6b006b6b6b006b6b, | 559 | 0x8000808080008080ULL, 0x2d002d2d2d002d2dULL, 0x6b006b6b6b006b6bULL, |
| 560 | 0xa800a8a8a800a8a8, 0x2b002b2b2b002b2b, 0x3600363636003636, | 560 | 0xa800a8a8a800a8a8ULL, 0x2b002b2b2b002b2bULL, 0x3600363636003636ULL, |
| 561 | 0xa600a6a6a600a6a6, 0xc500c5c5c500c5c5, 0x8600868686008686, | 561 | 0xa600a6a6a600a6a6ULL, 0xc500c5c5c500c5c5ULL, 0x8600868686008686ULL, |
| 562 | 0x4d004d4d4d004d4d, 0x3300333333003333, 0xfd00fdfdfd00fdfd, | 562 | 0x4d004d4d4d004d4dULL, 0x3300333333003333ULL, 0xfd00fdfdfd00fdfdULL, |
| 563 | 0x6600666666006666, 0x5800585858005858, 0x9600969696009696, | 563 | 0x6600666666006666ULL, 0x5800585858005858ULL, 0x9600969696009696ULL, |
| 564 | 0x3a003a3a3a003a3a, 0x0900090909000909, 0x9500959595009595, | 564 | 0x3a003a3a3a003a3aULL, 0x0900090909000909ULL, 0x9500959595009595ULL, |
| 565 | 0x1000101010001010, 0x7800787878007878, 0xd800d8d8d800d8d8, | 565 | 0x1000101010001010ULL, 0x7800787878007878ULL, 0xd800d8d8d800d8d8ULL, |
| 566 | 0x4200424242004242, 0xcc00cccccc00cccc, 0xef00efefef00efef, | 566 | 0x4200424242004242ULL, 0xcc00cccccc00ccccULL, 0xef00efefef00efefULL, |
| 567 | 0x2600262626002626, 0xe500e5e5e500e5e5, 0x6100616161006161, | 567 | 0x2600262626002626ULL, 0xe500e5e5e500e5e5ULL, 0x6100616161006161ULL, |
| 568 | 0x1a001a1a1a001a1a, 0x3f003f3f3f003f3f, 0x3b003b3b3b003b3b, | 568 | 0x1a001a1a1a001a1aULL, 0x3f003f3f3f003f3fULL, 0x3b003b3b3b003b3bULL, |
| 569 | 0x8200828282008282, 0xb600b6b6b600b6b6, 0xdb00dbdbdb00dbdb, | 569 | 0x8200828282008282ULL, 0xb600b6b6b600b6b6ULL, 0xdb00dbdbdb00dbdbULL, |
| 570 | 0xd400d4d4d400d4d4, 0x9800989898009898, 0xe800e8e8e800e8e8, | 570 | 0xd400d4d4d400d4d4ULL, 0x9800989898009898ULL, 0xe800e8e8e800e8e8ULL, |
| 571 | 0x8b008b8b8b008b8b, 0x0200020202000202, 0xeb00ebebeb00ebeb, | 571 | 0x8b008b8b8b008b8bULL, 0x0200020202000202ULL, 0xeb00ebebeb00ebebULL, |
| 572 | 0x0a000a0a0a000a0a, 0x2c002c2c2c002c2c, 0x1d001d1d1d001d1d, | 572 | 0x0a000a0a0a000a0aULL, 0x2c002c2c2c002c2cULL, 0x1d001d1d1d001d1dULL, |
| 573 | 0xb000b0b0b000b0b0, 0x6f006f6f6f006f6f, 0x8d008d8d8d008d8d, | 573 | 0xb000b0b0b000b0b0ULL, 0x6f006f6f6f006f6fULL, 0x8d008d8d8d008d8dULL, |
| 574 | 0x8800888888008888, 0x0e000e0e0e000e0e, 0x1900191919001919, | 574 | 0x8800888888008888ULL, 0x0e000e0e0e000e0eULL, 0x1900191919001919ULL, |
| 575 | 0x8700878787008787, 0x4e004e4e4e004e4e, 0x0b000b0b0b000b0b, | 575 | 0x8700878787008787ULL, 0x4e004e4e4e004e4eULL, 0x0b000b0b0b000b0bULL, |
| 576 | 0xa900a9a9a900a9a9, 0x0c000c0c0c000c0c, 0x7900797979007979, | 576 | 0xa900a9a9a900a9a9ULL, 0x0c000c0c0c000c0cULL, 0x7900797979007979ULL, |
| 577 | 0x1100111111001111, 0x7f007f7f7f007f7f, 0x2200222222002222, | 577 | 0x1100111111001111ULL, 0x7f007f7f7f007f7fULL, 0x2200222222002222ULL, |
| 578 | 0xe700e7e7e700e7e7, 0x5900595959005959, 0xe100e1e1e100e1e1, | 578 | 0xe700e7e7e700e7e7ULL, 0x5900595959005959ULL, 0xe100e1e1e100e1e1ULL, |
| 579 | 0xda00dadada00dada, 0x3d003d3d3d003d3d, 0xc800c8c8c800c8c8, | 579 | 0xda00dadada00dadaULL, 0x3d003d3d3d003d3dULL, 0xc800c8c8c800c8c8ULL, |
| 580 | 0x1200121212001212, 0x0400040404000404, 0x7400747474007474, | 580 | 0x1200121212001212ULL, 0x0400040404000404ULL, 0x7400747474007474ULL, |
| 581 | 0x5400545454005454, 0x3000303030003030, 0x7e007e7e7e007e7e, | 581 | 0x5400545454005454ULL, 0x3000303030003030ULL, 0x7e007e7e7e007e7eULL, |
| 582 | 0xb400b4b4b400b4b4, 0x2800282828002828, 0x5500555555005555, | 582 | 0xb400b4b4b400b4b4ULL, 0x2800282828002828ULL, 0x5500555555005555ULL, |
| 583 | 0x6800686868006868, 0x5000505050005050, 0xbe00bebebe00bebe, | 583 | 0x6800686868006868ULL, 0x5000505050005050ULL, 0xbe00bebebe00bebeULL, |
| 584 | 0xd000d0d0d000d0d0, 0xc400c4c4c400c4c4, 0x3100313131003131, | 584 | 0xd000d0d0d000d0d0ULL, 0xc400c4c4c400c4c4ULL, 0x3100313131003131ULL, |
| 585 | 0xcb00cbcbcb00cbcb, 0x2a002a2a2a002a2a, 0xad00adadad00adad, | 585 | 0xcb00cbcbcb00cbcbULL, 0x2a002a2a2a002a2aULL, 0xad00adadad00adadULL, |
| 586 | 0x0f000f0f0f000f0f, 0xca00cacaca00caca, 0x7000707070007070, | 586 | 0x0f000f0f0f000f0fULL, 0xca00cacaca00cacaULL, 0x7000707070007070ULL, |
| 587 | 0xff00ffffff00ffff, 0x3200323232003232, 0x6900696969006969, | 587 | 0xff00ffffff00ffffULL, 0x3200323232003232ULL, 0x6900696969006969ULL, |
| 588 | 0x0800080808000808, 0x6200626262006262, 0x0000000000000000, | 588 | 0x0800080808000808ULL, 0x6200626262006262ULL, 0x0000000000000000ULL, |
| 589 | 0x2400242424002424, 0xd100d1d1d100d1d1, 0xfb00fbfbfb00fbfb, | 589 | 0x2400242424002424ULL, 0xd100d1d1d100d1d1ULL, 0xfb00fbfbfb00fbfbULL, |
| 590 | 0xba00bababa00baba, 0xed00ededed00eded, 0x4500454545004545, | 590 | 0xba00bababa00babaULL, 0xed00ededed00ededULL, 0x4500454545004545ULL, |
| 591 | 0x8100818181008181, 0x7300737373007373, 0x6d006d6d6d006d6d, | 591 | 0x8100818181008181ULL, 0x7300737373007373ULL, 0x6d006d6d6d006d6dULL, |
| 592 | 0x8400848484008484, 0x9f009f9f9f009f9f, 0xee00eeeeee00eeee, | 592 | 0x8400848484008484ULL, 0x9f009f9f9f009f9fULL, 0xee00eeeeee00eeeeULL, |
| 593 | 0x4a004a4a4a004a4a, 0xc300c3c3c300c3c3, 0x2e002e2e2e002e2e, | 593 | 0x4a004a4a4a004a4aULL, 0xc300c3c3c300c3c3ULL, 0x2e002e2e2e002e2eULL, |
| 594 | 0xc100c1c1c100c1c1, 0x0100010101000101, 0xe600e6e6e600e6e6, | 594 | 0xc100c1c1c100c1c1ULL, 0x0100010101000101ULL, 0xe600e6e6e600e6e6ULL, |
| 595 | 0x2500252525002525, 0x4800484848004848, 0x9900999999009999, | 595 | 0x2500252525002525ULL, 0x4800484848004848ULL, 0x9900999999009999ULL, |
| 596 | 0xb900b9b9b900b9b9, 0xb300b3b3b300b3b3, 0x7b007b7b7b007b7b, | 596 | 0xb900b9b9b900b9b9ULL, 0xb300b3b3b300b3b3ULL, 0x7b007b7b7b007b7bULL, |
| 597 | 0xf900f9f9f900f9f9, 0xce00cecece00cece, 0xbf00bfbfbf00bfbf, | 597 | 0xf900f9f9f900f9f9ULL, 0xce00cecece00ceceULL, 0xbf00bfbfbf00bfbfULL, |
| 598 | 0xdf00dfdfdf00dfdf, 0x7100717171007171, 0x2900292929002929, | 598 | 0xdf00dfdfdf00dfdfULL, 0x7100717171007171ULL, 0x2900292929002929ULL, |
| 599 | 0xcd00cdcdcd00cdcd, 0x6c006c6c6c006c6c, 0x1300131313001313, | 599 | 0xcd00cdcdcd00cdcdULL, 0x6c006c6c6c006c6cULL, 0x1300131313001313ULL, |
| 600 | 0x6400646464006464, 0x9b009b9b9b009b9b, 0x6300636363006363, | 600 | 0x6400646464006464ULL, 0x9b009b9b9b009b9bULL, 0x6300636363006363ULL, |
| 601 | 0x9d009d9d9d009d9d, 0xc000c0c0c000c0c0, 0x4b004b4b4b004b4b, | 601 | 0x9d009d9d9d009d9dULL, 0xc000c0c0c000c0c0ULL, 0x4b004b4b4b004b4bULL, |
| 602 | 0xb700b7b7b700b7b7, 0xa500a5a5a500a5a5, 0x8900898989008989, | 602 | 0xb700b7b7b700b7b7ULL, 0xa500a5a5a500a5a5ULL, 0x8900898989008989ULL, |
| 603 | 0x5f005f5f5f005f5f, 0xb100b1b1b100b1b1, 0x1700171717001717, | 603 | 0x5f005f5f5f005f5fULL, 0xb100b1b1b100b1b1ULL, 0x1700171717001717ULL, |
| 604 | 0xf400f4f4f400f4f4, 0xbc00bcbcbc00bcbc, 0xd300d3d3d300d3d3, | 604 | 0xf400f4f4f400f4f4ULL, 0xbc00bcbcbc00bcbcULL, 0xd300d3d3d300d3d3ULL, |
| 605 | 0x4600464646004646, 0xcf00cfcfcf00cfcf, 0x3700373737003737, | 605 | 0x4600464646004646ULL, 0xcf00cfcfcf00cfcfULL, 0x3700373737003737ULL, |
| 606 | 0x5e005e5e5e005e5e, 0x4700474747004747, 0x9400949494009494, | 606 | 0x5e005e5e5e005e5eULL, 0x4700474747004747ULL, 0x9400949494009494ULL, |
| 607 | 0xfa00fafafa00fafa, 0xfc00fcfcfc00fcfc, 0x5b005b5b5b005b5b, | 607 | 0xfa00fafafa00fafaULL, 0xfc00fcfcfc00fcfcULL, 0x5b005b5b5b005b5bULL, |
| 608 | 0x9700979797009797, 0xfe00fefefe00fefe, 0x5a005a5a5a005a5a, | 608 | 0x9700979797009797ULL, 0xfe00fefefe00fefeULL, 0x5a005a5a5a005a5aULL, |
| 609 | 0xac00acacac00acac, 0x3c003c3c3c003c3c, 0x4c004c4c4c004c4c, | 609 | 0xac00acacac00acacULL, 0x3c003c3c3c003c3cULL, 0x4c004c4c4c004c4cULL, |
| 610 | 0x0300030303000303, 0x3500353535003535, 0xf300f3f3f300f3f3, | 610 | 0x0300030303000303ULL, 0x3500353535003535ULL, 0xf300f3f3f300f3f3ULL, |
| 611 | 0x2300232323002323, 0xb800b8b8b800b8b8, 0x5d005d5d5d005d5d, | 611 | 0x2300232323002323ULL, 0xb800b8b8b800b8b8ULL, 0x5d005d5d5d005d5dULL, |
| 612 | 0x6a006a6a6a006a6a, 0x9200929292009292, 0xd500d5d5d500d5d5, | 612 | 0x6a006a6a6a006a6aULL, 0x9200929292009292ULL, 0xd500d5d5d500d5d5ULL, |
| 613 | 0x2100212121002121, 0x4400444444004444, 0x5100515151005151, | 613 | 0x2100212121002121ULL, 0x4400444444004444ULL, 0x5100515151005151ULL, |
| 614 | 0xc600c6c6c600c6c6, 0x7d007d7d7d007d7d, 0x3900393939003939, | 614 | 0xc600c6c6c600c6c6ULL, 0x7d007d7d7d007d7dULL, 0x3900393939003939ULL, |
| 615 | 0x8300838383008383, 0xdc00dcdcdc00dcdc, 0xaa00aaaaaa00aaaa, | 615 | 0x8300838383008383ULL, 0xdc00dcdcdc00dcdcULL, 0xaa00aaaaaa00aaaaULL, |
| 616 | 0x7c007c7c7c007c7c, 0x7700777777007777, 0x5600565656005656, | 616 | 0x7c007c7c7c007c7cULL, 0x7700777777007777ULL, 0x5600565656005656ULL, |
| 617 | 0x0500050505000505, 0x1b001b1b1b001b1b, 0xa400a4a4a400a4a4, | 617 | 0x0500050505000505ULL, 0x1b001b1b1b001b1bULL, 0xa400a4a4a400a4a4ULL, |
| 618 | 0x1500151515001515, 0x3400343434003434, 0x1e001e1e1e001e1e, | 618 | 0x1500151515001515ULL, 0x3400343434003434ULL, 0x1e001e1e1e001e1eULL, |
| 619 | 0x1c001c1c1c001c1c, 0xf800f8f8f800f8f8, 0x5200525252005252, | 619 | 0x1c001c1c1c001c1cULL, 0xf800f8f8f800f8f8ULL, 0x5200525252005252ULL, |
| 620 | 0x2000202020002020, 0x1400141414001414, 0xe900e9e9e900e9e9, | 620 | 0x2000202020002020ULL, 0x1400141414001414ULL, 0xe900e9e9e900e9e9ULL, |
| 621 | 0xbd00bdbdbd00bdbd, 0xdd00dddddd00dddd, 0xe400e4e4e400e4e4, | 621 | 0xbd00bdbdbd00bdbdULL, 0xdd00dddddd00ddddULL, 0xe400e4e4e400e4e4ULL, |
| 622 | 0xa100a1a1a100a1a1, 0xe000e0e0e000e0e0, 0x8a008a8a8a008a8a, | 622 | 0xa100a1a1a100a1a1ULL, 0xe000e0e0e000e0e0ULL, 0x8a008a8a8a008a8aULL, |
| 623 | 0xf100f1f1f100f1f1, 0xd600d6d6d600d6d6, 0x7a007a7a7a007a7a, | 623 | 0xf100f1f1f100f1f1ULL, 0xd600d6d6d600d6d6ULL, 0x7a007a7a7a007a7aULL, |
| 624 | 0xbb00bbbbbb00bbbb, 0xe300e3e3e300e3e3, 0x4000404040004040, | 624 | 0xbb00bbbbbb00bbbbULL, 0xe300e3e3e300e3e3ULL, 0x4000404040004040ULL, |
| 625 | 0x4f004f4f4f004f4f, | 625 | 0x4f004f4f4f004f4fULL, |
| 626 | }; | 626 | }; |
| 627 | 627 | ||
| 628 | const u64 camellia_sp44044404[256] = { | 628 | const u64 camellia_sp44044404[256] = { |
| 629 | 0x7070007070700070, 0x2c2c002c2c2c002c, 0xb3b300b3b3b300b3, | 629 | 0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL, |
| 630 | 0xc0c000c0c0c000c0, 0xe4e400e4e4e400e4, 0x5757005757570057, | 630 | 0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL, |
| 631 | 0xeaea00eaeaea00ea, 0xaeae00aeaeae00ae, 0x2323002323230023, | 631 | 0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL, |
| 632 | 0x6b6b006b6b6b006b, 0x4545004545450045, 0xa5a500a5a5a500a5, | 632 | 0x6b6b006b6b6b006bULL, 0x4545004545450045ULL, 0xa5a500a5a5a500a5ULL, |
| 633 | 0xeded00ededed00ed, 0x4f4f004f4f4f004f, 0x1d1d001d1d1d001d, | 633 | 0xeded00ededed00edULL, 0x4f4f004f4f4f004fULL, 0x1d1d001d1d1d001dULL, |
| 634 | 0x9292009292920092, 0x8686008686860086, 0xafaf00afafaf00af, | 634 | 0x9292009292920092ULL, 0x8686008686860086ULL, 0xafaf00afafaf00afULL, |
| 635 | 0x7c7c007c7c7c007c, 0x1f1f001f1f1f001f, 0x3e3e003e3e3e003e, | 635 | 0x7c7c007c7c7c007cULL, 0x1f1f001f1f1f001fULL, 0x3e3e003e3e3e003eULL, |
| 636 | 0xdcdc00dcdcdc00dc, 0x5e5e005e5e5e005e, 0x0b0b000b0b0b000b, | 636 | 0xdcdc00dcdcdc00dcULL, 0x5e5e005e5e5e005eULL, 0x0b0b000b0b0b000bULL, |
| 637 | 0xa6a600a6a6a600a6, 0x3939003939390039, 0xd5d500d5d5d500d5, | 637 | 0xa6a600a6a6a600a6ULL, 0x3939003939390039ULL, 0xd5d500d5d5d500d5ULL, |
| 638 | 0x5d5d005d5d5d005d, 0xd9d900d9d9d900d9, 0x5a5a005a5a5a005a, | 638 | 0x5d5d005d5d5d005dULL, 0xd9d900d9d9d900d9ULL, 0x5a5a005a5a5a005aULL, |
| 639 | 0x5151005151510051, 0x6c6c006c6c6c006c, 0x8b8b008b8b8b008b, | 639 | 0x5151005151510051ULL, 0x6c6c006c6c6c006cULL, 0x8b8b008b8b8b008bULL, |
| 640 | 0x9a9a009a9a9a009a, 0xfbfb00fbfbfb00fb, 0xb0b000b0b0b000b0, | 640 | 0x9a9a009a9a9a009aULL, 0xfbfb00fbfbfb00fbULL, 0xb0b000b0b0b000b0ULL, |
| 641 | 0x7474007474740074, 0x2b2b002b2b2b002b, 0xf0f000f0f0f000f0, | 641 | 0x7474007474740074ULL, 0x2b2b002b2b2b002bULL, 0xf0f000f0f0f000f0ULL, |
| 642 | 0x8484008484840084, 0xdfdf00dfdfdf00df, 0xcbcb00cbcbcb00cb, | 642 | 0x8484008484840084ULL, 0xdfdf00dfdfdf00dfULL, 0xcbcb00cbcbcb00cbULL, |
| 643 | 0x3434003434340034, 0x7676007676760076, 0x6d6d006d6d6d006d, | 643 | 0x3434003434340034ULL, 0x7676007676760076ULL, 0x6d6d006d6d6d006dULL, |
| 644 | 0xa9a900a9a9a900a9, 0xd1d100d1d1d100d1, 0x0404000404040004, | 644 | 0xa9a900a9a9a900a9ULL, 0xd1d100d1d1d100d1ULL, 0x0404000404040004ULL, |
| 645 | 0x1414001414140014, 0x3a3a003a3a3a003a, 0xdede00dedede00de, | 645 | 0x1414001414140014ULL, 0x3a3a003a3a3a003aULL, 0xdede00dedede00deULL, |
| 646 | 0x1111001111110011, 0x3232003232320032, 0x9c9c009c9c9c009c, | 646 | 0x1111001111110011ULL, 0x3232003232320032ULL, 0x9c9c009c9c9c009cULL, |
| 647 | 0x5353005353530053, 0xf2f200f2f2f200f2, 0xfefe00fefefe00fe, | 647 | 0x5353005353530053ULL, 0xf2f200f2f2f200f2ULL, 0xfefe00fefefe00feULL, |
| 648 | 0xcfcf00cfcfcf00cf, 0xc3c300c3c3c300c3, 0x7a7a007a7a7a007a, | 648 | 0xcfcf00cfcfcf00cfULL, 0xc3c300c3c3c300c3ULL, 0x7a7a007a7a7a007aULL, |
| 649 | 0x2424002424240024, 0xe8e800e8e8e800e8, 0x6060006060600060, | 649 | 0x2424002424240024ULL, 0xe8e800e8e8e800e8ULL, 0x6060006060600060ULL, |
| 650 | 0x6969006969690069, 0xaaaa00aaaaaa00aa, 0xa0a000a0a0a000a0, | 650 | 0x6969006969690069ULL, 0xaaaa00aaaaaa00aaULL, 0xa0a000a0a0a000a0ULL, |
| 651 | 0xa1a100a1a1a100a1, 0x6262006262620062, 0x5454005454540054, | 651 | 0xa1a100a1a1a100a1ULL, 0x6262006262620062ULL, 0x5454005454540054ULL, |
| 652 | 0x1e1e001e1e1e001e, 0xe0e000e0e0e000e0, 0x6464006464640064, | 652 | 0x1e1e001e1e1e001eULL, 0xe0e000e0e0e000e0ULL, 0x6464006464640064ULL, |
| 653 | 0x1010001010100010, 0x0000000000000000, 0xa3a300a3a3a300a3, | 653 | 0x1010001010100010ULL, 0x0000000000000000ULL, 0xa3a300a3a3a300a3ULL, |
| 654 | 0x7575007575750075, 0x8a8a008a8a8a008a, 0xe6e600e6e6e600e6, | 654 | 0x7575007575750075ULL, 0x8a8a008a8a8a008aULL, 0xe6e600e6e6e600e6ULL, |
| 655 | 0x0909000909090009, 0xdddd00dddddd00dd, 0x8787008787870087, | 655 | 0x0909000909090009ULL, 0xdddd00dddddd00ddULL, 0x8787008787870087ULL, |
| 656 | 0x8383008383830083, 0xcdcd00cdcdcd00cd, 0x9090009090900090, | 656 | 0x8383008383830083ULL, 0xcdcd00cdcdcd00cdULL, 0x9090009090900090ULL, |
| 657 | 0x7373007373730073, 0xf6f600f6f6f600f6, 0x9d9d009d9d9d009d, | 657 | 0x7373007373730073ULL, 0xf6f600f6f6f600f6ULL, 0x9d9d009d9d9d009dULL, |
| 658 | 0xbfbf00bfbfbf00bf, 0x5252005252520052, 0xd8d800d8d8d800d8, | 658 | 0xbfbf00bfbfbf00bfULL, 0x5252005252520052ULL, 0xd8d800d8d8d800d8ULL, |
| 659 | 0xc8c800c8c8c800c8, 0xc6c600c6c6c600c6, 0x8181008181810081, | 659 | 0xc8c800c8c8c800c8ULL, 0xc6c600c6c6c600c6ULL, 0x8181008181810081ULL, |
| 660 | 0x6f6f006f6f6f006f, 0x1313001313130013, 0x6363006363630063, | 660 | 0x6f6f006f6f6f006fULL, 0x1313001313130013ULL, 0x6363006363630063ULL, |
| 661 | 0xe9e900e9e9e900e9, 0xa7a700a7a7a700a7, 0x9f9f009f9f9f009f, | 661 | 0xe9e900e9e9e900e9ULL, 0xa7a700a7a7a700a7ULL, 0x9f9f009f9f9f009fULL, |
| 662 | 0xbcbc00bcbcbc00bc, 0x2929002929290029, 0xf9f900f9f9f900f9, | 662 | 0xbcbc00bcbcbc00bcULL, 0x2929002929290029ULL, 0xf9f900f9f9f900f9ULL, |
| 663 | 0x2f2f002f2f2f002f, 0xb4b400b4b4b400b4, 0x7878007878780078, | 663 | 0x2f2f002f2f2f002fULL, 0xb4b400b4b4b400b4ULL, 0x7878007878780078ULL, |
| 664 | 0x0606000606060006, 0xe7e700e7e7e700e7, 0x7171007171710071, | 664 | 0x0606000606060006ULL, 0xe7e700e7e7e700e7ULL, 0x7171007171710071ULL, |
| 665 | 0xd4d400d4d4d400d4, 0xabab00ababab00ab, 0x8888008888880088, | 665 | 0xd4d400d4d4d400d4ULL, 0xabab00ababab00abULL, 0x8888008888880088ULL, |
| 666 | 0x8d8d008d8d8d008d, 0x7272007272720072, 0xb9b900b9b9b900b9, | 666 | 0x8d8d008d8d8d008dULL, 0x7272007272720072ULL, 0xb9b900b9b9b900b9ULL, |
| 667 | 0xf8f800f8f8f800f8, 0xacac00acacac00ac, 0x3636003636360036, | 667 | 0xf8f800f8f8f800f8ULL, 0xacac00acacac00acULL, 0x3636003636360036ULL, |
| 668 | 0x2a2a002a2a2a002a, 0x3c3c003c3c3c003c, 0xf1f100f1f1f100f1, | 668 | 0x2a2a002a2a2a002aULL, 0x3c3c003c3c3c003cULL, 0xf1f100f1f1f100f1ULL, |
| 669 | 0x4040004040400040, 0xd3d300d3d3d300d3, 0xbbbb00bbbbbb00bb, | 669 | 0x4040004040400040ULL, 0xd3d300d3d3d300d3ULL, 0xbbbb00bbbbbb00bbULL, |
| 670 | 0x4343004343430043, 0x1515001515150015, 0xadad00adadad00ad, | 670 | 0x4343004343430043ULL, 0x1515001515150015ULL, 0xadad00adadad00adULL, |
| 671 | 0x7777007777770077, 0x8080008080800080, 0x8282008282820082, | 671 | 0x7777007777770077ULL, 0x8080008080800080ULL, 0x8282008282820082ULL, |
| 672 | 0xecec00ececec00ec, 0x2727002727270027, 0xe5e500e5e5e500e5, | 672 | 0xecec00ececec00ecULL, 0x2727002727270027ULL, 0xe5e500e5e5e500e5ULL, |
| 673 | 0x8585008585850085, 0x3535003535350035, 0x0c0c000c0c0c000c, | 673 | 0x8585008585850085ULL, 0x3535003535350035ULL, 0x0c0c000c0c0c000cULL, |
| 674 | 0x4141004141410041, 0xefef00efefef00ef, 0x9393009393930093, | 674 | 0x4141004141410041ULL, 0xefef00efefef00efULL, 0x9393009393930093ULL, |
| 675 | 0x1919001919190019, 0x2121002121210021, 0x0e0e000e0e0e000e, | 675 | 0x1919001919190019ULL, 0x2121002121210021ULL, 0x0e0e000e0e0e000eULL, |
| 676 | 0x4e4e004e4e4e004e, 0x6565006565650065, 0xbdbd00bdbdbd00bd, | 676 | 0x4e4e004e4e4e004eULL, 0x6565006565650065ULL, 0xbdbd00bdbdbd00bdULL, |
| 677 | 0xb8b800b8b8b800b8, 0x8f8f008f8f8f008f, 0xebeb00ebebeb00eb, | 677 | 0xb8b800b8b8b800b8ULL, 0x8f8f008f8f8f008fULL, 0xebeb00ebebeb00ebULL, |
| 678 | 0xcece00cecece00ce, 0x3030003030300030, 0x5f5f005f5f5f005f, | 678 | 0xcece00cecece00ceULL, 0x3030003030300030ULL, 0x5f5f005f5f5f005fULL, |
| 679 | 0xc5c500c5c5c500c5, 0x1a1a001a1a1a001a, 0xe1e100e1e1e100e1, | 679 | 0xc5c500c5c5c500c5ULL, 0x1a1a001a1a1a001aULL, 0xe1e100e1e1e100e1ULL, |
| 680 | 0xcaca00cacaca00ca, 0x4747004747470047, 0x3d3d003d3d3d003d, | 680 | 0xcaca00cacaca00caULL, 0x4747004747470047ULL, 0x3d3d003d3d3d003dULL, |
| 681 | 0x0101000101010001, 0xd6d600d6d6d600d6, 0x5656005656560056, | 681 | 0x0101000101010001ULL, 0xd6d600d6d6d600d6ULL, 0x5656005656560056ULL, |
| 682 | 0x4d4d004d4d4d004d, 0x0d0d000d0d0d000d, 0x6666006666660066, | 682 | 0x4d4d004d4d4d004dULL, 0x0d0d000d0d0d000dULL, 0x6666006666660066ULL, |
| 683 | 0xcccc00cccccc00cc, 0x2d2d002d2d2d002d, 0x1212001212120012, | 683 | 0xcccc00cccccc00ccULL, 0x2d2d002d2d2d002dULL, 0x1212001212120012ULL, |
| 684 | 0x2020002020200020, 0xb1b100b1b1b100b1, 0x9999009999990099, | 684 | 0x2020002020200020ULL, 0xb1b100b1b1b100b1ULL, 0x9999009999990099ULL, |
| 685 | 0x4c4c004c4c4c004c, 0xc2c200c2c2c200c2, 0x7e7e007e7e7e007e, | 685 | 0x4c4c004c4c4c004cULL, 0xc2c200c2c2c200c2ULL, 0x7e7e007e7e7e007eULL, |
| 686 | 0x0505000505050005, 0xb7b700b7b7b700b7, 0x3131003131310031, | 686 | 0x0505000505050005ULL, 0xb7b700b7b7b700b7ULL, 0x3131003131310031ULL, |
| 687 | 0x1717001717170017, 0xd7d700d7d7d700d7, 0x5858005858580058, | 687 | 0x1717001717170017ULL, 0xd7d700d7d7d700d7ULL, 0x5858005858580058ULL, |
| 688 | 0x6161006161610061, 0x1b1b001b1b1b001b, 0x1c1c001c1c1c001c, | 688 | 0x6161006161610061ULL, 0x1b1b001b1b1b001bULL, 0x1c1c001c1c1c001cULL, |
| 689 | 0x0f0f000f0f0f000f, 0x1616001616160016, 0x1818001818180018, | 689 | 0x0f0f000f0f0f000fULL, 0x1616001616160016ULL, 0x1818001818180018ULL, |
| 690 | 0x2222002222220022, 0x4444004444440044, 0xb2b200b2b2b200b2, | 690 | 0x2222002222220022ULL, 0x4444004444440044ULL, 0xb2b200b2b2b200b2ULL, |
| 691 | 0xb5b500b5b5b500b5, 0x9191009191910091, 0x0808000808080008, | 691 | 0xb5b500b5b5b500b5ULL, 0x9191009191910091ULL, 0x0808000808080008ULL, |
| 692 | 0xa8a800a8a8a800a8, 0xfcfc00fcfcfc00fc, 0x5050005050500050, | 692 | 0xa8a800a8a8a800a8ULL, 0xfcfc00fcfcfc00fcULL, 0x5050005050500050ULL, |
| 693 | 0xd0d000d0d0d000d0, 0x7d7d007d7d7d007d, 0x8989008989890089, | 693 | 0xd0d000d0d0d000d0ULL, 0x7d7d007d7d7d007dULL, 0x8989008989890089ULL, |
| 694 | 0x9797009797970097, 0x5b5b005b5b5b005b, 0x9595009595950095, | 694 | 0x9797009797970097ULL, 0x5b5b005b5b5b005bULL, 0x9595009595950095ULL, |
| 695 | 0xffff00ffffff00ff, 0xd2d200d2d2d200d2, 0xc4c400c4c4c400c4, | 695 | 0xffff00ffffff00ffULL, 0xd2d200d2d2d200d2ULL, 0xc4c400c4c4c400c4ULL, |
| 696 | 0x4848004848480048, 0xf7f700f7f7f700f7, 0xdbdb00dbdbdb00db, | 696 | 0x4848004848480048ULL, 0xf7f700f7f7f700f7ULL, 0xdbdb00dbdbdb00dbULL, |
| 697 | 0x0303000303030003, 0xdada00dadada00da, 0x3f3f003f3f3f003f, | 697 | 0x0303000303030003ULL, 0xdada00dadada00daULL, 0x3f3f003f3f3f003fULL, |
| 698 | 0x9494009494940094, 0x5c5c005c5c5c005c, 0x0202000202020002, | 698 | 0x9494009494940094ULL, 0x5c5c005c5c5c005cULL, 0x0202000202020002ULL, |
| 699 | 0x4a4a004a4a4a004a, 0x3333003333330033, 0x6767006767670067, | 699 | 0x4a4a004a4a4a004aULL, 0x3333003333330033ULL, 0x6767006767670067ULL, |
| 700 | 0xf3f300f3f3f300f3, 0x7f7f007f7f7f007f, 0xe2e200e2e2e200e2, | 700 | 0xf3f300f3f3f300f3ULL, 0x7f7f007f7f7f007fULL, 0xe2e200e2e2e200e2ULL, |
| 701 | 0x9b9b009b9b9b009b, 0x2626002626260026, 0x3737003737370037, | 701 | 0x9b9b009b9b9b009bULL, 0x2626002626260026ULL, 0x3737003737370037ULL, |
| 702 | 0x3b3b003b3b3b003b, 0x9696009696960096, 0x4b4b004b4b4b004b, | 702 | 0x3b3b003b3b3b003bULL, 0x9696009696960096ULL, 0x4b4b004b4b4b004bULL, |
| 703 | 0xbebe00bebebe00be, 0x2e2e002e2e2e002e, 0x7979007979790079, | 703 | 0xbebe00bebebe00beULL, 0x2e2e002e2e2e002eULL, 0x7979007979790079ULL, |
| 704 | 0x8c8c008c8c8c008c, 0x6e6e006e6e6e006e, 0x8e8e008e8e8e008e, | 704 | 0x8c8c008c8c8c008cULL, 0x6e6e006e6e6e006eULL, 0x8e8e008e8e8e008eULL, |
| 705 | 0xf5f500f5f5f500f5, 0xb6b600b6b6b600b6, 0xfdfd00fdfdfd00fd, | 705 | 0xf5f500f5f5f500f5ULL, 0xb6b600b6b6b600b6ULL, 0xfdfd00fdfdfd00fdULL, |
| 706 | 0x5959005959590059, 0x9898009898980098, 0x6a6a006a6a6a006a, | 706 | 0x5959005959590059ULL, 0x9898009898980098ULL, 0x6a6a006a6a6a006aULL, |
| 707 | 0x4646004646460046, 0xbaba00bababa00ba, 0x2525002525250025, | 707 | 0x4646004646460046ULL, 0xbaba00bababa00baULL, 0x2525002525250025ULL, |
| 708 | 0x4242004242420042, 0xa2a200a2a2a200a2, 0xfafa00fafafa00fa, | 708 | 0x4242004242420042ULL, 0xa2a200a2a2a200a2ULL, 0xfafa00fafafa00faULL, |
| 709 | 0x0707000707070007, 0x5555005555550055, 0xeeee00eeeeee00ee, | 709 | 0x0707000707070007ULL, 0x5555005555550055ULL, 0xeeee00eeeeee00eeULL, |
| 710 | 0x0a0a000a0a0a000a, 0x4949004949490049, 0x6868006868680068, | 710 | 0x0a0a000a0a0a000aULL, 0x4949004949490049ULL, 0x6868006868680068ULL, |
| 711 | 0x3838003838380038, 0xa4a400a4a4a400a4, 0x2828002828280028, | 711 | 0x3838003838380038ULL, 0xa4a400a4a4a400a4ULL, 0x2828002828280028ULL, |
| 712 | 0x7b7b007b7b7b007b, 0xc9c900c9c9c900c9, 0xc1c100c1c1c100c1, | 712 | 0x7b7b007b7b7b007bULL, 0xc9c900c9c9c900c9ULL, 0xc1c100c1c1c100c1ULL, |
| 713 | 0xe3e300e3e3e300e3, 0xf4f400f4f4f400f4, 0xc7c700c7c7c700c7, | 713 | 0xe3e300e3e3e300e3ULL, 0xf4f400f4f4f400f4ULL, 0xc7c700c7c7c700c7ULL, |
| 714 | 0x9e9e009e9e9e009e, | 714 | 0x9e9e009e9e9e009eULL, |
| 715 | }; | 715 | }; |
| 716 | 716 | ||
| 717 | const u64 camellia_sp11101110[256] = { | 717 | const u64 camellia_sp11101110[256] = { |
| 718 | 0x7070700070707000, 0x8282820082828200, 0x2c2c2c002c2c2c00, | 718 | 0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL, |
| 719 | 0xececec00ececec00, 0xb3b3b300b3b3b300, 0x2727270027272700, | 719 | 0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL, |
| 720 | 0xc0c0c000c0c0c000, 0xe5e5e500e5e5e500, 0xe4e4e400e4e4e400, | 720 | 0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL, |
| 721 | 0x8585850085858500, 0x5757570057575700, 0x3535350035353500, | 721 | 0x8585850085858500ULL, 0x5757570057575700ULL, 0x3535350035353500ULL, |
| 722 | 0xeaeaea00eaeaea00, 0x0c0c0c000c0c0c00, 0xaeaeae00aeaeae00, | 722 | 0xeaeaea00eaeaea00ULL, 0x0c0c0c000c0c0c00ULL, 0xaeaeae00aeaeae00ULL, |
| 723 | 0x4141410041414100, 0x2323230023232300, 0xefefef00efefef00, | 723 | 0x4141410041414100ULL, 0x2323230023232300ULL, 0xefefef00efefef00ULL, |
| 724 | 0x6b6b6b006b6b6b00, 0x9393930093939300, 0x4545450045454500, | 724 | 0x6b6b6b006b6b6b00ULL, 0x9393930093939300ULL, 0x4545450045454500ULL, |
| 725 | 0x1919190019191900, 0xa5a5a500a5a5a500, 0x2121210021212100, | 725 | 0x1919190019191900ULL, 0xa5a5a500a5a5a500ULL, 0x2121210021212100ULL, |
| 726 | 0xededed00ededed00, 0x0e0e0e000e0e0e00, 0x4f4f4f004f4f4f00, | 726 | 0xededed00ededed00ULL, 0x0e0e0e000e0e0e00ULL, 0x4f4f4f004f4f4f00ULL, |
| 727 | 0x4e4e4e004e4e4e00, 0x1d1d1d001d1d1d00, 0x6565650065656500, | 727 | 0x4e4e4e004e4e4e00ULL, 0x1d1d1d001d1d1d00ULL, 0x6565650065656500ULL, |
| 728 | 0x9292920092929200, 0xbdbdbd00bdbdbd00, 0x8686860086868600, | 728 | 0x9292920092929200ULL, 0xbdbdbd00bdbdbd00ULL, 0x8686860086868600ULL, |
| 729 | 0xb8b8b800b8b8b800, 0xafafaf00afafaf00, 0x8f8f8f008f8f8f00, | 729 | 0xb8b8b800b8b8b800ULL, 0xafafaf00afafaf00ULL, 0x8f8f8f008f8f8f00ULL, |
| 730 | 0x7c7c7c007c7c7c00, 0xebebeb00ebebeb00, 0x1f1f1f001f1f1f00, | 730 | 0x7c7c7c007c7c7c00ULL, 0xebebeb00ebebeb00ULL, 0x1f1f1f001f1f1f00ULL, |
| 731 | 0xcecece00cecece00, 0x3e3e3e003e3e3e00, 0x3030300030303000, | 731 | 0xcecece00cecece00ULL, 0x3e3e3e003e3e3e00ULL, 0x3030300030303000ULL, |
| 732 | 0xdcdcdc00dcdcdc00, 0x5f5f5f005f5f5f00, 0x5e5e5e005e5e5e00, | 732 | 0xdcdcdc00dcdcdc00ULL, 0x5f5f5f005f5f5f00ULL, 0x5e5e5e005e5e5e00ULL, |
| 733 | 0xc5c5c500c5c5c500, 0x0b0b0b000b0b0b00, 0x1a1a1a001a1a1a00, | 733 | 0xc5c5c500c5c5c500ULL, 0x0b0b0b000b0b0b00ULL, 0x1a1a1a001a1a1a00ULL, |
| 734 | 0xa6a6a600a6a6a600, 0xe1e1e100e1e1e100, 0x3939390039393900, | 734 | 0xa6a6a600a6a6a600ULL, 0xe1e1e100e1e1e100ULL, 0x3939390039393900ULL, |
| 735 | 0xcacaca00cacaca00, 0xd5d5d500d5d5d500, 0x4747470047474700, | 735 | 0xcacaca00cacaca00ULL, 0xd5d5d500d5d5d500ULL, 0x4747470047474700ULL, |
| 736 | 0x5d5d5d005d5d5d00, 0x3d3d3d003d3d3d00, 0xd9d9d900d9d9d900, | 736 | 0x5d5d5d005d5d5d00ULL, 0x3d3d3d003d3d3d00ULL, 0xd9d9d900d9d9d900ULL, |
| 737 | 0x0101010001010100, 0x5a5a5a005a5a5a00, 0xd6d6d600d6d6d600, | 737 | 0x0101010001010100ULL, 0x5a5a5a005a5a5a00ULL, 0xd6d6d600d6d6d600ULL, |
| 738 | 0x5151510051515100, 0x5656560056565600, 0x6c6c6c006c6c6c00, | 738 | 0x5151510051515100ULL, 0x5656560056565600ULL, 0x6c6c6c006c6c6c00ULL, |
| 739 | 0x4d4d4d004d4d4d00, 0x8b8b8b008b8b8b00, 0x0d0d0d000d0d0d00, | 739 | 0x4d4d4d004d4d4d00ULL, 0x8b8b8b008b8b8b00ULL, 0x0d0d0d000d0d0d00ULL, |
| 740 | 0x9a9a9a009a9a9a00, 0x6666660066666600, 0xfbfbfb00fbfbfb00, | 740 | 0x9a9a9a009a9a9a00ULL, 0x6666660066666600ULL, 0xfbfbfb00fbfbfb00ULL, |
| 741 | 0xcccccc00cccccc00, 0xb0b0b000b0b0b000, 0x2d2d2d002d2d2d00, | 741 | 0xcccccc00cccccc00ULL, 0xb0b0b000b0b0b000ULL, 0x2d2d2d002d2d2d00ULL, |
| 742 | 0x7474740074747400, 0x1212120012121200, 0x2b2b2b002b2b2b00, | 742 | 0x7474740074747400ULL, 0x1212120012121200ULL, 0x2b2b2b002b2b2b00ULL, |
| 743 | 0x2020200020202000, 0xf0f0f000f0f0f000, 0xb1b1b100b1b1b100, | 743 | 0x2020200020202000ULL, 0xf0f0f000f0f0f000ULL, 0xb1b1b100b1b1b100ULL, |
| 744 | 0x8484840084848400, 0x9999990099999900, 0xdfdfdf00dfdfdf00, | 744 | 0x8484840084848400ULL, 0x9999990099999900ULL, 0xdfdfdf00dfdfdf00ULL, |
| 745 | 0x4c4c4c004c4c4c00, 0xcbcbcb00cbcbcb00, 0xc2c2c200c2c2c200, | 745 | 0x4c4c4c004c4c4c00ULL, 0xcbcbcb00cbcbcb00ULL, 0xc2c2c200c2c2c200ULL, |
| 746 | 0x3434340034343400, 0x7e7e7e007e7e7e00, 0x7676760076767600, | 746 | 0x3434340034343400ULL, 0x7e7e7e007e7e7e00ULL, 0x7676760076767600ULL, |
| 747 | 0x0505050005050500, 0x6d6d6d006d6d6d00, 0xb7b7b700b7b7b700, | 747 | 0x0505050005050500ULL, 0x6d6d6d006d6d6d00ULL, 0xb7b7b700b7b7b700ULL, |
| 748 | 0xa9a9a900a9a9a900, 0x3131310031313100, 0xd1d1d100d1d1d100, | 748 | 0xa9a9a900a9a9a900ULL, 0x3131310031313100ULL, 0xd1d1d100d1d1d100ULL, |
| 749 | 0x1717170017171700, 0x0404040004040400, 0xd7d7d700d7d7d700, | 749 | 0x1717170017171700ULL, 0x0404040004040400ULL, 0xd7d7d700d7d7d700ULL, |
| 750 | 0x1414140014141400, 0x5858580058585800, 0x3a3a3a003a3a3a00, | 750 | 0x1414140014141400ULL, 0x5858580058585800ULL, 0x3a3a3a003a3a3a00ULL, |
| 751 | 0x6161610061616100, 0xdedede00dedede00, 0x1b1b1b001b1b1b00, | 751 | 0x6161610061616100ULL, 0xdedede00dedede00ULL, 0x1b1b1b001b1b1b00ULL, |
| 752 | 0x1111110011111100, 0x1c1c1c001c1c1c00, 0x3232320032323200, | 752 | 0x1111110011111100ULL, 0x1c1c1c001c1c1c00ULL, 0x3232320032323200ULL, |
| 753 | 0x0f0f0f000f0f0f00, 0x9c9c9c009c9c9c00, 0x1616160016161600, | 753 | 0x0f0f0f000f0f0f00ULL, 0x9c9c9c009c9c9c00ULL, 0x1616160016161600ULL, |
| 754 | 0x5353530053535300, 0x1818180018181800, 0xf2f2f200f2f2f200, | 754 | 0x5353530053535300ULL, 0x1818180018181800ULL, 0xf2f2f200f2f2f200ULL, |
| 755 | 0x2222220022222200, 0xfefefe00fefefe00, 0x4444440044444400, | 755 | 0x2222220022222200ULL, 0xfefefe00fefefe00ULL, 0x4444440044444400ULL, |
| 756 | 0xcfcfcf00cfcfcf00, 0xb2b2b200b2b2b200, 0xc3c3c300c3c3c300, | 756 | 0xcfcfcf00cfcfcf00ULL, 0xb2b2b200b2b2b200ULL, 0xc3c3c300c3c3c300ULL, |
| 757 | 0xb5b5b500b5b5b500, 0x7a7a7a007a7a7a00, 0x9191910091919100, | 757 | 0xb5b5b500b5b5b500ULL, 0x7a7a7a007a7a7a00ULL, 0x9191910091919100ULL, |
| 758 | 0x2424240024242400, 0x0808080008080800, 0xe8e8e800e8e8e800, | 758 | 0x2424240024242400ULL, 0x0808080008080800ULL, 0xe8e8e800e8e8e800ULL, |
| 759 | 0xa8a8a800a8a8a800, 0x6060600060606000, 0xfcfcfc00fcfcfc00, | 759 | 0xa8a8a800a8a8a800ULL, 0x6060600060606000ULL, 0xfcfcfc00fcfcfc00ULL, |
| 760 | 0x6969690069696900, 0x5050500050505000, 0xaaaaaa00aaaaaa00, | 760 | 0x6969690069696900ULL, 0x5050500050505000ULL, 0xaaaaaa00aaaaaa00ULL, |
| 761 | 0xd0d0d000d0d0d000, 0xa0a0a000a0a0a000, 0x7d7d7d007d7d7d00, | 761 | 0xd0d0d000d0d0d000ULL, 0xa0a0a000a0a0a000ULL, 0x7d7d7d007d7d7d00ULL, |
| 762 | 0xa1a1a100a1a1a100, 0x8989890089898900, 0x6262620062626200, | 762 | 0xa1a1a100a1a1a100ULL, 0x8989890089898900ULL, 0x6262620062626200ULL, |
| 763 | 0x9797970097979700, 0x5454540054545400, 0x5b5b5b005b5b5b00, | 763 | 0x9797970097979700ULL, 0x5454540054545400ULL, 0x5b5b5b005b5b5b00ULL, |
| 764 | 0x1e1e1e001e1e1e00, 0x9595950095959500, 0xe0e0e000e0e0e000, | 764 | 0x1e1e1e001e1e1e00ULL, 0x9595950095959500ULL, 0xe0e0e000e0e0e000ULL, |
| 765 | 0xffffff00ffffff00, 0x6464640064646400, 0xd2d2d200d2d2d200, | 765 | 0xffffff00ffffff00ULL, 0x6464640064646400ULL, 0xd2d2d200d2d2d200ULL, |
| 766 | 0x1010100010101000, 0xc4c4c400c4c4c400, 0x0000000000000000, | 766 | 0x1010100010101000ULL, 0xc4c4c400c4c4c400ULL, 0x0000000000000000ULL, |
| 767 | 0x4848480048484800, 0xa3a3a300a3a3a300, 0xf7f7f700f7f7f700, | 767 | 0x4848480048484800ULL, 0xa3a3a300a3a3a300ULL, 0xf7f7f700f7f7f700ULL, |
| 768 | 0x7575750075757500, 0xdbdbdb00dbdbdb00, 0x8a8a8a008a8a8a00, | 768 | 0x7575750075757500ULL, 0xdbdbdb00dbdbdb00ULL, 0x8a8a8a008a8a8a00ULL, |
| 769 | 0x0303030003030300, 0xe6e6e600e6e6e600, 0xdadada00dadada00, | 769 | 0x0303030003030300ULL, 0xe6e6e600e6e6e600ULL, 0xdadada00dadada00ULL, |
| 770 | 0x0909090009090900, 0x3f3f3f003f3f3f00, 0xdddddd00dddddd00, | 770 | 0x0909090009090900ULL, 0x3f3f3f003f3f3f00ULL, 0xdddddd00dddddd00ULL, |
| 771 | 0x9494940094949400, 0x8787870087878700, 0x5c5c5c005c5c5c00, | 771 | 0x9494940094949400ULL, 0x8787870087878700ULL, 0x5c5c5c005c5c5c00ULL, |
| 772 | 0x8383830083838300, 0x0202020002020200, 0xcdcdcd00cdcdcd00, | 772 | 0x8383830083838300ULL, 0x0202020002020200ULL, 0xcdcdcd00cdcdcd00ULL, |
| 773 | 0x4a4a4a004a4a4a00, 0x9090900090909000, 0x3333330033333300, | 773 | 0x4a4a4a004a4a4a00ULL, 0x9090900090909000ULL, 0x3333330033333300ULL, |
| 774 | 0x7373730073737300, 0x6767670067676700, 0xf6f6f600f6f6f600, | 774 | 0x7373730073737300ULL, 0x6767670067676700ULL, 0xf6f6f600f6f6f600ULL, |
| 775 | 0xf3f3f300f3f3f300, 0x9d9d9d009d9d9d00, 0x7f7f7f007f7f7f00, | 775 | 0xf3f3f300f3f3f300ULL, 0x9d9d9d009d9d9d00ULL, 0x7f7f7f007f7f7f00ULL, |
| 776 | 0xbfbfbf00bfbfbf00, 0xe2e2e200e2e2e200, 0x5252520052525200, | 776 | 0xbfbfbf00bfbfbf00ULL, 0xe2e2e200e2e2e200ULL, 0x5252520052525200ULL, |
| 777 | 0x9b9b9b009b9b9b00, 0xd8d8d800d8d8d800, 0x2626260026262600, | 777 | 0x9b9b9b009b9b9b00ULL, 0xd8d8d800d8d8d800ULL, 0x2626260026262600ULL, |
| 778 | 0xc8c8c800c8c8c800, 0x3737370037373700, 0xc6c6c600c6c6c600, | 778 | 0xc8c8c800c8c8c800ULL, 0x3737370037373700ULL, 0xc6c6c600c6c6c600ULL, |
| 779 | 0x3b3b3b003b3b3b00, 0x8181810081818100, 0x9696960096969600, | 779 | 0x3b3b3b003b3b3b00ULL, 0x8181810081818100ULL, 0x9696960096969600ULL, |
| 780 | 0x6f6f6f006f6f6f00, 0x4b4b4b004b4b4b00, 0x1313130013131300, | 780 | 0x6f6f6f006f6f6f00ULL, 0x4b4b4b004b4b4b00ULL, 0x1313130013131300ULL, |
| 781 | 0xbebebe00bebebe00, 0x6363630063636300, 0x2e2e2e002e2e2e00, | 781 | 0xbebebe00bebebe00ULL, 0x6363630063636300ULL, 0x2e2e2e002e2e2e00ULL, |
| 782 | 0xe9e9e900e9e9e900, 0x7979790079797900, 0xa7a7a700a7a7a700, | 782 | 0xe9e9e900e9e9e900ULL, 0x7979790079797900ULL, 0xa7a7a700a7a7a700ULL, |
| 783 | 0x8c8c8c008c8c8c00, 0x9f9f9f009f9f9f00, 0x6e6e6e006e6e6e00, | 783 | 0x8c8c8c008c8c8c00ULL, 0x9f9f9f009f9f9f00ULL, 0x6e6e6e006e6e6e00ULL, |
| 784 | 0xbcbcbc00bcbcbc00, 0x8e8e8e008e8e8e00, 0x2929290029292900, | 784 | 0xbcbcbc00bcbcbc00ULL, 0x8e8e8e008e8e8e00ULL, 0x2929290029292900ULL, |
| 785 | 0xf5f5f500f5f5f500, 0xf9f9f900f9f9f900, 0xb6b6b600b6b6b600, | 785 | 0xf5f5f500f5f5f500ULL, 0xf9f9f900f9f9f900ULL, 0xb6b6b600b6b6b600ULL, |
| 786 | 0x2f2f2f002f2f2f00, 0xfdfdfd00fdfdfd00, 0xb4b4b400b4b4b400, | 786 | 0x2f2f2f002f2f2f00ULL, 0xfdfdfd00fdfdfd00ULL, 0xb4b4b400b4b4b400ULL, |
| 787 | 0x5959590059595900, 0x7878780078787800, 0x9898980098989800, | 787 | 0x5959590059595900ULL, 0x7878780078787800ULL, 0x9898980098989800ULL, |
| 788 | 0x0606060006060600, 0x6a6a6a006a6a6a00, 0xe7e7e700e7e7e700, | 788 | 0x0606060006060600ULL, 0x6a6a6a006a6a6a00ULL, 0xe7e7e700e7e7e700ULL, |
| 789 | 0x4646460046464600, 0x7171710071717100, 0xbababa00bababa00, | 789 | 0x4646460046464600ULL, 0x7171710071717100ULL, 0xbababa00bababa00ULL, |
| 790 | 0xd4d4d400d4d4d400, 0x2525250025252500, 0xababab00ababab00, | 790 | 0xd4d4d400d4d4d400ULL, 0x2525250025252500ULL, 0xababab00ababab00ULL, |
| 791 | 0x4242420042424200, 0x8888880088888800, 0xa2a2a200a2a2a200, | 791 | 0x4242420042424200ULL, 0x8888880088888800ULL, 0xa2a2a200a2a2a200ULL, |
| 792 | 0x8d8d8d008d8d8d00, 0xfafafa00fafafa00, 0x7272720072727200, | 792 | 0x8d8d8d008d8d8d00ULL, 0xfafafa00fafafa00ULL, 0x7272720072727200ULL, |
| 793 | 0x0707070007070700, 0xb9b9b900b9b9b900, 0x5555550055555500, | 793 | 0x0707070007070700ULL, 0xb9b9b900b9b9b900ULL, 0x5555550055555500ULL, |
| 794 | 0xf8f8f800f8f8f800, 0xeeeeee00eeeeee00, 0xacacac00acacac00, | 794 | 0xf8f8f800f8f8f800ULL, 0xeeeeee00eeeeee00ULL, 0xacacac00acacac00ULL, |
| 795 | 0x0a0a0a000a0a0a00, 0x3636360036363600, 0x4949490049494900, | 795 | 0x0a0a0a000a0a0a00ULL, 0x3636360036363600ULL, 0x4949490049494900ULL, |
| 796 | 0x2a2a2a002a2a2a00, 0x6868680068686800, 0x3c3c3c003c3c3c00, | 796 | 0x2a2a2a002a2a2a00ULL, 0x6868680068686800ULL, 0x3c3c3c003c3c3c00ULL, |
| 797 | 0x3838380038383800, 0xf1f1f100f1f1f100, 0xa4a4a400a4a4a400, | 797 | 0x3838380038383800ULL, 0xf1f1f100f1f1f100ULL, 0xa4a4a400a4a4a400ULL, |
| 798 | 0x4040400040404000, 0x2828280028282800, 0xd3d3d300d3d3d300, | 798 | 0x4040400040404000ULL, 0x2828280028282800ULL, 0xd3d3d300d3d3d300ULL, |
| 799 | 0x7b7b7b007b7b7b00, 0xbbbbbb00bbbbbb00, 0xc9c9c900c9c9c900, | 799 | 0x7b7b7b007b7b7b00ULL, 0xbbbbbb00bbbbbb00ULL, 0xc9c9c900c9c9c900ULL, |
| 800 | 0x4343430043434300, 0xc1c1c100c1c1c100, 0x1515150015151500, | 800 | 0x4343430043434300ULL, 0xc1c1c100c1c1c100ULL, 0x1515150015151500ULL, |
| 801 | 0xe3e3e300e3e3e300, 0xadadad00adadad00, 0xf4f4f400f4f4f400, | 801 | 0xe3e3e300e3e3e300ULL, 0xadadad00adadad00ULL, 0xf4f4f400f4f4f400ULL, |
| 802 | 0x7777770077777700, 0xc7c7c700c7c7c700, 0x8080800080808000, | 802 | 0x7777770077777700ULL, 0xc7c7c700c7c7c700ULL, 0x8080800080808000ULL, |
| 803 | 0x9e9e9e009e9e9e00, | 803 | 0x9e9e9e009e9e9e00ULL, |
| 804 | }; | 804 | }; |
| 805 | 805 | ||
| 806 | /* key constants */ | 806 | /* key constants */ |
| @@ -1601,7 +1601,6 @@ static struct crypto_alg camellia_algs[6] = { { | |||
| 1601 | .cra_ctxsize = sizeof(struct camellia_ctx), | 1601 | .cra_ctxsize = sizeof(struct camellia_ctx), |
| 1602 | .cra_alignmask = 0, | 1602 | .cra_alignmask = 0, |
| 1603 | .cra_module = THIS_MODULE, | 1603 | .cra_module = THIS_MODULE, |
| 1604 | .cra_list = LIST_HEAD_INIT(camellia_algs[0].cra_list), | ||
| 1605 | .cra_u = { | 1604 | .cra_u = { |
| 1606 | .cipher = { | 1605 | .cipher = { |
| 1607 | .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE, | 1606 | .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE, |
| @@ -1621,7 +1620,6 @@ static struct crypto_alg camellia_algs[6] = { { | |||
| 1621 | .cra_alignmask = 0, | 1620 | .cra_alignmask = 0, |
| 1622 | .cra_type = &crypto_blkcipher_type, | 1621 | .cra_type = &crypto_blkcipher_type, |
| 1623 | .cra_module = THIS_MODULE, | 1622 | .cra_module = THIS_MODULE, |
| 1624 | .cra_list = LIST_HEAD_INIT(camellia_algs[1].cra_list), | ||
| 1625 | .cra_u = { | 1623 | .cra_u = { |
| 1626 | .blkcipher = { | 1624 | .blkcipher = { |
| 1627 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, | 1625 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, |
| @@ -1641,7 +1639,6 @@ static struct crypto_alg camellia_algs[6] = { { | |||
| 1641 | .cra_alignmask = 0, | 1639 | .cra_alignmask = 0, |
| 1642 | .cra_type = &crypto_blkcipher_type, | 1640 | .cra_type = &crypto_blkcipher_type, |
| 1643 | .cra_module = THIS_MODULE, | 1641 | .cra_module = THIS_MODULE, |
| 1644 | .cra_list = LIST_HEAD_INIT(camellia_algs[2].cra_list), | ||
| 1645 | .cra_u = { | 1642 | .cra_u = { |
| 1646 | .blkcipher = { | 1643 | .blkcipher = { |
| 1647 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, | 1644 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, |
| @@ -1662,7 +1659,6 @@ static struct crypto_alg camellia_algs[6] = { { | |||
| 1662 | .cra_alignmask = 0, | 1659 | .cra_alignmask = 0, |
| 1663 | .cra_type = &crypto_blkcipher_type, | 1660 | .cra_type = &crypto_blkcipher_type, |
| 1664 | .cra_module = THIS_MODULE, | 1661 | .cra_module = THIS_MODULE, |
| 1665 | .cra_list = LIST_HEAD_INIT(camellia_algs[3].cra_list), | ||
| 1666 | .cra_u = { | 1662 | .cra_u = { |
| 1667 | .blkcipher = { | 1663 | .blkcipher = { |
| 1668 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, | 1664 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, |
| @@ -1683,7 +1679,6 @@ static struct crypto_alg camellia_algs[6] = { { | |||
| 1683 | .cra_alignmask = 0, | 1679 | .cra_alignmask = 0, |
| 1684 | .cra_type = &crypto_blkcipher_type, | 1680 | .cra_type = &crypto_blkcipher_type, |
| 1685 | .cra_module = THIS_MODULE, | 1681 | .cra_module = THIS_MODULE, |
| 1686 | .cra_list = LIST_HEAD_INIT(camellia_algs[4].cra_list), | ||
| 1687 | .cra_exit = lrw_exit_tfm, | 1682 | .cra_exit = lrw_exit_tfm, |
| 1688 | .cra_u = { | 1683 | .cra_u = { |
| 1689 | .blkcipher = { | 1684 | .blkcipher = { |
| @@ -1707,7 +1702,6 @@ static struct crypto_alg camellia_algs[6] = { { | |||
| 1707 | .cra_alignmask = 0, | 1702 | .cra_alignmask = 0, |
| 1708 | .cra_type = &crypto_blkcipher_type, | 1703 | .cra_type = &crypto_blkcipher_type, |
| 1709 | .cra_module = THIS_MODULE, | 1704 | .cra_module = THIS_MODULE, |
| 1710 | .cra_list = LIST_HEAD_INIT(camellia_algs[5].cra_list), | ||
| 1711 | .cra_u = { | 1705 | .cra_u = { |
| 1712 | .blkcipher = { | 1706 | .blkcipher = { |
| 1713 | .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, | 1707 | .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, |
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S new file mode 100644 index 000000000000..a41a3aaba220 --- /dev/null +++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S | |||
| @@ -0,0 +1,376 @@ | |||
| 1 | /* | ||
| 2 | * Cast5 Cipher 16-way parallel algorithm (AVX/x86_64) | ||
| 3 | * | ||
| 4 | * Copyright (C) 2012 Johannes Goetzfried | ||
| 5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | ||
| 6 | * | ||
| 7 | * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> | ||
| 8 | * | ||
| 9 | * This program is free software; you can redistribute it and/or modify | ||
| 10 | * it under the terms of the GNU General Public License as published by | ||
| 11 | * the Free Software Foundation; either version 2 of the License, or | ||
| 12 | * (at your option) any later version. | ||
| 13 | * | ||
| 14 | * This program is distributed in the hope that it will be useful, | ||
| 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 17 | * GNU General Public License for more details. | ||
| 18 | * | ||
| 19 | * You should have received a copy of the GNU General Public License | ||
| 20 | * along with this program; if not, write to the Free Software | ||
| 21 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | ||
| 22 | * USA | ||
| 23 | * | ||
| 24 | */ | ||
| 25 | |||
| 26 | .file "cast5-avx-x86_64-asm_64.S" | ||
| 27 | |||
| 28 | .extern cast5_s1 | ||
| 29 | .extern cast5_s2 | ||
| 30 | .extern cast5_s3 | ||
| 31 | .extern cast5_s4 | ||
| 32 | |||
| 33 | /* structure of crypto context */ | ||
| 34 | #define km 0 | ||
| 35 | #define kr (16*4) | ||
| 36 | #define rr ((16*4)+16) | ||
| 37 | |||
| 38 | /* s-boxes */ | ||
| 39 | #define s1 cast5_s1 | ||
| 40 | #define s2 cast5_s2 | ||
| 41 | #define s3 cast5_s3 | ||
| 42 | #define s4 cast5_s4 | ||
| 43 | |||
| 44 | /********************************************************************** | ||
| 45 | 16-way AVX cast5 | ||
| 46 | **********************************************************************/ | ||
| 47 | #define CTX %rdi | ||
| 48 | |||
| 49 | #define RL1 %xmm0 | ||
| 50 | #define RR1 %xmm1 | ||
| 51 | #define RL2 %xmm2 | ||
| 52 | #define RR2 %xmm3 | ||
| 53 | #define RL3 %xmm4 | ||
| 54 | #define RR3 %xmm5 | ||
| 55 | #define RL4 %xmm6 | ||
| 56 | #define RR4 %xmm7 | ||
| 57 | |||
| 58 | #define RX %xmm8 | ||
| 59 | |||
| 60 | #define RKM %xmm9 | ||
| 61 | #define RKR %xmm10 | ||
| 62 | #define RKRF %xmm11 | ||
| 63 | #define RKRR %xmm12 | ||
| 64 | |||
| 65 | #define R32 %xmm13 | ||
| 66 | #define R1ST %xmm14 | ||
| 67 | |||
| 68 | #define RTMP %xmm15 | ||
| 69 | |||
| 70 | #define RID1 %rbp | ||
| 71 | #define RID1d %ebp | ||
| 72 | #define RID2 %rsi | ||
| 73 | #define RID2d %esi | ||
| 74 | |||
| 75 | #define RGI1 %rdx | ||
| 76 | #define RGI1bl %dl | ||
| 77 | #define RGI1bh %dh | ||
| 78 | #define RGI2 %rcx | ||
| 79 | #define RGI2bl %cl | ||
| 80 | #define RGI2bh %ch | ||
| 81 | |||
| 82 | #define RGI3 %rax | ||
| 83 | #define RGI3bl %al | ||
| 84 | #define RGI3bh %ah | ||
| 85 | #define RGI4 %rbx | ||
| 86 | #define RGI4bl %bl | ||
| 87 | #define RGI4bh %bh | ||
| 88 | |||
| 89 | #define RFS1 %r8 | ||
| 90 | #define RFS1d %r8d | ||
| 91 | #define RFS2 %r9 | ||
| 92 | #define RFS2d %r9d | ||
| 93 | #define RFS3 %r10 | ||
| 94 | #define RFS3d %r10d | ||
| 95 | |||
| 96 | |||
| 97 | #define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \ | ||
| 98 | movzbl src ## bh, RID1d; \ | ||
| 99 | movzbl src ## bl, RID2d; \ | ||
| 100 | shrq $16, src; \ | ||
| 101 | movl s1(, RID1, 4), dst ## d; \ | ||
| 102 | op1 s2(, RID2, 4), dst ## d; \ | ||
| 103 | movzbl src ## bh, RID1d; \ | ||
| 104 | movzbl src ## bl, RID2d; \ | ||
| 105 | interleave_op(il_reg); \ | ||
| 106 | op2 s3(, RID1, 4), dst ## d; \ | ||
| 107 | op3 s4(, RID2, 4), dst ## d; | ||
| 108 | |||
| 109 | #define dummy(d) /* do nothing */ | ||
| 110 | |||
| 111 | #define shr_next(reg) \ | ||
| 112 | shrq $16, reg; | ||
| 113 | |||
| 114 | #define F_head(a, x, gi1, gi2, op0) \ | ||
| 115 | op0 a, RKM, x; \ | ||
| 116 | vpslld RKRF, x, RTMP; \ | ||
| 117 | vpsrld RKRR, x, x; \ | ||
| 118 | vpor RTMP, x, x; \ | ||
| 119 | \ | ||
| 120 | vmovq x, gi1; \ | ||
| 121 | vpextrq $1, x, gi2; | ||
| 122 | |||
| 123 | #define F_tail(a, x, gi1, gi2, op1, op2, op3) \ | ||
| 124 | lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \ | ||
| 125 | lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \ | ||
| 126 | \ | ||
| 127 | lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none); \ | ||
| 128 | shlq $32, RFS2; \ | ||
| 129 | orq RFS1, RFS2; \ | ||
| 130 | lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none); \ | ||
| 131 | shlq $32, RFS1; \ | ||
| 132 | orq RFS1, RFS3; \ | ||
| 133 | \ | ||
| 134 | vmovq RFS2, x; \ | ||
| 135 | vpinsrq $1, RFS3, x, x; | ||
| 136 | |||
| 137 | #define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \ | ||
| 138 | F_head(b1, RX, RGI1, RGI2, op0); \ | ||
| 139 | F_head(b2, RX, RGI3, RGI4, op0); \ | ||
| 140 | \ | ||
| 141 | F_tail(b1, RX, RGI1, RGI2, op1, op2, op3); \ | ||
| 142 | F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3); \ | ||
| 143 | \ | ||
| 144 | vpxor a1, RX, a1; \ | ||
| 145 | vpxor a2, RTMP, a2; | ||
| 146 | |||
| 147 | #define F1_2(a1, b1, a2, b2) \ | ||
| 148 | F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl) | ||
| 149 | #define F2_2(a1, b1, a2, b2) \ | ||
| 150 | F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl) | ||
| 151 | #define F3_2(a1, b1, a2, b2) \ | ||
| 152 | F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl) | ||
| 153 | |||
| 154 | #define subround(a1, b1, a2, b2, f) \ | ||
| 155 | F ## f ## _2(a1, b1, a2, b2); | ||
| 156 | |||
| 157 | #define round(l, r, n, f) \ | ||
| 158 | vbroadcastss (km+(4*n))(CTX), RKM; \ | ||
| 159 | vpand R1ST, RKR, RKRF; \ | ||
| 160 | vpsubq RKRF, R32, RKRR; \ | ||
| 161 | vpsrldq $1, RKR, RKR; \ | ||
| 162 | subround(l ## 1, r ## 1, l ## 2, r ## 2, f); \ | ||
| 163 | subround(l ## 3, r ## 3, l ## 4, r ## 4, f); | ||
| 164 | |||
| 165 | #define enc_preload_rkr() \ | ||
| 166 | vbroadcastss .L16_mask, RKR; \ | ||
| 167 | /* add 16-bit rotation to key rotations (mod 32) */ \ | ||
| 168 | vpxor kr(CTX), RKR, RKR; | ||
| 169 | |||
| 170 | #define dec_preload_rkr() \ | ||
| 171 | vbroadcastss .L16_mask, RKR; \ | ||
| 172 | /* add 16-bit rotation to key rotations (mod 32) */ \ | ||
| 173 | vpxor kr(CTX), RKR, RKR; \ | ||
| 174 | vpshufb .Lbswap128_mask, RKR, RKR; | ||
| 175 | |||
| 176 | #define transpose_2x4(x0, x1, t0, t1) \ | ||
| 177 | vpunpckldq x1, x0, t0; \ | ||
| 178 | vpunpckhdq x1, x0, t1; \ | ||
| 179 | \ | ||
| 180 | vpunpcklqdq t1, t0, x0; \ | ||
| 181 | vpunpckhqdq t1, t0, x1; | ||
| 182 | |||
| 183 | #define inpack_blocks(in, x0, x1, t0, t1, rmask) \ | ||
| 184 | vmovdqu (0*4*4)(in), x0; \ | ||
| 185 | vmovdqu (1*4*4)(in), x1; \ | ||
| 186 | vpshufb rmask, x0, x0; \ | ||
| 187 | vpshufb rmask, x1, x1; \ | ||
| 188 | \ | ||
| 189 | transpose_2x4(x0, x1, t0, t1) | ||
| 190 | |||
| 191 | #define outunpack_blocks(out, x0, x1, t0, t1, rmask) \ | ||
| 192 | transpose_2x4(x0, x1, t0, t1) \ | ||
| 193 | \ | ||
| 194 | vpshufb rmask, x0, x0; \ | ||
| 195 | vpshufb rmask, x1, x1; \ | ||
| 196 | vmovdqu x0, (0*4*4)(out); \ | ||
| 197 | vmovdqu x1, (1*4*4)(out); | ||
| 198 | |||
| 199 | #define outunpack_xor_blocks(out, x0, x1, t0, t1, rmask) \ | ||
| 200 | transpose_2x4(x0, x1, t0, t1) \ | ||
| 201 | \ | ||
| 202 | vpshufb rmask, x0, x0; \ | ||
| 203 | vpshufb rmask, x1, x1; \ | ||
| 204 | vpxor (0*4*4)(out), x0, x0; \ | ||
| 205 | vmovdqu x0, (0*4*4)(out); \ | ||
| 206 | vpxor (1*4*4)(out), x1, x1; \ | ||
| 207 | vmovdqu x1, (1*4*4)(out); | ||
| 208 | |||
| 209 | .data | ||
| 210 | |||
| 211 | .align 16 | ||
| 212 | .Lbswap_mask: | ||
| 213 | .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 | ||
| 214 | .Lbswap128_mask: | ||
| 215 | .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 | ||
| 216 | .L16_mask: | ||
| 217 | .byte 16, 16, 16, 16 | ||
| 218 | .L32_mask: | ||
| 219 | .byte 32, 0, 0, 0 | ||
| 220 | .Lfirst_mask: | ||
| 221 | .byte 0x1f, 0, 0, 0 | ||
| 222 | |||
| 223 | .text | ||
| 224 | |||
| 225 | .align 16 | ||
| 226 | .global __cast5_enc_blk_16way | ||
| 227 | .type __cast5_enc_blk_16way,@function; | ||
| 228 | |||
| 229 | __cast5_enc_blk_16way: | ||
| 230 | /* input: | ||
| 231 | * %rdi: ctx, CTX | ||
| 232 | * %rsi: dst | ||
| 233 | * %rdx: src | ||
| 234 | * %rcx: bool, if true: xor output | ||
| 235 | */ | ||
| 236 | |||
| 237 | pushq %rbp; | ||
| 238 | pushq %rbx; | ||
| 239 | pushq %rcx; | ||
| 240 | |||
| 241 | vmovdqa .Lbswap_mask, RKM; | ||
| 242 | vmovd .Lfirst_mask, R1ST; | ||
| 243 | vmovd .L32_mask, R32; | ||
| 244 | enc_preload_rkr(); | ||
| 245 | |||
| 246 | leaq 1*(2*4*4)(%rdx), %rax; | ||
| 247 | inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM); | ||
| 248 | inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM); | ||
| 249 | leaq 2*(2*4*4)(%rdx), %rax; | ||
| 250 | inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM); | ||
| 251 | leaq 3*(2*4*4)(%rdx), %rax; | ||
| 252 | inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM); | ||
| 253 | |||
| 254 | movq %rsi, %r11; | ||
| 255 | |||
| 256 | round(RL, RR, 0, 1); | ||
| 257 | round(RR, RL, 1, 2); | ||
| 258 | round(RL, RR, 2, 3); | ||
| 259 | round(RR, RL, 3, 1); | ||
| 260 | round(RL, RR, 4, 2); | ||
| 261 | round(RR, RL, 5, 3); | ||
| 262 | round(RL, RR, 6, 1); | ||
| 263 | round(RR, RL, 7, 2); | ||
| 264 | round(RL, RR, 8, 3); | ||
| 265 | round(RR, RL, 9, 1); | ||
| 266 | round(RL, RR, 10, 2); | ||
| 267 | round(RR, RL, 11, 3); | ||
| 268 | |||
| 269 | movzbl rr(CTX), %eax; | ||
| 270 | testl %eax, %eax; | ||
| 271 | jnz __skip_enc; | ||
| 272 | |||
| 273 | round(RL, RR, 12, 1); | ||
| 274 | round(RR, RL, 13, 2); | ||
| 275 | round(RL, RR, 14, 3); | ||
| 276 | round(RR, RL, 15, 1); | ||
| 277 | |||
| 278 | __skip_enc: | ||
| 279 | popq %rcx; | ||
| 280 | popq %rbx; | ||
| 281 | popq %rbp; | ||
| 282 | |||
| 283 | vmovdqa .Lbswap_mask, RKM; | ||
| 284 | leaq 1*(2*4*4)(%r11), %rax; | ||
| 285 | |||
| 286 | testb %cl, %cl; | ||
| 287 | jnz __enc_xor16; | ||
| 288 | |||
| 289 | outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM); | ||
| 290 | outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM); | ||
| 291 | leaq 2*(2*4*4)(%r11), %rax; | ||
| 292 | outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM); | ||
| 293 | leaq 3*(2*4*4)(%r11), %rax; | ||
| 294 | outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM); | ||
| 295 | |||
| 296 | ret; | ||
| 297 | |||
| 298 | __enc_xor16: | ||
| 299 | outunpack_xor_blocks(%r11, RR1, RL1, RTMP, RX, RKM); | ||
| 300 | outunpack_xor_blocks(%rax, RR2, RL2, RTMP, RX, RKM); | ||
| 301 | leaq 2*(2*4*4)(%r11), %rax; | ||
| 302 | outunpack_xor_blocks(%rax, RR3, RL3, RTMP, RX, RKM); | ||
| 303 | leaq 3*(2*4*4)(%r11), %rax; | ||
| 304 | outunpack_xor_blocks(%rax, RR4, RL4, RTMP, RX, RKM); | ||
| 305 | |||
| 306 | ret; | ||
| 307 | |||
| 308 | .align 16 | ||
| 309 | .global cast5_dec_blk_16way | ||
| 310 | .type cast5_dec_blk_16way,@function; | ||
| 311 | |||
| 312 | cast5_dec_blk_16way: | ||
| 313 | /* input: | ||
| 314 | * %rdi: ctx, CTX | ||
| 315 | * %rsi: dst | ||
| 316 | * %rdx: src | ||
| 317 | */ | ||
| 318 | |||
| 319 | pushq %rbp; | ||
| 320 | pushq %rbx; | ||
| 321 | |||
| 322 | vmovdqa .Lbswap_mask, RKM; | ||
| 323 | vmovd .Lfirst_mask, R1ST; | ||
| 324 | vmovd .L32_mask, R32; | ||
| 325 | dec_preload_rkr(); | ||
| 326 | |||
| 327 | leaq 1*(2*4*4)(%rdx), %rax; | ||
| 328 | inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM); | ||
| 329 | inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM); | ||
| 330 | leaq 2*(2*4*4)(%rdx), %rax; | ||
| 331 | inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM); | ||
| 332 | leaq 3*(2*4*4)(%rdx), %rax; | ||
| 333 | inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM); | ||
| 334 | |||
| 335 | movq %rsi, %r11; | ||
| 336 | |||
| 337 | movzbl rr(CTX), %eax; | ||
| 338 | testl %eax, %eax; | ||
| 339 | jnz __skip_dec; | ||
| 340 | |||
| 341 | round(RL, RR, 15, 1); | ||
| 342 | round(RR, RL, 14, 3); | ||
| 343 | round(RL, RR, 13, 2); | ||
| 344 | round(RR, RL, 12, 1); | ||
| 345 | |||
| 346 | __dec_tail: | ||
| 347 | round(RL, RR, 11, 3); | ||
| 348 | round(RR, RL, 10, 2); | ||
| 349 | round(RL, RR, 9, 1); | ||
| 350 | round(RR, RL, 8, 3); | ||
| 351 | round(RL, RR, 7, 2); | ||
| 352 | round(RR, RL, 6, 1); | ||
| 353 | round(RL, RR, 5, 3); | ||
| 354 | round(RR, RL, 4, 2); | ||
| 355 | round(RL, RR, 3, 1); | ||
| 356 | round(RR, RL, 2, 3); | ||
| 357 | round(RL, RR, 1, 2); | ||
| 358 | round(RR, RL, 0, 1); | ||
| 359 | |||
| 360 | vmovdqa .Lbswap_mask, RKM; | ||
| 361 | popq %rbx; | ||
| 362 | popq %rbp; | ||
| 363 | |||
| 364 | leaq 1*(2*4*4)(%r11), %rax; | ||
| 365 | outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM); | ||
| 366 | outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM); | ||
| 367 | leaq 2*(2*4*4)(%r11), %rax; | ||
| 368 | outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM); | ||
| 369 | leaq 3*(2*4*4)(%r11), %rax; | ||
| 370 | outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM); | ||
| 371 | |||
| 372 | ret; | ||
| 373 | |||
| 374 | __skip_dec: | ||
| 375 | vpsrldq $4, RKR, RKR; | ||
| 376 | jmp __dec_tail; | ||
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c new file mode 100644 index 000000000000..e0ea14f9547f --- /dev/null +++ b/arch/x86/crypto/cast5_avx_glue.c | |||
| @@ -0,0 +1,530 @@ | |||
| 1 | /* | ||
| 2 | * Glue Code for the AVX assembler implemention of the Cast5 Cipher | ||
| 3 | * | ||
| 4 | * Copyright (C) 2012 Johannes Goetzfried | ||
| 5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | ||
| 6 | * | ||
| 7 | * This program is free software; you can redistribute it and/or modify | ||
| 8 | * it under the terms of the GNU General Public License as published by | ||
| 9 | * the Free Software Foundation; either version 2 of the License, or | ||
| 10 | * (at your option) any later version. | ||
| 11 | * | ||
| 12 | * This program is distributed in the hope that it will be useful, | ||
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 15 | * GNU General Public License for more details. | ||
| 16 | * | ||
| 17 | * You should have received a copy of the GNU General Public License | ||
| 18 | * along with this program; if not, write to the Free Software | ||
| 19 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | ||
| 20 | * USA | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/module.h> | ||
| 25 | #include <linux/hardirq.h> | ||
| 26 | #include <linux/types.h> | ||
| 27 | #include <linux/crypto.h> | ||
| 28 | #include <linux/err.h> | ||
| 29 | #include <crypto/algapi.h> | ||
| 30 | #include <crypto/cast5.h> | ||
| 31 | #include <crypto/cryptd.h> | ||
| 32 | #include <crypto/ctr.h> | ||
| 33 | #include <asm/xcr.h> | ||
| 34 | #include <asm/xsave.h> | ||
| 35 | #include <asm/crypto/ablk_helper.h> | ||
| 36 | #include <asm/crypto/glue_helper.h> | ||
| 37 | |||
| 38 | #define CAST5_PARALLEL_BLOCKS 16 | ||
| 39 | |||
| 40 | asmlinkage void __cast5_enc_blk_16way(struct cast5_ctx *ctx, u8 *dst, | ||
| 41 | const u8 *src, bool xor); | ||
| 42 | asmlinkage void cast5_dec_blk_16way(struct cast5_ctx *ctx, u8 *dst, | ||
| 43 | const u8 *src); | ||
| 44 | |||
| 45 | static inline void cast5_enc_blk_xway(struct cast5_ctx *ctx, u8 *dst, | ||
| 46 | const u8 *src) | ||
| 47 | { | ||
| 48 | __cast5_enc_blk_16way(ctx, dst, src, false); | ||
| 49 | } | ||
| 50 | |||
| 51 | static inline void cast5_enc_blk_xway_xor(struct cast5_ctx *ctx, u8 *dst, | ||
| 52 | const u8 *src) | ||
| 53 | { | ||
| 54 | __cast5_enc_blk_16way(ctx, dst, src, true); | ||
| 55 | } | ||
| 56 | |||
| 57 | static inline void cast5_dec_blk_xway(struct cast5_ctx *ctx, u8 *dst, | ||
| 58 | const u8 *src) | ||
| 59 | { | ||
| 60 | cast5_dec_blk_16way(ctx, dst, src); | ||
| 61 | } | ||
| 62 | |||
| 63 | |||
| 64 | static inline bool cast5_fpu_begin(bool fpu_enabled, unsigned int nbytes) | ||
| 65 | { | ||
| 66 | return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS, | ||
| 67 | NULL, fpu_enabled, nbytes); | ||
| 68 | } | ||
| 69 | |||
| 70 | static inline void cast5_fpu_end(bool fpu_enabled) | ||
| 71 | { | ||
| 72 | return glue_fpu_end(fpu_enabled); | ||
| 73 | } | ||
| 74 | |||
| 75 | static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, | ||
| 76 | bool enc) | ||
| 77 | { | ||
| 78 | bool fpu_enabled = false; | ||
| 79 | struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 80 | const unsigned int bsize = CAST5_BLOCK_SIZE; | ||
| 81 | unsigned int nbytes; | ||
| 82 | int err; | ||
| 83 | |||
| 84 | err = blkcipher_walk_virt(desc, walk); | ||
| 85 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 86 | |||
| 87 | while ((nbytes = walk->nbytes)) { | ||
| 88 | u8 *wsrc = walk->src.virt.addr; | ||
| 89 | u8 *wdst = walk->dst.virt.addr; | ||
| 90 | |||
| 91 | fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes); | ||
| 92 | |||
| 93 | /* Process multi-block batch */ | ||
| 94 | if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { | ||
| 95 | do { | ||
| 96 | if (enc) | ||
| 97 | cast5_enc_blk_xway(ctx, wdst, wsrc); | ||
| 98 | else | ||
| 99 | cast5_dec_blk_xway(ctx, wdst, wsrc); | ||
| 100 | |||
| 101 | wsrc += bsize * CAST5_PARALLEL_BLOCKS; | ||
| 102 | wdst += bsize * CAST5_PARALLEL_BLOCKS; | ||
| 103 | nbytes -= bsize * CAST5_PARALLEL_BLOCKS; | ||
| 104 | } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); | ||
| 105 | |||
| 106 | if (nbytes < bsize) | ||
| 107 | goto done; | ||
| 108 | } | ||
| 109 | |||
| 110 | /* Handle leftovers */ | ||
| 111 | do { | ||
| 112 | if (enc) | ||
| 113 | __cast5_encrypt(ctx, wdst, wsrc); | ||
| 114 | else | ||
| 115 | __cast5_decrypt(ctx, wdst, wsrc); | ||
| 116 | |||
| 117 | wsrc += bsize; | ||
| 118 | wdst += bsize; | ||
| 119 | nbytes -= bsize; | ||
| 120 | } while (nbytes >= bsize); | ||
| 121 | |||
| 122 | done: | ||
| 123 | err = blkcipher_walk_done(desc, walk, nbytes); | ||
| 124 | } | ||
| 125 | |||
| 126 | cast5_fpu_end(fpu_enabled); | ||
| 127 | return err; | ||
| 128 | } | ||
| 129 | |||
| 130 | static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 131 | struct scatterlist *src, unsigned int nbytes) | ||
| 132 | { | ||
| 133 | struct blkcipher_walk walk; | ||
| 134 | |||
| 135 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
| 136 | return ecb_crypt(desc, &walk, true); | ||
| 137 | } | ||
| 138 | |||
| 139 | static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 140 | struct scatterlist *src, unsigned int nbytes) | ||
| 141 | { | ||
| 142 | struct blkcipher_walk walk; | ||
| 143 | |||
| 144 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
| 145 | return ecb_crypt(desc, &walk, false); | ||
| 146 | } | ||
| 147 | |||
| 148 | static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, | ||
| 149 | struct blkcipher_walk *walk) | ||
| 150 | { | ||
| 151 | struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 152 | const unsigned int bsize = CAST5_BLOCK_SIZE; | ||
| 153 | unsigned int nbytes = walk->nbytes; | ||
| 154 | u64 *src = (u64 *)walk->src.virt.addr; | ||
| 155 | u64 *dst = (u64 *)walk->dst.virt.addr; | ||
| 156 | u64 *iv = (u64 *)walk->iv; | ||
| 157 | |||
| 158 | do { | ||
| 159 | *dst = *src ^ *iv; | ||
| 160 | __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst); | ||
| 161 | iv = dst; | ||
| 162 | |||
| 163 | src += 1; | ||
| 164 | dst += 1; | ||
| 165 | nbytes -= bsize; | ||
| 166 | } while (nbytes >= bsize); | ||
| 167 | |||
| 168 | *(u64 *)walk->iv = *iv; | ||
| 169 | return nbytes; | ||
| 170 | } | ||
| 171 | |||
| 172 | static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 173 | struct scatterlist *src, unsigned int nbytes) | ||
| 174 | { | ||
| 175 | struct blkcipher_walk walk; | ||
| 176 | int err; | ||
| 177 | |||
| 178 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
| 179 | err = blkcipher_walk_virt(desc, &walk); | ||
| 180 | |||
| 181 | while ((nbytes = walk.nbytes)) { | ||
| 182 | nbytes = __cbc_encrypt(desc, &walk); | ||
| 183 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
| 184 | } | ||
| 185 | |||
| 186 | return err; | ||
| 187 | } | ||
| 188 | |||
| 189 | static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, | ||
| 190 | struct blkcipher_walk *walk) | ||
| 191 | { | ||
| 192 | struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 193 | const unsigned int bsize = CAST5_BLOCK_SIZE; | ||
| 194 | unsigned int nbytes = walk->nbytes; | ||
| 195 | u64 *src = (u64 *)walk->src.virt.addr; | ||
| 196 | u64 *dst = (u64 *)walk->dst.virt.addr; | ||
| 197 | u64 ivs[CAST5_PARALLEL_BLOCKS - 1]; | ||
| 198 | u64 last_iv; | ||
| 199 | int i; | ||
| 200 | |||
| 201 | /* Start of the last block. */ | ||
| 202 | src += nbytes / bsize - 1; | ||
| 203 | dst += nbytes / bsize - 1; | ||
| 204 | |||
| 205 | last_iv = *src; | ||
| 206 | |||
| 207 | /* Process multi-block batch */ | ||
| 208 | if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { | ||
| 209 | do { | ||
| 210 | nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1); | ||
| 211 | src -= CAST5_PARALLEL_BLOCKS - 1; | ||
| 212 | dst -= CAST5_PARALLEL_BLOCKS - 1; | ||
| 213 | |||
| 214 | for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++) | ||
| 215 | ivs[i] = src[i]; | ||
| 216 | |||
| 217 | cast5_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); | ||
| 218 | |||
| 219 | for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++) | ||
| 220 | *(dst + (i + 1)) ^= *(ivs + i); | ||
| 221 | |||
| 222 | nbytes -= bsize; | ||
| 223 | if (nbytes < bsize) | ||
| 224 | goto done; | ||
| 225 | |||
| 226 | *dst ^= *(src - 1); | ||
| 227 | src -= 1; | ||
| 228 | dst -= 1; | ||
| 229 | } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); | ||
| 230 | |||
| 231 | if (nbytes < bsize) | ||
| 232 | goto done; | ||
| 233 | } | ||
| 234 | |||
| 235 | /* Handle leftovers */ | ||
| 236 | for (;;) { | ||
| 237 | __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src); | ||
| 238 | |||
| 239 | nbytes -= bsize; | ||
| 240 | if (nbytes < bsize) | ||
| 241 | break; | ||
| 242 | |||
| 243 | *dst ^= *(src - 1); | ||
| 244 | src -= 1; | ||
| 245 | dst -= 1; | ||
| 246 | } | ||
| 247 | |||
| 248 | done: | ||
| 249 | *dst ^= *(u64 *)walk->iv; | ||
| 250 | *(u64 *)walk->iv = last_iv; | ||
| 251 | |||
| 252 | return nbytes; | ||
| 253 | } | ||
| 254 | |||
| 255 | static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 256 | struct scatterlist *src, unsigned int nbytes) | ||
| 257 | { | ||
| 258 | bool fpu_enabled = false; | ||
| 259 | struct blkcipher_walk walk; | ||
| 260 | int err; | ||
| 261 | |||
| 262 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
| 263 | err = blkcipher_walk_virt(desc, &walk); | ||
| 264 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 265 | |||
| 266 | while ((nbytes = walk.nbytes)) { | ||
| 267 | fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes); | ||
| 268 | nbytes = __cbc_decrypt(desc, &walk); | ||
| 269 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
| 270 | } | ||
| 271 | |||
| 272 | cast5_fpu_end(fpu_enabled); | ||
| 273 | return err; | ||
| 274 | } | ||
| 275 | |||
| 276 | static void ctr_crypt_final(struct blkcipher_desc *desc, | ||
| 277 | struct blkcipher_walk *walk) | ||
| 278 | { | ||
| 279 | struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 280 | u8 *ctrblk = walk->iv; | ||
| 281 | u8 keystream[CAST5_BLOCK_SIZE]; | ||
| 282 | u8 *src = walk->src.virt.addr; | ||
| 283 | u8 *dst = walk->dst.virt.addr; | ||
| 284 | unsigned int nbytes = walk->nbytes; | ||
| 285 | |||
| 286 | __cast5_encrypt(ctx, keystream, ctrblk); | ||
| 287 | crypto_xor(keystream, src, nbytes); | ||
| 288 | memcpy(dst, keystream, nbytes); | ||
| 289 | |||
| 290 | crypto_inc(ctrblk, CAST5_BLOCK_SIZE); | ||
| 291 | } | ||
| 292 | |||
| 293 | static unsigned int __ctr_crypt(struct blkcipher_desc *desc, | ||
| 294 | struct blkcipher_walk *walk) | ||
| 295 | { | ||
| 296 | struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 297 | const unsigned int bsize = CAST5_BLOCK_SIZE; | ||
| 298 | unsigned int nbytes = walk->nbytes; | ||
| 299 | u64 *src = (u64 *)walk->src.virt.addr; | ||
| 300 | u64 *dst = (u64 *)walk->dst.virt.addr; | ||
| 301 | u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); | ||
| 302 | __be64 ctrblocks[CAST5_PARALLEL_BLOCKS]; | ||
| 303 | int i; | ||
| 304 | |||
| 305 | /* Process multi-block batch */ | ||
| 306 | if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { | ||
| 307 | do { | ||
| 308 | /* create ctrblks for parallel encrypt */ | ||
| 309 | for (i = 0; i < CAST5_PARALLEL_BLOCKS; i++) { | ||
| 310 | if (dst != src) | ||
| 311 | dst[i] = src[i]; | ||
| 312 | |||
| 313 | ctrblocks[i] = cpu_to_be64(ctrblk++); | ||
| 314 | } | ||
| 315 | |||
| 316 | cast5_enc_blk_xway_xor(ctx, (u8 *)dst, | ||
| 317 | (u8 *)ctrblocks); | ||
| 318 | |||
| 319 | src += CAST5_PARALLEL_BLOCKS; | ||
| 320 | dst += CAST5_PARALLEL_BLOCKS; | ||
| 321 | nbytes -= bsize * CAST5_PARALLEL_BLOCKS; | ||
| 322 | } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); | ||
| 323 | |||
| 324 | if (nbytes < bsize) | ||
| 325 | goto done; | ||
| 326 | } | ||
| 327 | |||
| 328 | /* Handle leftovers */ | ||
| 329 | do { | ||
| 330 | if (dst != src) | ||
| 331 | *dst = *src; | ||
| 332 | |||
| 333 | ctrblocks[0] = cpu_to_be64(ctrblk++); | ||
| 334 | |||
| 335 | __cast5_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); | ||
| 336 | *dst ^= ctrblocks[0]; | ||
| 337 | |||
| 338 | src += 1; | ||
| 339 | dst += 1; | ||
| 340 | nbytes -= bsize; | ||
| 341 | } while (nbytes >= bsize); | ||
| 342 | |||
| 343 | done: | ||
| 344 | *(__be64 *)walk->iv = cpu_to_be64(ctrblk); | ||
| 345 | return nbytes; | ||
| 346 | } | ||
| 347 | |||
| 348 | static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 349 | struct scatterlist *src, unsigned int nbytes) | ||
| 350 | { | ||
| 351 | bool fpu_enabled = false; | ||
| 352 | struct blkcipher_walk walk; | ||
| 353 | int err; | ||
| 354 | |||
| 355 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
| 356 | err = blkcipher_walk_virt_block(desc, &walk, CAST5_BLOCK_SIZE); | ||
| 357 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 358 | |||
| 359 | while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) { | ||
| 360 | fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes); | ||
| 361 | nbytes = __ctr_crypt(desc, &walk); | ||
| 362 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
| 363 | } | ||
| 364 | |||
| 365 | cast5_fpu_end(fpu_enabled); | ||
| 366 | |||
| 367 | if (walk.nbytes) { | ||
| 368 | ctr_crypt_final(desc, &walk); | ||
| 369 | err = blkcipher_walk_done(desc, &walk, 0); | ||
| 370 | } | ||
| 371 | |||
| 372 | return err; | ||
| 373 | } | ||
| 374 | |||
| 375 | |||
| 376 | static struct crypto_alg cast5_algs[6] = { { | ||
| 377 | .cra_name = "__ecb-cast5-avx", | ||
| 378 | .cra_driver_name = "__driver-ecb-cast5-avx", | ||
| 379 | .cra_priority = 0, | ||
| 380 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 381 | .cra_blocksize = CAST5_BLOCK_SIZE, | ||
| 382 | .cra_ctxsize = sizeof(struct cast5_ctx), | ||
| 383 | .cra_alignmask = 0, | ||
| 384 | .cra_type = &crypto_blkcipher_type, | ||
| 385 | .cra_module = THIS_MODULE, | ||
| 386 | .cra_u = { | ||
| 387 | .blkcipher = { | ||
| 388 | .min_keysize = CAST5_MIN_KEY_SIZE, | ||
| 389 | .max_keysize = CAST5_MAX_KEY_SIZE, | ||
| 390 | .setkey = cast5_setkey, | ||
| 391 | .encrypt = ecb_encrypt, | ||
| 392 | .decrypt = ecb_decrypt, | ||
| 393 | }, | ||
| 394 | }, | ||
| 395 | }, { | ||
| 396 | .cra_name = "__cbc-cast5-avx", | ||
| 397 | .cra_driver_name = "__driver-cbc-cast5-avx", | ||
| 398 | .cra_priority = 0, | ||
| 399 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 400 | .cra_blocksize = CAST5_BLOCK_SIZE, | ||
| 401 | .cra_ctxsize = sizeof(struct cast5_ctx), | ||
| 402 | .cra_alignmask = 0, | ||
| 403 | .cra_type = &crypto_blkcipher_type, | ||
| 404 | .cra_module = THIS_MODULE, | ||
| 405 | .cra_u = { | ||
| 406 | .blkcipher = { | ||
| 407 | .min_keysize = CAST5_MIN_KEY_SIZE, | ||
| 408 | .max_keysize = CAST5_MAX_KEY_SIZE, | ||
| 409 | .setkey = cast5_setkey, | ||
| 410 | .encrypt = cbc_encrypt, | ||
| 411 | .decrypt = cbc_decrypt, | ||
| 412 | }, | ||
| 413 | }, | ||
| 414 | }, { | ||
| 415 | .cra_name = "__ctr-cast5-avx", | ||
| 416 | .cra_driver_name = "__driver-ctr-cast5-avx", | ||
| 417 | .cra_priority = 0, | ||
| 418 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 419 | .cra_blocksize = 1, | ||
| 420 | .cra_ctxsize = sizeof(struct cast5_ctx), | ||
| 421 | .cra_alignmask = 0, | ||
| 422 | .cra_type = &crypto_blkcipher_type, | ||
| 423 | .cra_module = THIS_MODULE, | ||
| 424 | .cra_u = { | ||
| 425 | .blkcipher = { | ||
| 426 | .min_keysize = CAST5_MIN_KEY_SIZE, | ||
| 427 | .max_keysize = CAST5_MAX_KEY_SIZE, | ||
| 428 | .ivsize = CAST5_BLOCK_SIZE, | ||
| 429 | .setkey = cast5_setkey, | ||
| 430 | .encrypt = ctr_crypt, | ||
| 431 | .decrypt = ctr_crypt, | ||
| 432 | }, | ||
| 433 | }, | ||
| 434 | }, { | ||
| 435 | .cra_name = "ecb(cast5)", | ||
| 436 | .cra_driver_name = "ecb-cast5-avx", | ||
| 437 | .cra_priority = 200, | ||
| 438 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 439 | .cra_blocksize = CAST5_BLOCK_SIZE, | ||
| 440 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 441 | .cra_alignmask = 0, | ||
| 442 | .cra_type = &crypto_ablkcipher_type, | ||
| 443 | .cra_module = THIS_MODULE, | ||
| 444 | .cra_init = ablk_init, | ||
| 445 | .cra_exit = ablk_exit, | ||
| 446 | .cra_u = { | ||
| 447 | .ablkcipher = { | ||
| 448 | .min_keysize = CAST5_MIN_KEY_SIZE, | ||
| 449 | .max_keysize = CAST5_MAX_KEY_SIZE, | ||
| 450 | .setkey = ablk_set_key, | ||
| 451 | .encrypt = ablk_encrypt, | ||
| 452 | .decrypt = ablk_decrypt, | ||
| 453 | }, | ||
| 454 | }, | ||
| 455 | }, { | ||
| 456 | .cra_name = "cbc(cast5)", | ||
| 457 | .cra_driver_name = "cbc-cast5-avx", | ||
| 458 | .cra_priority = 200, | ||
| 459 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 460 | .cra_blocksize = CAST5_BLOCK_SIZE, | ||
| 461 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 462 | .cra_alignmask = 0, | ||
| 463 | .cra_type = &crypto_ablkcipher_type, | ||
| 464 | .cra_module = THIS_MODULE, | ||
| 465 | .cra_init = ablk_init, | ||
| 466 | .cra_exit = ablk_exit, | ||
| 467 | .cra_u = { | ||
| 468 | .ablkcipher = { | ||
| 469 | .min_keysize = CAST5_MIN_KEY_SIZE, | ||
| 470 | .max_keysize = CAST5_MAX_KEY_SIZE, | ||
| 471 | .ivsize = CAST5_BLOCK_SIZE, | ||
| 472 | .setkey = ablk_set_key, | ||
| 473 | .encrypt = __ablk_encrypt, | ||
| 474 | .decrypt = ablk_decrypt, | ||
| 475 | }, | ||
| 476 | }, | ||
| 477 | }, { | ||
| 478 | .cra_name = "ctr(cast5)", | ||
| 479 | .cra_driver_name = "ctr-cast5-avx", | ||
| 480 | .cra_priority = 200, | ||
| 481 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 482 | .cra_blocksize = 1, | ||
| 483 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 484 | .cra_alignmask = 0, | ||
| 485 | .cra_type = &crypto_ablkcipher_type, | ||
| 486 | .cra_module = THIS_MODULE, | ||
| 487 | .cra_init = ablk_init, | ||
| 488 | .cra_exit = ablk_exit, | ||
| 489 | .cra_u = { | ||
| 490 | .ablkcipher = { | ||
| 491 | .min_keysize = CAST5_MIN_KEY_SIZE, | ||
| 492 | .max_keysize = CAST5_MAX_KEY_SIZE, | ||
| 493 | .ivsize = CAST5_BLOCK_SIZE, | ||
| 494 | .setkey = ablk_set_key, | ||
| 495 | .encrypt = ablk_encrypt, | ||
| 496 | .decrypt = ablk_encrypt, | ||
| 497 | .geniv = "chainiv", | ||
| 498 | }, | ||
| 499 | }, | ||
| 500 | } }; | ||
| 501 | |||
| 502 | static int __init cast5_init(void) | ||
| 503 | { | ||
| 504 | u64 xcr0; | ||
| 505 | |||
| 506 | if (!cpu_has_avx || !cpu_has_osxsave) { | ||
| 507 | pr_info("AVX instructions are not detected.\n"); | ||
| 508 | return -ENODEV; | ||
| 509 | } | ||
| 510 | |||
| 511 | xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); | ||
| 512 | if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { | ||
| 513 | pr_info("AVX detected but unusable.\n"); | ||
| 514 | return -ENODEV; | ||
| 515 | } | ||
| 516 | |||
| 517 | return crypto_register_algs(cast5_algs, ARRAY_SIZE(cast5_algs)); | ||
| 518 | } | ||
| 519 | |||
| 520 | static void __exit cast5_exit(void) | ||
| 521 | { | ||
| 522 | crypto_unregister_algs(cast5_algs, ARRAY_SIZE(cast5_algs)); | ||
| 523 | } | ||
| 524 | |||
| 525 | module_init(cast5_init); | ||
| 526 | module_exit(cast5_exit); | ||
| 527 | |||
| 528 | MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized"); | ||
| 529 | MODULE_LICENSE("GPL"); | ||
| 530 | MODULE_ALIAS("cast5"); | ||
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S new file mode 100644 index 000000000000..218d283772f4 --- /dev/null +++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S | |||
| @@ -0,0 +1,383 @@ | |||
| 1 | /* | ||
| 2 | * Cast6 Cipher 8-way parallel algorithm (AVX/x86_64) | ||
| 3 | * | ||
| 4 | * Copyright (C) 2012 Johannes Goetzfried | ||
| 5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | ||
| 6 | * | ||
| 7 | * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> | ||
| 8 | * | ||
| 9 | * This program is free software; you can redistribute it and/or modify | ||
| 10 | * it under the terms of the GNU General Public License as published by | ||
| 11 | * the Free Software Foundation; either version 2 of the License, or | ||
| 12 | * (at your option) any later version. | ||
| 13 | * | ||
| 14 | * This program is distributed in the hope that it will be useful, | ||
| 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 17 | * GNU General Public License for more details. | ||
| 18 | * | ||
| 19 | * You should have received a copy of the GNU General Public License | ||
| 20 | * along with this program; if not, write to the Free Software | ||
| 21 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | ||
| 22 | * USA | ||
| 23 | * | ||
| 24 | */ | ||
| 25 | |||
| 26 | .file "cast6-avx-x86_64-asm_64.S" | ||
| 27 | |||
| 28 | .extern cast6_s1 | ||
| 29 | .extern cast6_s2 | ||
| 30 | .extern cast6_s3 | ||
| 31 | .extern cast6_s4 | ||
| 32 | |||
| 33 | /* structure of crypto context */ | ||
| 34 | #define km 0 | ||
| 35 | #define kr (12*4*4) | ||
| 36 | |||
| 37 | /* s-boxes */ | ||
| 38 | #define s1 cast6_s1 | ||
| 39 | #define s2 cast6_s2 | ||
| 40 | #define s3 cast6_s3 | ||
| 41 | #define s4 cast6_s4 | ||
| 42 | |||
| 43 | /********************************************************************** | ||
| 44 | 8-way AVX cast6 | ||
| 45 | **********************************************************************/ | ||
| 46 | #define CTX %rdi | ||
| 47 | |||
| 48 | #define RA1 %xmm0 | ||
| 49 | #define RB1 %xmm1 | ||
| 50 | #define RC1 %xmm2 | ||
| 51 | #define RD1 %xmm3 | ||
| 52 | |||
| 53 | #define RA2 %xmm4 | ||
| 54 | #define RB2 %xmm5 | ||
| 55 | #define RC2 %xmm6 | ||
| 56 | #define RD2 %xmm7 | ||
| 57 | |||
| 58 | #define RX %xmm8 | ||
| 59 | |||
| 60 | #define RKM %xmm9 | ||
| 61 | #define RKR %xmm10 | ||
| 62 | #define RKRF %xmm11 | ||
| 63 | #define RKRR %xmm12 | ||
| 64 | #define R32 %xmm13 | ||
| 65 | #define R1ST %xmm14 | ||
| 66 | |||
| 67 | #define RTMP %xmm15 | ||
| 68 | |||
| 69 | #define RID1 %rbp | ||
| 70 | #define RID1d %ebp | ||
| 71 | #define RID2 %rsi | ||
| 72 | #define RID2d %esi | ||
| 73 | |||
| 74 | #define RGI1 %rdx | ||
| 75 | #define RGI1bl %dl | ||
| 76 | #define RGI1bh %dh | ||
| 77 | #define RGI2 %rcx | ||
| 78 | #define RGI2bl %cl | ||
| 79 | #define RGI2bh %ch | ||
| 80 | |||
| 81 | #define RGI3 %rax | ||
| 82 | #define RGI3bl %al | ||
| 83 | #define RGI3bh %ah | ||
| 84 | #define RGI4 %rbx | ||
| 85 | #define RGI4bl %bl | ||
| 86 | #define RGI4bh %bh | ||
| 87 | |||
| 88 | #define RFS1 %r8 | ||
| 89 | #define RFS1d %r8d | ||
| 90 | #define RFS2 %r9 | ||
| 91 | #define RFS2d %r9d | ||
| 92 | #define RFS3 %r10 | ||
| 93 | #define RFS3d %r10d | ||
| 94 | |||
| 95 | |||
| 96 | #define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \ | ||
| 97 | movzbl src ## bh, RID1d; \ | ||
| 98 | movzbl src ## bl, RID2d; \ | ||
| 99 | shrq $16, src; \ | ||
| 100 | movl s1(, RID1, 4), dst ## d; \ | ||
| 101 | op1 s2(, RID2, 4), dst ## d; \ | ||
| 102 | movzbl src ## bh, RID1d; \ | ||
| 103 | movzbl src ## bl, RID2d; \ | ||
| 104 | interleave_op(il_reg); \ | ||
| 105 | op2 s3(, RID1, 4), dst ## d; \ | ||
| 106 | op3 s4(, RID2, 4), dst ## d; | ||
| 107 | |||
| 108 | #define dummy(d) /* do nothing */ | ||
| 109 | |||
| 110 | #define shr_next(reg) \ | ||
| 111 | shrq $16, reg; | ||
| 112 | |||
| 113 | #define F_head(a, x, gi1, gi2, op0) \ | ||
| 114 | op0 a, RKM, x; \ | ||
| 115 | vpslld RKRF, x, RTMP; \ | ||
| 116 | vpsrld RKRR, x, x; \ | ||
| 117 | vpor RTMP, x, x; \ | ||
| 118 | \ | ||
| 119 | vmovq x, gi1; \ | ||
| 120 | vpextrq $1, x, gi2; | ||
| 121 | |||
| 122 | #define F_tail(a, x, gi1, gi2, op1, op2, op3) \ | ||
| 123 | lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \ | ||
| 124 | lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \ | ||
| 125 | \ | ||
| 126 | lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none); \ | ||
| 127 | shlq $32, RFS2; \ | ||
| 128 | orq RFS1, RFS2; \ | ||
| 129 | lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none); \ | ||
| 130 | shlq $32, RFS1; \ | ||
| 131 | orq RFS1, RFS3; \ | ||
| 132 | \ | ||
| 133 | vmovq RFS2, x; \ | ||
| 134 | vpinsrq $1, RFS3, x, x; | ||
| 135 | |||
| 136 | #define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \ | ||
| 137 | F_head(b1, RX, RGI1, RGI2, op0); \ | ||
| 138 | F_head(b2, RX, RGI3, RGI4, op0); \ | ||
| 139 | \ | ||
| 140 | F_tail(b1, RX, RGI1, RGI2, op1, op2, op3); \ | ||
| 141 | F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3); \ | ||
| 142 | \ | ||
| 143 | vpxor a1, RX, a1; \ | ||
| 144 | vpxor a2, RTMP, a2; | ||
| 145 | |||
| 146 | #define F1_2(a1, b1, a2, b2) \ | ||
| 147 | F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl) | ||
| 148 | #define F2_2(a1, b1, a2, b2) \ | ||
| 149 | F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl) | ||
| 150 | #define F3_2(a1, b1, a2, b2) \ | ||
| 151 | F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl) | ||
| 152 | |||
| 153 | #define qop(in, out, f) \ | ||
| 154 | F ## f ## _2(out ## 1, in ## 1, out ## 2, in ## 2); | ||
| 155 | |||
| 156 | #define get_round_keys(nn) \ | ||
| 157 | vbroadcastss (km+(4*(nn)))(CTX), RKM; \ | ||
| 158 | vpand R1ST, RKR, RKRF; \ | ||
| 159 | vpsubq RKRF, R32, RKRR; \ | ||
| 160 | vpsrldq $1, RKR, RKR; | ||
| 161 | |||
| 162 | #define Q(n) \ | ||
| 163 | get_round_keys(4*n+0); \ | ||
| 164 | qop(RD, RC, 1); \ | ||
| 165 | \ | ||
| 166 | get_round_keys(4*n+1); \ | ||
| 167 | qop(RC, RB, 2); \ | ||
| 168 | \ | ||
| 169 | get_round_keys(4*n+2); \ | ||
| 170 | qop(RB, RA, 3); \ | ||
| 171 | \ | ||
| 172 | get_round_keys(4*n+3); \ | ||
| 173 | qop(RA, RD, 1); | ||
| 174 | |||
| 175 | #define QBAR(n) \ | ||
| 176 | get_round_keys(4*n+3); \ | ||
| 177 | qop(RA, RD, 1); \ | ||
| 178 | \ | ||
| 179 | get_round_keys(4*n+2); \ | ||
| 180 | qop(RB, RA, 3); \ | ||
| 181 | \ | ||
| 182 | get_round_keys(4*n+1); \ | ||
| 183 | qop(RC, RB, 2); \ | ||
| 184 | \ | ||
| 185 | get_round_keys(4*n+0); \ | ||
| 186 | qop(RD, RC, 1); | ||
| 187 | |||
| 188 | #define shuffle(mask) \ | ||
| 189 | vpshufb mask, RKR, RKR; | ||
| 190 | |||
| 191 | #define preload_rkr(n, do_mask, mask) \ | ||
| 192 | vbroadcastss .L16_mask, RKR; \ | ||
| 193 | /* add 16-bit rotation to key rotations (mod 32) */ \ | ||
| 194 | vpxor (kr+n*16)(CTX), RKR, RKR; \ | ||
| 195 | do_mask(mask); | ||
| 196 | |||
| 197 | #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ | ||
| 198 | vpunpckldq x1, x0, t0; \ | ||
| 199 | vpunpckhdq x1, x0, t2; \ | ||
| 200 | vpunpckldq x3, x2, t1; \ | ||
| 201 | vpunpckhdq x3, x2, x3; \ | ||
| 202 | \ | ||
| 203 | vpunpcklqdq t1, t0, x0; \ | ||
| 204 | vpunpckhqdq t1, t0, x1; \ | ||
| 205 | vpunpcklqdq x3, t2, x2; \ | ||
| 206 | vpunpckhqdq x3, t2, x3; | ||
| 207 | |||
| 208 | #define inpack_blocks(in, x0, x1, x2, x3, t0, t1, t2, rmask) \ | ||
| 209 | vmovdqu (0*4*4)(in), x0; \ | ||
| 210 | vmovdqu (1*4*4)(in), x1; \ | ||
| 211 | vmovdqu (2*4*4)(in), x2; \ | ||
| 212 | vmovdqu (3*4*4)(in), x3; \ | ||
| 213 | vpshufb rmask, x0, x0; \ | ||
| 214 | vpshufb rmask, x1, x1; \ | ||
| 215 | vpshufb rmask, x2, x2; \ | ||
| 216 | vpshufb rmask, x3, x3; \ | ||
| 217 | \ | ||
| 218 | transpose_4x4(x0, x1, x2, x3, t0, t1, t2) | ||
| 219 | |||
| 220 | #define outunpack_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \ | ||
| 221 | transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ | ||
| 222 | \ | ||
| 223 | vpshufb rmask, x0, x0; \ | ||
| 224 | vpshufb rmask, x1, x1; \ | ||
| 225 | vpshufb rmask, x2, x2; \ | ||
| 226 | vpshufb rmask, x3, x3; \ | ||
| 227 | vmovdqu x0, (0*4*4)(out); \ | ||
| 228 | vmovdqu x1, (1*4*4)(out); \ | ||
| 229 | vmovdqu x2, (2*4*4)(out); \ | ||
| 230 | vmovdqu x3, (3*4*4)(out); | ||
| 231 | |||
| 232 | #define outunpack_xor_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \ | ||
| 233 | transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ | ||
| 234 | \ | ||
| 235 | vpshufb rmask, x0, x0; \ | ||
| 236 | vpshufb rmask, x1, x1; \ | ||
| 237 | vpshufb rmask, x2, x2; \ | ||
| 238 | vpshufb rmask, x3, x3; \ | ||
| 239 | vpxor (0*4*4)(out), x0, x0; \ | ||
| 240 | vmovdqu x0, (0*4*4)(out); \ | ||
| 241 | vpxor (1*4*4)(out), x1, x1; \ | ||
| 242 | vmovdqu x1, (1*4*4)(out); \ | ||
| 243 | vpxor (2*4*4)(out), x2, x2; \ | ||
| 244 | vmovdqu x2, (2*4*4)(out); \ | ||
| 245 | vpxor (3*4*4)(out), x3, x3; \ | ||
| 246 | vmovdqu x3, (3*4*4)(out); | ||
| 247 | |||
| 248 | .data | ||
| 249 | |||
| 250 | .align 16 | ||
| 251 | .Lbswap_mask: | ||
| 252 | .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 | ||
| 253 | .Lrkr_enc_Q_Q_QBAR_QBAR: | ||
| 254 | .byte 0, 1, 2, 3, 4, 5, 6, 7, 11, 10, 9, 8, 15, 14, 13, 12 | ||
| 255 | .Lrkr_enc_QBAR_QBAR_QBAR_QBAR: | ||
| 256 | .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 | ||
| 257 | .Lrkr_dec_Q_Q_Q_Q: | ||
| 258 | .byte 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3 | ||
| 259 | .Lrkr_dec_Q_Q_QBAR_QBAR: | ||
| 260 | .byte 12, 13, 14, 15, 8, 9, 10, 11, 7, 6, 5, 4, 3, 2, 1, 0 | ||
| 261 | .Lrkr_dec_QBAR_QBAR_QBAR_QBAR: | ||
| 262 | .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 | ||
| 263 | .L16_mask: | ||
| 264 | .byte 16, 16, 16, 16 | ||
| 265 | .L32_mask: | ||
| 266 | .byte 32, 0, 0, 0 | ||
| 267 | .Lfirst_mask: | ||
| 268 | .byte 0x1f, 0, 0, 0 | ||
| 269 | |||
| 270 | .text | ||
| 271 | |||
| 272 | .align 16 | ||
| 273 | .global __cast6_enc_blk_8way | ||
| 274 | .type __cast6_enc_blk_8way,@function; | ||
| 275 | |||
| 276 | __cast6_enc_blk_8way: | ||
| 277 | /* input: | ||
| 278 | * %rdi: ctx, CTX | ||
| 279 | * %rsi: dst | ||
| 280 | * %rdx: src | ||
| 281 | * %rcx: bool, if true: xor output | ||
| 282 | */ | ||
| 283 | |||
| 284 | pushq %rbp; | ||
| 285 | pushq %rbx; | ||
| 286 | pushq %rcx; | ||
| 287 | |||
| 288 | vmovdqa .Lbswap_mask, RKM; | ||
| 289 | vmovd .Lfirst_mask, R1ST; | ||
| 290 | vmovd .L32_mask, R32; | ||
| 291 | |||
| 292 | leaq (4*4*4)(%rdx), %rax; | ||
| 293 | inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); | ||
| 294 | inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); | ||
| 295 | |||
| 296 | movq %rsi, %r11; | ||
| 297 | |||
| 298 | preload_rkr(0, dummy, none); | ||
| 299 | Q(0); | ||
| 300 | Q(1); | ||
| 301 | Q(2); | ||
| 302 | Q(3); | ||
| 303 | preload_rkr(1, shuffle, .Lrkr_enc_Q_Q_QBAR_QBAR); | ||
| 304 | Q(4); | ||
| 305 | Q(5); | ||
| 306 | QBAR(6); | ||
| 307 | QBAR(7); | ||
| 308 | preload_rkr(2, shuffle, .Lrkr_enc_QBAR_QBAR_QBAR_QBAR); | ||
| 309 | QBAR(8); | ||
| 310 | QBAR(9); | ||
| 311 | QBAR(10); | ||
| 312 | QBAR(11); | ||
| 313 | |||
| 314 | popq %rcx; | ||
| 315 | popq %rbx; | ||
| 316 | popq %rbp; | ||
| 317 | |||
| 318 | vmovdqa .Lbswap_mask, RKM; | ||
| 319 | leaq (4*4*4)(%r11), %rax; | ||
| 320 | |||
| 321 | testb %cl, %cl; | ||
| 322 | jnz __enc_xor8; | ||
| 323 | |||
| 324 | outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); | ||
| 325 | outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); | ||
| 326 | |||
| 327 | ret; | ||
| 328 | |||
| 329 | __enc_xor8: | ||
| 330 | outunpack_xor_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); | ||
| 331 | outunpack_xor_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); | ||
| 332 | |||
| 333 | ret; | ||
| 334 | |||
| 335 | .align 16 | ||
| 336 | .global cast6_dec_blk_8way | ||
| 337 | .type cast6_dec_blk_8way,@function; | ||
| 338 | |||
| 339 | cast6_dec_blk_8way: | ||
| 340 | /* input: | ||
| 341 | * %rdi: ctx, CTX | ||
| 342 | * %rsi: dst | ||
| 343 | * %rdx: src | ||
| 344 | */ | ||
| 345 | |||
| 346 | pushq %rbp; | ||
| 347 | pushq %rbx; | ||
| 348 | |||
| 349 | vmovdqa .Lbswap_mask, RKM; | ||
| 350 | vmovd .Lfirst_mask, R1ST; | ||
| 351 | vmovd .L32_mask, R32; | ||
| 352 | |||
| 353 | leaq (4*4*4)(%rdx), %rax; | ||
| 354 | inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); | ||
| 355 | inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); | ||
| 356 | |||
| 357 | movq %rsi, %r11; | ||
| 358 | |||
| 359 | preload_rkr(2, shuffle, .Lrkr_dec_Q_Q_Q_Q); | ||
| 360 | Q(11); | ||
| 361 | Q(10); | ||
| 362 | Q(9); | ||
| 363 | Q(8); | ||
| 364 | preload_rkr(1, shuffle, .Lrkr_dec_Q_Q_QBAR_QBAR); | ||
| 365 | Q(7); | ||
| 366 | Q(6); | ||
| 367 | QBAR(5); | ||
| 368 | QBAR(4); | ||
| 369 | preload_rkr(0, shuffle, .Lrkr_dec_QBAR_QBAR_QBAR_QBAR); | ||
| 370 | QBAR(3); | ||
| 371 | QBAR(2); | ||
| 372 | QBAR(1); | ||
| 373 | QBAR(0); | ||
| 374 | |||
| 375 | popq %rbx; | ||
| 376 | popq %rbp; | ||
| 377 | |||
| 378 | vmovdqa .Lbswap_mask, RKM; | ||
| 379 | leaq (4*4*4)(%r11), %rax; | ||
| 380 | outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); | ||
| 381 | outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); | ||
| 382 | |||
| 383 | ret; | ||
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c new file mode 100644 index 000000000000..15e5f85a5011 --- /dev/null +++ b/arch/x86/crypto/cast6_avx_glue.c | |||
| @@ -0,0 +1,648 @@ | |||
| 1 | /* | ||
| 2 | * Glue Code for the AVX assembler implemention of the Cast6 Cipher | ||
| 3 | * | ||
| 4 | * Copyright (C) 2012 Johannes Goetzfried | ||
| 5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | ||
| 6 | * | ||
| 7 | * This program is free software; you can redistribute it and/or modify | ||
| 8 | * it under the terms of the GNU General Public License as published by | ||
| 9 | * the Free Software Foundation; either version 2 of the License, or | ||
| 10 | * (at your option) any later version. | ||
| 11 | * | ||
| 12 | * This program is distributed in the hope that it will be useful, | ||
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 15 | * GNU General Public License for more details. | ||
| 16 | * | ||
| 17 | * You should have received a copy of the GNU General Public License | ||
| 18 | * along with this program; if not, write to the Free Software | ||
| 19 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | ||
| 20 | * USA | ||
| 21 | * | ||
| 22 | */ | ||
| 23 | |||
| 24 | #include <linux/module.h> | ||
| 25 | #include <linux/hardirq.h> | ||
| 26 | #include <linux/types.h> | ||
| 27 | #include <linux/crypto.h> | ||
| 28 | #include <linux/err.h> | ||
| 29 | #include <crypto/algapi.h> | ||
| 30 | #include <crypto/cast6.h> | ||
| 31 | #include <crypto/cryptd.h> | ||
| 32 | #include <crypto/b128ops.h> | ||
| 33 | #include <crypto/ctr.h> | ||
| 34 | #include <crypto/lrw.h> | ||
| 35 | #include <crypto/xts.h> | ||
| 36 | #include <asm/xcr.h> | ||
| 37 | #include <asm/xsave.h> | ||
| 38 | #include <asm/crypto/ablk_helper.h> | ||
| 39 | #include <asm/crypto/glue_helper.h> | ||
| 40 | |||
| 41 | #define CAST6_PARALLEL_BLOCKS 8 | ||
| 42 | |||
| 43 | asmlinkage void __cast6_enc_blk_8way(struct cast6_ctx *ctx, u8 *dst, | ||
| 44 | const u8 *src, bool xor); | ||
| 45 | asmlinkage void cast6_dec_blk_8way(struct cast6_ctx *ctx, u8 *dst, | ||
| 46 | const u8 *src); | ||
| 47 | |||
| 48 | static inline void cast6_enc_blk_xway(struct cast6_ctx *ctx, u8 *dst, | ||
| 49 | const u8 *src) | ||
| 50 | { | ||
| 51 | __cast6_enc_blk_8way(ctx, dst, src, false); | ||
| 52 | } | ||
| 53 | |||
| 54 | static inline void cast6_enc_blk_xway_xor(struct cast6_ctx *ctx, u8 *dst, | ||
| 55 | const u8 *src) | ||
| 56 | { | ||
| 57 | __cast6_enc_blk_8way(ctx, dst, src, true); | ||
| 58 | } | ||
| 59 | |||
| 60 | static inline void cast6_dec_blk_xway(struct cast6_ctx *ctx, u8 *dst, | ||
| 61 | const u8 *src) | ||
| 62 | { | ||
| 63 | cast6_dec_blk_8way(ctx, dst, src); | ||
| 64 | } | ||
| 65 | |||
| 66 | |||
| 67 | static void cast6_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) | ||
| 68 | { | ||
| 69 | u128 ivs[CAST6_PARALLEL_BLOCKS - 1]; | ||
| 70 | unsigned int j; | ||
| 71 | |||
| 72 | for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++) | ||
| 73 | ivs[j] = src[j]; | ||
| 74 | |||
| 75 | cast6_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); | ||
| 76 | |||
| 77 | for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++) | ||
| 78 | u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); | ||
| 79 | } | ||
| 80 | |||
| 81 | static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) | ||
| 82 | { | ||
| 83 | be128 ctrblk; | ||
| 84 | |||
| 85 | u128_to_be128(&ctrblk, iv); | ||
| 86 | u128_inc(iv); | ||
| 87 | |||
| 88 | __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); | ||
| 89 | u128_xor(dst, src, (u128 *)&ctrblk); | ||
| 90 | } | ||
| 91 | |||
| 92 | static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, | ||
| 93 | u128 *iv) | ||
| 94 | { | ||
| 95 | be128 ctrblks[CAST6_PARALLEL_BLOCKS]; | ||
| 96 | unsigned int i; | ||
| 97 | |||
| 98 | for (i = 0; i < CAST6_PARALLEL_BLOCKS; i++) { | ||
| 99 | if (dst != src) | ||
| 100 | dst[i] = src[i]; | ||
| 101 | |||
| 102 | u128_to_be128(&ctrblks[i], iv); | ||
| 103 | u128_inc(iv); | ||
| 104 | } | ||
| 105 | |||
| 106 | cast6_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); | ||
| 107 | } | ||
| 108 | |||
| 109 | static const struct common_glue_ctx cast6_enc = { | ||
| 110 | .num_funcs = 2, | ||
| 111 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | ||
| 112 | |||
| 113 | .funcs = { { | ||
| 114 | .num_blocks = CAST6_PARALLEL_BLOCKS, | ||
| 115 | .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_enc_blk_xway) } | ||
| 116 | }, { | ||
| 117 | .num_blocks = 1, | ||
| 118 | .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) } | ||
| 119 | } } | ||
| 120 | }; | ||
| 121 | |||
| 122 | static const struct common_glue_ctx cast6_ctr = { | ||
| 123 | .num_funcs = 2, | ||
| 124 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | ||
| 125 | |||
| 126 | .funcs = { { | ||
| 127 | .num_blocks = CAST6_PARALLEL_BLOCKS, | ||
| 128 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr_xway) } | ||
| 129 | }, { | ||
| 130 | .num_blocks = 1, | ||
| 131 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) } | ||
| 132 | } } | ||
| 133 | }; | ||
| 134 | |||
| 135 | static const struct common_glue_ctx cast6_dec = { | ||
| 136 | .num_funcs = 2, | ||
| 137 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | ||
| 138 | |||
| 139 | .funcs = { { | ||
| 140 | .num_blocks = CAST6_PARALLEL_BLOCKS, | ||
| 141 | .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_dec_blk_xway) } | ||
| 142 | }, { | ||
| 143 | .num_blocks = 1, | ||
| 144 | .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) } | ||
| 145 | } } | ||
| 146 | }; | ||
| 147 | |||
| 148 | static const struct common_glue_ctx cast6_dec_cbc = { | ||
| 149 | .num_funcs = 2, | ||
| 150 | .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, | ||
| 151 | |||
| 152 | .funcs = { { | ||
| 153 | .num_blocks = CAST6_PARALLEL_BLOCKS, | ||
| 154 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_decrypt_cbc_xway) } | ||
| 155 | }, { | ||
| 156 | .num_blocks = 1, | ||
| 157 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) } | ||
| 158 | } } | ||
| 159 | }; | ||
| 160 | |||
| 161 | static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 162 | struct scatterlist *src, unsigned int nbytes) | ||
| 163 | { | ||
| 164 | return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes); | ||
| 165 | } | ||
| 166 | |||
| 167 | static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 168 | struct scatterlist *src, unsigned int nbytes) | ||
| 169 | { | ||
| 170 | return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes); | ||
| 171 | } | ||
| 172 | |||
| 173 | static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 174 | struct scatterlist *src, unsigned int nbytes) | ||
| 175 | { | ||
| 176 | return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc, | ||
| 177 | dst, src, nbytes); | ||
| 178 | } | ||
| 179 | |||
| 180 | static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 181 | struct scatterlist *src, unsigned int nbytes) | ||
| 182 | { | ||
| 183 | return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src, | ||
| 184 | nbytes); | ||
| 185 | } | ||
| 186 | |||
| 187 | static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 188 | struct scatterlist *src, unsigned int nbytes) | ||
| 189 | { | ||
| 190 | return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes); | ||
| 191 | } | ||
| 192 | |||
| 193 | static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes) | ||
| 194 | { | ||
| 195 | return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS, | ||
| 196 | NULL, fpu_enabled, nbytes); | ||
| 197 | } | ||
| 198 | |||
| 199 | static inline void cast6_fpu_end(bool fpu_enabled) | ||
| 200 | { | ||
| 201 | glue_fpu_end(fpu_enabled); | ||
| 202 | } | ||
| 203 | |||
| 204 | struct crypt_priv { | ||
| 205 | struct cast6_ctx *ctx; | ||
| 206 | bool fpu_enabled; | ||
| 207 | }; | ||
| 208 | |||
| 209 | static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) | ||
| 210 | { | ||
| 211 | const unsigned int bsize = CAST6_BLOCK_SIZE; | ||
| 212 | struct crypt_priv *ctx = priv; | ||
| 213 | int i; | ||
| 214 | |||
| 215 | ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes); | ||
| 216 | |||
| 217 | if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) { | ||
| 218 | cast6_enc_blk_xway(ctx->ctx, srcdst, srcdst); | ||
| 219 | return; | ||
| 220 | } | ||
| 221 | |||
| 222 | for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) | ||
| 223 | __cast6_encrypt(ctx->ctx, srcdst, srcdst); | ||
| 224 | } | ||
| 225 | |||
| 226 | static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) | ||
| 227 | { | ||
| 228 | const unsigned int bsize = CAST6_BLOCK_SIZE; | ||
| 229 | struct crypt_priv *ctx = priv; | ||
| 230 | int i; | ||
| 231 | |||
| 232 | ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes); | ||
| 233 | |||
| 234 | if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) { | ||
| 235 | cast6_dec_blk_xway(ctx->ctx, srcdst, srcdst); | ||
| 236 | return; | ||
| 237 | } | ||
| 238 | |||
| 239 | for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) | ||
| 240 | __cast6_decrypt(ctx->ctx, srcdst, srcdst); | ||
| 241 | } | ||
| 242 | |||
| 243 | struct cast6_lrw_ctx { | ||
| 244 | struct lrw_table_ctx lrw_table; | ||
| 245 | struct cast6_ctx cast6_ctx; | ||
| 246 | }; | ||
| 247 | |||
| 248 | static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key, | ||
| 249 | unsigned int keylen) | ||
| 250 | { | ||
| 251 | struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm); | ||
| 252 | int err; | ||
| 253 | |||
| 254 | err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE, | ||
| 255 | &tfm->crt_flags); | ||
| 256 | if (err) | ||
| 257 | return err; | ||
| 258 | |||
| 259 | return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE); | ||
| 260 | } | ||
| 261 | |||
| 262 | static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 263 | struct scatterlist *src, unsigned int nbytes) | ||
| 264 | { | ||
| 265 | struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 266 | be128 buf[CAST6_PARALLEL_BLOCKS]; | ||
| 267 | struct crypt_priv crypt_ctx = { | ||
| 268 | .ctx = &ctx->cast6_ctx, | ||
| 269 | .fpu_enabled = false, | ||
| 270 | }; | ||
| 271 | struct lrw_crypt_req req = { | ||
| 272 | .tbuf = buf, | ||
| 273 | .tbuflen = sizeof(buf), | ||
| 274 | |||
| 275 | .table_ctx = &ctx->lrw_table, | ||
| 276 | .crypt_ctx = &crypt_ctx, | ||
| 277 | .crypt_fn = encrypt_callback, | ||
| 278 | }; | ||
| 279 | int ret; | ||
| 280 | |||
| 281 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 282 | ret = lrw_crypt(desc, dst, src, nbytes, &req); | ||
| 283 | cast6_fpu_end(crypt_ctx.fpu_enabled); | ||
| 284 | |||
| 285 | return ret; | ||
| 286 | } | ||
| 287 | |||
| 288 | static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 289 | struct scatterlist *src, unsigned int nbytes) | ||
| 290 | { | ||
| 291 | struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 292 | be128 buf[CAST6_PARALLEL_BLOCKS]; | ||
| 293 | struct crypt_priv crypt_ctx = { | ||
| 294 | .ctx = &ctx->cast6_ctx, | ||
| 295 | .fpu_enabled = false, | ||
| 296 | }; | ||
| 297 | struct lrw_crypt_req req = { | ||
| 298 | .tbuf = buf, | ||
| 299 | .tbuflen = sizeof(buf), | ||
| 300 | |||
| 301 | .table_ctx = &ctx->lrw_table, | ||
| 302 | .crypt_ctx = &crypt_ctx, | ||
| 303 | .crypt_fn = decrypt_callback, | ||
| 304 | }; | ||
| 305 | int ret; | ||
| 306 | |||
| 307 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 308 | ret = lrw_crypt(desc, dst, src, nbytes, &req); | ||
| 309 | cast6_fpu_end(crypt_ctx.fpu_enabled); | ||
| 310 | |||
| 311 | return ret; | ||
| 312 | } | ||
| 313 | |||
| 314 | static void lrw_exit_tfm(struct crypto_tfm *tfm) | ||
| 315 | { | ||
| 316 | struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm); | ||
| 317 | |||
| 318 | lrw_free_table(&ctx->lrw_table); | ||
| 319 | } | ||
| 320 | |||
| 321 | struct cast6_xts_ctx { | ||
| 322 | struct cast6_ctx tweak_ctx; | ||
| 323 | struct cast6_ctx crypt_ctx; | ||
| 324 | }; | ||
| 325 | |||
| 326 | static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key, | ||
| 327 | unsigned int keylen) | ||
| 328 | { | ||
| 329 | struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm); | ||
| 330 | u32 *flags = &tfm->crt_flags; | ||
| 331 | int err; | ||
| 332 | |||
| 333 | /* key consists of keys of equal size concatenated, therefore | ||
| 334 | * the length must be even | ||
| 335 | */ | ||
| 336 | if (keylen % 2) { | ||
| 337 | *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | ||
| 338 | return -EINVAL; | ||
| 339 | } | ||
| 340 | |||
| 341 | /* first half of xts-key is for crypt */ | ||
| 342 | err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); | ||
| 343 | if (err) | ||
| 344 | return err; | ||
| 345 | |||
| 346 | /* second half of xts-key is for tweak */ | ||
| 347 | return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, | ||
| 348 | flags); | ||
| 349 | } | ||
| 350 | |||
| 351 | static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 352 | struct scatterlist *src, unsigned int nbytes) | ||
| 353 | { | ||
| 354 | struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 355 | be128 buf[CAST6_PARALLEL_BLOCKS]; | ||
| 356 | struct crypt_priv crypt_ctx = { | ||
| 357 | .ctx = &ctx->crypt_ctx, | ||
| 358 | .fpu_enabled = false, | ||
| 359 | }; | ||
| 360 | struct xts_crypt_req req = { | ||
| 361 | .tbuf = buf, | ||
| 362 | .tbuflen = sizeof(buf), | ||
| 363 | |||
| 364 | .tweak_ctx = &ctx->tweak_ctx, | ||
| 365 | .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt), | ||
| 366 | .crypt_ctx = &crypt_ctx, | ||
| 367 | .crypt_fn = encrypt_callback, | ||
| 368 | }; | ||
| 369 | int ret; | ||
| 370 | |||
| 371 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 372 | ret = xts_crypt(desc, dst, src, nbytes, &req); | ||
| 373 | cast6_fpu_end(crypt_ctx.fpu_enabled); | ||
| 374 | |||
| 375 | return ret; | ||
| 376 | } | ||
| 377 | |||
| 378 | static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | ||
| 379 | struct scatterlist *src, unsigned int nbytes) | ||
| 380 | { | ||
| 381 | struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | ||
| 382 | be128 buf[CAST6_PARALLEL_BLOCKS]; | ||
| 383 | struct crypt_priv crypt_ctx = { | ||
| 384 | .ctx = &ctx->crypt_ctx, | ||
| 385 | .fpu_enabled = false, | ||
| 386 | }; | ||
| 387 | struct xts_crypt_req req = { | ||
| 388 | .tbuf = buf, | ||
| 389 | .tbuflen = sizeof(buf), | ||
| 390 | |||
| 391 | .tweak_ctx = &ctx->tweak_ctx, | ||
| 392 | .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt), | ||
| 393 | .crypt_ctx = &crypt_ctx, | ||
| 394 | .crypt_fn = decrypt_callback, | ||
| 395 | }; | ||
| 396 | int ret; | ||
| 397 | |||
| 398 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | ||
| 399 | ret = xts_crypt(desc, dst, src, nbytes, &req); | ||
| 400 | cast6_fpu_end(crypt_ctx.fpu_enabled); | ||
| 401 | |||
| 402 | return ret; | ||
| 403 | } | ||
| 404 | |||
| 405 | static struct crypto_alg cast6_algs[10] = { { | ||
| 406 | .cra_name = "__ecb-cast6-avx", | ||
| 407 | .cra_driver_name = "__driver-ecb-cast6-avx", | ||
| 408 | .cra_priority = 0, | ||
| 409 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 410 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 411 | .cra_ctxsize = sizeof(struct cast6_ctx), | ||
| 412 | .cra_alignmask = 0, | ||
| 413 | .cra_type = &crypto_blkcipher_type, | ||
| 414 | .cra_module = THIS_MODULE, | ||
| 415 | .cra_u = { | ||
| 416 | .blkcipher = { | ||
| 417 | .min_keysize = CAST6_MIN_KEY_SIZE, | ||
| 418 | .max_keysize = CAST6_MAX_KEY_SIZE, | ||
| 419 | .setkey = cast6_setkey, | ||
| 420 | .encrypt = ecb_encrypt, | ||
| 421 | .decrypt = ecb_decrypt, | ||
| 422 | }, | ||
| 423 | }, | ||
| 424 | }, { | ||
| 425 | .cra_name = "__cbc-cast6-avx", | ||
| 426 | .cra_driver_name = "__driver-cbc-cast6-avx", | ||
| 427 | .cra_priority = 0, | ||
| 428 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 429 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 430 | .cra_ctxsize = sizeof(struct cast6_ctx), | ||
| 431 | .cra_alignmask = 0, | ||
| 432 | .cra_type = &crypto_blkcipher_type, | ||
| 433 | .cra_module = THIS_MODULE, | ||
| 434 | .cra_u = { | ||
| 435 | .blkcipher = { | ||
| 436 | .min_keysize = CAST6_MIN_KEY_SIZE, | ||
| 437 | .max_keysize = CAST6_MAX_KEY_SIZE, | ||
| 438 | .setkey = cast6_setkey, | ||
| 439 | .encrypt = cbc_encrypt, | ||
| 440 | .decrypt = cbc_decrypt, | ||
| 441 | }, | ||
| 442 | }, | ||
| 443 | }, { | ||
| 444 | .cra_name = "__ctr-cast6-avx", | ||
| 445 | .cra_driver_name = "__driver-ctr-cast6-avx", | ||
| 446 | .cra_priority = 0, | ||
| 447 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 448 | .cra_blocksize = 1, | ||
| 449 | .cra_ctxsize = sizeof(struct cast6_ctx), | ||
| 450 | .cra_alignmask = 0, | ||
| 451 | .cra_type = &crypto_blkcipher_type, | ||
| 452 | .cra_module = THIS_MODULE, | ||
| 453 | .cra_u = { | ||
| 454 | .blkcipher = { | ||
| 455 | .min_keysize = CAST6_MIN_KEY_SIZE, | ||
| 456 | .max_keysize = CAST6_MAX_KEY_SIZE, | ||
| 457 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 458 | .setkey = cast6_setkey, | ||
| 459 | .encrypt = ctr_crypt, | ||
| 460 | .decrypt = ctr_crypt, | ||
| 461 | }, | ||
| 462 | }, | ||
| 463 | }, { | ||
| 464 | .cra_name = "__lrw-cast6-avx", | ||
| 465 | .cra_driver_name = "__driver-lrw-cast6-avx", | ||
| 466 | .cra_priority = 0, | ||
| 467 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 468 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 469 | .cra_ctxsize = sizeof(struct cast6_lrw_ctx), | ||
| 470 | .cra_alignmask = 0, | ||
| 471 | .cra_type = &crypto_blkcipher_type, | ||
| 472 | .cra_module = THIS_MODULE, | ||
| 473 | .cra_exit = lrw_exit_tfm, | ||
| 474 | .cra_u = { | ||
| 475 | .blkcipher = { | ||
| 476 | .min_keysize = CAST6_MIN_KEY_SIZE + | ||
| 477 | CAST6_BLOCK_SIZE, | ||
| 478 | .max_keysize = CAST6_MAX_KEY_SIZE + | ||
| 479 | CAST6_BLOCK_SIZE, | ||
| 480 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 481 | .setkey = lrw_cast6_setkey, | ||
| 482 | .encrypt = lrw_encrypt, | ||
| 483 | .decrypt = lrw_decrypt, | ||
| 484 | }, | ||
| 485 | }, | ||
| 486 | }, { | ||
| 487 | .cra_name = "__xts-cast6-avx", | ||
| 488 | .cra_driver_name = "__driver-xts-cast6-avx", | ||
| 489 | .cra_priority = 0, | ||
| 490 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
| 491 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 492 | .cra_ctxsize = sizeof(struct cast6_xts_ctx), | ||
| 493 | .cra_alignmask = 0, | ||
| 494 | .cra_type = &crypto_blkcipher_type, | ||
| 495 | .cra_module = THIS_MODULE, | ||
| 496 | .cra_u = { | ||
| 497 | .blkcipher = { | ||
| 498 | .min_keysize = CAST6_MIN_KEY_SIZE * 2, | ||
| 499 | .max_keysize = CAST6_MAX_KEY_SIZE * 2, | ||
| 500 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 501 | .setkey = xts_cast6_setkey, | ||
| 502 | .encrypt = xts_encrypt, | ||
| 503 | .decrypt = xts_decrypt, | ||
| 504 | }, | ||
| 505 | }, | ||
| 506 | }, { | ||
| 507 | .cra_name = "ecb(cast6)", | ||
| 508 | .cra_driver_name = "ecb-cast6-avx", | ||
| 509 | .cra_priority = 200, | ||
| 510 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 511 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 512 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 513 | .cra_alignmask = 0, | ||
| 514 | .cra_type = &crypto_ablkcipher_type, | ||
| 515 | .cra_module = THIS_MODULE, | ||
| 516 | .cra_init = ablk_init, | ||
| 517 | .cra_exit = ablk_exit, | ||
| 518 | .cra_u = { | ||
| 519 | .ablkcipher = { | ||
| 520 | .min_keysize = CAST6_MIN_KEY_SIZE, | ||
| 521 | .max_keysize = CAST6_MAX_KEY_SIZE, | ||
| 522 | .setkey = ablk_set_key, | ||
| 523 | .encrypt = ablk_encrypt, | ||
| 524 | .decrypt = ablk_decrypt, | ||
| 525 | }, | ||
| 526 | }, | ||
| 527 | }, { | ||
| 528 | .cra_name = "cbc(cast6)", | ||
| 529 | .cra_driver_name = "cbc-cast6-avx", | ||
| 530 | .cra_priority = 200, | ||
| 531 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 532 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 533 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 534 | .cra_alignmask = 0, | ||
| 535 | .cra_type = &crypto_ablkcipher_type, | ||
| 536 | .cra_module = THIS_MODULE, | ||
| 537 | .cra_init = ablk_init, | ||
| 538 | .cra_exit = ablk_exit, | ||
| 539 | .cra_u = { | ||
| 540 | .ablkcipher = { | ||
| 541 | .min_keysize = CAST6_MIN_KEY_SIZE, | ||
| 542 | .max_keysize = CAST6_MAX_KEY_SIZE, | ||
| 543 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 544 | .setkey = ablk_set_key, | ||
| 545 | .encrypt = __ablk_encrypt, | ||
| 546 | .decrypt = ablk_decrypt, | ||
| 547 | }, | ||
| 548 | }, | ||
| 549 | }, { | ||
| 550 | .cra_name = "ctr(cast6)", | ||
| 551 | .cra_driver_name = "ctr-cast6-avx", | ||
| 552 | .cra_priority = 200, | ||
| 553 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 554 | .cra_blocksize = 1, | ||
| 555 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 556 | .cra_alignmask = 0, | ||
| 557 | .cra_type = &crypto_ablkcipher_type, | ||
| 558 | .cra_module = THIS_MODULE, | ||
| 559 | .cra_init = ablk_init, | ||
| 560 | .cra_exit = ablk_exit, | ||
| 561 | .cra_u = { | ||
| 562 | .ablkcipher = { | ||
| 563 | .min_keysize = CAST6_MIN_KEY_SIZE, | ||
| 564 | .max_keysize = CAST6_MAX_KEY_SIZE, | ||
| 565 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 566 | .setkey = ablk_set_key, | ||
| 567 | .encrypt = ablk_encrypt, | ||
| 568 | .decrypt = ablk_encrypt, | ||
| 569 | .geniv = "chainiv", | ||
| 570 | }, | ||
| 571 | }, | ||
| 572 | }, { | ||
| 573 | .cra_name = "lrw(cast6)", | ||
| 574 | .cra_driver_name = "lrw-cast6-avx", | ||
| 575 | .cra_priority = 200, | ||
| 576 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 577 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 578 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 579 | .cra_alignmask = 0, | ||
| 580 | .cra_type = &crypto_ablkcipher_type, | ||
| 581 | .cra_module = THIS_MODULE, | ||
| 582 | .cra_init = ablk_init, | ||
| 583 | .cra_exit = ablk_exit, | ||
| 584 | .cra_u = { | ||
| 585 | .ablkcipher = { | ||
| 586 | .min_keysize = CAST6_MIN_KEY_SIZE + | ||
| 587 | CAST6_BLOCK_SIZE, | ||
| 588 | .max_keysize = CAST6_MAX_KEY_SIZE + | ||
| 589 | CAST6_BLOCK_SIZE, | ||
| 590 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 591 | .setkey = ablk_set_key, | ||
| 592 | .encrypt = ablk_encrypt, | ||
| 593 | .decrypt = ablk_decrypt, | ||
| 594 | }, | ||
| 595 | }, | ||
| 596 | }, { | ||
| 597 | .cra_name = "xts(cast6)", | ||
| 598 | .cra_driver_name = "xts-cast6-avx", | ||
| 599 | .cra_priority = 200, | ||
| 600 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | ||
| 601 | .cra_blocksize = CAST6_BLOCK_SIZE, | ||
| 602 | .cra_ctxsize = sizeof(struct async_helper_ctx), | ||
| 603 | .cra_alignmask = 0, | ||
| 604 | .cra_type = &crypto_ablkcipher_type, | ||
| 605 | .cra_module = THIS_MODULE, | ||
| 606 | .cra_init = ablk_init, | ||
| 607 | .cra_exit = ablk_exit, | ||
| 608 | .cra_u = { | ||
| 609 | .ablkcipher = { | ||
| 610 | .min_keysize = CAST6_MIN_KEY_SIZE * 2, | ||
| 611 | .max_keysize = CAST6_MAX_KEY_SIZE * 2, | ||
| 612 | .ivsize = CAST6_BLOCK_SIZE, | ||
| 613 | .setkey = ablk_set_key, | ||
| 614 | .encrypt = ablk_encrypt, | ||
| 615 | .decrypt = ablk_decrypt, | ||
| 616 | }, | ||
| 617 | }, | ||
| 618 | } }; | ||
| 619 | |||
| 620 | static int __init cast6_init(void) | ||
| 621 | { | ||
| 622 | u64 xcr0; | ||
| 623 | |||
| 624 | if (!cpu_has_avx || !cpu_has_osxsave) { | ||
| 625 | pr_info("AVX instructions are not detected.\n"); | ||
| 626 | return -ENODEV; | ||
| 627 | } | ||
| 628 | |||
| 629 | xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); | ||
| 630 | if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { | ||
| 631 | pr_info("AVX detected but unusable.\n"); | ||
| 632 | return -ENODEV; | ||
| 633 | } | ||
| 634 | |||
| 635 | return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs)); | ||
| 636 | } | ||
| 637 | |||
| 638 | static void __exit cast6_exit(void) | ||
| 639 | { | ||
| 640 | crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs)); | ||
| 641 | } | ||
| 642 | |||
| 643 | module_init(cast6_init); | ||
| 644 | module_exit(cast6_exit); | ||
| 645 | |||
| 646 | MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized"); | ||
| 647 | MODULE_LICENSE("GPL"); | ||
| 648 | MODULE_ALIAS("cast6"); | ||
diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index b4bf0a63b520..6759dd1135be 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c | |||
| @@ -150,7 +150,6 @@ static struct shash_alg ghash_alg = { | |||
| 150 | .cra_blocksize = GHASH_BLOCK_SIZE, | 150 | .cra_blocksize = GHASH_BLOCK_SIZE, |
| 151 | .cra_ctxsize = sizeof(struct ghash_ctx), | 151 | .cra_ctxsize = sizeof(struct ghash_ctx), |
| 152 | .cra_module = THIS_MODULE, | 152 | .cra_module = THIS_MODULE, |
| 153 | .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list), | ||
| 154 | }, | 153 | }, |
| 155 | }; | 154 | }; |
| 156 | 155 | ||
| @@ -288,7 +287,6 @@ static struct ahash_alg ghash_async_alg = { | |||
| 288 | .cra_blocksize = GHASH_BLOCK_SIZE, | 287 | .cra_blocksize = GHASH_BLOCK_SIZE, |
| 289 | .cra_type = &crypto_ahash_type, | 288 | .cra_type = &crypto_ahash_type, |
| 290 | .cra_module = THIS_MODULE, | 289 | .cra_module = THIS_MODULE, |
| 291 | .cra_list = LIST_HEAD_INIT(ghash_async_alg.halg.base.cra_list), | ||
| 292 | .cra_init = ghash_async_init_tfm, | 290 | .cra_init = ghash_async_init_tfm, |
| 293 | .cra_exit = ghash_async_exit_tfm, | 291 | .cra_exit = ghash_async_exit_tfm, |
| 294 | }, | 292 | }, |
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c index 4854f0f31e4f..30b3927bd733 100644 --- a/arch/x86/crypto/glue_helper.c +++ b/arch/x86/crypto/glue_helper.c | |||
| @@ -110,7 +110,7 @@ static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn, | |||
| 110 | nbytes -= bsize; | 110 | nbytes -= bsize; |
| 111 | } while (nbytes >= bsize); | 111 | } while (nbytes >= bsize); |
| 112 | 112 | ||
| 113 | u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); | 113 | *(u128 *)walk->iv = *iv; |
| 114 | return nbytes; | 114 | return nbytes; |
| 115 | } | 115 | } |
| 116 | 116 | ||
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c index bccb76d80987..a3a3c0205c16 100644 --- a/arch/x86/crypto/salsa20_glue.c +++ b/arch/x86/crypto/salsa20_glue.c | |||
| @@ -97,7 +97,6 @@ static struct crypto_alg alg = { | |||
| 97 | .cra_ctxsize = sizeof(struct salsa20_ctx), | 97 | .cra_ctxsize = sizeof(struct salsa20_ctx), |
| 98 | .cra_alignmask = 3, | 98 | .cra_alignmask = 3, |
| 99 | .cra_module = THIS_MODULE, | 99 | .cra_module = THIS_MODULE, |
| 100 | .cra_list = LIST_HEAD_INIT(alg.cra_list), | ||
| 101 | .cra_u = { | 100 | .cra_u = { |
| 102 | .blkcipher = { | 101 | .blkcipher = { |
| 103 | .setkey = setkey, | 102 | .setkey = setkey, |
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index b36bdac237eb..3f543a04cf1e 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c | |||
| @@ -390,7 +390,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 390 | .cra_alignmask = 0, | 390 | .cra_alignmask = 0, |
| 391 | .cra_type = &crypto_blkcipher_type, | 391 | .cra_type = &crypto_blkcipher_type, |
| 392 | .cra_module = THIS_MODULE, | 392 | .cra_module = THIS_MODULE, |
| 393 | .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list), | ||
| 394 | .cra_u = { | 393 | .cra_u = { |
| 395 | .blkcipher = { | 394 | .blkcipher = { |
| 396 | .min_keysize = SERPENT_MIN_KEY_SIZE, | 395 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
| @@ -410,7 +409,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 410 | .cra_alignmask = 0, | 409 | .cra_alignmask = 0, |
| 411 | .cra_type = &crypto_blkcipher_type, | 410 | .cra_type = &crypto_blkcipher_type, |
| 412 | .cra_module = THIS_MODULE, | 411 | .cra_module = THIS_MODULE, |
| 413 | .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list), | ||
| 414 | .cra_u = { | 412 | .cra_u = { |
| 415 | .blkcipher = { | 413 | .blkcipher = { |
| 416 | .min_keysize = SERPENT_MIN_KEY_SIZE, | 414 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
| @@ -430,7 +428,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 430 | .cra_alignmask = 0, | 428 | .cra_alignmask = 0, |
| 431 | .cra_type = &crypto_blkcipher_type, | 429 | .cra_type = &crypto_blkcipher_type, |
| 432 | .cra_module = THIS_MODULE, | 430 | .cra_module = THIS_MODULE, |
| 433 | .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list), | ||
| 434 | .cra_u = { | 431 | .cra_u = { |
| 435 | .blkcipher = { | 432 | .blkcipher = { |
| 436 | .min_keysize = SERPENT_MIN_KEY_SIZE, | 433 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
| @@ -451,7 +448,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 451 | .cra_alignmask = 0, | 448 | .cra_alignmask = 0, |
| 452 | .cra_type = &crypto_blkcipher_type, | 449 | .cra_type = &crypto_blkcipher_type, |
| 453 | .cra_module = THIS_MODULE, | 450 | .cra_module = THIS_MODULE, |
| 454 | .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list), | ||
| 455 | .cra_exit = lrw_exit_tfm, | 451 | .cra_exit = lrw_exit_tfm, |
| 456 | .cra_u = { | 452 | .cra_u = { |
| 457 | .blkcipher = { | 453 | .blkcipher = { |
| @@ -475,7 +471,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 475 | .cra_alignmask = 0, | 471 | .cra_alignmask = 0, |
| 476 | .cra_type = &crypto_blkcipher_type, | 472 | .cra_type = &crypto_blkcipher_type, |
| 477 | .cra_module = THIS_MODULE, | 473 | .cra_module = THIS_MODULE, |
| 478 | .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list), | ||
| 479 | .cra_u = { | 474 | .cra_u = { |
| 480 | .blkcipher = { | 475 | .blkcipher = { |
| 481 | .min_keysize = SERPENT_MIN_KEY_SIZE * 2, | 476 | .min_keysize = SERPENT_MIN_KEY_SIZE * 2, |
| @@ -496,7 +491,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 496 | .cra_alignmask = 0, | 491 | .cra_alignmask = 0, |
| 497 | .cra_type = &crypto_ablkcipher_type, | 492 | .cra_type = &crypto_ablkcipher_type, |
| 498 | .cra_module = THIS_MODULE, | 493 | .cra_module = THIS_MODULE, |
| 499 | .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list), | ||
| 500 | .cra_init = ablk_init, | 494 | .cra_init = ablk_init, |
| 501 | .cra_exit = ablk_exit, | 495 | .cra_exit = ablk_exit, |
| 502 | .cra_u = { | 496 | .cra_u = { |
| @@ -518,7 +512,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 518 | .cra_alignmask = 0, | 512 | .cra_alignmask = 0, |
| 519 | .cra_type = &crypto_ablkcipher_type, | 513 | .cra_type = &crypto_ablkcipher_type, |
| 520 | .cra_module = THIS_MODULE, | 514 | .cra_module = THIS_MODULE, |
| 521 | .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list), | ||
| 522 | .cra_init = ablk_init, | 515 | .cra_init = ablk_init, |
| 523 | .cra_exit = ablk_exit, | 516 | .cra_exit = ablk_exit, |
| 524 | .cra_u = { | 517 | .cra_u = { |
| @@ -541,7 +534,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 541 | .cra_alignmask = 0, | 534 | .cra_alignmask = 0, |
| 542 | .cra_type = &crypto_ablkcipher_type, | 535 | .cra_type = &crypto_ablkcipher_type, |
| 543 | .cra_module = THIS_MODULE, | 536 | .cra_module = THIS_MODULE, |
| 544 | .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list), | ||
| 545 | .cra_init = ablk_init, | 537 | .cra_init = ablk_init, |
| 546 | .cra_exit = ablk_exit, | 538 | .cra_exit = ablk_exit, |
| 547 | .cra_u = { | 539 | .cra_u = { |
| @@ -565,7 +557,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 565 | .cra_alignmask = 0, | 557 | .cra_alignmask = 0, |
| 566 | .cra_type = &crypto_ablkcipher_type, | 558 | .cra_type = &crypto_ablkcipher_type, |
| 567 | .cra_module = THIS_MODULE, | 559 | .cra_module = THIS_MODULE, |
| 568 | .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list), | ||
| 569 | .cra_init = ablk_init, | 560 | .cra_init = ablk_init, |
| 570 | .cra_exit = ablk_exit, | 561 | .cra_exit = ablk_exit, |
| 571 | .cra_u = { | 562 | .cra_u = { |
| @@ -590,7 +581,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 590 | .cra_alignmask = 0, | 581 | .cra_alignmask = 0, |
| 591 | .cra_type = &crypto_ablkcipher_type, | 582 | .cra_type = &crypto_ablkcipher_type, |
| 592 | .cra_module = THIS_MODULE, | 583 | .cra_module = THIS_MODULE, |
| 593 | .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list), | ||
| 594 | .cra_init = ablk_init, | 584 | .cra_init = ablk_init, |
| 595 | .cra_exit = ablk_exit, | 585 | .cra_exit = ablk_exit, |
| 596 | .cra_u = { | 586 | .cra_u = { |
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index d679c8675f4a..9107a9908c41 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c | |||
| @@ -393,7 +393,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 393 | .cra_alignmask = 0, | 393 | .cra_alignmask = 0, |
| 394 | .cra_type = &crypto_blkcipher_type, | 394 | .cra_type = &crypto_blkcipher_type, |
| 395 | .cra_module = THIS_MODULE, | 395 | .cra_module = THIS_MODULE, |
| 396 | .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list), | ||
| 397 | .cra_u = { | 396 | .cra_u = { |
| 398 | .blkcipher = { | 397 | .blkcipher = { |
| 399 | .min_keysize = SERPENT_MIN_KEY_SIZE, | 398 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
| @@ -413,7 +412,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 413 | .cra_alignmask = 0, | 412 | .cra_alignmask = 0, |
| 414 | .cra_type = &crypto_blkcipher_type, | 413 | .cra_type = &crypto_blkcipher_type, |
| 415 | .cra_module = THIS_MODULE, | 414 | .cra_module = THIS_MODULE, |
| 416 | .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list), | ||
| 417 | .cra_u = { | 415 | .cra_u = { |
| 418 | .blkcipher = { | 416 | .blkcipher = { |
| 419 | .min_keysize = SERPENT_MIN_KEY_SIZE, | 417 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
| @@ -433,7 +431,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 433 | .cra_alignmask = 0, | 431 | .cra_alignmask = 0, |
| 434 | .cra_type = &crypto_blkcipher_type, | 432 | .cra_type = &crypto_blkcipher_type, |
| 435 | .cra_module = THIS_MODULE, | 433 | .cra_module = THIS_MODULE, |
| 436 | .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list), | ||
| 437 | .cra_u = { | 434 | .cra_u = { |
| 438 | .blkcipher = { | 435 | .blkcipher = { |
| 439 | .min_keysize = SERPENT_MIN_KEY_SIZE, | 436 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
| @@ -454,7 +451,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 454 | .cra_alignmask = 0, | 451 | .cra_alignmask = 0, |
| 455 | .cra_type = &crypto_blkcipher_type, | 452 | .cra_type = &crypto_blkcipher_type, |
| 456 | .cra_module = THIS_MODULE, | 453 | .cra_module = THIS_MODULE, |
| 457 | .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list), | ||
| 458 | .cra_exit = lrw_exit_tfm, | 454 | .cra_exit = lrw_exit_tfm, |
| 459 | .cra_u = { | 455 | .cra_u = { |
| 460 | .blkcipher = { | 456 | .blkcipher = { |
| @@ -478,7 +474,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 478 | .cra_alignmask = 0, | 474 | .cra_alignmask = 0, |
| 479 | .cra_type = &crypto_blkcipher_type, | 475 | .cra_type = &crypto_blkcipher_type, |
| 480 | .cra_module = THIS_MODULE, | 476 | .cra_module = THIS_MODULE, |
| 481 | .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list), | ||
| 482 | .cra_u = { | 477 | .cra_u = { |
| 483 | .blkcipher = { | 478 | .blkcipher = { |
| 484 | .min_keysize = SERPENT_MIN_KEY_SIZE * 2, | 479 | .min_keysize = SERPENT_MIN_KEY_SIZE * 2, |
| @@ -499,7 +494,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 499 | .cra_alignmask = 0, | 494 | .cra_alignmask = 0, |
| 500 | .cra_type = &crypto_ablkcipher_type, | 495 | .cra_type = &crypto_ablkcipher_type, |
| 501 | .cra_module = THIS_MODULE, | 496 | .cra_module = THIS_MODULE, |
| 502 | .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list), | ||
| 503 | .cra_init = ablk_init, | 497 | .cra_init = ablk_init, |
| 504 | .cra_exit = ablk_exit, | 498 | .cra_exit = ablk_exit, |
| 505 | .cra_u = { | 499 | .cra_u = { |
| @@ -521,7 +515,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 521 | .cra_alignmask = 0, | 515 | .cra_alignmask = 0, |
| 522 | .cra_type = &crypto_ablkcipher_type, | 516 | .cra_type = &crypto_ablkcipher_type, |
| 523 | .cra_module = THIS_MODULE, | 517 | .cra_module = THIS_MODULE, |
| 524 | .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list), | ||
| 525 | .cra_init = ablk_init, | 518 | .cra_init = ablk_init, |
| 526 | .cra_exit = ablk_exit, | 519 | .cra_exit = ablk_exit, |
| 527 | .cra_u = { | 520 | .cra_u = { |
| @@ -544,7 +537,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 544 | .cra_alignmask = 0, | 537 | .cra_alignmask = 0, |
| 545 | .cra_type = &crypto_ablkcipher_type, | 538 | .cra_type = &crypto_ablkcipher_type, |
| 546 | .cra_module = THIS_MODULE, | 539 | .cra_module = THIS_MODULE, |
| 547 | .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list), | ||
| 548 | .cra_init = ablk_init, | 540 | .cra_init = ablk_init, |
| 549 | .cra_exit = ablk_exit, | 541 | .cra_exit = ablk_exit, |
| 550 | .cra_u = { | 542 | .cra_u = { |
| @@ -568,7 +560,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 568 | .cra_alignmask = 0, | 560 | .cra_alignmask = 0, |
| 569 | .cra_type = &crypto_ablkcipher_type, | 561 | .cra_type = &crypto_ablkcipher_type, |
| 570 | .cra_module = THIS_MODULE, | 562 | .cra_module = THIS_MODULE, |
| 571 | .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list), | ||
| 572 | .cra_init = ablk_init, | 563 | .cra_init = ablk_init, |
| 573 | .cra_exit = ablk_exit, | 564 | .cra_exit = ablk_exit, |
| 574 | .cra_u = { | 565 | .cra_u = { |
| @@ -593,7 +584,6 @@ static struct crypto_alg serpent_algs[10] = { { | |||
| 593 | .cra_alignmask = 0, | 584 | .cra_alignmask = 0, |
| 594 | .cra_type = &crypto_ablkcipher_type, | 585 | .cra_type = &crypto_ablkcipher_type, |
| 595 | .cra_module = THIS_MODULE, | 586 | .cra_module = THIS_MODULE, |
| 596 | .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list), | ||
| 597 | .cra_init = ablk_init, | 587 | .cra_init = ablk_init, |
| 598 | .cra_exit = ablk_exit, | 588 | .cra_exit = ablk_exit, |
| 599 | .cra_u = { | 589 | .cra_u = { |
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S index 35f45574390d..1585abb13dde 100644 --- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S | |||
| @@ -4,6 +4,8 @@ | |||
| 4 | * Copyright (C) 2012 Johannes Goetzfried | 4 | * Copyright (C) 2012 Johannes Goetzfried |
| 5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | 5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> |
| 6 | * | 6 | * |
| 7 | * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> | ||
| 8 | * | ||
| 7 | * This program is free software; you can redistribute it and/or modify | 9 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License as published by | 10 | * it under the terms of the GNU General Public License as published by |
| 9 | * the Free Software Foundation; either version 2 of the License, or | 11 | * the Free Software Foundation; either version 2 of the License, or |
| @@ -47,16 +49,22 @@ | |||
| 47 | #define RC2 %xmm6 | 49 | #define RC2 %xmm6 |
| 48 | #define RD2 %xmm7 | 50 | #define RD2 %xmm7 |
| 49 | 51 | ||
| 50 | #define RX %xmm8 | 52 | #define RX0 %xmm8 |
| 51 | #define RY %xmm9 | 53 | #define RY0 %xmm9 |
| 54 | |||
| 55 | #define RX1 %xmm10 | ||
| 56 | #define RY1 %xmm11 | ||
| 52 | 57 | ||
| 53 | #define RK1 %xmm10 | 58 | #define RK1 %xmm12 |
| 54 | #define RK2 %xmm11 | 59 | #define RK2 %xmm13 |
| 55 | 60 | ||
| 56 | #define RID1 %rax | 61 | #define RT %xmm14 |
| 57 | #define RID1b %al | 62 | #define RR %xmm15 |
| 58 | #define RID2 %rbx | 63 | |
| 59 | #define RID2b %bl | 64 | #define RID1 %rbp |
| 65 | #define RID1d %ebp | ||
| 66 | #define RID2 %rsi | ||
| 67 | #define RID2d %esi | ||
| 60 | 68 | ||
| 61 | #define RGI1 %rdx | 69 | #define RGI1 %rdx |
| 62 | #define RGI1bl %dl | 70 | #define RGI1bl %dl |
| @@ -65,6 +73,13 @@ | |||
| 65 | #define RGI2bl %cl | 73 | #define RGI2bl %cl |
| 66 | #define RGI2bh %ch | 74 | #define RGI2bh %ch |
| 67 | 75 | ||
| 76 | #define RGI3 %rax | ||
| 77 | #define RGI3bl %al | ||
| 78 | #define RGI3bh %ah | ||
| 79 | #define RGI4 %rbx | ||
| 80 | #define RGI4bl %bl | ||
| 81 | #define RGI4bh %bh | ||
| 82 | |||
| 68 | #define RGS1 %r8 | 83 | #define RGS1 %r8 |
| 69 | #define RGS1d %r8d | 84 | #define RGS1d %r8d |
| 70 | #define RGS2 %r9 | 85 | #define RGS2 %r9 |
| @@ -73,89 +88,123 @@ | |||
| 73 | #define RGS3d %r10d | 88 | #define RGS3d %r10d |
| 74 | 89 | ||
| 75 | 90 | ||
| 76 | #define lookup_32bit(t0, t1, t2, t3, src, dst) \ | 91 | #define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \ |
| 77 | movb src ## bl, RID1b; \ | 92 | movzbl src ## bl, RID1d; \ |
| 78 | movb src ## bh, RID2b; \ | 93 | movzbl src ## bh, RID2d; \ |
| 79 | movl t0(CTX, RID1, 4), dst ## d; \ | ||
| 80 | xorl t1(CTX, RID2, 4), dst ## d; \ | ||
| 81 | shrq $16, src; \ | 94 | shrq $16, src; \ |
| 82 | movb src ## bl, RID1b; \ | 95 | movl t0(CTX, RID1, 4), dst ## d; \ |
| 83 | movb src ## bh, RID2b; \ | 96 | movl t1(CTX, RID2, 4), RID2d; \ |
| 97 | movzbl src ## bl, RID1d; \ | ||
| 98 | xorl RID2d, dst ## d; \ | ||
| 99 | movzbl src ## bh, RID2d; \ | ||
| 100 | interleave_op(il_reg); \ | ||
| 84 | xorl t2(CTX, RID1, 4), dst ## d; \ | 101 | xorl t2(CTX, RID1, 4), dst ## d; \ |
| 85 | xorl t3(CTX, RID2, 4), dst ## d; | 102 | xorl t3(CTX, RID2, 4), dst ## d; |
| 86 | 103 | ||
| 87 | #define G(a, x, t0, t1, t2, t3) \ | 104 | #define dummy(d) /* do nothing */ |
| 88 | vmovq a, RGI1; \ | 105 | |
| 89 | vpsrldq $8, a, x; \ | 106 | #define shr_next(reg) \ |
| 90 | vmovq x, RGI2; \ | 107 | shrq $16, reg; |
| 108 | |||
| 109 | #define G(gi1, gi2, x, t0, t1, t2, t3) \ | ||
| 110 | lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1); \ | ||
| 111 | lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2); \ | ||
| 112 | \ | ||
| 113 | lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none); \ | ||
| 114 | shlq $32, RGS2; \ | ||
| 115 | orq RGS1, RGS2; \ | ||
| 116 | lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none); \ | ||
| 117 | shlq $32, RGS1; \ | ||
| 118 | orq RGS1, RGS3; | ||
| 119 | |||
| 120 | #define round_head_2(a, b, x1, y1, x2, y2) \ | ||
| 121 | vmovq b ## 1, RGI3; \ | ||
| 122 | vpextrq $1, b ## 1, RGI4; \ | ||
| 91 | \ | 123 | \ |
| 92 | lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \ | 124 | G(RGI1, RGI2, x1, s0, s1, s2, s3); \ |
| 93 | shrq $16, RGI1; \ | 125 | vmovq a ## 2, RGI1; \ |
| 94 | lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \ | 126 | vpextrq $1, a ## 2, RGI2; \ |
| 95 | shlq $32, RGS2; \ | 127 | vmovq RGS2, x1; \ |
| 96 | orq RGS1, RGS2; \ | 128 | vpinsrq $1, RGS3, x1, x1; \ |
| 97 | \ | 129 | \ |
| 98 | lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \ | 130 | G(RGI3, RGI4, y1, s1, s2, s3, s0); \ |
| 99 | shrq $16, RGI2; \ | 131 | vmovq b ## 2, RGI3; \ |
| 100 | lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \ | 132 | vpextrq $1, b ## 2, RGI4; \ |
| 101 | shlq $32, RGS3; \ | 133 | vmovq RGS2, y1; \ |
| 102 | orq RGS1, RGS3; \ | 134 | vpinsrq $1, RGS3, y1, y1; \ |
| 103 | \ | 135 | \ |
| 104 | vmovq RGS2, x; \ | 136 | G(RGI1, RGI2, x2, s0, s1, s2, s3); \ |
| 105 | vpinsrq $1, RGS3, x, x; | 137 | vmovq RGS2, x2; \ |
| 138 | vpinsrq $1, RGS3, x2, x2; \ | ||
| 139 | \ | ||
| 140 | G(RGI3, RGI4, y2, s1, s2, s3, s0); \ | ||
| 141 | vmovq RGS2, y2; \ | ||
| 142 | vpinsrq $1, RGS3, y2, y2; | ||
| 106 | 143 | ||
| 107 | #define encround(a, b, c, d, x, y) \ | 144 | #define encround_tail(a, b, c, d, x, y, prerotate) \ |
| 108 | G(a, x, s0, s1, s2, s3); \ | ||
| 109 | G(b, y, s1, s2, s3, s0); \ | ||
| 110 | vpaddd x, y, x; \ | 145 | vpaddd x, y, x; \ |
| 146 | vpaddd x, RK1, RT;\ | ||
| 147 | prerotate(b); \ | ||
| 148 | vpxor RT, c, c; \ | ||
| 111 | vpaddd y, x, y; \ | 149 | vpaddd y, x, y; \ |
| 112 | vpaddd x, RK1, x; \ | ||
| 113 | vpaddd y, RK2, y; \ | 150 | vpaddd y, RK2, y; \ |
| 114 | vpxor x, c, c; \ | 151 | vpsrld $1, c, RT; \ |
| 115 | vpsrld $1, c, x; \ | ||
| 116 | vpslld $(32 - 1), c, c; \ | 152 | vpslld $(32 - 1), c, c; \ |
| 117 | vpor c, x, c; \ | 153 | vpor c, RT, c; \ |
| 118 | vpslld $1, d, x; \ | 154 | vpxor d, y, d; \ |
| 119 | vpsrld $(32 - 1), d, d; \ | 155 | |
| 120 | vpor d, x, d; \ | 156 | #define decround_tail(a, b, c, d, x, y, prerotate) \ |
| 121 | vpxor d, y, d; | ||
| 122 | |||
| 123 | #define decround(a, b, c, d, x, y) \ | ||
| 124 | G(a, x, s0, s1, s2, s3); \ | ||
| 125 | G(b, y, s1, s2, s3, s0); \ | ||
| 126 | vpaddd x, y, x; \ | 157 | vpaddd x, y, x; \ |
| 158 | vpaddd x, RK1, RT;\ | ||
| 159 | prerotate(a); \ | ||
| 160 | vpxor RT, c, c; \ | ||
| 127 | vpaddd y, x, y; \ | 161 | vpaddd y, x, y; \ |
| 128 | vpaddd y, RK2, y; \ | 162 | vpaddd y, RK2, y; \ |
| 129 | vpxor d, y, d; \ | 163 | vpxor d, y, d; \ |
| 130 | vpsrld $1, d, y; \ | 164 | vpsrld $1, d, y; \ |
| 131 | vpslld $(32 - 1), d, d; \ | 165 | vpslld $(32 - 1), d, d; \ |
| 132 | vpor d, y, d; \ | 166 | vpor d, y, d; \ |
| 133 | vpslld $1, c, y; \ | 167 | |
| 134 | vpsrld $(32 - 1), c, c; \ | 168 | #define rotate_1l(x) \ |
| 135 | vpor c, y, c; \ | 169 | vpslld $1, x, RR; \ |
| 136 | vpaddd x, RK1, x; \ | 170 | vpsrld $(32 - 1), x, x; \ |
| 137 | vpxor x, c, c; | 171 | vpor x, RR, x; |
| 138 | 172 | ||
| 139 | #define encrypt_round(n, a, b, c, d) \ | 173 | #define preload_rgi(c) \ |
| 140 | vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ | 174 | vmovq c, RGI1; \ |
| 141 | vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ | 175 | vpextrq $1, c, RGI2; |
| 142 | encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ | 176 | |
| 143 | encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); | 177 | #define encrypt_round(n, a, b, c, d, preload, prerotate) \ |
| 144 | 178 | vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ | |
| 145 | #define decrypt_round(n, a, b, c, d) \ | 179 | vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ |
| 146 | vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ | 180 | round_head_2(a, b, RX0, RY0, RX1, RY1); \ |
| 147 | vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ | 181 | encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \ |
| 148 | decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ | 182 | preload(c ## 1); \ |
| 149 | decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); | 183 | encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate); |
| 184 | |||
| 185 | #define decrypt_round(n, a, b, c, d, preload, prerotate) \ | ||
| 186 | vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ | ||
| 187 | vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ | ||
| 188 | round_head_2(a, b, RX0, RY0, RX1, RY1); \ | ||
| 189 | decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \ | ||
| 190 | preload(c ## 1); \ | ||
| 191 | decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate); | ||
| 150 | 192 | ||
| 151 | #define encrypt_cycle(n) \ | 193 | #define encrypt_cycle(n) \ |
| 152 | encrypt_round((2*n), RA, RB, RC, RD); \ | 194 | encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \ |
| 153 | encrypt_round(((2*n) + 1), RC, RD, RA, RB); | 195 | encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); |
| 196 | |||
| 197 | #define encrypt_cycle_last(n) \ | ||
| 198 | encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \ | ||
| 199 | encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy); | ||
| 154 | 200 | ||
| 155 | #define decrypt_cycle(n) \ | 201 | #define decrypt_cycle(n) \ |
| 156 | decrypt_round(((2*n) + 1), RC, RD, RA, RB); \ | 202 | decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \ |
| 157 | decrypt_round((2*n), RA, RB, RC, RD); | 203 | decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); |
| 158 | 204 | ||
| 205 | #define decrypt_cycle_last(n) \ | ||
| 206 | decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \ | ||
| 207 | decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy); | ||
| 159 | 208 | ||
| 160 | #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ | 209 | #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ |
| 161 | vpunpckldq x1, x0, t0; \ | 210 | vpunpckldq x1, x0, t0; \ |
| @@ -216,17 +265,20 @@ __twofish_enc_blk_8way: | |||
| 216 | * %rcx: bool, if true: xor output | 265 | * %rcx: bool, if true: xor output |
| 217 | */ | 266 | */ |
| 218 | 267 | ||
| 268 | pushq %rbp; | ||
| 219 | pushq %rbx; | 269 | pushq %rbx; |
| 220 | pushq %rcx; | 270 | pushq %rcx; |
| 221 | 271 | ||
| 222 | vmovdqu w(CTX), RK1; | 272 | vmovdqu w(CTX), RK1; |
| 223 | 273 | ||
| 224 | leaq (4*4*4)(%rdx), %rax; | 274 | leaq (4*4*4)(%rdx), %rax; |
| 225 | inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); | 275 | inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); |
| 226 | inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); | 276 | preload_rgi(RA1); |
| 277 | rotate_1l(RD1); | ||
| 278 | inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); | ||
| 279 | rotate_1l(RD2); | ||
| 227 | 280 | ||
| 228 | xorq RID1, RID1; | 281 | movq %rsi, %r11; |
| 229 | xorq RID2, RID2; | ||
| 230 | 282 | ||
| 231 | encrypt_cycle(0); | 283 | encrypt_cycle(0); |
| 232 | encrypt_cycle(1); | 284 | encrypt_cycle(1); |
| @@ -235,26 +287,27 @@ __twofish_enc_blk_8way: | |||
| 235 | encrypt_cycle(4); | 287 | encrypt_cycle(4); |
| 236 | encrypt_cycle(5); | 288 | encrypt_cycle(5); |
| 237 | encrypt_cycle(6); | 289 | encrypt_cycle(6); |
| 238 | encrypt_cycle(7); | 290 | encrypt_cycle_last(7); |
| 239 | 291 | ||
| 240 | vmovdqu (w+4*4)(CTX), RK1; | 292 | vmovdqu (w+4*4)(CTX), RK1; |
| 241 | 293 | ||
| 242 | popq %rcx; | 294 | popq %rcx; |
| 243 | popq %rbx; | 295 | popq %rbx; |
| 296 | popq %rbp; | ||
| 244 | 297 | ||
| 245 | leaq (4*4*4)(%rsi), %rax; | 298 | leaq (4*4*4)(%r11), %rax; |
| 246 | 299 | ||
| 247 | testb %cl, %cl; | 300 | testb %cl, %cl; |
| 248 | jnz __enc_xor8; | 301 | jnz __enc_xor8; |
| 249 | 302 | ||
| 250 | outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); | 303 | outunpack_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); |
| 251 | outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); | 304 | outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); |
| 252 | 305 | ||
| 253 | ret; | 306 | ret; |
| 254 | 307 | ||
| 255 | __enc_xor8: | 308 | __enc_xor8: |
| 256 | outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); | 309 | outunpack_xor_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); |
| 257 | outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); | 310 | outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); |
| 258 | 311 | ||
| 259 | ret; | 312 | ret; |
| 260 | 313 | ||
| @@ -269,16 +322,19 @@ twofish_dec_blk_8way: | |||
| 269 | * %rdx: src | 322 | * %rdx: src |
| 270 | */ | 323 | */ |
| 271 | 324 | ||
| 325 | pushq %rbp; | ||
| 272 | pushq %rbx; | 326 | pushq %rbx; |
| 273 | 327 | ||
| 274 | vmovdqu (w+4*4)(CTX), RK1; | 328 | vmovdqu (w+4*4)(CTX), RK1; |
| 275 | 329 | ||
| 276 | leaq (4*4*4)(%rdx), %rax; | 330 | leaq (4*4*4)(%rdx), %rax; |
| 277 | inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); | 331 | inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); |
| 278 | inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); | 332 | preload_rgi(RC1); |
| 333 | rotate_1l(RA1); | ||
| 334 | inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); | ||
| 335 | rotate_1l(RA2); | ||
| 279 | 336 | ||
| 280 | xorq RID1, RID1; | 337 | movq %rsi, %r11; |
| 281 | xorq RID2, RID2; | ||
| 282 | 338 | ||
| 283 | decrypt_cycle(7); | 339 | decrypt_cycle(7); |
| 284 | decrypt_cycle(6); | 340 | decrypt_cycle(6); |
| @@ -287,14 +343,15 @@ twofish_dec_blk_8way: | |||
| 287 | decrypt_cycle(3); | 343 | decrypt_cycle(3); |
| 288 | decrypt_cycle(2); | 344 | decrypt_cycle(2); |
| 289 | decrypt_cycle(1); | 345 | decrypt_cycle(1); |
| 290 | decrypt_cycle(0); | 346 | decrypt_cycle_last(0); |
| 291 | 347 | ||
| 292 | vmovdqu (w)(CTX), RK1; | 348 | vmovdqu (w)(CTX), RK1; |
| 293 | 349 | ||
| 294 | popq %rbx; | 350 | popq %rbx; |
| 351 | popq %rbp; | ||
| 295 | 352 | ||
| 296 | leaq (4*4*4)(%rsi), %rax; | 353 | leaq (4*4*4)(%r11), %rax; |
| 297 | outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); | 354 | outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); |
| 298 | outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); | 355 | outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); |
| 299 | 356 | ||
| 300 | ret; | 357 | ret; |
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 782b67ddaf6a..e7708b5442e0 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c | |||
| @@ -378,7 +378,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 378 | .cra_alignmask = 0, | 378 | .cra_alignmask = 0, |
| 379 | .cra_type = &crypto_blkcipher_type, | 379 | .cra_type = &crypto_blkcipher_type, |
| 380 | .cra_module = THIS_MODULE, | 380 | .cra_module = THIS_MODULE, |
| 381 | .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list), | ||
| 382 | .cra_u = { | 381 | .cra_u = { |
| 383 | .blkcipher = { | 382 | .blkcipher = { |
| 384 | .min_keysize = TF_MIN_KEY_SIZE, | 383 | .min_keysize = TF_MIN_KEY_SIZE, |
| @@ -398,7 +397,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 398 | .cra_alignmask = 0, | 397 | .cra_alignmask = 0, |
| 399 | .cra_type = &crypto_blkcipher_type, | 398 | .cra_type = &crypto_blkcipher_type, |
| 400 | .cra_module = THIS_MODULE, | 399 | .cra_module = THIS_MODULE, |
| 401 | .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list), | ||
| 402 | .cra_u = { | 400 | .cra_u = { |
| 403 | .blkcipher = { | 401 | .blkcipher = { |
| 404 | .min_keysize = TF_MIN_KEY_SIZE, | 402 | .min_keysize = TF_MIN_KEY_SIZE, |
| @@ -418,7 +416,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 418 | .cra_alignmask = 0, | 416 | .cra_alignmask = 0, |
| 419 | .cra_type = &crypto_blkcipher_type, | 417 | .cra_type = &crypto_blkcipher_type, |
| 420 | .cra_module = THIS_MODULE, | 418 | .cra_module = THIS_MODULE, |
| 421 | .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list), | ||
| 422 | .cra_u = { | 419 | .cra_u = { |
| 423 | .blkcipher = { | 420 | .blkcipher = { |
| 424 | .min_keysize = TF_MIN_KEY_SIZE, | 421 | .min_keysize = TF_MIN_KEY_SIZE, |
| @@ -439,7 +436,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 439 | .cra_alignmask = 0, | 436 | .cra_alignmask = 0, |
| 440 | .cra_type = &crypto_blkcipher_type, | 437 | .cra_type = &crypto_blkcipher_type, |
| 441 | .cra_module = THIS_MODULE, | 438 | .cra_module = THIS_MODULE, |
| 442 | .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list), | ||
| 443 | .cra_exit = lrw_twofish_exit_tfm, | 439 | .cra_exit = lrw_twofish_exit_tfm, |
| 444 | .cra_u = { | 440 | .cra_u = { |
| 445 | .blkcipher = { | 441 | .blkcipher = { |
| @@ -463,7 +459,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 463 | .cra_alignmask = 0, | 459 | .cra_alignmask = 0, |
| 464 | .cra_type = &crypto_blkcipher_type, | 460 | .cra_type = &crypto_blkcipher_type, |
| 465 | .cra_module = THIS_MODULE, | 461 | .cra_module = THIS_MODULE, |
| 466 | .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list), | ||
| 467 | .cra_u = { | 462 | .cra_u = { |
| 468 | .blkcipher = { | 463 | .blkcipher = { |
| 469 | .min_keysize = TF_MIN_KEY_SIZE * 2, | 464 | .min_keysize = TF_MIN_KEY_SIZE * 2, |
| @@ -484,7 +479,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 484 | .cra_alignmask = 0, | 479 | .cra_alignmask = 0, |
| 485 | .cra_type = &crypto_ablkcipher_type, | 480 | .cra_type = &crypto_ablkcipher_type, |
| 486 | .cra_module = THIS_MODULE, | 481 | .cra_module = THIS_MODULE, |
| 487 | .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list), | ||
| 488 | .cra_init = ablk_init, | 482 | .cra_init = ablk_init, |
| 489 | .cra_exit = ablk_exit, | 483 | .cra_exit = ablk_exit, |
| 490 | .cra_u = { | 484 | .cra_u = { |
| @@ -506,7 +500,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 506 | .cra_alignmask = 0, | 500 | .cra_alignmask = 0, |
| 507 | .cra_type = &crypto_ablkcipher_type, | 501 | .cra_type = &crypto_ablkcipher_type, |
| 508 | .cra_module = THIS_MODULE, | 502 | .cra_module = THIS_MODULE, |
| 509 | .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list), | ||
| 510 | .cra_init = ablk_init, | 503 | .cra_init = ablk_init, |
| 511 | .cra_exit = ablk_exit, | 504 | .cra_exit = ablk_exit, |
| 512 | .cra_u = { | 505 | .cra_u = { |
| @@ -529,7 +522,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 529 | .cra_alignmask = 0, | 522 | .cra_alignmask = 0, |
| 530 | .cra_type = &crypto_ablkcipher_type, | 523 | .cra_type = &crypto_ablkcipher_type, |
| 531 | .cra_module = THIS_MODULE, | 524 | .cra_module = THIS_MODULE, |
| 532 | .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list), | ||
| 533 | .cra_init = ablk_init, | 525 | .cra_init = ablk_init, |
| 534 | .cra_exit = ablk_exit, | 526 | .cra_exit = ablk_exit, |
| 535 | .cra_u = { | 527 | .cra_u = { |
| @@ -553,7 +545,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 553 | .cra_alignmask = 0, | 545 | .cra_alignmask = 0, |
| 554 | .cra_type = &crypto_ablkcipher_type, | 546 | .cra_type = &crypto_ablkcipher_type, |
| 555 | .cra_module = THIS_MODULE, | 547 | .cra_module = THIS_MODULE, |
| 556 | .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list), | ||
| 557 | .cra_init = ablk_init, | 548 | .cra_init = ablk_init, |
| 558 | .cra_exit = ablk_exit, | 549 | .cra_exit = ablk_exit, |
| 559 | .cra_u = { | 550 | .cra_u = { |
| @@ -578,7 +569,6 @@ static struct crypto_alg twofish_algs[10] = { { | |||
| 578 | .cra_alignmask = 0, | 569 | .cra_alignmask = 0, |
| 579 | .cra_type = &crypto_ablkcipher_type, | 570 | .cra_type = &crypto_ablkcipher_type, |
| 580 | .cra_module = THIS_MODULE, | 571 | .cra_module = THIS_MODULE, |
| 581 | .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list), | ||
| 582 | .cra_init = ablk_init, | 572 | .cra_init = ablk_init, |
| 583 | .cra_exit = ablk_exit, | 573 | .cra_exit = ablk_exit, |
| 584 | .cra_u = { | 574 | .cra_u = { |
diff --git a/arch/x86/crypto/twofish_glue.c b/arch/x86/crypto/twofish_glue.c index 359ae084275c..0a5202303501 100644 --- a/arch/x86/crypto/twofish_glue.c +++ b/arch/x86/crypto/twofish_glue.c | |||
| @@ -70,7 +70,6 @@ static struct crypto_alg alg = { | |||
| 70 | .cra_ctxsize = sizeof(struct twofish_ctx), | 70 | .cra_ctxsize = sizeof(struct twofish_ctx), |
| 71 | .cra_alignmask = 0, | 71 | .cra_alignmask = 0, |
| 72 | .cra_module = THIS_MODULE, | 72 | .cra_module = THIS_MODULE, |
| 73 | .cra_list = LIST_HEAD_INIT(alg.cra_list), | ||
| 74 | .cra_u = { | 73 | .cra_u = { |
| 75 | .cipher = { | 74 | .cipher = { |
| 76 | .cia_min_keysize = TF_MIN_KEY_SIZE, | 75 | .cia_min_keysize = TF_MIN_KEY_SIZE, |
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 15f9347316c8..aa3eb358b7e8 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c | |||
| @@ -342,7 +342,6 @@ static struct crypto_alg tf_algs[5] = { { | |||
| 342 | .cra_alignmask = 0, | 342 | .cra_alignmask = 0, |
| 343 | .cra_type = &crypto_blkcipher_type, | 343 | .cra_type = &crypto_blkcipher_type, |
| 344 | .cra_module = THIS_MODULE, | 344 | .cra_module = THIS_MODULE, |
| 345 | .cra_list = LIST_HEAD_INIT(tf_algs[0].cra_list), | ||
| 346 | .cra_u = { | 345 | .cra_u = { |
| 347 | .blkcipher = { | 346 | .blkcipher = { |
| 348 | .min_keysize = TF_MIN_KEY_SIZE, | 347 | .min_keysize = TF_MIN_KEY_SIZE, |
| @@ -362,7 +361,6 @@ static struct crypto_alg tf_algs[5] = { { | |||
| 362 | .cra_alignmask = 0, | 361 | .cra_alignmask = 0, |
| 363 | .cra_type = &crypto_blkcipher_type, | 362 | .cra_type = &crypto_blkcipher_type, |
| 364 | .cra_module = THIS_MODULE, | 363 | .cra_module = THIS_MODULE, |
| 365 | .cra_list = LIST_HEAD_INIT(tf_algs[1].cra_list), | ||
| 366 | .cra_u = { | 364 | .cra_u = { |
| 367 | .blkcipher = { | 365 | .blkcipher = { |
| 368 | .min_keysize = TF_MIN_KEY_SIZE, | 366 | .min_keysize = TF_MIN_KEY_SIZE, |
| @@ -383,7 +381,6 @@ static struct crypto_alg tf_algs[5] = { { | |||
| 383 | .cra_alignmask = 0, | 381 | .cra_alignmask = 0, |
| 384 | .cra_type = &crypto_blkcipher_type, | 382 | .cra_type = &crypto_blkcipher_type, |
| 385 | .cra_module = THIS_MODULE, | 383 | .cra_module = THIS_MODULE, |
| 386 | .cra_list = LIST_HEAD_INIT(tf_algs[2].cra_list), | ||
| 387 | .cra_u = { | 384 | .cra_u = { |
| 388 | .blkcipher = { | 385 | .blkcipher = { |
| 389 | .min_keysize = TF_MIN_KEY_SIZE, | 386 | .min_keysize = TF_MIN_KEY_SIZE, |
| @@ -404,7 +401,6 @@ static struct crypto_alg tf_algs[5] = { { | |||
| 404 | .cra_alignmask = 0, | 401 | .cra_alignmask = 0, |
| 405 | .cra_type = &crypto_blkcipher_type, | 402 | .cra_type = &crypto_blkcipher_type, |
| 406 | .cra_module = THIS_MODULE, | 403 | .cra_module = THIS_MODULE, |
| 407 | .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list), | ||
| 408 | .cra_exit = lrw_twofish_exit_tfm, | 404 | .cra_exit = lrw_twofish_exit_tfm, |
| 409 | .cra_u = { | 405 | .cra_u = { |
| 410 | .blkcipher = { | 406 | .blkcipher = { |
| @@ -426,7 +422,6 @@ static struct crypto_alg tf_algs[5] = { { | |||
| 426 | .cra_alignmask = 0, | 422 | .cra_alignmask = 0, |
| 427 | .cra_type = &crypto_blkcipher_type, | 423 | .cra_type = &crypto_blkcipher_type, |
| 428 | .cra_module = THIS_MODULE, | 424 | .cra_module = THIS_MODULE, |
| 429 | .cra_list = LIST_HEAD_INIT(tf_algs[4].cra_list), | ||
| 430 | .cra_u = { | 425 | .cra_u = { |
| 431 | .blkcipher = { | 426 | .blkcipher = { |
| 432 | .min_keysize = TF_MIN_KEY_SIZE * 2, | 427 | .min_keysize = TF_MIN_KEY_SIZE * 2, |
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild index f9c0d3ba9e84..1595d6813432 100644 --- a/arch/x86/include/asm/Kbuild +++ b/arch/x86/include/asm/Kbuild | |||
| @@ -22,7 +22,3 @@ header-y += sigcontext32.h | |||
| 22 | header-y += ucontext.h | 22 | header-y += ucontext.h |
| 23 | header-y += vm86.h | 23 | header-y += vm86.h |
| 24 | header-y += vsyscall.h | 24 | header-y += vsyscall.h |
| 25 | |||
| 26 | genhdr-y += unistd_32.h | ||
| 27 | genhdr-y += unistd_64.h | ||
| 28 | genhdr-y += unistd_x32.h | ||
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index f34261296ffb..338803422239 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h | |||
| @@ -409,7 +409,7 @@ extern struct apic *apic; | |||
| 409 | * to enforce the order with in them. | 409 | * to enforce the order with in them. |
| 410 | */ | 410 | */ |
| 411 | #define apic_driver(sym) \ | 411 | #define apic_driver(sym) \ |
| 412 | static struct apic *__apicdrivers_##sym __used \ | 412 | static const struct apic *__apicdrivers_##sym __used \ |
| 413 | __aligned(sizeof(struct apic *)) \ | 413 | __aligned(sizeof(struct apic *)) \ |
| 414 | __section(.apicdrivers) = { &sym } | 414 | __section(.apicdrivers) = { &sym } |
| 415 | 415 | ||
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index 58cb6d4085f7..250b8774c158 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h | |||
| @@ -309,9 +309,9 @@ static inline void atomic_or_long(unsigned long *v1, unsigned long v2) | |||
| 309 | #define smp_mb__after_atomic_inc() barrier() | 309 | #define smp_mb__after_atomic_inc() barrier() |
| 310 | 310 | ||
| 311 | #ifdef CONFIG_X86_32 | 311 | #ifdef CONFIG_X86_32 |
| 312 | # include "atomic64_32.h" | 312 | # include <asm/atomic64_32.h> |
| 313 | #else | 313 | #else |
| 314 | # include "atomic64_64.h" | 314 | # include <asm/atomic64_64.h> |
| 315 | #endif | 315 | #endif |
| 316 | 316 | ||
| 317 | #endif /* _ASM_X86_ATOMIC_H */ | 317 | #endif /* _ASM_X86_ATOMIC_H */ |
diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h index 7f8422a28a46..0fa675033912 100644 --- a/arch/x86/include/asm/calling.h +++ b/arch/x86/include/asm/calling.h | |||
| @@ -46,7 +46,7 @@ For 32-bit we have the following conventions - kernel is built with | |||
| 46 | 46 | ||
| 47 | */ | 47 | */ |
| 48 | 48 | ||
| 49 | #include "dwarf2.h" | 49 | #include <asm/dwarf2.h> |
| 50 | 50 | ||
| 51 | /* | 51 | /* |
| 52 | * 64-bit system call stack frame layout defines and helpers, | 52 | * 64-bit system call stack frame layout defines and helpers, |
diff --git a/arch/x86/include/asm/checksum.h b/arch/x86/include/asm/checksum.h index 848850fd7d62..5f5bb0f97361 100644 --- a/arch/x86/include/asm/checksum.h +++ b/arch/x86/include/asm/checksum.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_X86_32 | 1 | #ifdef CONFIG_X86_32 |
| 2 | # include "checksum_32.h" | 2 | # include <asm/checksum_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "checksum_64.h" | 4 | # include <asm/checksum_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h index 99480e55973d..8d871eaddb66 100644 --- a/arch/x86/include/asm/cmpxchg.h +++ b/arch/x86/include/asm/cmpxchg.h | |||
| @@ -138,9 +138,9 @@ extern void __add_wrong_size(void) | |||
| 138 | __raw_cmpxchg((ptr), (old), (new), (size), "") | 138 | __raw_cmpxchg((ptr), (old), (new), (size), "") |
| 139 | 139 | ||
| 140 | #ifdef CONFIG_X86_32 | 140 | #ifdef CONFIG_X86_32 |
| 141 | # include "cmpxchg_32.h" | 141 | # include <asm/cmpxchg_32.h> |
| 142 | #else | 142 | #else |
| 143 | # include "cmpxchg_64.h" | 143 | # include <asm/cmpxchg_64.h> |
| 144 | #endif | 144 | #endif |
| 145 | 145 | ||
| 146 | #ifdef __HAVE_ARCH_CMPXCHG | 146 | #ifdef __HAVE_ARCH_CMPXCHG |
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index fedf32b73e65..59c6c401f79f 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h | |||
| @@ -41,6 +41,7 @@ typedef s64 __attribute__((aligned(4))) compat_s64; | |||
| 41 | typedef u32 compat_uint_t; | 41 | typedef u32 compat_uint_t; |
| 42 | typedef u32 compat_ulong_t; | 42 | typedef u32 compat_ulong_t; |
| 43 | typedef u64 __attribute__((aligned(4))) compat_u64; | 43 | typedef u64 __attribute__((aligned(4))) compat_u64; |
| 44 | typedef u32 compat_uptr_t; | ||
| 44 | 45 | ||
| 45 | struct compat_timespec { | 46 | struct compat_timespec { |
| 46 | compat_time_t tv_sec; | 47 | compat_time_t tv_sec; |
| @@ -124,6 +125,78 @@ typedef u32 compat_old_sigset_t; /* at least 32 bits */ | |||
| 124 | 125 | ||
| 125 | typedef u32 compat_sigset_word; | 126 | typedef u32 compat_sigset_word; |
| 126 | 127 | ||
| 128 | typedef union compat_sigval { | ||
| 129 | compat_int_t sival_int; | ||
| 130 | compat_uptr_t sival_ptr; | ||
| 131 | } compat_sigval_t; | ||
| 132 | |||
| 133 | typedef struct compat_siginfo { | ||
| 134 | int si_signo; | ||
| 135 | int si_errno; | ||
| 136 | int si_code; | ||
| 137 | |||
| 138 | union { | ||
| 139 | int _pad[128/sizeof(int) - 3]; | ||
| 140 | |||
| 141 | /* kill() */ | ||
| 142 | struct { | ||
| 143 | unsigned int _pid; /* sender's pid */ | ||
| 144 | unsigned int _uid; /* sender's uid */ | ||
| 145 | } _kill; | ||
| 146 | |||
| 147 | /* POSIX.1b timers */ | ||
| 148 | struct { | ||
| 149 | compat_timer_t _tid; /* timer id */ | ||
| 150 | int _overrun; /* overrun count */ | ||
| 151 | compat_sigval_t _sigval; /* same as below */ | ||
| 152 | int _sys_private; /* not to be passed to user */ | ||
| 153 | int _overrun_incr; /* amount to add to overrun */ | ||
| 154 | } _timer; | ||
| 155 | |||
| 156 | /* POSIX.1b signals */ | ||
| 157 | struct { | ||
| 158 | unsigned int _pid; /* sender's pid */ | ||
| 159 | unsigned int _uid; /* sender's uid */ | ||
| 160 | compat_sigval_t _sigval; | ||
| 161 | } _rt; | ||
| 162 | |||
| 163 | /* SIGCHLD */ | ||
| 164 | struct { | ||
| 165 | unsigned int _pid; /* which child */ | ||
| 166 | unsigned int _uid; /* sender's uid */ | ||
| 167 | int _status; /* exit code */ | ||
| 168 | compat_clock_t _utime; | ||
| 169 | compat_clock_t _stime; | ||
| 170 | } _sigchld; | ||
| 171 | |||
| 172 | /* SIGCHLD (x32 version) */ | ||
| 173 | struct { | ||
| 174 | unsigned int _pid; /* which child */ | ||
| 175 | unsigned int _uid; /* sender's uid */ | ||
| 176 | int _status; /* exit code */ | ||
| 177 | compat_s64 _utime; | ||
| 178 | compat_s64 _stime; | ||
| 179 | } _sigchld_x32; | ||
| 180 | |||
| 181 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 182 | struct { | ||
| 183 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 184 | } _sigfault; | ||
| 185 | |||
| 186 | /* SIGPOLL */ | ||
| 187 | struct { | ||
| 188 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 189 | int _fd; | ||
| 190 | } _sigpoll; | ||
| 191 | |||
| 192 | struct { | ||
| 193 | unsigned int _call_addr; /* calling insn */ | ||
| 194 | int _syscall; /* triggering system call number */ | ||
| 195 | unsigned int _arch; /* AUDIT_ARCH_* of syscall */ | ||
| 196 | } _sigsys; | ||
| 197 | } _sifields; | ||
| 198 | } compat_siginfo_t; | ||
| 199 | |||
| 127 | #define COMPAT_OFF_T_MAX 0x7fffffff | 200 | #define COMPAT_OFF_T_MAX 0x7fffffff |
| 128 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL | 201 | #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL |
| 129 | 202 | ||
| @@ -209,7 +282,6 @@ typedef struct user_regs_struct32 compat_elf_gregset_t; | |||
| 209 | * as pointers because the syscall entry code will have | 282 | * as pointers because the syscall entry code will have |
| 210 | * appropriately converted them already. | 283 | * appropriately converted them already. |
| 211 | */ | 284 | */ |
| 212 | typedef u32 compat_uptr_t; | ||
| 213 | 285 | ||
| 214 | static inline void __user *compat_ptr(compat_uptr_t uptr) | 286 | static inline void __user *compat_ptr(compat_uptr_t uptr) |
| 215 | { | 287 | { |
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 16cae425d1f8..8c297aa53eef 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h | |||
| @@ -4,7 +4,9 @@ | |||
| 4 | #ifndef _ASM_X86_CPUFEATURE_H | 4 | #ifndef _ASM_X86_CPUFEATURE_H |
| 5 | #define _ASM_X86_CPUFEATURE_H | 5 | #define _ASM_X86_CPUFEATURE_H |
| 6 | 6 | ||
| 7 | #ifndef _ASM_X86_REQUIRED_FEATURES_H | ||
| 7 | #include <asm/required-features.h> | 8 | #include <asm/required-features.h> |
| 9 | #endif | ||
| 8 | 10 | ||
| 9 | #define NCAPINTS 10 /* N 32-bit words worth of info */ | 11 | #define NCAPINTS 10 /* N 32-bit words worth of info */ |
| 10 | 12 | ||
diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h index b04cbdb138cd..e6232773ce49 100644 --- a/arch/x86/include/asm/ia32.h +++ b/arch/x86/include/asm/ia32.h | |||
| @@ -86,73 +86,6 @@ struct stat64 { | |||
| 86 | unsigned long long st_ino; | 86 | unsigned long long st_ino; |
| 87 | } __attribute__((packed)); | 87 | } __attribute__((packed)); |
| 88 | 88 | ||
| 89 | typedef struct compat_siginfo { | ||
| 90 | int si_signo; | ||
| 91 | int si_errno; | ||
| 92 | int si_code; | ||
| 93 | |||
| 94 | union { | ||
| 95 | int _pad[((128 / sizeof(int)) - 3)]; | ||
| 96 | |||
| 97 | /* kill() */ | ||
| 98 | struct { | ||
| 99 | unsigned int _pid; /* sender's pid */ | ||
| 100 | unsigned int _uid; /* sender's uid */ | ||
| 101 | } _kill; | ||
| 102 | |||
| 103 | /* POSIX.1b timers */ | ||
| 104 | struct { | ||
| 105 | compat_timer_t _tid; /* timer id */ | ||
| 106 | int _overrun; /* overrun count */ | ||
| 107 | compat_sigval_t _sigval; /* same as below */ | ||
| 108 | int _sys_private; /* not to be passed to user */ | ||
| 109 | int _overrun_incr; /* amount to add to overrun */ | ||
| 110 | } _timer; | ||
| 111 | |||
| 112 | /* POSIX.1b signals */ | ||
| 113 | struct { | ||
| 114 | unsigned int _pid; /* sender's pid */ | ||
| 115 | unsigned int _uid; /* sender's uid */ | ||
| 116 | compat_sigval_t _sigval; | ||
| 117 | } _rt; | ||
| 118 | |||
| 119 | /* SIGCHLD */ | ||
| 120 | struct { | ||
| 121 | unsigned int _pid; /* which child */ | ||
| 122 | unsigned int _uid; /* sender's uid */ | ||
| 123 | int _status; /* exit code */ | ||
| 124 | compat_clock_t _utime; | ||
| 125 | compat_clock_t _stime; | ||
| 126 | } _sigchld; | ||
| 127 | |||
| 128 | /* SIGCHLD (x32 version) */ | ||
| 129 | struct { | ||
| 130 | unsigned int _pid; /* which child */ | ||
| 131 | unsigned int _uid; /* sender's uid */ | ||
| 132 | int _status; /* exit code */ | ||
| 133 | compat_s64 _utime; | ||
| 134 | compat_s64 _stime; | ||
| 135 | } _sigchld_x32; | ||
| 136 | |||
| 137 | /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ | ||
| 138 | struct { | ||
| 139 | unsigned int _addr; /* faulting insn/memory ref. */ | ||
| 140 | } _sigfault; | ||
| 141 | |||
| 142 | /* SIGPOLL */ | ||
| 143 | struct { | ||
| 144 | int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ | ||
| 145 | int _fd; | ||
| 146 | } _sigpoll; | ||
| 147 | |||
| 148 | struct { | ||
| 149 | unsigned int _call_addr; /* calling insn */ | ||
| 150 | int _syscall; /* triggering system call number */ | ||
| 151 | unsigned int _arch; /* AUDIT_ARCH_* of syscall */ | ||
| 152 | } _sigsys; | ||
| 153 | } _sifields; | ||
| 154 | } compat_siginfo_t; | ||
| 155 | |||
| 156 | #define IA32_STACK_TOP IA32_PAGE_OFFSET | 89 | #define IA32_STACK_TOP IA32_PAGE_OFFSET |
| 157 | 90 | ||
| 158 | #ifdef __KERNEL__ | 91 | #ifdef __KERNEL__ |
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h index 41e08cb6a092..a65ec29e6ffb 100644 --- a/arch/x86/include/asm/kvm.h +++ b/arch/x86/include/asm/kvm.h | |||
| @@ -41,6 +41,7 @@ | |||
| 41 | #define __KVM_HAVE_DEBUGREGS | 41 | #define __KVM_HAVE_DEBUGREGS |
| 42 | #define __KVM_HAVE_XSAVE | 42 | #define __KVM_HAVE_XSAVE |
| 43 | #define __KVM_HAVE_XCRS | 43 | #define __KVM_HAVE_XCRS |
| 44 | #define __KVM_HAVE_READONLY_MEM | ||
| 44 | 45 | ||
| 45 | /* Architectural interrupt line count. */ | 46 | /* Architectural interrupt line count. */ |
| 46 | #define KVM_NR_INTERRUPTS 256 | 47 | #define KVM_NR_INTERRUPTS 256 |
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h index c764f43b71c5..15f960c06ff7 100644 --- a/arch/x86/include/asm/kvm_emulate.h +++ b/arch/x86/include/asm/kvm_emulate.h | |||
| @@ -86,6 +86,19 @@ struct x86_instruction_info { | |||
| 86 | 86 | ||
| 87 | struct x86_emulate_ops { | 87 | struct x86_emulate_ops { |
| 88 | /* | 88 | /* |
| 89 | * read_gpr: read a general purpose register (rax - r15) | ||
| 90 | * | ||
| 91 | * @reg: gpr number. | ||
| 92 | */ | ||
| 93 | ulong (*read_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg); | ||
| 94 | /* | ||
| 95 | * write_gpr: write a general purpose register (rax - r15) | ||
| 96 | * | ||
| 97 | * @reg: gpr number. | ||
| 98 | * @val: value to write. | ||
| 99 | */ | ||
| 100 | void (*write_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val); | ||
| 101 | /* | ||
| 89 | * read_std: Read bytes of standard (non-emulated/special) memory. | 102 | * read_std: Read bytes of standard (non-emulated/special) memory. |
| 90 | * Used for descriptor reading. | 103 | * Used for descriptor reading. |
| 91 | * @addr: [IN ] Linear address from which to read. | 104 | * @addr: [IN ] Linear address from which to read. |
| @@ -200,8 +213,9 @@ typedef u32 __attribute__((vector_size(16))) sse128_t; | |||
| 200 | 213 | ||
| 201 | /* Type, address-of, and value of an instruction's operand. */ | 214 | /* Type, address-of, and value of an instruction's operand. */ |
| 202 | struct operand { | 215 | struct operand { |
| 203 | enum { OP_REG, OP_MEM, OP_IMM, OP_XMM, OP_MM, OP_NONE } type; | 216 | enum { OP_REG, OP_MEM, OP_MEM_STR, OP_IMM, OP_XMM, OP_MM, OP_NONE } type; |
| 204 | unsigned int bytes; | 217 | unsigned int bytes; |
| 218 | unsigned int count; | ||
| 205 | union { | 219 | union { |
| 206 | unsigned long orig_val; | 220 | unsigned long orig_val; |
| 207 | u64 orig_val64; | 221 | u64 orig_val64; |
| @@ -221,6 +235,7 @@ struct operand { | |||
| 221 | char valptr[sizeof(unsigned long) + 2]; | 235 | char valptr[sizeof(unsigned long) + 2]; |
| 222 | sse128_t vec_val; | 236 | sse128_t vec_val; |
| 223 | u64 mm_val; | 237 | u64 mm_val; |
| 238 | void *data; | ||
| 224 | }; | 239 | }; |
| 225 | }; | 240 | }; |
| 226 | 241 | ||
| @@ -236,14 +251,23 @@ struct read_cache { | |||
| 236 | unsigned long end; | 251 | unsigned long end; |
| 237 | }; | 252 | }; |
| 238 | 253 | ||
| 254 | /* Execution mode, passed to the emulator. */ | ||
| 255 | enum x86emul_mode { | ||
| 256 | X86EMUL_MODE_REAL, /* Real mode. */ | ||
| 257 | X86EMUL_MODE_VM86, /* Virtual 8086 mode. */ | ||
| 258 | X86EMUL_MODE_PROT16, /* 16-bit protected mode. */ | ||
| 259 | X86EMUL_MODE_PROT32, /* 32-bit protected mode. */ | ||
| 260 | X86EMUL_MODE_PROT64, /* 64-bit (long) mode. */ | ||
| 261 | }; | ||
| 262 | |||
| 239 | struct x86_emulate_ctxt { | 263 | struct x86_emulate_ctxt { |
| 240 | struct x86_emulate_ops *ops; | 264 | const struct x86_emulate_ops *ops; |
| 241 | 265 | ||
| 242 | /* Register state before/after emulation. */ | 266 | /* Register state before/after emulation. */ |
| 243 | unsigned long eflags; | 267 | unsigned long eflags; |
| 244 | unsigned long eip; /* eip before instruction emulation */ | 268 | unsigned long eip; /* eip before instruction emulation */ |
| 245 | /* Emulated execution mode, represented by an X86EMUL_MODE value. */ | 269 | /* Emulated execution mode, represented by an X86EMUL_MODE value. */ |
| 246 | int mode; | 270 | enum x86emul_mode mode; |
| 247 | 271 | ||
| 248 | /* interruptibility state, as a result of execution of STI or MOV SS */ | 272 | /* interruptibility state, as a result of execution of STI or MOV SS */ |
| 249 | int interruptibility; | 273 | int interruptibility; |
| @@ -281,8 +305,10 @@ struct x86_emulate_ctxt { | |||
| 281 | bool rip_relative; | 305 | bool rip_relative; |
| 282 | unsigned long _eip; | 306 | unsigned long _eip; |
| 283 | struct operand memop; | 307 | struct operand memop; |
| 308 | u32 regs_valid; /* bitmaps of registers in _regs[] that can be read */ | ||
| 309 | u32 regs_dirty; /* bitmaps of registers in _regs[] that have been written */ | ||
| 284 | /* Fields above regs are cleared together. */ | 310 | /* Fields above regs are cleared together. */ |
| 285 | unsigned long regs[NR_VCPU_REGS]; | 311 | unsigned long _regs[NR_VCPU_REGS]; |
| 286 | struct operand *memopp; | 312 | struct operand *memopp; |
| 287 | struct fetch_cache fetch; | 313 | struct fetch_cache fetch; |
| 288 | struct read_cache io_read; | 314 | struct read_cache io_read; |
| @@ -293,17 +319,6 @@ struct x86_emulate_ctxt { | |||
| 293 | #define REPE_PREFIX 0xf3 | 319 | #define REPE_PREFIX 0xf3 |
| 294 | #define REPNE_PREFIX 0xf2 | 320 | #define REPNE_PREFIX 0xf2 |
| 295 | 321 | ||
| 296 | /* Execution mode, passed to the emulator. */ | ||
| 297 | #define X86EMUL_MODE_REAL 0 /* Real mode. */ | ||
| 298 | #define X86EMUL_MODE_VM86 1 /* Virtual 8086 mode. */ | ||
| 299 | #define X86EMUL_MODE_PROT16 2 /* 16-bit protected mode. */ | ||
| 300 | #define X86EMUL_MODE_PROT32 4 /* 32-bit protected mode. */ | ||
| 301 | #define X86EMUL_MODE_PROT64 8 /* 64-bit (long) mode. */ | ||
| 302 | |||
| 303 | /* any protected mode */ | ||
| 304 | #define X86EMUL_MODE_PROT (X86EMUL_MODE_PROT16|X86EMUL_MODE_PROT32| \ | ||
| 305 | X86EMUL_MODE_PROT64) | ||
| 306 | |||
| 307 | /* CPUID vendors */ | 322 | /* CPUID vendors */ |
| 308 | #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541 | 323 | #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541 |
| 309 | #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163 | 324 | #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163 |
| @@ -394,4 +409,7 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt, | |||
| 394 | u16 tss_selector, int idt_index, int reason, | 409 | u16 tss_selector, int idt_index, int reason, |
| 395 | bool has_error_code, u32 error_code); | 410 | bool has_error_code, u32 error_code); |
| 396 | int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq); | 411 | int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq); |
| 412 | void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt); | ||
| 413 | void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt); | ||
| 414 | |||
| 397 | #endif /* _ASM_X86_KVM_X86_EMULATE_H */ | 415 | #endif /* _ASM_X86_KVM_X86_EMULATE_H */ |
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1eaa6b056670..b2e11f452435 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h | |||
| @@ -271,10 +271,24 @@ struct kvm_mmu { | |||
| 271 | union kvm_mmu_page_role base_role; | 271 | union kvm_mmu_page_role base_role; |
| 272 | bool direct_map; | 272 | bool direct_map; |
| 273 | 273 | ||
| 274 | /* | ||
| 275 | * Bitmap; bit set = permission fault | ||
| 276 | * Byte index: page fault error code [4:1] | ||
| 277 | * Bit index: pte permissions in ACC_* format | ||
| 278 | */ | ||
| 279 | u8 permissions[16]; | ||
| 280 | |||
| 274 | u64 *pae_root; | 281 | u64 *pae_root; |
| 275 | u64 *lm_root; | 282 | u64 *lm_root; |
| 276 | u64 rsvd_bits_mask[2][4]; | 283 | u64 rsvd_bits_mask[2][4]; |
| 277 | 284 | ||
| 285 | /* | ||
| 286 | * Bitmap: bit set = last pte in walk | ||
| 287 | * index[0:1]: level (zero-based) | ||
| 288 | * index[2]: pte.ps | ||
| 289 | */ | ||
| 290 | u8 last_pte_bitmap; | ||
| 291 | |||
| 278 | bool nx; | 292 | bool nx; |
| 279 | 293 | ||
| 280 | u64 pdptrs[4]; /* pae */ | 294 | u64 pdptrs[4]; /* pae */ |
| @@ -398,12 +412,15 @@ struct kvm_vcpu_arch { | |||
| 398 | struct x86_emulate_ctxt emulate_ctxt; | 412 | struct x86_emulate_ctxt emulate_ctxt; |
| 399 | bool emulate_regs_need_sync_to_vcpu; | 413 | bool emulate_regs_need_sync_to_vcpu; |
| 400 | bool emulate_regs_need_sync_from_vcpu; | 414 | bool emulate_regs_need_sync_from_vcpu; |
| 415 | int (*complete_userspace_io)(struct kvm_vcpu *vcpu); | ||
| 401 | 416 | ||
| 402 | gpa_t time; | 417 | gpa_t time; |
| 403 | struct pvclock_vcpu_time_info hv_clock; | 418 | struct pvclock_vcpu_time_info hv_clock; |
| 404 | unsigned int hw_tsc_khz; | 419 | unsigned int hw_tsc_khz; |
| 405 | unsigned int time_offset; | 420 | unsigned int time_offset; |
| 406 | struct page *time_page; | 421 | struct page *time_page; |
| 422 | /* set guest stopped flag in pvclock flags field */ | ||
| 423 | bool pvclock_set_guest_stopped_request; | ||
| 407 | 424 | ||
| 408 | struct { | 425 | struct { |
| 409 | u64 msr_val; | 426 | u64 msr_val; |
| @@ -438,6 +455,7 @@ struct kvm_vcpu_arch { | |||
| 438 | unsigned long dr6; | 455 | unsigned long dr6; |
| 439 | unsigned long dr7; | 456 | unsigned long dr7; |
| 440 | unsigned long eff_db[KVM_NR_DB_REGS]; | 457 | unsigned long eff_db[KVM_NR_DB_REGS]; |
| 458 | unsigned long guest_debug_dr7; | ||
| 441 | 459 | ||
| 442 | u64 mcg_cap; | 460 | u64 mcg_cap; |
| 443 | u64 mcg_status; | 461 | u64 mcg_status; |
| @@ -484,14 +502,24 @@ struct kvm_vcpu_arch { | |||
| 484 | }; | 502 | }; |
| 485 | 503 | ||
| 486 | struct kvm_lpage_info { | 504 | struct kvm_lpage_info { |
| 487 | unsigned long rmap_pde; | ||
| 488 | int write_count; | 505 | int write_count; |
| 489 | }; | 506 | }; |
| 490 | 507 | ||
| 491 | struct kvm_arch_memory_slot { | 508 | struct kvm_arch_memory_slot { |
| 509 | unsigned long *rmap[KVM_NR_PAGE_SIZES]; | ||
| 492 | struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; | 510 | struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; |
| 493 | }; | 511 | }; |
| 494 | 512 | ||
| 513 | struct kvm_apic_map { | ||
| 514 | struct rcu_head rcu; | ||
| 515 | u8 ldr_bits; | ||
| 516 | /* fields bellow are used to decode ldr values in different modes */ | ||
| 517 | u32 cid_shift, cid_mask, lid_mask; | ||
| 518 | struct kvm_lapic *phys_map[256]; | ||
| 519 | /* first index is cluster id second is cpu id in a cluster */ | ||
| 520 | struct kvm_lapic *logical_map[16][16]; | ||
| 521 | }; | ||
| 522 | |||
| 495 | struct kvm_arch { | 523 | struct kvm_arch { |
| 496 | unsigned int n_used_mmu_pages; | 524 | unsigned int n_used_mmu_pages; |
| 497 | unsigned int n_requested_mmu_pages; | 525 | unsigned int n_requested_mmu_pages; |
| @@ -509,6 +537,8 @@ struct kvm_arch { | |||
| 509 | struct kvm_ioapic *vioapic; | 537 | struct kvm_ioapic *vioapic; |
| 510 | struct kvm_pit *vpit; | 538 | struct kvm_pit *vpit; |
| 511 | int vapics_in_nmi_mode; | 539 | int vapics_in_nmi_mode; |
| 540 | struct mutex apic_map_lock; | ||
| 541 | struct kvm_apic_map *apic_map; | ||
| 512 | 542 | ||
| 513 | unsigned int tss_addr; | 543 | unsigned int tss_addr; |
| 514 | struct page *apic_access_page; | 544 | struct page *apic_access_page; |
| @@ -602,8 +632,7 @@ struct kvm_x86_ops { | |||
| 602 | void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); | 632 | void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); |
| 603 | void (*vcpu_put)(struct kvm_vcpu *vcpu); | 633 | void (*vcpu_put)(struct kvm_vcpu *vcpu); |
| 604 | 634 | ||
| 605 | void (*set_guest_debug)(struct kvm_vcpu *vcpu, | 635 | void (*update_db_bp_intercept)(struct kvm_vcpu *vcpu); |
| 606 | struct kvm_guest_debug *dbg); | ||
| 607 | int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); | 636 | int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); |
| 608 | int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); | 637 | int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); |
| 609 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); | 638 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); |
| @@ -941,6 +970,7 @@ extern bool kvm_rebooting; | |||
| 941 | 970 | ||
| 942 | #define KVM_ARCH_WANT_MMU_NOTIFIER | 971 | #define KVM_ARCH_WANT_MMU_NOTIFIER |
| 943 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); | 972 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); |
| 973 | int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end); | ||
| 944 | int kvm_age_hva(struct kvm *kvm, unsigned long hva); | 974 | int kvm_age_hva(struct kvm *kvm, unsigned long hva); |
| 945 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); | 975 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); |
| 946 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); | 976 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); |
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 2f7712e08b1e..eb3e9d85e1f1 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h | |||
| @@ -102,21 +102,21 @@ struct kvm_vcpu_pv_apf_data { | |||
| 102 | extern void kvmclock_init(void); | 102 | extern void kvmclock_init(void); |
| 103 | extern int kvm_register_clock(char *txt); | 103 | extern int kvm_register_clock(char *txt); |
| 104 | 104 | ||
| 105 | #ifdef CONFIG_KVM_CLOCK | 105 | #ifdef CONFIG_KVM_GUEST |
| 106 | bool kvm_check_and_clear_guest_paused(void); | 106 | bool kvm_check_and_clear_guest_paused(void); |
| 107 | #else | 107 | #else |
| 108 | static inline bool kvm_check_and_clear_guest_paused(void) | 108 | static inline bool kvm_check_and_clear_guest_paused(void) |
| 109 | { | 109 | { |
| 110 | return false; | 110 | return false; |
| 111 | } | 111 | } |
| 112 | #endif /* CONFIG_KVMCLOCK */ | 112 | #endif /* CONFIG_KVM_GUEST */ |
| 113 | 113 | ||
| 114 | /* This instruction is vmcall. On non-VT architectures, it will generate a | 114 | /* This instruction is vmcall. On non-VT architectures, it will generate a |
| 115 | * trap that we will then rewrite to the appropriate instruction. | 115 | * trap that we will then rewrite to the appropriate instruction. |
| 116 | */ | 116 | */ |
| 117 | #define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" | 117 | #define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" |
| 118 | 118 | ||
| 119 | /* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun | 119 | /* For KVM hypercalls, a three-byte sequence of either the vmcall or the vmmcall |
| 120 | * instruction. The hypervisor may replace it with something else but only the | 120 | * instruction. The hypervisor may replace it with something else but only the |
| 121 | * instructions are guaranteed to be supported. | 121 | * instructions are guaranteed to be supported. |
| 122 | * | 122 | * |
diff --git a/arch/x86/include/asm/mmzone.h b/arch/x86/include/asm/mmzone.h index 64217ea16a36..d497bc425cae 100644 --- a/arch/x86/include/asm/mmzone.h +++ b/arch/x86/include/asm/mmzone.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_X86_32 | 1 | #ifdef CONFIG_X86_32 |
| 2 | # include "mmzone_32.h" | 2 | # include <asm/mmzone_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "mmzone_64.h" | 4 | # include <asm/mmzone_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 957ec87385af..fbee9714d9ab 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h | |||
| @@ -248,6 +248,9 @@ | |||
| 248 | 248 | ||
| 249 | #define MSR_IA32_PERF_STATUS 0x00000198 | 249 | #define MSR_IA32_PERF_STATUS 0x00000198 |
| 250 | #define MSR_IA32_PERF_CTL 0x00000199 | 250 | #define MSR_IA32_PERF_CTL 0x00000199 |
| 251 | #define MSR_AMD_PSTATE_DEF_BASE 0xc0010064 | ||
| 252 | #define MSR_AMD_PERF_STATUS 0xc0010063 | ||
| 253 | #define MSR_AMD_PERF_CTL 0xc0010062 | ||
| 251 | 254 | ||
| 252 | #define MSR_IA32_MPERF 0x000000e7 | 255 | #define MSR_IA32_MPERF 0x000000e7 |
| 253 | #define MSR_IA32_APERF 0x000000e8 | 256 | #define MSR_IA32_APERF 0x000000e8 |
diff --git a/arch/x86/include/asm/mutex.h b/arch/x86/include/asm/mutex.h index a731b9c573a6..7d3a48275394 100644 --- a/arch/x86/include/asm/mutex.h +++ b/arch/x86/include/asm/mutex.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_X86_32 | 1 | #ifdef CONFIG_X86_32 |
| 2 | # include "mutex_32.h" | 2 | # include <asm/mutex_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "mutex_64.h" | 4 | # include <asm/mutex_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h index bfacd2ccf651..49119fcea2dc 100644 --- a/arch/x86/include/asm/numa.h +++ b/arch/x86/include/asm/numa.h | |||
| @@ -53,9 +53,9 @@ static inline int numa_cpu_node(int cpu) | |||
| 53 | #endif /* CONFIG_NUMA */ | 53 | #endif /* CONFIG_NUMA */ |
| 54 | 54 | ||
| 55 | #ifdef CONFIG_X86_32 | 55 | #ifdef CONFIG_X86_32 |
| 56 | # include "numa_32.h" | 56 | # include <asm/numa_32.h> |
| 57 | #else | 57 | #else |
| 58 | # include "numa_64.h" | 58 | # include <asm/numa_64.h> |
| 59 | #endif | 59 | #endif |
| 60 | 60 | ||
| 61 | #ifdef CONFIG_NUMA | 61 | #ifdef CONFIG_NUMA |
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h index df75d07571ce..6e41b9343928 100644 --- a/arch/x86/include/asm/pci.h +++ b/arch/x86/include/asm/pci.h | |||
| @@ -141,7 +141,7 @@ void default_restore_msi_irqs(struct pci_dev *dev, int irq); | |||
| 141 | #endif /* __KERNEL__ */ | 141 | #endif /* __KERNEL__ */ |
| 142 | 142 | ||
| 143 | #ifdef CONFIG_X86_64 | 143 | #ifdef CONFIG_X86_64 |
| 144 | #include "pci_64.h" | 144 | #include <asm/pci_64.h> |
| 145 | #endif | 145 | #endif |
| 146 | 146 | ||
| 147 | /* implement the pci_ DMA API in terms of the generic device dma_ one */ | 147 | /* implement the pci_ DMA API in terms of the generic device dma_ one */ |
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 49afb3f41eb6..fc9948465293 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h | |||
| @@ -384,9 +384,9 @@ pte_t *populate_extra_pte(unsigned long vaddr); | |||
| 384 | #endif /* __ASSEMBLY__ */ | 384 | #endif /* __ASSEMBLY__ */ |
| 385 | 385 | ||
| 386 | #ifdef CONFIG_X86_32 | 386 | #ifdef CONFIG_X86_32 |
| 387 | # include "pgtable_32.h" | 387 | # include <asm/pgtable_32.h> |
| 388 | #else | 388 | #else |
| 389 | # include "pgtable_64.h" | 389 | # include <asm/pgtable_64.h> |
| 390 | #endif | 390 | #endif |
| 391 | 391 | ||
| 392 | #ifndef __ASSEMBLY__ | 392 | #ifndef __ASSEMBLY__ |
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index db8fec6d2953..ec8a1fc9505d 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h | |||
| @@ -174,9 +174,9 @@ | |||
| 174 | #endif | 174 | #endif |
| 175 | 175 | ||
| 176 | #ifdef CONFIG_X86_32 | 176 | #ifdef CONFIG_X86_32 |
| 177 | # include "pgtable_32_types.h" | 177 | # include <asm/pgtable_32_types.h> |
| 178 | #else | 178 | #else |
| 179 | # include "pgtable_64_types.h" | 179 | # include <asm/pgtable_64_types.h> |
| 180 | #endif | 180 | #endif |
| 181 | 181 | ||
| 182 | #ifndef __ASSEMBLY__ | 182 | #ifndef __ASSEMBLY__ |
diff --git a/arch/x86/include/asm/posix_types.h b/arch/x86/include/asm/posix_types.h index 7ef7c3020e5c..bad3665c25fc 100644 --- a/arch/x86/include/asm/posix_types.h +++ b/arch/x86/include/asm/posix_types.h | |||
| @@ -1,15 +1,15 @@ | |||
| 1 | #ifdef __KERNEL__ | 1 | #ifdef __KERNEL__ |
| 2 | # ifdef CONFIG_X86_32 | 2 | # ifdef CONFIG_X86_32 |
| 3 | # include "posix_types_32.h" | 3 | # include <asm/posix_types_32.h> |
| 4 | # else | 4 | # else |
| 5 | # include "posix_types_64.h" | 5 | # include <asm/posix_types_64.h> |
| 6 | # endif | 6 | # endif |
| 7 | #else | 7 | #else |
| 8 | # ifdef __i386__ | 8 | # ifdef __i386__ |
| 9 | # include "posix_types_32.h" | 9 | # include <asm/posix_types_32.h> |
| 10 | # elif defined(__ILP32__) | 10 | # elif defined(__ILP32__) |
| 11 | # include "posix_types_x32.h" | 11 | # include <asm/posix_types_x32.h> |
| 12 | # else | 12 | # else |
| 13 | # include "posix_types_64.h" | 13 | # include <asm/posix_types_64.h> |
| 14 | # endif | 14 | # endif |
| 15 | #endif | 15 | #endif |
diff --git a/arch/x86/include/asm/seccomp.h b/arch/x86/include/asm/seccomp.h index c62e58a5a90d..0f3d7f099224 100644 --- a/arch/x86/include/asm/seccomp.h +++ b/arch/x86/include/asm/seccomp.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_X86_32 | 1 | #ifdef CONFIG_X86_32 |
| 2 | # include "seccomp_32.h" | 2 | # include <asm/seccomp_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "seccomp_64.h" | 4 | # include <asm/seccomp_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/x86/include/asm/string.h b/arch/x86/include/asm/string.h index 6dfd6d9373a0..09224d7a5862 100644 --- a/arch/x86/include/asm/string.h +++ b/arch/x86/include/asm/string.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_X86_32 | 1 | #ifdef CONFIG_X86_32 |
| 2 | # include "string_32.h" | 2 | # include <asm/string_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "string_64.h" | 4 | # include <asm/string_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/x86/include/asm/suspend.h b/arch/x86/include/asm/suspend.h index 9bd521fe4570..2fab6c2c3575 100644 --- a/arch/x86/include/asm/suspend.h +++ b/arch/x86/include/asm/suspend.h | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | #ifdef CONFIG_X86_32 | 1 | #ifdef CONFIG_X86_32 |
| 2 | # include "suspend_32.h" | 2 | # include <asm/suspend_32.h> |
| 3 | #else | 3 | #else |
| 4 | # include "suspend_64.h" | 4 | # include <asm/suspend_64.h> |
| 5 | #endif | 5 | #endif |
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index a91acfbb1a98..7ccf8d131535 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h | |||
| @@ -589,9 +589,9 @@ extern struct movsl_mask { | |||
| 589 | #define ARCH_HAS_NOCACHE_UACCESS 1 | 589 | #define ARCH_HAS_NOCACHE_UACCESS 1 |
| 590 | 590 | ||
| 591 | #ifdef CONFIG_X86_32 | 591 | #ifdef CONFIG_X86_32 |
| 592 | # include "uaccess_32.h" | 592 | # include <asm/uaccess_32.h> |
| 593 | #else | 593 | #else |
| 594 | # include "uaccess_64.h" | 594 | # include <asm/uaccess_64.h> |
| 595 | #endif | 595 | #endif |
| 596 | 596 | ||
| 597 | #endif /* _ASM_X86_UACCESS_H */ | 597 | #endif /* _ASM_X86_UACCESS_H */ |
diff --git a/arch/x86/include/asm/user.h b/arch/x86/include/asm/user.h index 24532c7da3d6..ccab4af1646d 100644 --- a/arch/x86/include/asm/user.h +++ b/arch/x86/include/asm/user.h | |||
| @@ -2,9 +2,9 @@ | |||
| 2 | #define _ASM_X86_USER_H | 2 | #define _ASM_X86_USER_H |
| 3 | 3 | ||
| 4 | #ifdef CONFIG_X86_32 | 4 | #ifdef CONFIG_X86_32 |
| 5 | # include "user_32.h" | 5 | # include <asm/user_32.h> |
| 6 | #else | 6 | #else |
| 7 | # include "user_64.h" | 7 | # include <asm/user_64.h> |
| 8 | #endif | 8 | #endif |
| 9 | 9 | ||
| 10 | #include <asm/types.h> | 10 | #include <asm/types.h> |
diff --git a/arch/x86/include/asm/xen/interface.h b/arch/x86/include/asm/xen/interface.h index cbf0c9d50b92..1707cfa928fb 100644 --- a/arch/x86/include/asm/xen/interface.h +++ b/arch/x86/include/asm/xen/interface.h | |||
| @@ -47,6 +47,10 @@ | |||
| 47 | #endif | 47 | #endif |
| 48 | 48 | ||
| 49 | #ifndef __ASSEMBLY__ | 49 | #ifndef __ASSEMBLY__ |
| 50 | /* Explicitly size integers that represent pfns in the public interface | ||
| 51 | * with Xen so that on ARM we can have one ABI that works for 32 and 64 | ||
| 52 | * bit guests. */ | ||
| 53 | typedef unsigned long xen_pfn_t; | ||
| 50 | /* Guest handles for primitive C types. */ | 54 | /* Guest handles for primitive C types. */ |
| 51 | __DEFINE_GUEST_HANDLE(uchar, unsigned char); | 55 | __DEFINE_GUEST_HANDLE(uchar, unsigned char); |
| 52 | __DEFINE_GUEST_HANDLE(uint, unsigned int); | 56 | __DEFINE_GUEST_HANDLE(uint, unsigned int); |
| @@ -57,6 +61,7 @@ DEFINE_GUEST_HANDLE(long); | |||
| 57 | DEFINE_GUEST_HANDLE(void); | 61 | DEFINE_GUEST_HANDLE(void); |
| 58 | DEFINE_GUEST_HANDLE(uint64_t); | 62 | DEFINE_GUEST_HANDLE(uint64_t); |
| 59 | DEFINE_GUEST_HANDLE(uint32_t); | 63 | DEFINE_GUEST_HANDLE(uint32_t); |
| 64 | DEFINE_GUEST_HANDLE(xen_pfn_t); | ||
| 60 | #endif | 65 | #endif |
| 61 | 66 | ||
| 62 | #ifndef HYPERVISOR_VIRT_START | 67 | #ifndef HYPERVISOR_VIRT_START |
| @@ -116,11 +121,13 @@ struct arch_shared_info { | |||
| 116 | #endif /* !__ASSEMBLY__ */ | 121 | #endif /* !__ASSEMBLY__ */ |
| 117 | 122 | ||
| 118 | #ifdef CONFIG_X86_32 | 123 | #ifdef CONFIG_X86_32 |
| 119 | #include "interface_32.h" | 124 | #include <asm/xen/interface_32.h> |
| 120 | #else | 125 | #else |
| 121 | #include "interface_64.h" | 126 | #include <asm/xen/interface_64.h> |
| 122 | #endif | 127 | #endif |
| 123 | 128 | ||
| 129 | #include <asm/pvclock-abi.h> | ||
| 130 | |||
| 124 | #ifndef __ASSEMBLY__ | 131 | #ifndef __ASSEMBLY__ |
| 125 | /* | 132 | /* |
| 126 | * The following is all CPU context. Note that the fpu_ctxt block is filled | 133 | * The following is all CPU context. Note that the fpu_ctxt block is filled |
diff --git a/arch/x86/include/asm/xen/swiotlb-xen.h b/arch/x86/include/asm/xen/swiotlb-xen.h index 1be1ab7d6a41..ee52fcac6f72 100644 --- a/arch/x86/include/asm/xen/swiotlb-xen.h +++ b/arch/x86/include/asm/xen/swiotlb-xen.h | |||
| @@ -5,10 +5,12 @@ | |||
| 5 | extern int xen_swiotlb; | 5 | extern int xen_swiotlb; |
| 6 | extern int __init pci_xen_swiotlb_detect(void); | 6 | extern int __init pci_xen_swiotlb_detect(void); |
| 7 | extern void __init pci_xen_swiotlb_init(void); | 7 | extern void __init pci_xen_swiotlb_init(void); |
| 8 | extern int pci_xen_swiotlb_init_late(void); | ||
| 8 | #else | 9 | #else |
| 9 | #define xen_swiotlb (0) | 10 | #define xen_swiotlb (0) |
| 10 | static inline int __init pci_xen_swiotlb_detect(void) { return 0; } | 11 | static inline int __init pci_xen_swiotlb_detect(void) { return 0; } |
| 11 | static inline void __init pci_xen_swiotlb_init(void) { } | 12 | static inline void __init pci_xen_swiotlb_init(void) { } |
| 13 | static inline int pci_xen_swiotlb_init_late(void) { return -ENXIO; } | ||
| 12 | #endif | 14 | #endif |
| 13 | 15 | ||
| 14 | #endif /* _ASM_X86_SWIOTLB_XEN_H */ | 16 | #endif /* _ASM_X86_SWIOTLB_XEN_H */ |
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h index 7fcf6f3dbcc3..f8fde90bc45e 100644 --- a/arch/x86/include/asm/xor.h +++ b/arch/x86/include/asm/xor.h | |||
| @@ -3,8 +3,8 @@ | |||
| 3 | # include <asm-generic/xor.h> | 3 | # include <asm-generic/xor.h> |
| 4 | #else | 4 | #else |
| 5 | #ifdef CONFIG_X86_32 | 5 | #ifdef CONFIG_X86_32 |
| 6 | # include "xor_32.h" | 6 | # include <asm/xor_32.h> |
| 7 | #else | 7 | #else |
| 8 | # include "xor_64.h" | 8 | # include <asm/xor_64.h> |
| 9 | #endif | 9 | #endif |
| 10 | #endif | 10 | #endif |
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h index aabd5850bdb9..f79cb7ec0e06 100644 --- a/arch/x86/include/asm/xor_32.h +++ b/arch/x86/include/asm/xor_32.h | |||
| @@ -822,7 +822,7 @@ static struct xor_block_template xor_block_pIII_sse = { | |||
| 822 | }; | 822 | }; |
| 823 | 823 | ||
| 824 | /* Also try the AVX routines */ | 824 | /* Also try the AVX routines */ |
| 825 | #include "xor_avx.h" | 825 | #include <asm/xor_avx.h> |
| 826 | 826 | ||
| 827 | /* Also try the generic routines. */ | 827 | /* Also try the generic routines. */ |
| 828 | #include <asm-generic/xor.h> | 828 | #include <asm-generic/xor.h> |
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h index 5fc06d0b7eb5..87ac522c4af5 100644 --- a/arch/x86/include/asm/xor_64.h +++ b/arch/x86/include/asm/xor_64.h | |||
| @@ -306,7 +306,7 @@ static struct xor_block_template xor_block_sse = { | |||
| 306 | 306 | ||
| 307 | 307 | ||
| 308 | /* Also try the AVX routines */ | 308 | /* Also try the AVX routines */ |
| 309 | #include "xor_avx.h" | 309 | #include <asm/xor_avx.h> |
| 310 | 310 | ||
| 311 | #undef XOR_TRY_TEMPLATES | 311 | #undef XOR_TRY_TEMPLATES |
| 312 | #define XOR_TRY_TEMPLATES \ | 312 | #define XOR_TRY_TEMPLATES \ |
diff --git a/arch/x86/include/uapi/asm/Kbuild b/arch/x86/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..83b6e9a0dce4 --- /dev/null +++ b/arch/x86/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,6 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||
| 4 | genhdr-y += unistd_32.h | ||
| 5 | genhdr-y += unistd_64.h | ||
| 6 | genhdr-y += unistd_x32.h | ||
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 8d7a619718b5..a48ea05157d3 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile | |||
| @@ -81,8 +81,7 @@ obj-$(CONFIG_DEBUG_RODATA_TEST) += test_rodata.o | |||
| 81 | obj-$(CONFIG_DEBUG_NX_TEST) += test_nx.o | 81 | obj-$(CONFIG_DEBUG_NX_TEST) += test_nx.o |
| 82 | obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o | 82 | obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o |
| 83 | 83 | ||
| 84 | obj-$(CONFIG_KVM_GUEST) += kvm.o | 84 | obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o |
| 85 | obj-$(CONFIG_KVM_CLOCK) += kvmclock.o | ||
| 86 | obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o | 85 | obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o |
| 87 | obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o | 86 | obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o |
| 88 | obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o | 87 | obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o |
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index bc552cff2578..a65829ac2b9a 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c | |||
| @@ -30,7 +30,7 @@ | |||
| 30 | 30 | ||
| 31 | static int numachip_system __read_mostly; | 31 | static int numachip_system __read_mostly; |
| 32 | 32 | ||
| 33 | static struct apic apic_numachip __read_mostly; | 33 | static const struct apic apic_numachip __read_mostly; |
| 34 | 34 | ||
| 35 | static unsigned int get_apic_id(unsigned long x) | 35 | static unsigned int get_apic_id(unsigned long x) |
| 36 | { | 36 | { |
| @@ -199,7 +199,7 @@ static int numachip_acpi_madt_oem_check(char *oem_id, char *oem_table_id) | |||
| 199 | return 0; | 199 | return 0; |
| 200 | } | 200 | } |
| 201 | 201 | ||
| 202 | static struct apic apic_numachip __refconst = { | 202 | static const struct apic apic_numachip __refconst = { |
| 203 | 203 | ||
| 204 | .name = "NumaConnect system", | 204 | .name = "NumaConnect system", |
| 205 | .probe = numachip_probe, | 205 | .probe = numachip_probe, |
diff --git a/arch/x86/kernel/cpu/mkcapflags.pl b/arch/x86/kernel/cpu/mkcapflags.pl index c7b3fe2d72e0..091972ef49de 100644 --- a/arch/x86/kernel/cpu/mkcapflags.pl +++ b/arch/x86/kernel/cpu/mkcapflags.pl | |||
| @@ -8,7 +8,10 @@ | |||
| 8 | open(IN, "< $in\0") or die "$0: cannot open: $in: $!\n"; | 8 | open(IN, "< $in\0") or die "$0: cannot open: $in: $!\n"; |
| 9 | open(OUT, "> $out\0") or die "$0: cannot create: $out: $!\n"; | 9 | open(OUT, "> $out\0") or die "$0: cannot create: $out: $!\n"; |
| 10 | 10 | ||
| 11 | print OUT "#include <asm/cpufeature.h>\n\n"; | 11 | print OUT "#ifndef _ASM_X86_CPUFEATURE_H\n"; |
| 12 | print OUT "#include <asm/cpufeature.h>\n"; | ||
| 13 | print OUT "#endif\n"; | ||
| 14 | print OUT "\n"; | ||
| 12 | print OUT "const char * const x86_cap_flags[NCAPINTS*32] = {\n"; | 15 | print OUT "const char * const x86_cap_flags[NCAPINTS*32] = {\n"; |
| 13 | 16 | ||
| 14 | %features = (); | 17 | %features = (); |
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index c1d61ee4b4f1..b3e5e51bc907 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c | |||
| @@ -354,6 +354,7 @@ static void kvm_pv_guest_cpu_reboot(void *unused) | |||
| 354 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) | 354 | if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) |
| 355 | wrmsrl(MSR_KVM_PV_EOI_EN, 0); | 355 | wrmsrl(MSR_KVM_PV_EOI_EN, 0); |
| 356 | kvm_pv_disable_apf(); | 356 | kvm_pv_disable_apf(); |
| 357 | kvm_disable_steal_time(); | ||
| 357 | } | 358 | } |
| 358 | 359 | ||
| 359 | static int kvm_pv_reboot_notify(struct notifier_block *nb, | 360 | static int kvm_pv_reboot_notify(struct notifier_block *nb, |
| @@ -396,9 +397,7 @@ void kvm_disable_steal_time(void) | |||
| 396 | #ifdef CONFIG_SMP | 397 | #ifdef CONFIG_SMP |
| 397 | static void __init kvm_smp_prepare_boot_cpu(void) | 398 | static void __init kvm_smp_prepare_boot_cpu(void) |
| 398 | { | 399 | { |
| 399 | #ifdef CONFIG_KVM_CLOCK | ||
| 400 | WARN_ON(kvm_register_clock("primary cpu clock")); | 400 | WARN_ON(kvm_register_clock("primary cpu clock")); |
| 401 | #endif | ||
| 402 | kvm_guest_cpu_init(); | 401 | kvm_guest_cpu_init(); |
| 403 | native_smp_prepare_boot_cpu(); | 402 | native_smp_prepare_boot_cpu(); |
| 404 | } | 403 | } |
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index af6db6ec5b2a..4929c1be0ac0 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c | |||
| @@ -225,7 +225,7 @@ static struct platform_device rtc_device = { | |||
| 225 | static __init int add_rtc_cmos(void) | 225 | static __init int add_rtc_cmos(void) |
| 226 | { | 226 | { |
| 227 | #ifdef CONFIG_PNP | 227 | #ifdef CONFIG_PNP |
| 228 | static const char *ids[] __initconst = | 228 | static const char * const const ids[] __initconst = |
| 229 | { "PNP0b00", "PNP0b01", "PNP0b02", }; | 229 | { "PNP0b00", "PNP0b01", "PNP0b02", }; |
| 230 | struct pnp_dev *dev; | 230 | struct pnp_dev *dev; |
| 231 | struct pnp_id *id; | 231 | struct pnp_id *id; |
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 4f165479c453..d609be046b57 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c | |||
| @@ -957,7 +957,7 @@ void __init setup_arch(char **cmdline_p) | |||
| 957 | initmem_init(); | 957 | initmem_init(); |
| 958 | memblock_find_dma_reserve(); | 958 | memblock_find_dma_reserve(); |
| 959 | 959 | ||
| 960 | #ifdef CONFIG_KVM_CLOCK | 960 | #ifdef CONFIG_KVM_GUEST |
| 961 | kvmclock_init(); | 961 | kvmclock_init(); |
| 962 | #endif | 962 | #endif |
| 963 | 963 | ||
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index a28f338843ea..586f00059805 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig | |||
| @@ -20,6 +20,7 @@ if VIRTUALIZATION | |||
| 20 | config KVM | 20 | config KVM |
| 21 | tristate "Kernel-based Virtual Machine (KVM) support" | 21 | tristate "Kernel-based Virtual Machine (KVM) support" |
| 22 | depends on HAVE_KVM | 22 | depends on HAVE_KVM |
| 23 | depends on HIGH_RES_TIMERS | ||
| 23 | # for device assignment: | 24 | # for device assignment: |
| 24 | depends on PCI | 25 | depends on PCI |
| 25 | # for TASKSTATS/TASK_DELAY_ACCT: | 26 | # for TASKSTATS/TASK_DELAY_ACCT: |
| @@ -37,6 +38,7 @@ config KVM | |||
| 37 | select TASK_DELAY_ACCT | 38 | select TASK_DELAY_ACCT |
| 38 | select PERF_EVENTS | 39 | select PERF_EVENTS |
| 39 | select HAVE_KVM_MSI | 40 | select HAVE_KVM_MSI |
| 41 | select HAVE_KVM_CPU_RELAX_INTERCEPT | ||
| 40 | ---help--- | 42 | ---help--- |
| 41 | Support hosting fully virtualized guest machines using hardware | 43 | Support hosting fully virtualized guest machines using hardware |
| 42 | virtualization extensions. You will need a fairly recent | 44 | virtualization extensions. You will need a fairly recent |
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 4f579e8dcacf..04d30401c5cb 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile | |||
| @@ -12,7 +12,7 @@ kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o) | |||
| 12 | kvm-$(CONFIG_KVM_ASYNC_PF) += $(addprefix ../../../virt/kvm/, async_pf.o) | 12 | kvm-$(CONFIG_KVM_ASYNC_PF) += $(addprefix ../../../virt/kvm/, async_pf.o) |
| 13 | 13 | ||
| 14 | kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ | 14 | kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ |
| 15 | i8254.o timer.o cpuid.o pmu.o | 15 | i8254.o cpuid.o pmu.o |
| 16 | kvm-intel-y += vmx.o | 16 | kvm-intel-y += vmx.o |
| 17 | kvm-amd-y += svm.o | 17 | kvm-amd-y += svm.o |
| 18 | 18 | ||
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 0595f1397b7c..ec79e773342e 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c | |||
| @@ -316,7 +316,7 @@ static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, | |||
| 316 | } | 316 | } |
| 317 | case 7: { | 317 | case 7: { |
| 318 | entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; | 318 | entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; |
| 319 | /* Mask ebx against host capbability word 9 */ | 319 | /* Mask ebx against host capability word 9 */ |
| 320 | if (index == 0) { | 320 | if (index == 0) { |
| 321 | entry->ebx &= kvm_supported_word9_x86_features; | 321 | entry->ebx &= kvm_supported_word9_x86_features; |
| 322 | cpuid_mask(&entry->ebx, 9); | 322 | cpuid_mask(&entry->ebx, 9); |
| @@ -397,8 +397,8 @@ static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, | |||
| 397 | break; | 397 | break; |
| 398 | } | 398 | } |
| 399 | case KVM_CPUID_SIGNATURE: { | 399 | case KVM_CPUID_SIGNATURE: { |
| 400 | char signature[12] = "KVMKVMKVM\0\0"; | 400 | static const char signature[12] = "KVMKVMKVM\0\0"; |
| 401 | u32 *sigptr = (u32 *)signature; | 401 | const u32 *sigptr = (const u32 *)signature; |
| 402 | entry->eax = KVM_CPUID_FEATURES; | 402 | entry->eax = KVM_CPUID_FEATURES; |
| 403 | entry->ebx = sigptr[0]; | 403 | entry->ebx = sigptr[0]; |
| 404 | entry->ecx = sigptr[1]; | 404 | entry->ecx = sigptr[1]; |
| @@ -484,10 +484,10 @@ struct kvm_cpuid_param { | |||
| 484 | u32 func; | 484 | u32 func; |
| 485 | u32 idx; | 485 | u32 idx; |
| 486 | bool has_leaf_count; | 486 | bool has_leaf_count; |
| 487 | bool (*qualifier)(struct kvm_cpuid_param *param); | 487 | bool (*qualifier)(const struct kvm_cpuid_param *param); |
| 488 | }; | 488 | }; |
| 489 | 489 | ||
| 490 | static bool is_centaur_cpu(struct kvm_cpuid_param *param) | 490 | static bool is_centaur_cpu(const struct kvm_cpuid_param *param) |
| 491 | { | 491 | { |
| 492 | return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR; | 492 | return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR; |
| 493 | } | 493 | } |
| @@ -498,7 +498,7 @@ int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | |||
| 498 | struct kvm_cpuid_entry2 *cpuid_entries; | 498 | struct kvm_cpuid_entry2 *cpuid_entries; |
| 499 | int limit, nent = 0, r = -E2BIG, i; | 499 | int limit, nent = 0, r = -E2BIG, i; |
| 500 | u32 func; | 500 | u32 func; |
| 501 | static struct kvm_cpuid_param param[] = { | 501 | static const struct kvm_cpuid_param param[] = { |
| 502 | { .func = 0, .has_leaf_count = true }, | 502 | { .func = 0, .has_leaf_count = true }, |
| 503 | { .func = 0x80000000, .has_leaf_count = true }, | 503 | { .func = 0x80000000, .has_leaf_count = true }, |
| 504 | { .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true }, | 504 | { .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true }, |
| @@ -517,7 +517,7 @@ int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | |||
| 517 | 517 | ||
| 518 | r = 0; | 518 | r = 0; |
| 519 | for (i = 0; i < ARRAY_SIZE(param); i++) { | 519 | for (i = 0; i < ARRAY_SIZE(param); i++) { |
| 520 | struct kvm_cpuid_param *ent = ¶m[i]; | 520 | const struct kvm_cpuid_param *ent = ¶m[i]; |
| 521 | 521 | ||
| 522 | if (ent->qualifier && !ent->qualifier(ent)) | 522 | if (ent->qualifier && !ent->qualifier(ent)) |
| 523 | continue; | 523 | continue; |
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index a3b57a27be88..39171cb307ea 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c | |||
| @@ -161,9 +161,9 @@ struct opcode { | |||
| 161 | u64 intercept : 8; | 161 | u64 intercept : 8; |
| 162 | union { | 162 | union { |
| 163 | int (*execute)(struct x86_emulate_ctxt *ctxt); | 163 | int (*execute)(struct x86_emulate_ctxt *ctxt); |
| 164 | struct opcode *group; | 164 | const struct opcode *group; |
| 165 | struct group_dual *gdual; | 165 | const struct group_dual *gdual; |
| 166 | struct gprefix *gprefix; | 166 | const struct gprefix *gprefix; |
| 167 | } u; | 167 | } u; |
| 168 | int (*check_perm)(struct x86_emulate_ctxt *ctxt); | 168 | int (*check_perm)(struct x86_emulate_ctxt *ctxt); |
| 169 | }; | 169 | }; |
| @@ -202,6 +202,42 @@ struct gprefix { | |||
| 202 | #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a | 202 | #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a |
| 203 | #define EFLG_RESERVED_ONE_MASK 2 | 203 | #define EFLG_RESERVED_ONE_MASK 2 |
| 204 | 204 | ||
| 205 | static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr) | ||
| 206 | { | ||
| 207 | if (!(ctxt->regs_valid & (1 << nr))) { | ||
| 208 | ctxt->regs_valid |= 1 << nr; | ||
| 209 | ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr); | ||
| 210 | } | ||
| 211 | return ctxt->_regs[nr]; | ||
| 212 | } | ||
| 213 | |||
| 214 | static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr) | ||
| 215 | { | ||
| 216 | ctxt->regs_valid |= 1 << nr; | ||
| 217 | ctxt->regs_dirty |= 1 << nr; | ||
| 218 | return &ctxt->_regs[nr]; | ||
| 219 | } | ||
| 220 | |||
| 221 | static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr) | ||
| 222 | { | ||
| 223 | reg_read(ctxt, nr); | ||
| 224 | return reg_write(ctxt, nr); | ||
| 225 | } | ||
| 226 | |||
| 227 | static void writeback_registers(struct x86_emulate_ctxt *ctxt) | ||
| 228 | { | ||
| 229 | unsigned reg; | ||
| 230 | |||
| 231 | for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16) | ||
| 232 | ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]); | ||
| 233 | } | ||
| 234 | |||
| 235 | static void invalidate_registers(struct x86_emulate_ctxt *ctxt) | ||
| 236 | { | ||
| 237 | ctxt->regs_dirty = 0; | ||
| 238 | ctxt->regs_valid = 0; | ||
| 239 | } | ||
| 240 | |||
| 205 | /* | 241 | /* |
| 206 | * Instruction emulation: | 242 | * Instruction emulation: |
| 207 | * Most instructions are emulated directly via a fragment of inline assembly | 243 | * Most instructions are emulated directly via a fragment of inline assembly |
| @@ -374,8 +410,8 @@ struct gprefix { | |||
| 374 | #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \ | 410 | #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \ |
| 375 | do { \ | 411 | do { \ |
| 376 | unsigned long _tmp; \ | 412 | unsigned long _tmp; \ |
| 377 | ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \ | 413 | ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \ |
| 378 | ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \ | 414 | ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \ |
| 379 | \ | 415 | \ |
| 380 | __asm__ __volatile__ ( \ | 416 | __asm__ __volatile__ ( \ |
| 381 | _PRE_EFLAGS("0", "5", "1") \ | 417 | _PRE_EFLAGS("0", "5", "1") \ |
| @@ -494,7 +530,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, in | |||
| 494 | 530 | ||
| 495 | static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc) | 531 | static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc) |
| 496 | { | 532 | { |
| 497 | masked_increment(&ctxt->regs[VCPU_REGS_RSP], stack_mask(ctxt), inc); | 533 | masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc); |
| 498 | } | 534 | } |
| 499 | 535 | ||
| 500 | static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel) | 536 | static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel) |
| @@ -632,8 +668,6 @@ static int __linearize(struct x86_emulate_ctxt *ctxt, | |||
| 632 | 668 | ||
| 633 | la = seg_base(ctxt, addr.seg) + addr.ea; | 669 | la = seg_base(ctxt, addr.seg) + addr.ea; |
| 634 | switch (ctxt->mode) { | 670 | switch (ctxt->mode) { |
| 635 | case X86EMUL_MODE_REAL: | ||
| 636 | break; | ||
| 637 | case X86EMUL_MODE_PROT64: | 671 | case X86EMUL_MODE_PROT64: |
| 638 | if (((signed long)la << 16) >> 16 != la) | 672 | if (((signed long)la << 16) >> 16 != la) |
| 639 | return emulate_gp(ctxt, 0); | 673 | return emulate_gp(ctxt, 0); |
| @@ -655,7 +689,7 @@ static int __linearize(struct x86_emulate_ctxt *ctxt, | |||
| 655 | if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim) | 689 | if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim) |
| 656 | goto bad; | 690 | goto bad; |
| 657 | } else { | 691 | } else { |
| 658 | /* exapand-down segment */ | 692 | /* expand-down segment */ |
| 659 | if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim) | 693 | if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim) |
| 660 | goto bad; | 694 | goto bad; |
| 661 | lim = desc.d ? 0xffffffff : 0xffff; | 695 | lim = desc.d ? 0xffffffff : 0xffff; |
| @@ -663,7 +697,10 @@ static int __linearize(struct x86_emulate_ctxt *ctxt, | |||
| 663 | goto bad; | 697 | goto bad; |
| 664 | } | 698 | } |
| 665 | cpl = ctxt->ops->cpl(ctxt); | 699 | cpl = ctxt->ops->cpl(ctxt); |
| 666 | rpl = sel & 3; | 700 | if (ctxt->mode == X86EMUL_MODE_REAL) |
| 701 | rpl = 0; | ||
| 702 | else | ||
| 703 | rpl = sel & 3; | ||
| 667 | cpl = max(cpl, rpl); | 704 | cpl = max(cpl, rpl); |
| 668 | if (!(desc.type & 8)) { | 705 | if (!(desc.type & 8)) { |
| 669 | /* data segment */ | 706 | /* data segment */ |
| @@ -688,9 +725,9 @@ static int __linearize(struct x86_emulate_ctxt *ctxt, | |||
| 688 | return X86EMUL_CONTINUE; | 725 | return X86EMUL_CONTINUE; |
| 689 | bad: | 726 | bad: |
| 690 | if (addr.seg == VCPU_SREG_SS) | 727 | if (addr.seg == VCPU_SREG_SS) |
| 691 | return emulate_ss(ctxt, addr.seg); | 728 | return emulate_ss(ctxt, sel); |
| 692 | else | 729 | else |
| 693 | return emulate_gp(ctxt, addr.seg); | 730 | return emulate_gp(ctxt, sel); |
| 694 | } | 731 | } |
| 695 | 732 | ||
| 696 | static int linearize(struct x86_emulate_ctxt *ctxt, | 733 | static int linearize(struct x86_emulate_ctxt *ctxt, |
| @@ -786,14 +823,15 @@ static int do_insn_fetch(struct x86_emulate_ctxt *ctxt, | |||
| 786 | * pointer into the block that addresses the relevant register. | 823 | * pointer into the block that addresses the relevant register. |
| 787 | * @highbyte_regs specifies whether to decode AH,CH,DH,BH. | 824 | * @highbyte_regs specifies whether to decode AH,CH,DH,BH. |
| 788 | */ | 825 | */ |
| 789 | static void *decode_register(u8 modrm_reg, unsigned long *regs, | 826 | static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg, |
| 790 | int highbyte_regs) | 827 | int highbyte_regs) |
| 791 | { | 828 | { |
| 792 | void *p; | 829 | void *p; |
| 793 | 830 | ||
| 794 | p = ®s[modrm_reg]; | ||
| 795 | if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8) | 831 | if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8) |
| 796 | p = (unsigned char *)®s[modrm_reg & 3] + 1; | 832 | p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1; |
| 833 | else | ||
| 834 | p = reg_rmw(ctxt, modrm_reg); | ||
| 797 | return p; | 835 | return p; |
| 798 | } | 836 | } |
| 799 | 837 | ||
| @@ -871,23 +909,23 @@ static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg) | |||
| 871 | { | 909 | { |
| 872 | ctxt->ops->get_fpu(ctxt); | 910 | ctxt->ops->get_fpu(ctxt); |
| 873 | switch (reg) { | 911 | switch (reg) { |
| 874 | case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break; | 912 | case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break; |
| 875 | case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break; | 913 | case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break; |
| 876 | case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break; | 914 | case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break; |
| 877 | case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break; | 915 | case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break; |
| 878 | case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break; | 916 | case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break; |
| 879 | case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break; | 917 | case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break; |
| 880 | case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break; | 918 | case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break; |
| 881 | case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break; | 919 | case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break; |
| 882 | #ifdef CONFIG_X86_64 | 920 | #ifdef CONFIG_X86_64 |
| 883 | case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break; | 921 | case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break; |
| 884 | case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break; | 922 | case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break; |
| 885 | case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break; | 923 | case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break; |
| 886 | case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break; | 924 | case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break; |
| 887 | case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break; | 925 | case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break; |
| 888 | case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break; | 926 | case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break; |
| 889 | case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break; | 927 | case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break; |
| 890 | case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break; | 928 | case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break; |
| 891 | #endif | 929 | #endif |
| 892 | default: BUG(); | 930 | default: BUG(); |
| 893 | } | 931 | } |
| @@ -899,23 +937,23 @@ static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, | |||
| 899 | { | 937 | { |
| 900 | ctxt->ops->get_fpu(ctxt); | 938 | ctxt->ops->get_fpu(ctxt); |
| 901 | switch (reg) { | 939 | switch (reg) { |
| 902 | case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break; | 940 | case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break; |
| 903 | case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break; | 941 | case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break; |
| 904 | case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break; | 942 | case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break; |
| 905 | case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break; | 943 | case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break; |
| 906 | case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break; | 944 | case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break; |
| 907 | case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break; | 945 | case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break; |
| 908 | case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break; | 946 | case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break; |
| 909 | case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break; | 947 | case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break; |
| 910 | #ifdef CONFIG_X86_64 | 948 | #ifdef CONFIG_X86_64 |
| 911 | case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break; | 949 | case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break; |
| 912 | case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break; | 950 | case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break; |
| 913 | case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break; | 951 | case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break; |
| 914 | case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break; | 952 | case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break; |
| 915 | case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break; | 953 | case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break; |
| 916 | case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break; | 954 | case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break; |
| 917 | case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break; | 955 | case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break; |
| 918 | case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break; | 956 | case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break; |
| 919 | #endif | 957 | #endif |
| 920 | default: BUG(); | 958 | default: BUG(); |
| 921 | } | 959 | } |
| @@ -982,10 +1020,10 @@ static void decode_register_operand(struct x86_emulate_ctxt *ctxt, | |||
| 982 | 1020 | ||
| 983 | op->type = OP_REG; | 1021 | op->type = OP_REG; |
| 984 | if (ctxt->d & ByteOp) { | 1022 | if (ctxt->d & ByteOp) { |
| 985 | op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs); | 1023 | op->addr.reg = decode_register(ctxt, reg, highbyte_regs); |
| 986 | op->bytes = 1; | 1024 | op->bytes = 1; |
| 987 | } else { | 1025 | } else { |
| 988 | op->addr.reg = decode_register(reg, ctxt->regs, 0); | 1026 | op->addr.reg = decode_register(ctxt, reg, 0); |
| 989 | op->bytes = ctxt->op_bytes; | 1027 | op->bytes = ctxt->op_bytes; |
| 990 | } | 1028 | } |
| 991 | fetch_register_operand(op); | 1029 | fetch_register_operand(op); |
| @@ -1020,8 +1058,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, | |||
| 1020 | if (ctxt->modrm_mod == 3) { | 1058 | if (ctxt->modrm_mod == 3) { |
| 1021 | op->type = OP_REG; | 1059 | op->type = OP_REG; |
| 1022 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; | 1060 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; |
| 1023 | op->addr.reg = decode_register(ctxt->modrm_rm, | 1061 | op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp); |
| 1024 | ctxt->regs, ctxt->d & ByteOp); | ||
| 1025 | if (ctxt->d & Sse) { | 1062 | if (ctxt->d & Sse) { |
| 1026 | op->type = OP_XMM; | 1063 | op->type = OP_XMM; |
| 1027 | op->bytes = 16; | 1064 | op->bytes = 16; |
| @@ -1042,10 +1079,10 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, | |||
| 1042 | op->type = OP_MEM; | 1079 | op->type = OP_MEM; |
| 1043 | 1080 | ||
| 1044 | if (ctxt->ad_bytes == 2) { | 1081 | if (ctxt->ad_bytes == 2) { |
| 1045 | unsigned bx = ctxt->regs[VCPU_REGS_RBX]; | 1082 | unsigned bx = reg_read(ctxt, VCPU_REGS_RBX); |
| 1046 | unsigned bp = ctxt->regs[VCPU_REGS_RBP]; | 1083 | unsigned bp = reg_read(ctxt, VCPU_REGS_RBP); |
| 1047 | unsigned si = ctxt->regs[VCPU_REGS_RSI]; | 1084 | unsigned si = reg_read(ctxt, VCPU_REGS_RSI); |
| 1048 | unsigned di = ctxt->regs[VCPU_REGS_RDI]; | 1085 | unsigned di = reg_read(ctxt, VCPU_REGS_RDI); |
| 1049 | 1086 | ||
| 1050 | /* 16-bit ModR/M decode. */ | 1087 | /* 16-bit ModR/M decode. */ |
| 1051 | switch (ctxt->modrm_mod) { | 1088 | switch (ctxt->modrm_mod) { |
| @@ -1102,17 +1139,17 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, | |||
| 1102 | if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) | 1139 | if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) |
| 1103 | modrm_ea += insn_fetch(s32, ctxt); | 1140 | modrm_ea += insn_fetch(s32, ctxt); |
| 1104 | else { | 1141 | else { |
| 1105 | modrm_ea += ctxt->regs[base_reg]; | 1142 | modrm_ea += reg_read(ctxt, base_reg); |
| 1106 | adjust_modrm_seg(ctxt, base_reg); | 1143 | adjust_modrm_seg(ctxt, base_reg); |
| 1107 | } | 1144 | } |
| 1108 | if (index_reg != 4) | 1145 | if (index_reg != 4) |
| 1109 | modrm_ea += ctxt->regs[index_reg] << scale; | 1146 | modrm_ea += reg_read(ctxt, index_reg) << scale; |
| 1110 | } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { | 1147 | } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { |
| 1111 | if (ctxt->mode == X86EMUL_MODE_PROT64) | 1148 | if (ctxt->mode == X86EMUL_MODE_PROT64) |
| 1112 | ctxt->rip_relative = 1; | 1149 | ctxt->rip_relative = 1; |
| 1113 | } else { | 1150 | } else { |
| 1114 | base_reg = ctxt->modrm_rm; | 1151 | base_reg = ctxt->modrm_rm; |
| 1115 | modrm_ea += ctxt->regs[base_reg]; | 1152 | modrm_ea += reg_read(ctxt, base_reg); |
| 1116 | adjust_modrm_seg(ctxt, base_reg); | 1153 | adjust_modrm_seg(ctxt, base_reg); |
| 1117 | } | 1154 | } |
| 1118 | switch (ctxt->modrm_mod) { | 1155 | switch (ctxt->modrm_mod) { |
| @@ -1179,24 +1216,21 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt, | |||
| 1179 | int rc; | 1216 | int rc; |
| 1180 | struct read_cache *mc = &ctxt->mem_read; | 1217 | struct read_cache *mc = &ctxt->mem_read; |
| 1181 | 1218 | ||
| 1182 | while (size) { | 1219 | if (mc->pos < mc->end) |
| 1183 | int n = min(size, 8u); | 1220 | goto read_cached; |
| 1184 | size -= n; | ||
| 1185 | if (mc->pos < mc->end) | ||
| 1186 | goto read_cached; | ||
| 1187 | 1221 | ||
| 1188 | rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n, | 1222 | WARN_ON((mc->end + size) >= sizeof(mc->data)); |
| 1189 | &ctxt->exception); | ||
| 1190 | if (rc != X86EMUL_CONTINUE) | ||
| 1191 | return rc; | ||
| 1192 | mc->end += n; | ||
| 1193 | 1223 | ||
| 1194 | read_cached: | 1224 | rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size, |
| 1195 | memcpy(dest, mc->data + mc->pos, n); | 1225 | &ctxt->exception); |
| 1196 | mc->pos += n; | 1226 | if (rc != X86EMUL_CONTINUE) |
| 1197 | dest += n; | 1227 | return rc; |
| 1198 | addr += n; | 1228 | |
| 1199 | } | 1229 | mc->end += size; |
| 1230 | |||
| 1231 | read_cached: | ||
| 1232 | memcpy(dest, mc->data + mc->pos, size); | ||
| 1233 | mc->pos += size; | ||
| 1200 | return X86EMUL_CONTINUE; | 1234 | return X86EMUL_CONTINUE; |
| 1201 | } | 1235 | } |
| 1202 | 1236 | ||
| @@ -1253,10 +1287,10 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt, | |||
| 1253 | if (rc->pos == rc->end) { /* refill pio read ahead */ | 1287 | if (rc->pos == rc->end) { /* refill pio read ahead */ |
| 1254 | unsigned int in_page, n; | 1288 | unsigned int in_page, n; |
| 1255 | unsigned int count = ctxt->rep_prefix ? | 1289 | unsigned int count = ctxt->rep_prefix ? |
| 1256 | address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1; | 1290 | address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1; |
| 1257 | in_page = (ctxt->eflags & EFLG_DF) ? | 1291 | in_page = (ctxt->eflags & EFLG_DF) ? |
| 1258 | offset_in_page(ctxt->regs[VCPU_REGS_RDI]) : | 1292 | offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) : |
| 1259 | PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]); | 1293 | PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); |
| 1260 | n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size, | 1294 | n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size, |
| 1261 | count); | 1295 | count); |
| 1262 | if (n == 0) | 1296 | if (n == 0) |
| @@ -1267,8 +1301,15 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt, | |||
| 1267 | rc->end = n * size; | 1301 | rc->end = n * size; |
| 1268 | } | 1302 | } |
| 1269 | 1303 | ||
| 1270 | memcpy(dest, rc->data + rc->pos, size); | 1304 | if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) { |
| 1271 | rc->pos += size; | 1305 | ctxt->dst.data = rc->data + rc->pos; |
| 1306 | ctxt->dst.type = OP_MEM_STR; | ||
| 1307 | ctxt->dst.count = (rc->end - rc->pos) / size; | ||
| 1308 | rc->pos = rc->end; | ||
| 1309 | } else { | ||
| 1310 | memcpy(dest, rc->data + rc->pos, size); | ||
| 1311 | rc->pos += size; | ||
| 1312 | } | ||
| 1272 | return 1; | 1313 | return 1; |
| 1273 | } | 1314 | } |
| 1274 | 1315 | ||
| @@ -1291,7 +1332,7 @@ static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt, | |||
| 1291 | static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt, | 1332 | static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt, |
| 1292 | u16 selector, struct desc_ptr *dt) | 1333 | u16 selector, struct desc_ptr *dt) |
| 1293 | { | 1334 | { |
| 1294 | struct x86_emulate_ops *ops = ctxt->ops; | 1335 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 1295 | 1336 | ||
| 1296 | if (selector & 1 << 2) { | 1337 | if (selector & 1 << 2) { |
| 1297 | struct desc_struct desc; | 1338 | struct desc_struct desc; |
| @@ -1355,19 +1396,15 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, | |||
| 1355 | bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ | 1396 | bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ |
| 1356 | ulong desc_addr; | 1397 | ulong desc_addr; |
| 1357 | int ret; | 1398 | int ret; |
| 1399 | u16 dummy; | ||
| 1358 | 1400 | ||
| 1359 | memset(&seg_desc, 0, sizeof seg_desc); | 1401 | memset(&seg_desc, 0, sizeof seg_desc); |
| 1360 | 1402 | ||
| 1361 | if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) | 1403 | if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) |
| 1362 | || ctxt->mode == X86EMUL_MODE_REAL) { | 1404 | || ctxt->mode == X86EMUL_MODE_REAL) { |
| 1363 | /* set real mode segment descriptor */ | 1405 | /* set real mode segment descriptor */ |
| 1406 | ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg); | ||
| 1364 | set_desc_base(&seg_desc, selector << 4); | 1407 | set_desc_base(&seg_desc, selector << 4); |
| 1365 | set_desc_limit(&seg_desc, 0xffff); | ||
| 1366 | seg_desc.type = 3; | ||
| 1367 | seg_desc.p = 1; | ||
| 1368 | seg_desc.s = 1; | ||
| 1369 | if (ctxt->mode == X86EMUL_MODE_VM86) | ||
| 1370 | seg_desc.dpl = 3; | ||
| 1371 | goto load; | 1408 | goto load; |
| 1372 | } | 1409 | } |
| 1373 | 1410 | ||
| @@ -1396,7 +1433,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, | |||
| 1396 | err_code = selector & 0xfffc; | 1433 | err_code = selector & 0xfffc; |
| 1397 | err_vec = GP_VECTOR; | 1434 | err_vec = GP_VECTOR; |
| 1398 | 1435 | ||
| 1399 | /* can't load system descriptor into segment selecor */ | 1436 | /* can't load system descriptor into segment selector */ |
| 1400 | if (seg <= VCPU_SREG_GS && !seg_desc.s) | 1437 | if (seg <= VCPU_SREG_GS && !seg_desc.s) |
| 1401 | goto exception; | 1438 | goto exception; |
| 1402 | 1439 | ||
| @@ -1516,6 +1553,14 @@ static int writeback(struct x86_emulate_ctxt *ctxt) | |||
| 1516 | if (rc != X86EMUL_CONTINUE) | 1553 | if (rc != X86EMUL_CONTINUE) |
| 1517 | return rc; | 1554 | return rc; |
| 1518 | break; | 1555 | break; |
| 1556 | case OP_MEM_STR: | ||
| 1557 | rc = segmented_write(ctxt, | ||
| 1558 | ctxt->dst.addr.mem, | ||
| 1559 | ctxt->dst.data, | ||
| 1560 | ctxt->dst.bytes * ctxt->dst.count); | ||
| 1561 | if (rc != X86EMUL_CONTINUE) | ||
| 1562 | return rc; | ||
| 1563 | break; | ||
| 1519 | case OP_XMM: | 1564 | case OP_XMM: |
| 1520 | write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm); | 1565 | write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm); |
| 1521 | break; | 1566 | break; |
| @@ -1536,7 +1581,7 @@ static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes) | |||
| 1536 | struct segmented_address addr; | 1581 | struct segmented_address addr; |
| 1537 | 1582 | ||
| 1538 | rsp_increment(ctxt, -bytes); | 1583 | rsp_increment(ctxt, -bytes); |
| 1539 | addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt); | 1584 | addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt); |
| 1540 | addr.seg = VCPU_SREG_SS; | 1585 | addr.seg = VCPU_SREG_SS; |
| 1541 | 1586 | ||
| 1542 | return segmented_write(ctxt, addr, data, bytes); | 1587 | return segmented_write(ctxt, addr, data, bytes); |
| @@ -1555,7 +1600,7 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt, | |||
| 1555 | int rc; | 1600 | int rc; |
| 1556 | struct segmented_address addr; | 1601 | struct segmented_address addr; |
| 1557 | 1602 | ||
| 1558 | addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt); | 1603 | addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt); |
| 1559 | addr.seg = VCPU_SREG_SS; | 1604 | addr.seg = VCPU_SREG_SS; |
| 1560 | rc = segmented_read(ctxt, addr, dest, len); | 1605 | rc = segmented_read(ctxt, addr, dest, len); |
| 1561 | if (rc != X86EMUL_CONTINUE) | 1606 | if (rc != X86EMUL_CONTINUE) |
| @@ -1623,26 +1668,28 @@ static int em_enter(struct x86_emulate_ctxt *ctxt) | |||
| 1623 | int rc; | 1668 | int rc; |
| 1624 | unsigned frame_size = ctxt->src.val; | 1669 | unsigned frame_size = ctxt->src.val; |
| 1625 | unsigned nesting_level = ctxt->src2.val & 31; | 1670 | unsigned nesting_level = ctxt->src2.val & 31; |
| 1671 | ulong rbp; | ||
| 1626 | 1672 | ||
| 1627 | if (nesting_level) | 1673 | if (nesting_level) |
| 1628 | return X86EMUL_UNHANDLEABLE; | 1674 | return X86EMUL_UNHANDLEABLE; |
| 1629 | 1675 | ||
| 1630 | rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt)); | 1676 | rbp = reg_read(ctxt, VCPU_REGS_RBP); |
| 1677 | rc = push(ctxt, &rbp, stack_size(ctxt)); | ||
| 1631 | if (rc != X86EMUL_CONTINUE) | 1678 | if (rc != X86EMUL_CONTINUE) |
| 1632 | return rc; | 1679 | return rc; |
| 1633 | assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP], | 1680 | assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP), |
| 1634 | stack_mask(ctxt)); | 1681 | stack_mask(ctxt)); |
| 1635 | assign_masked(&ctxt->regs[VCPU_REGS_RSP], | 1682 | assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), |
| 1636 | ctxt->regs[VCPU_REGS_RSP] - frame_size, | 1683 | reg_read(ctxt, VCPU_REGS_RSP) - frame_size, |
| 1637 | stack_mask(ctxt)); | 1684 | stack_mask(ctxt)); |
| 1638 | return X86EMUL_CONTINUE; | 1685 | return X86EMUL_CONTINUE; |
| 1639 | } | 1686 | } |
| 1640 | 1687 | ||
| 1641 | static int em_leave(struct x86_emulate_ctxt *ctxt) | 1688 | static int em_leave(struct x86_emulate_ctxt *ctxt) |
| 1642 | { | 1689 | { |
| 1643 | assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP], | 1690 | assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP), |
| 1644 | stack_mask(ctxt)); | 1691 | stack_mask(ctxt)); |
| 1645 | return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes); | 1692 | return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes); |
| 1646 | } | 1693 | } |
| 1647 | 1694 | ||
| 1648 | static int em_push_sreg(struct x86_emulate_ctxt *ctxt) | 1695 | static int em_push_sreg(struct x86_emulate_ctxt *ctxt) |
| @@ -1670,13 +1717,13 @@ static int em_pop_sreg(struct x86_emulate_ctxt *ctxt) | |||
| 1670 | 1717 | ||
| 1671 | static int em_pusha(struct x86_emulate_ctxt *ctxt) | 1718 | static int em_pusha(struct x86_emulate_ctxt *ctxt) |
| 1672 | { | 1719 | { |
| 1673 | unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP]; | 1720 | unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP); |
| 1674 | int rc = X86EMUL_CONTINUE; | 1721 | int rc = X86EMUL_CONTINUE; |
| 1675 | int reg = VCPU_REGS_RAX; | 1722 | int reg = VCPU_REGS_RAX; |
| 1676 | 1723 | ||
| 1677 | while (reg <= VCPU_REGS_RDI) { | 1724 | while (reg <= VCPU_REGS_RDI) { |
| 1678 | (reg == VCPU_REGS_RSP) ? | 1725 | (reg == VCPU_REGS_RSP) ? |
| 1679 | (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]); | 1726 | (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg)); |
| 1680 | 1727 | ||
| 1681 | rc = em_push(ctxt); | 1728 | rc = em_push(ctxt); |
| 1682 | if (rc != X86EMUL_CONTINUE) | 1729 | if (rc != X86EMUL_CONTINUE) |
| @@ -1705,7 +1752,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt) | |||
| 1705 | --reg; | 1752 | --reg; |
| 1706 | } | 1753 | } |
| 1707 | 1754 | ||
| 1708 | rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes); | 1755 | rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes); |
| 1709 | if (rc != X86EMUL_CONTINUE) | 1756 | if (rc != X86EMUL_CONTINUE) |
| 1710 | break; | 1757 | break; |
| 1711 | --reg; | 1758 | --reg; |
| @@ -1713,9 +1760,9 @@ static int em_popa(struct x86_emulate_ctxt *ctxt) | |||
| 1713 | return rc; | 1760 | return rc; |
| 1714 | } | 1761 | } |
| 1715 | 1762 | ||
| 1716 | int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) | 1763 | static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) |
| 1717 | { | 1764 | { |
| 1718 | struct x86_emulate_ops *ops = ctxt->ops; | 1765 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 1719 | int rc; | 1766 | int rc; |
| 1720 | struct desc_ptr dt; | 1767 | struct desc_ptr dt; |
| 1721 | gva_t cs_addr; | 1768 | gva_t cs_addr; |
| @@ -1762,11 +1809,22 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) | |||
| 1762 | return rc; | 1809 | return rc; |
| 1763 | } | 1810 | } |
| 1764 | 1811 | ||
| 1812 | int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) | ||
| 1813 | { | ||
| 1814 | int rc; | ||
| 1815 | |||
| 1816 | invalidate_registers(ctxt); | ||
| 1817 | rc = __emulate_int_real(ctxt, irq); | ||
| 1818 | if (rc == X86EMUL_CONTINUE) | ||
| 1819 | writeback_registers(ctxt); | ||
| 1820 | return rc; | ||
| 1821 | } | ||
| 1822 | |||
| 1765 | static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq) | 1823 | static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq) |
| 1766 | { | 1824 | { |
| 1767 | switch(ctxt->mode) { | 1825 | switch(ctxt->mode) { |
| 1768 | case X86EMUL_MODE_REAL: | 1826 | case X86EMUL_MODE_REAL: |
| 1769 | return emulate_int_real(ctxt, irq); | 1827 | return __emulate_int_real(ctxt, irq); |
| 1770 | case X86EMUL_MODE_VM86: | 1828 | case X86EMUL_MODE_VM86: |
| 1771 | case X86EMUL_MODE_PROT16: | 1829 | case X86EMUL_MODE_PROT16: |
| 1772 | case X86EMUL_MODE_PROT32: | 1830 | case X86EMUL_MODE_PROT32: |
| @@ -1973,14 +2031,14 @@ static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt) | |||
| 1973 | { | 2031 | { |
| 1974 | u64 old = ctxt->dst.orig_val64; | 2032 | u64 old = ctxt->dst.orig_val64; |
| 1975 | 2033 | ||
| 1976 | if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) || | 2034 | if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) || |
| 1977 | ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) { | 2035 | ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) { |
| 1978 | ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0); | 2036 | *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0); |
| 1979 | ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32); | 2037 | *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32); |
| 1980 | ctxt->eflags &= ~EFLG_ZF; | 2038 | ctxt->eflags &= ~EFLG_ZF; |
| 1981 | } else { | 2039 | } else { |
| 1982 | ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) | | 2040 | ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | |
| 1983 | (u32) ctxt->regs[VCPU_REGS_RBX]; | 2041 | (u32) reg_read(ctxt, VCPU_REGS_RBX); |
| 1984 | 2042 | ||
| 1985 | ctxt->eflags |= EFLG_ZF; | 2043 | ctxt->eflags |= EFLG_ZF; |
| 1986 | } | 2044 | } |
| @@ -2016,7 +2074,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) | |||
| 2016 | { | 2074 | { |
| 2017 | /* Save real source value, then compare EAX against destination. */ | 2075 | /* Save real source value, then compare EAX against destination. */ |
| 2018 | ctxt->src.orig_val = ctxt->src.val; | 2076 | ctxt->src.orig_val = ctxt->src.val; |
| 2019 | ctxt->src.val = ctxt->regs[VCPU_REGS_RAX]; | 2077 | ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX); |
| 2020 | emulate_2op_SrcV(ctxt, "cmp"); | 2078 | emulate_2op_SrcV(ctxt, "cmp"); |
| 2021 | 2079 | ||
| 2022 | if (ctxt->eflags & EFLG_ZF) { | 2080 | if (ctxt->eflags & EFLG_ZF) { |
| @@ -2025,7 +2083,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) | |||
| 2025 | } else { | 2083 | } else { |
| 2026 | /* Failure: write the value we saw to EAX. */ | 2084 | /* Failure: write the value we saw to EAX. */ |
| 2027 | ctxt->dst.type = OP_REG; | 2085 | ctxt->dst.type = OP_REG; |
| 2028 | ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX]; | 2086 | ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); |
| 2029 | } | 2087 | } |
| 2030 | return X86EMUL_CONTINUE; | 2088 | return X86EMUL_CONTINUE; |
| 2031 | } | 2089 | } |
| @@ -2050,12 +2108,6 @@ static void | |||
| 2050 | setup_syscalls_segments(struct x86_emulate_ctxt *ctxt, | 2108 | setup_syscalls_segments(struct x86_emulate_ctxt *ctxt, |
| 2051 | struct desc_struct *cs, struct desc_struct *ss) | 2109 | struct desc_struct *cs, struct desc_struct *ss) |
| 2052 | { | 2110 | { |
| 2053 | u16 selector; | ||
| 2054 | |||
| 2055 | memset(cs, 0, sizeof(struct desc_struct)); | ||
| 2056 | ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS); | ||
| 2057 | memset(ss, 0, sizeof(struct desc_struct)); | ||
| 2058 | |||
| 2059 | cs->l = 0; /* will be adjusted later */ | 2111 | cs->l = 0; /* will be adjusted later */ |
| 2060 | set_desc_base(cs, 0); /* flat segment */ | 2112 | set_desc_base(cs, 0); /* flat segment */ |
| 2061 | cs->g = 1; /* 4kb granularity */ | 2113 | cs->g = 1; /* 4kb granularity */ |
| @@ -2065,6 +2117,7 @@ setup_syscalls_segments(struct x86_emulate_ctxt *ctxt, | |||
| 2065 | cs->dpl = 0; /* will be adjusted later */ | 2117 | cs->dpl = 0; /* will be adjusted later */ |
| 2066 | cs->p = 1; | 2118 | cs->p = 1; |
| 2067 | cs->d = 1; | 2119 | cs->d = 1; |
| 2120 | cs->avl = 0; | ||
| 2068 | 2121 | ||
| 2069 | set_desc_base(ss, 0); /* flat segment */ | 2122 | set_desc_base(ss, 0); /* flat segment */ |
| 2070 | set_desc_limit(ss, 0xfffff); /* 4GB limit */ | 2123 | set_desc_limit(ss, 0xfffff); /* 4GB limit */ |
| @@ -2074,6 +2127,8 @@ setup_syscalls_segments(struct x86_emulate_ctxt *ctxt, | |||
| 2074 | ss->d = 1; /* 32bit stack segment */ | 2127 | ss->d = 1; /* 32bit stack segment */ |
| 2075 | ss->dpl = 0; | 2128 | ss->dpl = 0; |
| 2076 | ss->p = 1; | 2129 | ss->p = 1; |
| 2130 | ss->l = 0; | ||
| 2131 | ss->avl = 0; | ||
| 2077 | } | 2132 | } |
| 2078 | 2133 | ||
| 2079 | static bool vendor_intel(struct x86_emulate_ctxt *ctxt) | 2134 | static bool vendor_intel(struct x86_emulate_ctxt *ctxt) |
| @@ -2089,7 +2144,7 @@ static bool vendor_intel(struct x86_emulate_ctxt *ctxt) | |||
| 2089 | 2144 | ||
| 2090 | static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) | 2145 | static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) |
| 2091 | { | 2146 | { |
| 2092 | struct x86_emulate_ops *ops = ctxt->ops; | 2147 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2093 | u32 eax, ebx, ecx, edx; | 2148 | u32 eax, ebx, ecx, edx; |
| 2094 | 2149 | ||
| 2095 | /* | 2150 | /* |
| @@ -2133,7 +2188,7 @@ static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) | |||
| 2133 | 2188 | ||
| 2134 | static int em_syscall(struct x86_emulate_ctxt *ctxt) | 2189 | static int em_syscall(struct x86_emulate_ctxt *ctxt) |
| 2135 | { | 2190 | { |
| 2136 | struct x86_emulate_ops *ops = ctxt->ops; | 2191 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2137 | struct desc_struct cs, ss; | 2192 | struct desc_struct cs, ss; |
| 2138 | u64 msr_data; | 2193 | u64 msr_data; |
| 2139 | u16 cs_sel, ss_sel; | 2194 | u16 cs_sel, ss_sel; |
| @@ -2165,10 +2220,10 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt) | |||
| 2165 | ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); | 2220 | ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); |
| 2166 | ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); | 2221 | ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); |
| 2167 | 2222 | ||
| 2168 | ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip; | 2223 | *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip; |
| 2169 | if (efer & EFER_LMA) { | 2224 | if (efer & EFER_LMA) { |
| 2170 | #ifdef CONFIG_X86_64 | 2225 | #ifdef CONFIG_X86_64 |
| 2171 | ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF; | 2226 | *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF; |
| 2172 | 2227 | ||
| 2173 | ops->get_msr(ctxt, | 2228 | ops->get_msr(ctxt, |
| 2174 | ctxt->mode == X86EMUL_MODE_PROT64 ? | 2229 | ctxt->mode == X86EMUL_MODE_PROT64 ? |
| @@ -2191,7 +2246,7 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt) | |||
| 2191 | 2246 | ||
| 2192 | static int em_sysenter(struct x86_emulate_ctxt *ctxt) | 2247 | static int em_sysenter(struct x86_emulate_ctxt *ctxt) |
| 2193 | { | 2248 | { |
| 2194 | struct x86_emulate_ops *ops = ctxt->ops; | 2249 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2195 | struct desc_struct cs, ss; | 2250 | struct desc_struct cs, ss; |
| 2196 | u64 msr_data; | 2251 | u64 msr_data; |
| 2197 | u16 cs_sel, ss_sel; | 2252 | u16 cs_sel, ss_sel; |
| @@ -2228,6 +2283,8 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt) | |||
| 2228 | if (msr_data == 0x0) | 2283 | if (msr_data == 0x0) |
| 2229 | return emulate_gp(ctxt, 0); | 2284 | return emulate_gp(ctxt, 0); |
| 2230 | break; | 2285 | break; |
| 2286 | default: | ||
| 2287 | break; | ||
| 2231 | } | 2288 | } |
| 2232 | 2289 | ||
| 2233 | ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF); | 2290 | ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF); |
| @@ -2247,14 +2304,14 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt) | |||
| 2247 | ctxt->_eip = msr_data; | 2304 | ctxt->_eip = msr_data; |
| 2248 | 2305 | ||
| 2249 | ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data); | 2306 | ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data); |
| 2250 | ctxt->regs[VCPU_REGS_RSP] = msr_data; | 2307 | *reg_write(ctxt, VCPU_REGS_RSP) = msr_data; |
| 2251 | 2308 | ||
| 2252 | return X86EMUL_CONTINUE; | 2309 | return X86EMUL_CONTINUE; |
| 2253 | } | 2310 | } |
| 2254 | 2311 | ||
| 2255 | static int em_sysexit(struct x86_emulate_ctxt *ctxt) | 2312 | static int em_sysexit(struct x86_emulate_ctxt *ctxt) |
| 2256 | { | 2313 | { |
| 2257 | struct x86_emulate_ops *ops = ctxt->ops; | 2314 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2258 | struct desc_struct cs, ss; | 2315 | struct desc_struct cs, ss; |
| 2259 | u64 msr_data; | 2316 | u64 msr_data; |
| 2260 | int usermode; | 2317 | int usermode; |
| @@ -2297,8 +2354,8 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt) | |||
| 2297 | ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); | 2354 | ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); |
| 2298 | ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); | 2355 | ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); |
| 2299 | 2356 | ||
| 2300 | ctxt->_eip = ctxt->regs[VCPU_REGS_RDX]; | 2357 | ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX); |
| 2301 | ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX]; | 2358 | *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX); |
| 2302 | 2359 | ||
| 2303 | return X86EMUL_CONTINUE; | 2360 | return X86EMUL_CONTINUE; |
| 2304 | } | 2361 | } |
| @@ -2317,7 +2374,7 @@ static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt) | |||
| 2317 | static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt, | 2374 | static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt, |
| 2318 | u16 port, u16 len) | 2375 | u16 port, u16 len) |
| 2319 | { | 2376 | { |
| 2320 | struct x86_emulate_ops *ops = ctxt->ops; | 2377 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2321 | struct desc_struct tr_seg; | 2378 | struct desc_struct tr_seg; |
| 2322 | u32 base3; | 2379 | u32 base3; |
| 2323 | int r; | 2380 | int r; |
| @@ -2367,14 +2424,14 @@ static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt, | |||
| 2367 | { | 2424 | { |
| 2368 | tss->ip = ctxt->_eip; | 2425 | tss->ip = ctxt->_eip; |
| 2369 | tss->flag = ctxt->eflags; | 2426 | tss->flag = ctxt->eflags; |
| 2370 | tss->ax = ctxt->regs[VCPU_REGS_RAX]; | 2427 | tss->ax = reg_read(ctxt, VCPU_REGS_RAX); |
| 2371 | tss->cx = ctxt->regs[VCPU_REGS_RCX]; | 2428 | tss->cx = reg_read(ctxt, VCPU_REGS_RCX); |
| 2372 | tss->dx = ctxt->regs[VCPU_REGS_RDX]; | 2429 | tss->dx = reg_read(ctxt, VCPU_REGS_RDX); |
| 2373 | tss->bx = ctxt->regs[VCPU_REGS_RBX]; | 2430 | tss->bx = reg_read(ctxt, VCPU_REGS_RBX); |
| 2374 | tss->sp = ctxt->regs[VCPU_REGS_RSP]; | 2431 | tss->sp = reg_read(ctxt, VCPU_REGS_RSP); |
| 2375 | tss->bp = ctxt->regs[VCPU_REGS_RBP]; | 2432 | tss->bp = reg_read(ctxt, VCPU_REGS_RBP); |
| 2376 | tss->si = ctxt->regs[VCPU_REGS_RSI]; | 2433 | tss->si = reg_read(ctxt, VCPU_REGS_RSI); |
| 2377 | tss->di = ctxt->regs[VCPU_REGS_RDI]; | 2434 | tss->di = reg_read(ctxt, VCPU_REGS_RDI); |
| 2378 | 2435 | ||
| 2379 | tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); | 2436 | tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); |
| 2380 | tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); | 2437 | tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); |
| @@ -2390,14 +2447,14 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt, | |||
| 2390 | 2447 | ||
| 2391 | ctxt->_eip = tss->ip; | 2448 | ctxt->_eip = tss->ip; |
| 2392 | ctxt->eflags = tss->flag | 2; | 2449 | ctxt->eflags = tss->flag | 2; |
| 2393 | ctxt->regs[VCPU_REGS_RAX] = tss->ax; | 2450 | *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax; |
| 2394 | ctxt->regs[VCPU_REGS_RCX] = tss->cx; | 2451 | *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx; |
| 2395 | ctxt->regs[VCPU_REGS_RDX] = tss->dx; | 2452 | *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx; |
| 2396 | ctxt->regs[VCPU_REGS_RBX] = tss->bx; | 2453 | *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx; |
| 2397 | ctxt->regs[VCPU_REGS_RSP] = tss->sp; | 2454 | *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp; |
| 2398 | ctxt->regs[VCPU_REGS_RBP] = tss->bp; | 2455 | *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp; |
| 2399 | ctxt->regs[VCPU_REGS_RSI] = tss->si; | 2456 | *reg_write(ctxt, VCPU_REGS_RSI) = tss->si; |
| 2400 | ctxt->regs[VCPU_REGS_RDI] = tss->di; | 2457 | *reg_write(ctxt, VCPU_REGS_RDI) = tss->di; |
| 2401 | 2458 | ||
| 2402 | /* | 2459 | /* |
| 2403 | * SDM says that segment selectors are loaded before segment | 2460 | * SDM says that segment selectors are loaded before segment |
| @@ -2410,7 +2467,7 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt, | |||
| 2410 | set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); | 2467 | set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); |
| 2411 | 2468 | ||
| 2412 | /* | 2469 | /* |
| 2413 | * Now load segment descriptors. If fault happenes at this stage | 2470 | * Now load segment descriptors. If fault happens at this stage |
| 2414 | * it is handled in a context of new task | 2471 | * it is handled in a context of new task |
| 2415 | */ | 2472 | */ |
| 2416 | ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR); | 2473 | ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR); |
| @@ -2436,7 +2493,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt, | |||
| 2436 | u16 tss_selector, u16 old_tss_sel, | 2493 | u16 tss_selector, u16 old_tss_sel, |
| 2437 | ulong old_tss_base, struct desc_struct *new_desc) | 2494 | ulong old_tss_base, struct desc_struct *new_desc) |
| 2438 | { | 2495 | { |
| 2439 | struct x86_emulate_ops *ops = ctxt->ops; | 2496 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2440 | struct tss_segment_16 tss_seg; | 2497 | struct tss_segment_16 tss_seg; |
| 2441 | int ret; | 2498 | int ret; |
| 2442 | u32 new_tss_base = get_desc_base(new_desc); | 2499 | u32 new_tss_base = get_desc_base(new_desc); |
| @@ -2482,14 +2539,14 @@ static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt, | |||
| 2482 | tss->cr3 = ctxt->ops->get_cr(ctxt, 3); | 2539 | tss->cr3 = ctxt->ops->get_cr(ctxt, 3); |
| 2483 | tss->eip = ctxt->_eip; | 2540 | tss->eip = ctxt->_eip; |
| 2484 | tss->eflags = ctxt->eflags; | 2541 | tss->eflags = ctxt->eflags; |
| 2485 | tss->eax = ctxt->regs[VCPU_REGS_RAX]; | 2542 | tss->eax = reg_read(ctxt, VCPU_REGS_RAX); |
| 2486 | tss->ecx = ctxt->regs[VCPU_REGS_RCX]; | 2543 | tss->ecx = reg_read(ctxt, VCPU_REGS_RCX); |
| 2487 | tss->edx = ctxt->regs[VCPU_REGS_RDX]; | 2544 | tss->edx = reg_read(ctxt, VCPU_REGS_RDX); |
| 2488 | tss->ebx = ctxt->regs[VCPU_REGS_RBX]; | 2545 | tss->ebx = reg_read(ctxt, VCPU_REGS_RBX); |
| 2489 | tss->esp = ctxt->regs[VCPU_REGS_RSP]; | 2546 | tss->esp = reg_read(ctxt, VCPU_REGS_RSP); |
| 2490 | tss->ebp = ctxt->regs[VCPU_REGS_RBP]; | 2547 | tss->ebp = reg_read(ctxt, VCPU_REGS_RBP); |
| 2491 | tss->esi = ctxt->regs[VCPU_REGS_RSI]; | 2548 | tss->esi = reg_read(ctxt, VCPU_REGS_RSI); |
| 2492 | tss->edi = ctxt->regs[VCPU_REGS_RDI]; | 2549 | tss->edi = reg_read(ctxt, VCPU_REGS_RDI); |
| 2493 | 2550 | ||
| 2494 | tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); | 2551 | tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); |
| 2495 | tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); | 2552 | tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); |
| @@ -2511,14 +2568,14 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt, | |||
| 2511 | ctxt->eflags = tss->eflags | 2; | 2568 | ctxt->eflags = tss->eflags | 2; |
| 2512 | 2569 | ||
| 2513 | /* General purpose registers */ | 2570 | /* General purpose registers */ |
| 2514 | ctxt->regs[VCPU_REGS_RAX] = tss->eax; | 2571 | *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax; |
| 2515 | ctxt->regs[VCPU_REGS_RCX] = tss->ecx; | 2572 | *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx; |
| 2516 | ctxt->regs[VCPU_REGS_RDX] = tss->edx; | 2573 | *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx; |
| 2517 | ctxt->regs[VCPU_REGS_RBX] = tss->ebx; | 2574 | *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx; |
| 2518 | ctxt->regs[VCPU_REGS_RSP] = tss->esp; | 2575 | *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp; |
| 2519 | ctxt->regs[VCPU_REGS_RBP] = tss->ebp; | 2576 | *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp; |
| 2520 | ctxt->regs[VCPU_REGS_RSI] = tss->esi; | 2577 | *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi; |
| 2521 | ctxt->regs[VCPU_REGS_RDI] = tss->edi; | 2578 | *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi; |
| 2522 | 2579 | ||
| 2523 | /* | 2580 | /* |
| 2524 | * SDM says that segment selectors are loaded before segment | 2581 | * SDM says that segment selectors are loaded before segment |
| @@ -2583,7 +2640,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt, | |||
| 2583 | u16 tss_selector, u16 old_tss_sel, | 2640 | u16 tss_selector, u16 old_tss_sel, |
| 2584 | ulong old_tss_base, struct desc_struct *new_desc) | 2641 | ulong old_tss_base, struct desc_struct *new_desc) |
| 2585 | { | 2642 | { |
| 2586 | struct x86_emulate_ops *ops = ctxt->ops; | 2643 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2587 | struct tss_segment_32 tss_seg; | 2644 | struct tss_segment_32 tss_seg; |
| 2588 | int ret; | 2645 | int ret; |
| 2589 | u32 new_tss_base = get_desc_base(new_desc); | 2646 | u32 new_tss_base = get_desc_base(new_desc); |
| @@ -2627,7 +2684,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt, | |||
| 2627 | u16 tss_selector, int idt_index, int reason, | 2684 | u16 tss_selector, int idt_index, int reason, |
| 2628 | bool has_error_code, u32 error_code) | 2685 | bool has_error_code, u32 error_code) |
| 2629 | { | 2686 | { |
| 2630 | struct x86_emulate_ops *ops = ctxt->ops; | 2687 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 2631 | struct desc_struct curr_tss_desc, next_tss_desc; | 2688 | struct desc_struct curr_tss_desc, next_tss_desc; |
| 2632 | int ret; | 2689 | int ret; |
| 2633 | u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR); | 2690 | u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR); |
| @@ -2652,7 +2709,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt, | |||
| 2652 | * | 2709 | * |
| 2653 | * 1. jmp/call/int to task gate: Check against DPL of the task gate | 2710 | * 1. jmp/call/int to task gate: Check against DPL of the task gate |
| 2654 | * 2. Exception/IRQ/iret: No check is performed | 2711 | * 2. Exception/IRQ/iret: No check is performed |
| 2655 | * 3. jmp/call to TSS: Check agains DPL of the TSS | 2712 | * 3. jmp/call to TSS: Check against DPL of the TSS |
| 2656 | */ | 2713 | */ |
| 2657 | if (reason == TASK_SWITCH_GATE) { | 2714 | if (reason == TASK_SWITCH_GATE) { |
| 2658 | if (idt_index != -1) { | 2715 | if (idt_index != -1) { |
| @@ -2693,7 +2750,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt, | |||
| 2693 | ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT; | 2750 | ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT; |
| 2694 | 2751 | ||
| 2695 | /* set back link to prev task only if NT bit is set in eflags | 2752 | /* set back link to prev task only if NT bit is set in eflags |
| 2696 | note that old_tss_sel is not used afetr this point */ | 2753 | note that old_tss_sel is not used after this point */ |
| 2697 | if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE) | 2754 | if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE) |
| 2698 | old_tss_sel = 0xffff; | 2755 | old_tss_sel = 0xffff; |
| 2699 | 2756 | ||
| @@ -2733,26 +2790,28 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt, | |||
| 2733 | { | 2790 | { |
| 2734 | int rc; | 2791 | int rc; |
| 2735 | 2792 | ||
| 2793 | invalidate_registers(ctxt); | ||
| 2736 | ctxt->_eip = ctxt->eip; | 2794 | ctxt->_eip = ctxt->eip; |
| 2737 | ctxt->dst.type = OP_NONE; | 2795 | ctxt->dst.type = OP_NONE; |
| 2738 | 2796 | ||
| 2739 | rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason, | 2797 | rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason, |
| 2740 | has_error_code, error_code); | 2798 | has_error_code, error_code); |
| 2741 | 2799 | ||
| 2742 | if (rc == X86EMUL_CONTINUE) | 2800 | if (rc == X86EMUL_CONTINUE) { |
| 2743 | ctxt->eip = ctxt->_eip; | 2801 | ctxt->eip = ctxt->_eip; |
| 2802 | writeback_registers(ctxt); | ||
| 2803 | } | ||
| 2744 | 2804 | ||
| 2745 | return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; | 2805 | return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; |
| 2746 | } | 2806 | } |
| 2747 | 2807 | ||
| 2748 | static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg, | 2808 | static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg, |
| 2749 | int reg, struct operand *op) | 2809 | struct operand *op) |
| 2750 | { | 2810 | { |
| 2751 | int df = (ctxt->eflags & EFLG_DF) ? -1 : 1; | 2811 | int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count; |
| 2752 | 2812 | ||
| 2753 | register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes); | 2813 | register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes); |
| 2754 | op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]); | 2814 | op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg)); |
| 2755 | op->addr.mem.seg = seg; | ||
| 2756 | } | 2815 | } |
| 2757 | 2816 | ||
| 2758 | static int em_das(struct x86_emulate_ctxt *ctxt) | 2817 | static int em_das(struct x86_emulate_ctxt *ctxt) |
| @@ -2927,7 +2986,7 @@ static int em_cwd(struct x86_emulate_ctxt *ctxt) | |||
| 2927 | { | 2986 | { |
| 2928 | ctxt->dst.type = OP_REG; | 2987 | ctxt->dst.type = OP_REG; |
| 2929 | ctxt->dst.bytes = ctxt->src.bytes; | 2988 | ctxt->dst.bytes = ctxt->src.bytes; |
| 2930 | ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX]; | 2989 | ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); |
| 2931 | ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1); | 2990 | ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1); |
| 2932 | 2991 | ||
| 2933 | return X86EMUL_CONTINUE; | 2992 | return X86EMUL_CONTINUE; |
| @@ -2938,8 +2997,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt) | |||
| 2938 | u64 tsc = 0; | 2997 | u64 tsc = 0; |
| 2939 | 2998 | ||
| 2940 | ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc); | 2999 | ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc); |
| 2941 | ctxt->regs[VCPU_REGS_RAX] = (u32)tsc; | 3000 | *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc; |
| 2942 | ctxt->regs[VCPU_REGS_RDX] = tsc >> 32; | 3001 | *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32; |
| 2943 | return X86EMUL_CONTINUE; | 3002 | return X86EMUL_CONTINUE; |
| 2944 | } | 3003 | } |
| 2945 | 3004 | ||
| @@ -2947,10 +3006,10 @@ static int em_rdpmc(struct x86_emulate_ctxt *ctxt) | |||
| 2947 | { | 3006 | { |
| 2948 | u64 pmc; | 3007 | u64 pmc; |
| 2949 | 3008 | ||
| 2950 | if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc)) | 3009 | if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc)) |
| 2951 | return emulate_gp(ctxt, 0); | 3010 | return emulate_gp(ctxt, 0); |
| 2952 | ctxt->regs[VCPU_REGS_RAX] = (u32)pmc; | 3011 | *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc; |
| 2953 | ctxt->regs[VCPU_REGS_RDX] = pmc >> 32; | 3012 | *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32; |
| 2954 | return X86EMUL_CONTINUE; | 3013 | return X86EMUL_CONTINUE; |
| 2955 | } | 3014 | } |
| 2956 | 3015 | ||
| @@ -2992,9 +3051,9 @@ static int em_wrmsr(struct x86_emulate_ctxt *ctxt) | |||
| 2992 | { | 3051 | { |
| 2993 | u64 msr_data; | 3052 | u64 msr_data; |
| 2994 | 3053 | ||
| 2995 | msr_data = (u32)ctxt->regs[VCPU_REGS_RAX] | 3054 | msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX) |
| 2996 | | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32); | 3055 | | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32); |
| 2997 | if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) | 3056 | if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data)) |
| 2998 | return emulate_gp(ctxt, 0); | 3057 | return emulate_gp(ctxt, 0); |
| 2999 | 3058 | ||
| 3000 | return X86EMUL_CONTINUE; | 3059 | return X86EMUL_CONTINUE; |
| @@ -3004,11 +3063,11 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt) | |||
| 3004 | { | 3063 | { |
| 3005 | u64 msr_data; | 3064 | u64 msr_data; |
| 3006 | 3065 | ||
| 3007 | if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) | 3066 | if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data)) |
| 3008 | return emulate_gp(ctxt, 0); | 3067 | return emulate_gp(ctxt, 0); |
| 3009 | 3068 | ||
| 3010 | ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data; | 3069 | *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data; |
| 3011 | ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32; | 3070 | *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32; |
| 3012 | return X86EMUL_CONTINUE; | 3071 | return X86EMUL_CONTINUE; |
| 3013 | } | 3072 | } |
| 3014 | 3073 | ||
| @@ -3188,8 +3247,8 @@ static int em_lmsw(struct x86_emulate_ctxt *ctxt) | |||
| 3188 | 3247 | ||
| 3189 | static int em_loop(struct x86_emulate_ctxt *ctxt) | 3248 | static int em_loop(struct x86_emulate_ctxt *ctxt) |
| 3190 | { | 3249 | { |
| 3191 | register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1); | 3250 | register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1); |
| 3192 | if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) && | 3251 | if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) && |
| 3193 | (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags))) | 3252 | (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags))) |
| 3194 | jmp_rel(ctxt, ctxt->src.val); | 3253 | jmp_rel(ctxt, ctxt->src.val); |
| 3195 | 3254 | ||
| @@ -3198,7 +3257,7 @@ static int em_loop(struct x86_emulate_ctxt *ctxt) | |||
| 3198 | 3257 | ||
| 3199 | static int em_jcxz(struct x86_emulate_ctxt *ctxt) | 3258 | static int em_jcxz(struct x86_emulate_ctxt *ctxt) |
| 3200 | { | 3259 | { |
| 3201 | if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) | 3260 | if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) |
| 3202 | jmp_rel(ctxt, ctxt->src.val); | 3261 | jmp_rel(ctxt, ctxt->src.val); |
| 3203 | 3262 | ||
| 3204 | return X86EMUL_CONTINUE; | 3263 | return X86EMUL_CONTINUE; |
| @@ -3286,20 +3345,20 @@ static int em_cpuid(struct x86_emulate_ctxt *ctxt) | |||
| 3286 | { | 3345 | { |
| 3287 | u32 eax, ebx, ecx, edx; | 3346 | u32 eax, ebx, ecx, edx; |
| 3288 | 3347 | ||
| 3289 | eax = ctxt->regs[VCPU_REGS_RAX]; | 3348 | eax = reg_read(ctxt, VCPU_REGS_RAX); |
| 3290 | ecx = ctxt->regs[VCPU_REGS_RCX]; | 3349 | ecx = reg_read(ctxt, VCPU_REGS_RCX); |
| 3291 | ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx); | 3350 | ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx); |
| 3292 | ctxt->regs[VCPU_REGS_RAX] = eax; | 3351 | *reg_write(ctxt, VCPU_REGS_RAX) = eax; |
| 3293 | ctxt->regs[VCPU_REGS_RBX] = ebx; | 3352 | *reg_write(ctxt, VCPU_REGS_RBX) = ebx; |
| 3294 | ctxt->regs[VCPU_REGS_RCX] = ecx; | 3353 | *reg_write(ctxt, VCPU_REGS_RCX) = ecx; |
| 3295 | ctxt->regs[VCPU_REGS_RDX] = edx; | 3354 | *reg_write(ctxt, VCPU_REGS_RDX) = edx; |
| 3296 | return X86EMUL_CONTINUE; | 3355 | return X86EMUL_CONTINUE; |
| 3297 | } | 3356 | } |
| 3298 | 3357 | ||
| 3299 | static int em_lahf(struct x86_emulate_ctxt *ctxt) | 3358 | static int em_lahf(struct x86_emulate_ctxt *ctxt) |
| 3300 | { | 3359 | { |
| 3301 | ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL; | 3360 | *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL; |
| 3302 | ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8; | 3361 | *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8; |
| 3303 | return X86EMUL_CONTINUE; | 3362 | return X86EMUL_CONTINUE; |
| 3304 | } | 3363 | } |
| 3305 | 3364 | ||
| @@ -3456,7 +3515,7 @@ static int check_svme(struct x86_emulate_ctxt *ctxt) | |||
| 3456 | 3515 | ||
| 3457 | static int check_svme_pa(struct x86_emulate_ctxt *ctxt) | 3516 | static int check_svme_pa(struct x86_emulate_ctxt *ctxt) |
| 3458 | { | 3517 | { |
| 3459 | u64 rax = ctxt->regs[VCPU_REGS_RAX]; | 3518 | u64 rax = reg_read(ctxt, VCPU_REGS_RAX); |
| 3460 | 3519 | ||
| 3461 | /* Valid physical address? */ | 3520 | /* Valid physical address? */ |
| 3462 | if (rax & 0xffff000000000000ULL) | 3521 | if (rax & 0xffff000000000000ULL) |
| @@ -3478,7 +3537,7 @@ static int check_rdtsc(struct x86_emulate_ctxt *ctxt) | |||
| 3478 | static int check_rdpmc(struct x86_emulate_ctxt *ctxt) | 3537 | static int check_rdpmc(struct x86_emulate_ctxt *ctxt) |
| 3479 | { | 3538 | { |
| 3480 | u64 cr4 = ctxt->ops->get_cr(ctxt, 4); | 3539 | u64 cr4 = ctxt->ops->get_cr(ctxt, 4); |
| 3481 | u64 rcx = ctxt->regs[VCPU_REGS_RCX]; | 3540 | u64 rcx = reg_read(ctxt, VCPU_REGS_RCX); |
| 3482 | 3541 | ||
| 3483 | if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || | 3542 | if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || |
| 3484 | (rcx > 3)) | 3543 | (rcx > 3)) |
| @@ -3531,13 +3590,13 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt) | |||
| 3531 | I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ | 3590 | I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ |
| 3532 | I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) | 3591 | I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) |
| 3533 | 3592 | ||
| 3534 | static struct opcode group7_rm1[] = { | 3593 | static const struct opcode group7_rm1[] = { |
| 3535 | DI(SrcNone | Priv, monitor), | 3594 | DI(SrcNone | Priv, monitor), |
| 3536 | DI(SrcNone | Priv, mwait), | 3595 | DI(SrcNone | Priv, mwait), |
| 3537 | N, N, N, N, N, N, | 3596 | N, N, N, N, N, N, |
| 3538 | }; | 3597 | }; |
| 3539 | 3598 | ||
| 3540 | static struct opcode group7_rm3[] = { | 3599 | static const struct opcode group7_rm3[] = { |
| 3541 | DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa), | 3600 | DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa), |
| 3542 | II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall), | 3601 | II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall), |
| 3543 | DIP(SrcNone | Prot | Priv, vmload, check_svme_pa), | 3602 | DIP(SrcNone | Prot | Priv, vmload, check_svme_pa), |
| @@ -3548,13 +3607,13 @@ static struct opcode group7_rm3[] = { | |||
| 3548 | DIP(SrcNone | Prot | Priv, invlpga, check_svme), | 3607 | DIP(SrcNone | Prot | Priv, invlpga, check_svme), |
| 3549 | }; | 3608 | }; |
| 3550 | 3609 | ||
| 3551 | static struct opcode group7_rm7[] = { | 3610 | static const struct opcode group7_rm7[] = { |
| 3552 | N, | 3611 | N, |
| 3553 | DIP(SrcNone, rdtscp, check_rdtsc), | 3612 | DIP(SrcNone, rdtscp, check_rdtsc), |
| 3554 | N, N, N, N, N, N, | 3613 | N, N, N, N, N, N, |
| 3555 | }; | 3614 | }; |
| 3556 | 3615 | ||
| 3557 | static struct opcode group1[] = { | 3616 | static const struct opcode group1[] = { |
| 3558 | I(Lock, em_add), | 3617 | I(Lock, em_add), |
| 3559 | I(Lock | PageTable, em_or), | 3618 | I(Lock | PageTable, em_or), |
| 3560 | I(Lock, em_adc), | 3619 | I(Lock, em_adc), |
| @@ -3565,11 +3624,11 @@ static struct opcode group1[] = { | |||
| 3565 | I(0, em_cmp), | 3624 | I(0, em_cmp), |
| 3566 | }; | 3625 | }; |
| 3567 | 3626 | ||
| 3568 | static struct opcode group1A[] = { | 3627 | static const struct opcode group1A[] = { |
| 3569 | I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N, | 3628 | I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N, |
| 3570 | }; | 3629 | }; |
| 3571 | 3630 | ||
| 3572 | static struct opcode group3[] = { | 3631 | static const struct opcode group3[] = { |
| 3573 | I(DstMem | SrcImm, em_test), | 3632 | I(DstMem | SrcImm, em_test), |
| 3574 | I(DstMem | SrcImm, em_test), | 3633 | I(DstMem | SrcImm, em_test), |
| 3575 | I(DstMem | SrcNone | Lock, em_not), | 3634 | I(DstMem | SrcNone | Lock, em_not), |
| @@ -3580,13 +3639,13 @@ static struct opcode group3[] = { | |||
| 3580 | I(SrcMem, em_idiv_ex), | 3639 | I(SrcMem, em_idiv_ex), |
| 3581 | }; | 3640 | }; |
| 3582 | 3641 | ||
| 3583 | static struct opcode group4[] = { | 3642 | static const struct opcode group4[] = { |
| 3584 | I(ByteOp | DstMem | SrcNone | Lock, em_grp45), | 3643 | I(ByteOp | DstMem | SrcNone | Lock, em_grp45), |
| 3585 | I(ByteOp | DstMem | SrcNone | Lock, em_grp45), | 3644 | I(ByteOp | DstMem | SrcNone | Lock, em_grp45), |
| 3586 | N, N, N, N, N, N, | 3645 | N, N, N, N, N, N, |
| 3587 | }; | 3646 | }; |
| 3588 | 3647 | ||
| 3589 | static struct opcode group5[] = { | 3648 | static const struct opcode group5[] = { |
| 3590 | I(DstMem | SrcNone | Lock, em_grp45), | 3649 | I(DstMem | SrcNone | Lock, em_grp45), |
| 3591 | I(DstMem | SrcNone | Lock, em_grp45), | 3650 | I(DstMem | SrcNone | Lock, em_grp45), |
| 3592 | I(SrcMem | Stack, em_grp45), | 3651 | I(SrcMem | Stack, em_grp45), |
| @@ -3596,7 +3655,7 @@ static struct opcode group5[] = { | |||
| 3596 | I(SrcMem | Stack, em_grp45), N, | 3655 | I(SrcMem | Stack, em_grp45), N, |
| 3597 | }; | 3656 | }; |
| 3598 | 3657 | ||
| 3599 | static struct opcode group6[] = { | 3658 | static const struct opcode group6[] = { |
| 3600 | DI(Prot, sldt), | 3659 | DI(Prot, sldt), |
| 3601 | DI(Prot, str), | 3660 | DI(Prot, str), |
| 3602 | II(Prot | Priv | SrcMem16, em_lldt, lldt), | 3661 | II(Prot | Priv | SrcMem16, em_lldt, lldt), |
| @@ -3604,7 +3663,7 @@ static struct opcode group6[] = { | |||
| 3604 | N, N, N, N, | 3663 | N, N, N, N, |
| 3605 | }; | 3664 | }; |
| 3606 | 3665 | ||
| 3607 | static struct group_dual group7 = { { | 3666 | static const struct group_dual group7 = { { |
| 3608 | II(Mov | DstMem | Priv, em_sgdt, sgdt), | 3667 | II(Mov | DstMem | Priv, em_sgdt, sgdt), |
| 3609 | II(Mov | DstMem | Priv, em_sidt, sidt), | 3668 | II(Mov | DstMem | Priv, em_sidt, sidt), |
| 3610 | II(SrcMem | Priv, em_lgdt, lgdt), | 3669 | II(SrcMem | Priv, em_lgdt, lgdt), |
| @@ -3621,7 +3680,7 @@ static struct group_dual group7 = { { | |||
| 3621 | EXT(0, group7_rm7), | 3680 | EXT(0, group7_rm7), |
| 3622 | } }; | 3681 | } }; |
| 3623 | 3682 | ||
| 3624 | static struct opcode group8[] = { | 3683 | static const struct opcode group8[] = { |
| 3625 | N, N, N, N, | 3684 | N, N, N, N, |
| 3626 | I(DstMem | SrcImmByte, em_bt), | 3685 | I(DstMem | SrcImmByte, em_bt), |
| 3627 | I(DstMem | SrcImmByte | Lock | PageTable, em_bts), | 3686 | I(DstMem | SrcImmByte | Lock | PageTable, em_bts), |
| @@ -3629,26 +3688,26 @@ static struct opcode group8[] = { | |||
| 3629 | I(DstMem | SrcImmByte | Lock | PageTable, em_btc), | 3688 | I(DstMem | SrcImmByte | Lock | PageTable, em_btc), |
| 3630 | }; | 3689 | }; |
| 3631 | 3690 | ||
| 3632 | static struct group_dual group9 = { { | 3691 | static const struct group_dual group9 = { { |
| 3633 | N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N, | 3692 | N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N, |
| 3634 | }, { | 3693 | }, { |
| 3635 | N, N, N, N, N, N, N, N, | 3694 | N, N, N, N, N, N, N, N, |
| 3636 | } }; | 3695 | } }; |
| 3637 | 3696 | ||
| 3638 | static struct opcode group11[] = { | 3697 | static const struct opcode group11[] = { |
| 3639 | I(DstMem | SrcImm | Mov | PageTable, em_mov), | 3698 | I(DstMem | SrcImm | Mov | PageTable, em_mov), |
| 3640 | X7(D(Undefined)), | 3699 | X7(D(Undefined)), |
| 3641 | }; | 3700 | }; |
| 3642 | 3701 | ||
| 3643 | static struct gprefix pfx_0f_6f_0f_7f = { | 3702 | static const struct gprefix pfx_0f_6f_0f_7f = { |
| 3644 | I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov), | 3703 | I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov), |
| 3645 | }; | 3704 | }; |
| 3646 | 3705 | ||
| 3647 | static struct gprefix pfx_vmovntpx = { | 3706 | static const struct gprefix pfx_vmovntpx = { |
| 3648 | I(0, em_mov), N, N, N, | 3707 | I(0, em_mov), N, N, N, |
| 3649 | }; | 3708 | }; |
| 3650 | 3709 | ||
| 3651 | static struct opcode opcode_table[256] = { | 3710 | static const struct opcode opcode_table[256] = { |
| 3652 | /* 0x00 - 0x07 */ | 3711 | /* 0x00 - 0x07 */ |
| 3653 | I6ALU(Lock, em_add), | 3712 | I6ALU(Lock, em_add), |
| 3654 | I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg), | 3713 | I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg), |
| @@ -3689,7 +3748,7 @@ static struct opcode opcode_table[256] = { | |||
| 3689 | I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op), | 3748 | I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op), |
| 3690 | I(SrcImmByte | Mov | Stack, em_push), | 3749 | I(SrcImmByte | Mov | Stack, em_push), |
| 3691 | I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op), | 3750 | I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op), |
| 3692 | I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */ | 3751 | I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */ |
| 3693 | I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */ | 3752 | I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */ |
| 3694 | /* 0x70 - 0x7F */ | 3753 | /* 0x70 - 0x7F */ |
| 3695 | X16(D(SrcImmByte)), | 3754 | X16(D(SrcImmByte)), |
| @@ -3765,7 +3824,7 @@ static struct opcode opcode_table[256] = { | |||
| 3765 | D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5), | 3824 | D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5), |
| 3766 | }; | 3825 | }; |
| 3767 | 3826 | ||
| 3768 | static struct opcode twobyte_table[256] = { | 3827 | static const struct opcode twobyte_table[256] = { |
| 3769 | /* 0x00 - 0x0F */ | 3828 | /* 0x00 - 0x0F */ |
| 3770 | G(0, group6), GD(0, &group7), N, N, | 3829 | G(0, group6), GD(0, &group7), N, N, |
| 3771 | N, I(ImplicitOps | VendorSpecific, em_syscall), | 3830 | N, I(ImplicitOps | VendorSpecific, em_syscall), |
| @@ -3936,7 +3995,7 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op, | |||
| 3936 | case OpAcc: | 3995 | case OpAcc: |
| 3937 | op->type = OP_REG; | 3996 | op->type = OP_REG; |
| 3938 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; | 3997 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; |
| 3939 | op->addr.reg = &ctxt->regs[VCPU_REGS_RAX]; | 3998 | op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); |
| 3940 | fetch_register_operand(op); | 3999 | fetch_register_operand(op); |
| 3941 | op->orig_val = op->val; | 4000 | op->orig_val = op->val; |
| 3942 | break; | 4001 | break; |
| @@ -3944,19 +4003,20 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op, | |||
| 3944 | op->type = OP_MEM; | 4003 | op->type = OP_MEM; |
| 3945 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; | 4004 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; |
| 3946 | op->addr.mem.ea = | 4005 | op->addr.mem.ea = |
| 3947 | register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]); | 4006 | register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI)); |
| 3948 | op->addr.mem.seg = VCPU_SREG_ES; | 4007 | op->addr.mem.seg = VCPU_SREG_ES; |
| 3949 | op->val = 0; | 4008 | op->val = 0; |
| 4009 | op->count = 1; | ||
| 3950 | break; | 4010 | break; |
| 3951 | case OpDX: | 4011 | case OpDX: |
| 3952 | op->type = OP_REG; | 4012 | op->type = OP_REG; |
| 3953 | op->bytes = 2; | 4013 | op->bytes = 2; |
| 3954 | op->addr.reg = &ctxt->regs[VCPU_REGS_RDX]; | 4014 | op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); |
| 3955 | fetch_register_operand(op); | 4015 | fetch_register_operand(op); |
| 3956 | break; | 4016 | break; |
| 3957 | case OpCL: | 4017 | case OpCL: |
| 3958 | op->bytes = 1; | 4018 | op->bytes = 1; |
| 3959 | op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff; | 4019 | op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff; |
| 3960 | break; | 4020 | break; |
| 3961 | case OpImmByte: | 4021 | case OpImmByte: |
| 3962 | rc = decode_imm(ctxt, op, 1, true); | 4022 | rc = decode_imm(ctxt, op, 1, true); |
| @@ -3987,9 +4047,10 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op, | |||
| 3987 | op->type = OP_MEM; | 4047 | op->type = OP_MEM; |
| 3988 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; | 4048 | op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; |
| 3989 | op->addr.mem.ea = | 4049 | op->addr.mem.ea = |
| 3990 | register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]); | 4050 | register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI)); |
| 3991 | op->addr.mem.seg = seg_override(ctxt); | 4051 | op->addr.mem.seg = seg_override(ctxt); |
| 3992 | op->val = 0; | 4052 | op->val = 0; |
| 4053 | op->count = 1; | ||
| 3993 | break; | 4054 | break; |
| 3994 | case OpImmFAddr: | 4055 | case OpImmFAddr: |
| 3995 | op->type = OP_IMM; | 4056 | op->type = OP_IMM; |
| @@ -4293,9 +4354,10 @@ static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt, | |||
| 4293 | read_mmx_reg(ctxt, &op->mm_val, op->addr.mm); | 4354 | read_mmx_reg(ctxt, &op->mm_val, op->addr.mm); |
| 4294 | } | 4355 | } |
| 4295 | 4356 | ||
| 4357 | |||
| 4296 | int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) | 4358 | int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) |
| 4297 | { | 4359 | { |
| 4298 | struct x86_emulate_ops *ops = ctxt->ops; | 4360 | const struct x86_emulate_ops *ops = ctxt->ops; |
| 4299 | int rc = X86EMUL_CONTINUE; | 4361 | int rc = X86EMUL_CONTINUE; |
| 4300 | int saved_dst_type = ctxt->dst.type; | 4362 | int saved_dst_type = ctxt->dst.type; |
| 4301 | 4363 | ||
| @@ -4356,7 +4418,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) | |||
| 4356 | } | 4418 | } |
| 4357 | 4419 | ||
| 4358 | /* Instruction can only be executed in protected mode */ | 4420 | /* Instruction can only be executed in protected mode */ |
| 4359 | if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) { | 4421 | if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) { |
| 4360 | rc = emulate_ud(ctxt); | 4422 | rc = emulate_ud(ctxt); |
| 4361 | goto done; | 4423 | goto done; |
| 4362 | } | 4424 | } |
| @@ -4377,7 +4439,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) | |||
| 4377 | 4439 | ||
| 4378 | if (ctxt->rep_prefix && (ctxt->d & String)) { | 4440 | if (ctxt->rep_prefix && (ctxt->d & String)) { |
| 4379 | /* All REP prefixes have the same first termination condition */ | 4441 | /* All REP prefixes have the same first termination condition */ |
| 4380 | if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) { | 4442 | if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) { |
| 4381 | ctxt->eip = ctxt->_eip; | 4443 | ctxt->eip = ctxt->_eip; |
| 4382 | goto done; | 4444 | goto done; |
| 4383 | } | 4445 | } |
| @@ -4450,7 +4512,7 @@ special_insn: | |||
| 4450 | ctxt->dst.val = ctxt->src.addr.mem.ea; | 4512 | ctxt->dst.val = ctxt->src.addr.mem.ea; |
| 4451 | break; | 4513 | break; |
| 4452 | case 0x90 ... 0x97: /* nop / xchg reg, rax */ | 4514 | case 0x90 ... 0x97: /* nop / xchg reg, rax */ |
| 4453 | if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX]) | 4515 | if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX)) |
| 4454 | break; | 4516 | break; |
| 4455 | rc = em_xchg(ctxt); | 4517 | rc = em_xchg(ctxt); |
| 4456 | break; | 4518 | break; |
| @@ -4478,7 +4540,7 @@ special_insn: | |||
| 4478 | rc = em_grp2(ctxt); | 4540 | rc = em_grp2(ctxt); |
| 4479 | break; | 4541 | break; |
| 4480 | case 0xd2 ... 0xd3: /* Grp2 */ | 4542 | case 0xd2 ... 0xd3: /* Grp2 */ |
| 4481 | ctxt->src.val = ctxt->regs[VCPU_REGS_RCX]; | 4543 | ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX); |
| 4482 | rc = em_grp2(ctxt); | 4544 | rc = em_grp2(ctxt); |
| 4483 | break; | 4545 | break; |
| 4484 | case 0xe9: /* jmp rel */ | 4546 | case 0xe9: /* jmp rel */ |
| @@ -4524,23 +4586,27 @@ writeback: | |||
| 4524 | ctxt->dst.type = saved_dst_type; | 4586 | ctxt->dst.type = saved_dst_type; |
| 4525 | 4587 | ||
| 4526 | if ((ctxt->d & SrcMask) == SrcSI) | 4588 | if ((ctxt->d & SrcMask) == SrcSI) |
| 4527 | string_addr_inc(ctxt, seg_override(ctxt), | 4589 | string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src); |
| 4528 | VCPU_REGS_RSI, &ctxt->src); | ||
| 4529 | 4590 | ||
| 4530 | if ((ctxt->d & DstMask) == DstDI) | 4591 | if ((ctxt->d & DstMask) == DstDI) |
| 4531 | string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI, | 4592 | string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst); |
| 4532 | &ctxt->dst); | ||
| 4533 | 4593 | ||
| 4534 | if (ctxt->rep_prefix && (ctxt->d & String)) { | 4594 | if (ctxt->rep_prefix && (ctxt->d & String)) { |
| 4595 | unsigned int count; | ||
| 4535 | struct read_cache *r = &ctxt->io_read; | 4596 | struct read_cache *r = &ctxt->io_read; |
| 4536 | register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1); | 4597 | if ((ctxt->d & SrcMask) == SrcSI) |
| 4598 | count = ctxt->src.count; | ||
| 4599 | else | ||
| 4600 | count = ctxt->dst.count; | ||
| 4601 | register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), | ||
| 4602 | -count); | ||
| 4537 | 4603 | ||
| 4538 | if (!string_insn_completed(ctxt)) { | 4604 | if (!string_insn_completed(ctxt)) { |
| 4539 | /* | 4605 | /* |
| 4540 | * Re-enter guest when pio read ahead buffer is empty | 4606 | * Re-enter guest when pio read ahead buffer is empty |
| 4541 | * or, if it is not used, after each 1024 iteration. | 4607 | * or, if it is not used, after each 1024 iteration. |
| 4542 | */ | 4608 | */ |
| 4543 | if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) && | 4609 | if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) && |
| 4544 | (r->end == 0 || r->end != r->pos)) { | 4610 | (r->end == 0 || r->end != r->pos)) { |
| 4545 | /* | 4611 | /* |
| 4546 | * Reset read cache. Usually happens before | 4612 | * Reset read cache. Usually happens before |
| @@ -4548,6 +4614,7 @@ writeback: | |||
| 4548 | * we have to do it here. | 4614 | * we have to do it here. |
| 4549 | */ | 4615 | */ |
| 4550 | ctxt->mem_read.end = 0; | 4616 | ctxt->mem_read.end = 0; |
| 4617 | writeback_registers(ctxt); | ||
| 4551 | return EMULATION_RESTART; | 4618 | return EMULATION_RESTART; |
| 4552 | } | 4619 | } |
| 4553 | goto done; /* skip rip writeback */ | 4620 | goto done; /* skip rip writeback */ |
| @@ -4562,6 +4629,9 @@ done: | |||
| 4562 | if (rc == X86EMUL_INTERCEPTED) | 4629 | if (rc == X86EMUL_INTERCEPTED) |
| 4563 | return EMULATION_INTERCEPTED; | 4630 | return EMULATION_INTERCEPTED; |
| 4564 | 4631 | ||
| 4632 | if (rc == X86EMUL_CONTINUE) | ||
| 4633 | writeback_registers(ctxt); | ||
| 4634 | |||
| 4565 | return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; | 4635 | return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; |
| 4566 | 4636 | ||
| 4567 | twobyte_insn: | 4637 | twobyte_insn: |
| @@ -4634,3 +4704,13 @@ twobyte_insn: | |||
| 4634 | cannot_emulate: | 4704 | cannot_emulate: |
| 4635 | return EMULATION_FAILED; | 4705 | return EMULATION_FAILED; |
| 4636 | } | 4706 | } |
| 4707 | |||
| 4708 | void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt) | ||
| 4709 | { | ||
| 4710 | invalidate_registers(ctxt); | ||
| 4711 | } | ||
| 4712 | |||
| 4713 | void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt) | ||
| 4714 | { | ||
| 4715 | writeback_registers(ctxt); | ||
| 4716 | } | ||
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index adba28f88d1a..11300d2fa714 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c | |||
| @@ -108,7 +108,7 @@ static s64 __kpit_elapsed(struct kvm *kvm) | |||
| 108 | ktime_t remaining; | 108 | ktime_t remaining; |
| 109 | struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state; | 109 | struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state; |
| 110 | 110 | ||
| 111 | if (!ps->pit_timer.period) | 111 | if (!ps->period) |
| 112 | return 0; | 112 | return 0; |
| 113 | 113 | ||
| 114 | /* | 114 | /* |
| @@ -120,9 +120,9 @@ static s64 __kpit_elapsed(struct kvm *kvm) | |||
| 120 | * itself with the initial count and continues counting | 120 | * itself with the initial count and continues counting |
| 121 | * from there. | 121 | * from there. |
| 122 | */ | 122 | */ |
| 123 | remaining = hrtimer_get_remaining(&ps->pit_timer.timer); | 123 | remaining = hrtimer_get_remaining(&ps->timer); |
| 124 | elapsed = ps->pit_timer.period - ktime_to_ns(remaining); | 124 | elapsed = ps->period - ktime_to_ns(remaining); |
| 125 | elapsed = mod_64(elapsed, ps->pit_timer.period); | 125 | elapsed = mod_64(elapsed, ps->period); |
| 126 | 126 | ||
| 127 | return elapsed; | 127 | return elapsed; |
| 128 | } | 128 | } |
| @@ -238,12 +238,12 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian) | |||
| 238 | int value; | 238 | int value; |
| 239 | 239 | ||
| 240 | spin_lock(&ps->inject_lock); | 240 | spin_lock(&ps->inject_lock); |
| 241 | value = atomic_dec_return(&ps->pit_timer.pending); | 241 | value = atomic_dec_return(&ps->pending); |
| 242 | if (value < 0) | 242 | if (value < 0) |
| 243 | /* spurious acks can be generated if, for example, the | 243 | /* spurious acks can be generated if, for example, the |
| 244 | * PIC is being reset. Handle it gracefully here | 244 | * PIC is being reset. Handle it gracefully here |
| 245 | */ | 245 | */ |
| 246 | atomic_inc(&ps->pit_timer.pending); | 246 | atomic_inc(&ps->pending); |
| 247 | else if (value > 0) | 247 | else if (value > 0) |
| 248 | /* in this case, we had multiple outstanding pit interrupts | 248 | /* in this case, we had multiple outstanding pit interrupts |
| 249 | * that we needed to inject. Reinject | 249 | * that we needed to inject. Reinject |
| @@ -261,28 +261,17 @@ void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu) | |||
| 261 | if (!kvm_vcpu_is_bsp(vcpu) || !pit) | 261 | if (!kvm_vcpu_is_bsp(vcpu) || !pit) |
| 262 | return; | 262 | return; |
| 263 | 263 | ||
| 264 | timer = &pit->pit_state.pit_timer.timer; | 264 | timer = &pit->pit_state.timer; |
| 265 | if (hrtimer_cancel(timer)) | 265 | if (hrtimer_cancel(timer)) |
| 266 | hrtimer_start_expires(timer, HRTIMER_MODE_ABS); | 266 | hrtimer_start_expires(timer, HRTIMER_MODE_ABS); |
| 267 | } | 267 | } |
| 268 | 268 | ||
| 269 | static void destroy_pit_timer(struct kvm_pit *pit) | 269 | static void destroy_pit_timer(struct kvm_pit *pit) |
| 270 | { | 270 | { |
| 271 | hrtimer_cancel(&pit->pit_state.pit_timer.timer); | 271 | hrtimer_cancel(&pit->pit_state.timer); |
| 272 | flush_kthread_work(&pit->expired); | 272 | flush_kthread_work(&pit->expired); |
| 273 | } | 273 | } |
| 274 | 274 | ||
| 275 | static bool kpit_is_periodic(struct kvm_timer *ktimer) | ||
| 276 | { | ||
| 277 | struct kvm_kpit_state *ps = container_of(ktimer, struct kvm_kpit_state, | ||
| 278 | pit_timer); | ||
| 279 | return ps->is_periodic; | ||
| 280 | } | ||
| 281 | |||
| 282 | static struct kvm_timer_ops kpit_ops = { | ||
| 283 | .is_periodic = kpit_is_periodic, | ||
| 284 | }; | ||
| 285 | |||
| 286 | static void pit_do_work(struct kthread_work *work) | 275 | static void pit_do_work(struct kthread_work *work) |
| 287 | { | 276 | { |
| 288 | struct kvm_pit *pit = container_of(work, struct kvm_pit, expired); | 277 | struct kvm_pit *pit = container_of(work, struct kvm_pit, expired); |
| @@ -322,16 +311,16 @@ static void pit_do_work(struct kthread_work *work) | |||
| 322 | 311 | ||
| 323 | static enum hrtimer_restart pit_timer_fn(struct hrtimer *data) | 312 | static enum hrtimer_restart pit_timer_fn(struct hrtimer *data) |
| 324 | { | 313 | { |
| 325 | struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); | 314 | struct kvm_kpit_state *ps = container_of(data, struct kvm_kpit_state, timer); |
| 326 | struct kvm_pit *pt = ktimer->kvm->arch.vpit; | 315 | struct kvm_pit *pt = ps->kvm->arch.vpit; |
| 327 | 316 | ||
| 328 | if (ktimer->reinject || !atomic_read(&ktimer->pending)) { | 317 | if (ps->reinject || !atomic_read(&ps->pending)) { |
| 329 | atomic_inc(&ktimer->pending); | 318 | atomic_inc(&ps->pending); |
| 330 | queue_kthread_work(&pt->worker, &pt->expired); | 319 | queue_kthread_work(&pt->worker, &pt->expired); |
| 331 | } | 320 | } |
| 332 | 321 | ||
| 333 | if (ktimer->t_ops->is_periodic(ktimer)) { | 322 | if (ps->is_periodic) { |
| 334 | hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); | 323 | hrtimer_add_expires_ns(&ps->timer, ps->period); |
| 335 | return HRTIMER_RESTART; | 324 | return HRTIMER_RESTART; |
| 336 | } else | 325 | } else |
| 337 | return HRTIMER_NORESTART; | 326 | return HRTIMER_NORESTART; |
| @@ -340,7 +329,6 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data) | |||
| 340 | static void create_pit_timer(struct kvm *kvm, u32 val, int is_period) | 329 | static void create_pit_timer(struct kvm *kvm, u32 val, int is_period) |
| 341 | { | 330 | { |
| 342 | struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state; | 331 | struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state; |
| 343 | struct kvm_timer *pt = &ps->pit_timer; | ||
| 344 | s64 interval; | 332 | s64 interval; |
| 345 | 333 | ||
| 346 | if (!irqchip_in_kernel(kvm) || ps->flags & KVM_PIT_FLAGS_HPET_LEGACY) | 334 | if (!irqchip_in_kernel(kvm) || ps->flags & KVM_PIT_FLAGS_HPET_LEGACY) |
| @@ -351,19 +339,18 @@ static void create_pit_timer(struct kvm *kvm, u32 val, int is_period) | |||
| 351 | pr_debug("create pit timer, interval is %llu nsec\n", interval); | 339 | pr_debug("create pit timer, interval is %llu nsec\n", interval); |
| 352 | 340 | ||
| 353 | /* TODO The new value only affected after the retriggered */ | 341 | /* TODO The new value only affected after the retriggered */ |
| 354 | hrtimer_cancel(&pt->timer); | 342 | hrtimer_cancel(&ps->timer); |
| 355 | flush_kthread_work(&ps->pit->expired); | 343 | flush_kthread_work(&ps->pit->expired); |
| 356 | pt->period = interval; | 344 | ps->period = interval; |
| 357 | ps->is_periodic = is_period; | 345 | ps->is_periodic = is_period; |
| 358 | 346 | ||
| 359 | pt->timer.function = pit_timer_fn; | 347 | ps->timer.function = pit_timer_fn; |
| 360 | pt->t_ops = &kpit_ops; | 348 | ps->kvm = ps->pit->kvm; |
| 361 | pt->kvm = ps->pit->kvm; | ||
| 362 | 349 | ||
| 363 | atomic_set(&pt->pending, 0); | 350 | atomic_set(&ps->pending, 0); |
| 364 | ps->irq_ack = 1; | 351 | ps->irq_ack = 1; |
| 365 | 352 | ||
| 366 | hrtimer_start(&pt->timer, ktime_add_ns(ktime_get(), interval), | 353 | hrtimer_start(&ps->timer, ktime_add_ns(ktime_get(), interval), |
| 367 | HRTIMER_MODE_ABS); | 354 | HRTIMER_MODE_ABS); |
| 368 | } | 355 | } |
| 369 | 356 | ||
| @@ -639,7 +626,7 @@ void kvm_pit_reset(struct kvm_pit *pit) | |||
| 639 | } | 626 | } |
| 640 | mutex_unlock(&pit->pit_state.lock); | 627 | mutex_unlock(&pit->pit_state.lock); |
| 641 | 628 | ||
| 642 | atomic_set(&pit->pit_state.pit_timer.pending, 0); | 629 | atomic_set(&pit->pit_state.pending, 0); |
| 643 | pit->pit_state.irq_ack = 1; | 630 | pit->pit_state.irq_ack = 1; |
| 644 | } | 631 | } |
| 645 | 632 | ||
| @@ -648,7 +635,7 @@ static void pit_mask_notifer(struct kvm_irq_mask_notifier *kimn, bool mask) | |||
| 648 | struct kvm_pit *pit = container_of(kimn, struct kvm_pit, mask_notifier); | 635 | struct kvm_pit *pit = container_of(kimn, struct kvm_pit, mask_notifier); |
| 649 | 636 | ||
| 650 | if (!mask) { | 637 | if (!mask) { |
| 651 | atomic_set(&pit->pit_state.pit_timer.pending, 0); | 638 | atomic_set(&pit->pit_state.pending, 0); |
| 652 | pit->pit_state.irq_ack = 1; | 639 | pit->pit_state.irq_ack = 1; |
| 653 | } | 640 | } |
| 654 | } | 641 | } |
| @@ -706,12 +693,11 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags) | |||
| 706 | 693 | ||
| 707 | pit_state = &pit->pit_state; | 694 | pit_state = &pit->pit_state; |
| 708 | pit_state->pit = pit; | 695 | pit_state->pit = pit; |
| 709 | hrtimer_init(&pit_state->pit_timer.timer, | 696 | hrtimer_init(&pit_state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); |
| 710 | CLOCK_MONOTONIC, HRTIMER_MODE_ABS); | ||
| 711 | pit_state->irq_ack_notifier.gsi = 0; | 697 | pit_state->irq_ack_notifier.gsi = 0; |
| 712 | pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq; | 698 | pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq; |
| 713 | kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier); | 699 | kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier); |
| 714 | pit_state->pit_timer.reinject = true; | 700 | pit_state->reinject = true; |
| 715 | mutex_unlock(&pit->pit_state.lock); | 701 | mutex_unlock(&pit->pit_state.lock); |
| 716 | 702 | ||
| 717 | kvm_pit_reset(pit); | 703 | kvm_pit_reset(pit); |
| @@ -761,7 +747,7 @@ void kvm_free_pit(struct kvm *kvm) | |||
| 761 | kvm_unregister_irq_ack_notifier(kvm, | 747 | kvm_unregister_irq_ack_notifier(kvm, |
| 762 | &kvm->arch.vpit->pit_state.irq_ack_notifier); | 748 | &kvm->arch.vpit->pit_state.irq_ack_notifier); |
| 763 | mutex_lock(&kvm->arch.vpit->pit_state.lock); | 749 | mutex_lock(&kvm->arch.vpit->pit_state.lock); |
| 764 | timer = &kvm->arch.vpit->pit_state.pit_timer.timer; | 750 | timer = &kvm->arch.vpit->pit_state.timer; |
| 765 | hrtimer_cancel(timer); | 751 | hrtimer_cancel(timer); |
| 766 | flush_kthread_work(&kvm->arch.vpit->expired); | 752 | flush_kthread_work(&kvm->arch.vpit->expired); |
| 767 | kthread_stop(kvm->arch.vpit->worker_task); | 753 | kthread_stop(kvm->arch.vpit->worker_task); |
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h index fdf40425ea1d..dd1b16b611b0 100644 --- a/arch/x86/kvm/i8254.h +++ b/arch/x86/kvm/i8254.h | |||
| @@ -24,8 +24,12 @@ struct kvm_kpit_channel_state { | |||
| 24 | struct kvm_kpit_state { | 24 | struct kvm_kpit_state { |
| 25 | struct kvm_kpit_channel_state channels[3]; | 25 | struct kvm_kpit_channel_state channels[3]; |
| 26 | u32 flags; | 26 | u32 flags; |
| 27 | struct kvm_timer pit_timer; | ||
| 28 | bool is_periodic; | 27 | bool is_periodic; |
| 28 | s64 period; /* unit: ns */ | ||
| 29 | struct hrtimer timer; | ||
| 30 | atomic_t pending; /* accumulated triggered timers */ | ||
| 31 | bool reinject; | ||
| 32 | struct kvm *kvm; | ||
| 29 | u32 speaker_data_on; | 33 | u32 speaker_data_on; |
| 30 | struct mutex lock; | 34 | struct mutex lock; |
| 31 | struct kvm_pit *pit; | 35 | struct kvm_pit *pit; |
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index 9fc9aa7ac703..848206df0967 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c | |||
| @@ -190,17 +190,17 @@ void kvm_pic_update_irq(struct kvm_pic *s) | |||
| 190 | 190 | ||
| 191 | int kvm_pic_set_irq(struct kvm_pic *s, int irq, int irq_source_id, int level) | 191 | int kvm_pic_set_irq(struct kvm_pic *s, int irq, int irq_source_id, int level) |
| 192 | { | 192 | { |
| 193 | int ret = -1; | 193 | int ret, irq_level; |
| 194 | |||
| 195 | BUG_ON(irq < 0 || irq >= PIC_NUM_PINS); | ||
| 194 | 196 | ||
| 195 | pic_lock(s); | 197 | pic_lock(s); |
| 196 | if (irq >= 0 && irq < PIC_NUM_PINS) { | 198 | irq_level = __kvm_irq_line_state(&s->irq_states[irq], |
| 197 | int irq_level = __kvm_irq_line_state(&s->irq_states[irq], | 199 | irq_source_id, level); |
| 198 | irq_source_id, level); | 200 | ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, irq_level); |
| 199 | ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, irq_level); | 201 | pic_update_irq(s); |
| 200 | pic_update_irq(s); | 202 | trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr, |
| 201 | trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr, | 203 | s->pics[irq >> 3].imr, ret == 0); |
| 202 | s->pics[irq >> 3].imr, ret == 0); | ||
| 203 | } | ||
| 204 | pic_unlock(s); | 204 | pic_unlock(s); |
| 205 | 205 | ||
| 206 | return ret; | 206 | return ret; |
| @@ -275,23 +275,20 @@ void kvm_pic_reset(struct kvm_kpic_state *s) | |||
| 275 | { | 275 | { |
| 276 | int irq, i; | 276 | int irq, i; |
| 277 | struct kvm_vcpu *vcpu; | 277 | struct kvm_vcpu *vcpu; |
| 278 | u8 irr = s->irr, isr = s->imr; | 278 | u8 edge_irr = s->irr & ~s->elcr; |
| 279 | bool found = false; | 279 | bool found = false; |
| 280 | 280 | ||
| 281 | s->last_irr = 0; | 281 | s->last_irr = 0; |
| 282 | s->irr = 0; | 282 | s->irr &= s->elcr; |
| 283 | s->imr = 0; | 283 | s->imr = 0; |
| 284 | s->isr = 0; | ||
| 285 | s->priority_add = 0; | 284 | s->priority_add = 0; |
| 286 | s->irq_base = 0; | ||
| 287 | s->read_reg_select = 0; | ||
| 288 | s->poll = 0; | ||
| 289 | s->special_mask = 0; | 285 | s->special_mask = 0; |
| 290 | s->init_state = 0; | 286 | s->read_reg_select = 0; |
| 291 | s->auto_eoi = 0; | 287 | if (!s->init4) { |
| 292 | s->rotate_on_auto_eoi = 0; | 288 | s->special_fully_nested_mode = 0; |
| 293 | s->special_fully_nested_mode = 0; | 289 | s->auto_eoi = 0; |
| 294 | s->init4 = 0; | 290 | } |
| 291 | s->init_state = 1; | ||
| 295 | 292 | ||
| 296 | kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm) | 293 | kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm) |
| 297 | if (kvm_apic_accept_pic_intr(vcpu)) { | 294 | if (kvm_apic_accept_pic_intr(vcpu)) { |
| @@ -304,7 +301,7 @@ void kvm_pic_reset(struct kvm_kpic_state *s) | |||
| 304 | return; | 301 | return; |
| 305 | 302 | ||
| 306 | for (irq = 0; irq < PIC_NUM_PINS/2; irq++) | 303 | for (irq = 0; irq < PIC_NUM_PINS/2; irq++) |
| 307 | if (irr & (1 << irq) || isr & (1 << irq)) | 304 | if (edge_irr & (1 << irq)) |
| 308 | pic_clear_isr(s, irq); | 305 | pic_clear_isr(s, irq); |
| 309 | } | 306 | } |
| 310 | 307 | ||
| @@ -316,40 +313,13 @@ static void pic_ioport_write(void *opaque, u32 addr, u32 val) | |||
| 316 | addr &= 1; | 313 | addr &= 1; |
| 317 | if (addr == 0) { | 314 | if (addr == 0) { |
| 318 | if (val & 0x10) { | 315 | if (val & 0x10) { |
| 319 | u8 edge_irr = s->irr & ~s->elcr; | ||
| 320 | int i; | ||
| 321 | bool found = false; | ||
| 322 | struct kvm_vcpu *vcpu; | ||
| 323 | |||
| 324 | s->init4 = val & 1; | 316 | s->init4 = val & 1; |
| 325 | s->last_irr = 0; | ||
| 326 | s->irr &= s->elcr; | ||
| 327 | s->imr = 0; | ||
| 328 | s->priority_add = 0; | ||
| 329 | s->special_mask = 0; | ||
| 330 | s->read_reg_select = 0; | ||
| 331 | if (!s->init4) { | ||
| 332 | s->special_fully_nested_mode = 0; | ||
| 333 | s->auto_eoi = 0; | ||
| 334 | } | ||
| 335 | s->init_state = 1; | ||
| 336 | if (val & 0x02) | 317 | if (val & 0x02) |
| 337 | pr_pic_unimpl("single mode not supported"); | 318 | pr_pic_unimpl("single mode not supported"); |
| 338 | if (val & 0x08) | 319 | if (val & 0x08) |
| 339 | pr_pic_unimpl( | 320 | pr_pic_unimpl( |
| 340 | "level sensitive irq not supported"); | 321 | "level sensitive irq not supported"); |
| 341 | 322 | kvm_pic_reset(s); | |
| 342 | kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm) | ||
| 343 | if (kvm_apic_accept_pic_intr(vcpu)) { | ||
| 344 | found = true; | ||
| 345 | break; | ||
| 346 | } | ||
| 347 | |||
| 348 | |||
| 349 | if (found) | ||
| 350 | for (irq = 0; irq < PIC_NUM_PINS/2; irq++) | ||
| 351 | if (edge_irr & (1 << irq)) | ||
| 352 | pic_clear_isr(s, irq); | ||
| 353 | } else if (val & 0x08) { | 323 | } else if (val & 0x08) { |
| 354 | if (val & 0x04) | 324 | if (val & 0x04) |
| 355 | s->poll = 1; | 325 | s->poll = 1; |
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 2086f2bfba33..2d03568e9498 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h | |||
| @@ -70,7 +70,7 @@ struct kvm_pic { | |||
| 70 | struct kvm_io_device dev_slave; | 70 | struct kvm_io_device dev_slave; |
| 71 | struct kvm_io_device dev_eclr; | 71 | struct kvm_io_device dev_eclr; |
| 72 | void (*ack_notifier)(void *opaque, int irq); | 72 | void (*ack_notifier)(void *opaque, int irq); |
| 73 | unsigned long irq_states[16]; | 73 | unsigned long irq_states[PIC_NUM_PINS]; |
| 74 | }; | 74 | }; |
| 75 | 75 | ||
| 76 | struct kvm_pic *kvm_create_pic(struct kvm *kvm); | 76 | struct kvm_pic *kvm_create_pic(struct kvm *kvm); |
diff --git a/arch/x86/kvm/kvm_timer.h b/arch/x86/kvm/kvm_timer.h deleted file mode 100644 index 497dbaa366d4..000000000000 --- a/arch/x86/kvm/kvm_timer.h +++ /dev/null | |||
| @@ -1,18 +0,0 @@ | |||
| 1 | |||
| 2 | struct kvm_timer { | ||
| 3 | struct hrtimer timer; | ||
| 4 | s64 period; /* unit: ns */ | ||
| 5 | u32 timer_mode_mask; | ||
| 6 | u64 tscdeadline; | ||
| 7 | atomic_t pending; /* accumulated triggered timers */ | ||
| 8 | bool reinject; | ||
| 9 | struct kvm_timer_ops *t_ops; | ||
| 10 | struct kvm *kvm; | ||
| 11 | struct kvm_vcpu *vcpu; | ||
| 12 | }; | ||
| 13 | |||
| 14 | struct kvm_timer_ops { | ||
| 15 | bool (*is_periodic)(struct kvm_timer *); | ||
| 16 | }; | ||
| 17 | |||
| 18 | enum hrtimer_restart kvm_timer_fn(struct hrtimer *data); | ||
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index ce878788a39f..c6e6b721b6ee 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c | |||
| @@ -34,6 +34,7 @@ | |||
| 34 | #include <asm/current.h> | 34 | #include <asm/current.h> |
| 35 | #include <asm/apicdef.h> | 35 | #include <asm/apicdef.h> |
| 36 | #include <linux/atomic.h> | 36 | #include <linux/atomic.h> |
| 37 | #include <linux/jump_label.h> | ||
| 37 | #include "kvm_cache_regs.h" | 38 | #include "kvm_cache_regs.h" |
| 38 | #include "irq.h" | 39 | #include "irq.h" |
| 39 | #include "trace.h" | 40 | #include "trace.h" |
| @@ -65,6 +66,7 @@ | |||
| 65 | #define APIC_DEST_NOSHORT 0x0 | 66 | #define APIC_DEST_NOSHORT 0x0 |
| 66 | #define APIC_DEST_MASK 0x800 | 67 | #define APIC_DEST_MASK 0x800 |
| 67 | #define MAX_APIC_VECTOR 256 | 68 | #define MAX_APIC_VECTOR 256 |
| 69 | #define APIC_VECTORS_PER_REG 32 | ||
| 68 | 70 | ||
| 69 | #define VEC_POS(v) ((v) & (32 - 1)) | 71 | #define VEC_POS(v) ((v) & (32 - 1)) |
| 70 | #define REG_POS(v) (((v) >> 5) << 4) | 72 | #define REG_POS(v) (((v) >> 5) << 4) |
| @@ -72,11 +74,6 @@ | |||
| 72 | static unsigned int min_timer_period_us = 500; | 74 | static unsigned int min_timer_period_us = 500; |
| 73 | module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR); | 75 | module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR); |
| 74 | 76 | ||
| 75 | static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off) | ||
| 76 | { | ||
| 77 | return *((u32 *) (apic->regs + reg_off)); | ||
| 78 | } | ||
| 79 | |||
| 80 | static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) | 77 | static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) |
| 81 | { | 78 | { |
| 82 | *((u32 *) (apic->regs + reg_off)) = val; | 79 | *((u32 *) (apic->regs + reg_off)) = val; |
| @@ -117,19 +114,23 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap) | |||
| 117 | return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); | 114 | return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); |
| 118 | } | 115 | } |
| 119 | 116 | ||
| 120 | static inline int apic_hw_enabled(struct kvm_lapic *apic) | 117 | struct static_key_deferred apic_hw_disabled __read_mostly; |
| 121 | { | 118 | struct static_key_deferred apic_sw_disabled __read_mostly; |
| 122 | return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; | ||
| 123 | } | ||
| 124 | 119 | ||
| 125 | static inline int apic_sw_enabled(struct kvm_lapic *apic) | 120 | static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) |
| 126 | { | 121 | { |
| 127 | return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED; | 122 | if ((kvm_apic_get_reg(apic, APIC_SPIV) ^ val) & APIC_SPIV_APIC_ENABLED) { |
| 123 | if (val & APIC_SPIV_APIC_ENABLED) | ||
| 124 | static_key_slow_dec_deferred(&apic_sw_disabled); | ||
| 125 | else | ||
| 126 | static_key_slow_inc(&apic_sw_disabled.key); | ||
| 127 | } | ||
| 128 | apic_set_reg(apic, APIC_SPIV, val); | ||
| 128 | } | 129 | } |
| 129 | 130 | ||
| 130 | static inline int apic_enabled(struct kvm_lapic *apic) | 131 | static inline int apic_enabled(struct kvm_lapic *apic) |
| 131 | { | 132 | { |
| 132 | return apic_sw_enabled(apic) && apic_hw_enabled(apic); | 133 | return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); |
| 133 | } | 134 | } |
| 134 | 135 | ||
| 135 | #define LVT_MASK \ | 136 | #define LVT_MASK \ |
| @@ -139,36 +140,135 @@ static inline int apic_enabled(struct kvm_lapic *apic) | |||
| 139 | (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ | 140 | (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ |
| 140 | APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) | 141 | APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) |
| 141 | 142 | ||
| 143 | static inline int apic_x2apic_mode(struct kvm_lapic *apic) | ||
| 144 | { | ||
| 145 | return apic->vcpu->arch.apic_base & X2APIC_ENABLE; | ||
| 146 | } | ||
| 147 | |||
| 142 | static inline int kvm_apic_id(struct kvm_lapic *apic) | 148 | static inline int kvm_apic_id(struct kvm_lapic *apic) |
| 143 | { | 149 | { |
| 144 | return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff; | 150 | return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff; |
| 151 | } | ||
| 152 | |||
| 153 | static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr) | ||
| 154 | { | ||
| 155 | u16 cid; | ||
| 156 | ldr >>= 32 - map->ldr_bits; | ||
| 157 | cid = (ldr >> map->cid_shift) & map->cid_mask; | ||
| 158 | |||
| 159 | BUG_ON(cid >= ARRAY_SIZE(map->logical_map)); | ||
| 160 | |||
| 161 | return cid; | ||
| 162 | } | ||
| 163 | |||
| 164 | static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr) | ||
| 165 | { | ||
| 166 | ldr >>= (32 - map->ldr_bits); | ||
| 167 | return ldr & map->lid_mask; | ||
| 168 | } | ||
| 169 | |||
| 170 | static void recalculate_apic_map(struct kvm *kvm) | ||
| 171 | { | ||
| 172 | struct kvm_apic_map *new, *old = NULL; | ||
| 173 | struct kvm_vcpu *vcpu; | ||
| 174 | int i; | ||
| 175 | |||
| 176 | new = kzalloc(sizeof(struct kvm_apic_map), GFP_KERNEL); | ||
| 177 | |||
| 178 | mutex_lock(&kvm->arch.apic_map_lock); | ||
| 179 | |||
| 180 | if (!new) | ||
| 181 | goto out; | ||
| 182 | |||
| 183 | new->ldr_bits = 8; | ||
| 184 | /* flat mode is default */ | ||
| 185 | new->cid_shift = 8; | ||
| 186 | new->cid_mask = 0; | ||
| 187 | new->lid_mask = 0xff; | ||
| 188 | |||
| 189 | kvm_for_each_vcpu(i, vcpu, kvm) { | ||
| 190 | struct kvm_lapic *apic = vcpu->arch.apic; | ||
| 191 | u16 cid, lid; | ||
| 192 | u32 ldr; | ||
| 193 | |||
| 194 | if (!kvm_apic_present(vcpu)) | ||
| 195 | continue; | ||
| 196 | |||
| 197 | /* | ||
| 198 | * All APICs have to be configured in the same mode by an OS. | ||
| 199 | * We take advatage of this while building logical id loockup | ||
| 200 | * table. After reset APICs are in xapic/flat mode, so if we | ||
| 201 | * find apic with different setting we assume this is the mode | ||
| 202 | * OS wants all apics to be in; build lookup table accordingly. | ||
| 203 | */ | ||
| 204 | if (apic_x2apic_mode(apic)) { | ||
| 205 | new->ldr_bits = 32; | ||
| 206 | new->cid_shift = 16; | ||
| 207 | new->cid_mask = new->lid_mask = 0xffff; | ||
| 208 | } else if (kvm_apic_sw_enabled(apic) && | ||
| 209 | !new->cid_mask /* flat mode */ && | ||
| 210 | kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_CLUSTER) { | ||
| 211 | new->cid_shift = 4; | ||
| 212 | new->cid_mask = 0xf; | ||
| 213 | new->lid_mask = 0xf; | ||
| 214 | } | ||
| 215 | |||
| 216 | new->phys_map[kvm_apic_id(apic)] = apic; | ||
| 217 | |||
| 218 | ldr = kvm_apic_get_reg(apic, APIC_LDR); | ||
| 219 | cid = apic_cluster_id(new, ldr); | ||
| 220 | lid = apic_logical_id(new, ldr); | ||
| 221 | |||
| 222 | if (lid) | ||
| 223 | new->logical_map[cid][ffs(lid) - 1] = apic; | ||
| 224 | } | ||
| 225 | out: | ||
| 226 | old = rcu_dereference_protected(kvm->arch.apic_map, | ||
| 227 | lockdep_is_held(&kvm->arch.apic_map_lock)); | ||
| 228 | rcu_assign_pointer(kvm->arch.apic_map, new); | ||
| 229 | mutex_unlock(&kvm->arch.apic_map_lock); | ||
| 230 | |||
| 231 | if (old) | ||
| 232 | kfree_rcu(old, rcu); | ||
| 233 | } | ||
| 234 | |||
| 235 | static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id) | ||
| 236 | { | ||
| 237 | apic_set_reg(apic, APIC_ID, id << 24); | ||
| 238 | recalculate_apic_map(apic->vcpu->kvm); | ||
| 239 | } | ||
| 240 | |||
| 241 | static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) | ||
| 242 | { | ||
| 243 | apic_set_reg(apic, APIC_LDR, id); | ||
| 244 | recalculate_apic_map(apic->vcpu->kvm); | ||
| 145 | } | 245 | } |
| 146 | 246 | ||
| 147 | static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) | 247 | static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) |
| 148 | { | 248 | { |
| 149 | return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); | 249 | return !(kvm_apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); |
| 150 | } | 250 | } |
| 151 | 251 | ||
| 152 | static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) | 252 | static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) |
| 153 | { | 253 | { |
| 154 | return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; | 254 | return kvm_apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; |
| 155 | } | 255 | } |
| 156 | 256 | ||
| 157 | static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) | 257 | static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) |
| 158 | { | 258 | { |
| 159 | return ((apic_get_reg(apic, APIC_LVTT) & | 259 | return ((kvm_apic_get_reg(apic, APIC_LVTT) & |
| 160 | apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT); | 260 | apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT); |
| 161 | } | 261 | } |
| 162 | 262 | ||
| 163 | static inline int apic_lvtt_period(struct kvm_lapic *apic) | 263 | static inline int apic_lvtt_period(struct kvm_lapic *apic) |
| 164 | { | 264 | { |
| 165 | return ((apic_get_reg(apic, APIC_LVTT) & | 265 | return ((kvm_apic_get_reg(apic, APIC_LVTT) & |
| 166 | apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC); | 266 | apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC); |
| 167 | } | 267 | } |
| 168 | 268 | ||
| 169 | static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) | 269 | static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) |
| 170 | { | 270 | { |
| 171 | return ((apic_get_reg(apic, APIC_LVTT) & | 271 | return ((kvm_apic_get_reg(apic, APIC_LVTT) & |
| 172 | apic->lapic_timer.timer_mode_mask) == | 272 | apic->lapic_timer.timer_mode_mask) == |
| 173 | APIC_LVT_TIMER_TSCDEADLINE); | 273 | APIC_LVT_TIMER_TSCDEADLINE); |
| 174 | } | 274 | } |
| @@ -184,7 +284,7 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu) | |||
| 184 | struct kvm_cpuid_entry2 *feat; | 284 | struct kvm_cpuid_entry2 *feat; |
| 185 | u32 v = APIC_VERSION; | 285 | u32 v = APIC_VERSION; |
| 186 | 286 | ||
| 187 | if (!irqchip_in_kernel(vcpu->kvm)) | 287 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 188 | return; | 288 | return; |
| 189 | 289 | ||
| 190 | feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); | 290 | feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); |
| @@ -193,12 +293,7 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu) | |||
| 193 | apic_set_reg(apic, APIC_LVR, v); | 293 | apic_set_reg(apic, APIC_LVR, v); |
| 194 | } | 294 | } |
| 195 | 295 | ||
| 196 | static inline int apic_x2apic_mode(struct kvm_lapic *apic) | 296 | static const unsigned int apic_lvt_mask[APIC_LVT_NUM] = { |
| 197 | { | ||
| 198 | return apic->vcpu->arch.apic_base & X2APIC_ENABLE; | ||
| 199 | } | ||
| 200 | |||
| 201 | static unsigned int apic_lvt_mask[APIC_LVT_NUM] = { | ||
| 202 | LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ | 297 | LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ |
| 203 | LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ | 298 | LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ |
| 204 | LVT_MASK | APIC_MODE_MASK, /* LVTPC */ | 299 | LVT_MASK | APIC_MODE_MASK, /* LVTPC */ |
| @@ -208,25 +303,30 @@ static unsigned int apic_lvt_mask[APIC_LVT_NUM] = { | |||
| 208 | 303 | ||
| 209 | static int find_highest_vector(void *bitmap) | 304 | static int find_highest_vector(void *bitmap) |
| 210 | { | 305 | { |
| 211 | u32 *word = bitmap; | 306 | int vec; |
| 212 | int word_offset = MAX_APIC_VECTOR >> 5; | 307 | u32 *reg; |
| 213 | 308 | ||
| 214 | while ((word_offset != 0) && (word[(--word_offset) << 2] == 0)) | 309 | for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; |
| 215 | continue; | 310 | vec >= 0; vec -= APIC_VECTORS_PER_REG) { |
| 311 | reg = bitmap + REG_POS(vec); | ||
| 312 | if (*reg) | ||
| 313 | return fls(*reg) - 1 + vec; | ||
| 314 | } | ||
| 216 | 315 | ||
| 217 | if (likely(!word_offset && !word[0])) | 316 | return -1; |
| 218 | return -1; | ||
| 219 | else | ||
| 220 | return fls(word[word_offset << 2]) - 1 + (word_offset << 5); | ||
| 221 | } | 317 | } |
| 222 | 318 | ||
| 223 | static u8 count_vectors(void *bitmap) | 319 | static u8 count_vectors(void *bitmap) |
| 224 | { | 320 | { |
| 225 | u32 *word = bitmap; | 321 | int vec; |
| 226 | int word_offset; | 322 | u32 *reg; |
| 227 | u8 count = 0; | 323 | u8 count = 0; |
| 228 | for (word_offset = 0; word_offset < MAX_APIC_VECTOR >> 5; ++word_offset) | 324 | |
| 229 | count += hweight32(word[word_offset << 2]); | 325 | for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { |
| 326 | reg = bitmap + REG_POS(vec); | ||
| 327 | count += hweight32(*reg); | ||
| 328 | } | ||
| 329 | |||
| 230 | return count; | 330 | return count; |
| 231 | } | 331 | } |
| 232 | 332 | ||
| @@ -285,7 +385,6 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) | |||
| 285 | 385 | ||
| 286 | int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) | 386 | int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) |
| 287 | { | 387 | { |
| 288 | struct kvm_lapic *apic = vcpu->arch.apic; | ||
| 289 | int highest_irr; | 388 | int highest_irr; |
| 290 | 389 | ||
| 291 | /* This may race with setting of irr in __apic_accept_irq() and | 390 | /* This may race with setting of irr in __apic_accept_irq() and |
| @@ -293,9 +392,9 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) | |||
| 293 | * will cause vmexit immediately and the value will be recalculated | 392 | * will cause vmexit immediately and the value will be recalculated |
| 294 | * on the next vmentry. | 393 | * on the next vmentry. |
| 295 | */ | 394 | */ |
| 296 | if (!apic) | 395 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 297 | return 0; | 396 | return 0; |
| 298 | highest_irr = apic_find_highest_irr(apic); | 397 | highest_irr = apic_find_highest_irr(vcpu->arch.apic); |
| 299 | 398 | ||
| 300 | return highest_irr; | 399 | return highest_irr; |
| 301 | } | 400 | } |
| @@ -378,8 +477,8 @@ static void apic_update_ppr(struct kvm_lapic *apic) | |||
| 378 | u32 tpr, isrv, ppr, old_ppr; | 477 | u32 tpr, isrv, ppr, old_ppr; |
| 379 | int isr; | 478 | int isr; |
| 380 | 479 | ||
| 381 | old_ppr = apic_get_reg(apic, APIC_PROCPRI); | 480 | old_ppr = kvm_apic_get_reg(apic, APIC_PROCPRI); |
| 382 | tpr = apic_get_reg(apic, APIC_TASKPRI); | 481 | tpr = kvm_apic_get_reg(apic, APIC_TASKPRI); |
| 383 | isr = apic_find_highest_isr(apic); | 482 | isr = apic_find_highest_isr(apic); |
| 384 | isrv = (isr != -1) ? isr : 0; | 483 | isrv = (isr != -1) ? isr : 0; |
| 385 | 484 | ||
| @@ -415,13 +514,13 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda) | |||
| 415 | u32 logical_id; | 514 | u32 logical_id; |
| 416 | 515 | ||
| 417 | if (apic_x2apic_mode(apic)) { | 516 | if (apic_x2apic_mode(apic)) { |
| 418 | logical_id = apic_get_reg(apic, APIC_LDR); | 517 | logical_id = kvm_apic_get_reg(apic, APIC_LDR); |
| 419 | return logical_id & mda; | 518 | return logical_id & mda; |
| 420 | } | 519 | } |
| 421 | 520 | ||
| 422 | logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR)); | 521 | logical_id = GET_APIC_LOGICAL_ID(kvm_apic_get_reg(apic, APIC_LDR)); |
| 423 | 522 | ||
| 424 | switch (apic_get_reg(apic, APIC_DFR)) { | 523 | switch (kvm_apic_get_reg(apic, APIC_DFR)) { |
| 425 | case APIC_DFR_FLAT: | 524 | case APIC_DFR_FLAT: |
| 426 | if (logical_id & mda) | 525 | if (logical_id & mda) |
| 427 | result = 1; | 526 | result = 1; |
| @@ -433,7 +532,7 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda) | |||
| 433 | break; | 532 | break; |
| 434 | default: | 533 | default: |
| 435 | apic_debug("Bad DFR vcpu %d: %08x\n", | 534 | apic_debug("Bad DFR vcpu %d: %08x\n", |
| 436 | apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR)); | 535 | apic->vcpu->vcpu_id, kvm_apic_get_reg(apic, APIC_DFR)); |
| 437 | break; | 536 | break; |
| 438 | } | 537 | } |
| 439 | 538 | ||
| @@ -478,6 +577,72 @@ int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, | |||
| 478 | return result; | 577 | return result; |
| 479 | } | 578 | } |
| 480 | 579 | ||
| 580 | bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, | ||
| 581 | struct kvm_lapic_irq *irq, int *r) | ||
| 582 | { | ||
| 583 | struct kvm_apic_map *map; | ||
| 584 | unsigned long bitmap = 1; | ||
| 585 | struct kvm_lapic **dst; | ||
| 586 | int i; | ||
| 587 | bool ret = false; | ||
| 588 | |||
| 589 | *r = -1; | ||
| 590 | |||
| 591 | if (irq->shorthand == APIC_DEST_SELF) { | ||
| 592 | *r = kvm_apic_set_irq(src->vcpu, irq); | ||
| 593 | return true; | ||
| 594 | } | ||
| 595 | |||
| 596 | if (irq->shorthand) | ||
| 597 | return false; | ||
| 598 | |||
| 599 | rcu_read_lock(); | ||
| 600 | map = rcu_dereference(kvm->arch.apic_map); | ||
| 601 | |||
| 602 | if (!map) | ||
| 603 | goto out; | ||
| 604 | |||
| 605 | if (irq->dest_mode == 0) { /* physical mode */ | ||
| 606 | if (irq->delivery_mode == APIC_DM_LOWEST || | ||
| 607 | irq->dest_id == 0xff) | ||
| 608 | goto out; | ||
| 609 | dst = &map->phys_map[irq->dest_id & 0xff]; | ||
| 610 | } else { | ||
| 611 | u32 mda = irq->dest_id << (32 - map->ldr_bits); | ||
| 612 | |||
| 613 | dst = map->logical_map[apic_cluster_id(map, mda)]; | ||
| 614 | |||
| 615 | bitmap = apic_logical_id(map, mda); | ||
| 616 | |||
| 617 | if (irq->delivery_mode == APIC_DM_LOWEST) { | ||
| 618 | int l = -1; | ||
| 619 | for_each_set_bit(i, &bitmap, 16) { | ||
| 620 | if (!dst[i]) | ||
| 621 | continue; | ||
| 622 | if (l < 0) | ||
| 623 | l = i; | ||
| 624 | else if (kvm_apic_compare_prio(dst[i]->vcpu, dst[l]->vcpu) < 0) | ||
| 625 | l = i; | ||
| 626 | } | ||
| 627 | |||
| 628 | bitmap = (l >= 0) ? 1 << l : 0; | ||
| 629 | } | ||
| 630 | } | ||
| 631 | |||
| 632 | for_each_set_bit(i, &bitmap, 16) { | ||
| 633 | if (!dst[i]) | ||
| 634 | continue; | ||
| 635 | if (*r < 0) | ||
| 636 | *r = 0; | ||
| 637 | *r += kvm_apic_set_irq(dst[i]->vcpu, irq); | ||
| 638 | } | ||
| 639 | |||
| 640 | ret = true; | ||
| 641 | out: | ||
| 642 | rcu_read_unlock(); | ||
| 643 | return ret; | ||
| 644 | } | ||
| 645 | |||
| 481 | /* | 646 | /* |
| 482 | * Add a pending IRQ into lapic. | 647 | * Add a pending IRQ into lapic. |
| 483 | * Return 1 if successfully added and 0 if discarded. | 648 | * Return 1 if successfully added and 0 if discarded. |
| @@ -591,7 +756,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) | |||
| 591 | apic_clear_isr(vector, apic); | 756 | apic_clear_isr(vector, apic); |
| 592 | apic_update_ppr(apic); | 757 | apic_update_ppr(apic); |
| 593 | 758 | ||
| 594 | if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) && | 759 | if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) && |
| 595 | kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) { | 760 | kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) { |
| 596 | int trigger_mode; | 761 | int trigger_mode; |
| 597 | if (apic_test_vector(vector, apic->regs + APIC_TMR)) | 762 | if (apic_test_vector(vector, apic->regs + APIC_TMR)) |
| @@ -606,8 +771,8 @@ static int apic_set_eoi(struct kvm_lapic *apic) | |||
| 606 | 771 | ||
| 607 | static void apic_send_ipi(struct kvm_lapic *apic) | 772 | static void apic_send_ipi(struct kvm_lapic *apic) |
| 608 | { | 773 | { |
| 609 | u32 icr_low = apic_get_reg(apic, APIC_ICR); | 774 | u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR); |
| 610 | u32 icr_high = apic_get_reg(apic, APIC_ICR2); | 775 | u32 icr_high = kvm_apic_get_reg(apic, APIC_ICR2); |
| 611 | struct kvm_lapic_irq irq; | 776 | struct kvm_lapic_irq irq; |
| 612 | 777 | ||
| 613 | irq.vector = icr_low & APIC_VECTOR_MASK; | 778 | irq.vector = icr_low & APIC_VECTOR_MASK; |
| @@ -642,7 +807,7 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic) | |||
| 642 | ASSERT(apic != NULL); | 807 | ASSERT(apic != NULL); |
| 643 | 808 | ||
| 644 | /* if initial count is 0, current count should also be 0 */ | 809 | /* if initial count is 0, current count should also be 0 */ |
| 645 | if (apic_get_reg(apic, APIC_TMICT) == 0) | 810 | if (kvm_apic_get_reg(apic, APIC_TMICT) == 0) |
| 646 | return 0; | 811 | return 0; |
| 647 | 812 | ||
| 648 | remaining = hrtimer_get_remaining(&apic->lapic_timer.timer); | 813 | remaining = hrtimer_get_remaining(&apic->lapic_timer.timer); |
| @@ -696,13 +861,15 @@ static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) | |||
| 696 | 861 | ||
| 697 | val = apic_get_tmcct(apic); | 862 | val = apic_get_tmcct(apic); |
| 698 | break; | 863 | break; |
| 699 | 864 | case APIC_PROCPRI: | |
| 865 | apic_update_ppr(apic); | ||
| 866 | val = kvm_apic_get_reg(apic, offset); | ||
| 867 | break; | ||
| 700 | case APIC_TASKPRI: | 868 | case APIC_TASKPRI: |
| 701 | report_tpr_access(apic, false); | 869 | report_tpr_access(apic, false); |
| 702 | /* fall thru */ | 870 | /* fall thru */ |
| 703 | default: | 871 | default: |
| 704 | apic_update_ppr(apic); | 872 | val = kvm_apic_get_reg(apic, offset); |
| 705 | val = apic_get_reg(apic, offset); | ||
| 706 | break; | 873 | break; |
| 707 | } | 874 | } |
| 708 | 875 | ||
| @@ -719,7 +886,7 @@ static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len, | |||
| 719 | { | 886 | { |
| 720 | unsigned char alignment = offset & 0xf; | 887 | unsigned char alignment = offset & 0xf; |
| 721 | u32 result; | 888 | u32 result; |
| 722 | /* this bitmask has a bit cleared for each reserver register */ | 889 | /* this bitmask has a bit cleared for each reserved register */ |
| 723 | static const u64 rmask = 0x43ff01ffffffe70cULL; | 890 | static const u64 rmask = 0x43ff01ffffffe70cULL; |
| 724 | 891 | ||
| 725 | if ((alignment + len) > 4) { | 892 | if ((alignment + len) > 4) { |
| @@ -754,7 +921,7 @@ static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len, | |||
| 754 | 921 | ||
| 755 | static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) | 922 | static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) |
| 756 | { | 923 | { |
| 757 | return apic_hw_enabled(apic) && | 924 | return kvm_apic_hw_enabled(apic) && |
| 758 | addr >= apic->base_address && | 925 | addr >= apic->base_address && |
| 759 | addr < apic->base_address + LAPIC_MMIO_LENGTH; | 926 | addr < apic->base_address + LAPIC_MMIO_LENGTH; |
| 760 | } | 927 | } |
| @@ -777,7 +944,7 @@ static void update_divide_count(struct kvm_lapic *apic) | |||
| 777 | { | 944 | { |
| 778 | u32 tmp1, tmp2, tdcr; | 945 | u32 tmp1, tmp2, tdcr; |
| 779 | 946 | ||
| 780 | tdcr = apic_get_reg(apic, APIC_TDCR); | 947 | tdcr = kvm_apic_get_reg(apic, APIC_TDCR); |
| 781 | tmp1 = tdcr & 0xf; | 948 | tmp1 = tdcr & 0xf; |
| 782 | tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; | 949 | tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; |
| 783 | apic->divide_count = 0x1 << (tmp2 & 0x7); | 950 | apic->divide_count = 0x1 << (tmp2 & 0x7); |
| @@ -792,9 +959,9 @@ static void start_apic_timer(struct kvm_lapic *apic) | |||
| 792 | atomic_set(&apic->lapic_timer.pending, 0); | 959 | atomic_set(&apic->lapic_timer.pending, 0); |
| 793 | 960 | ||
| 794 | if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { | 961 | if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { |
| 795 | /* lapic timer in oneshot or peroidic mode */ | 962 | /* lapic timer in oneshot or periodic mode */ |
| 796 | now = apic->lapic_timer.timer.base->get_time(); | 963 | now = apic->lapic_timer.timer.base->get_time(); |
| 797 | apic->lapic_timer.period = (u64)apic_get_reg(apic, APIC_TMICT) | 964 | apic->lapic_timer.period = (u64)kvm_apic_get_reg(apic, APIC_TMICT) |
| 798 | * APIC_BUS_CYCLE_NS * apic->divide_count; | 965 | * APIC_BUS_CYCLE_NS * apic->divide_count; |
| 799 | 966 | ||
| 800 | if (!apic->lapic_timer.period) | 967 | if (!apic->lapic_timer.period) |
| @@ -826,7 +993,7 @@ static void start_apic_timer(struct kvm_lapic *apic) | |||
| 826 | "timer initial count 0x%x, period %lldns, " | 993 | "timer initial count 0x%x, period %lldns, " |
| 827 | "expire @ 0x%016" PRIx64 ".\n", __func__, | 994 | "expire @ 0x%016" PRIx64 ".\n", __func__, |
| 828 | APIC_BUS_CYCLE_NS, ktime_to_ns(now), | 995 | APIC_BUS_CYCLE_NS, ktime_to_ns(now), |
| 829 | apic_get_reg(apic, APIC_TMICT), | 996 | kvm_apic_get_reg(apic, APIC_TMICT), |
| 830 | apic->lapic_timer.period, | 997 | apic->lapic_timer.period, |
| 831 | ktime_to_ns(ktime_add_ns(now, | 998 | ktime_to_ns(ktime_add_ns(now, |
| 832 | apic->lapic_timer.period))); | 999 | apic->lapic_timer.period))); |
| @@ -858,7 +1025,7 @@ static void start_apic_timer(struct kvm_lapic *apic) | |||
| 858 | 1025 | ||
| 859 | static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) | 1026 | static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) |
| 860 | { | 1027 | { |
| 861 | int nmi_wd_enabled = apic_lvt_nmi_mode(apic_get_reg(apic, APIC_LVT0)); | 1028 | int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0)); |
| 862 | 1029 | ||
| 863 | if (apic_lvt_nmi_mode(lvt0_val)) { | 1030 | if (apic_lvt_nmi_mode(lvt0_val)) { |
| 864 | if (!nmi_wd_enabled) { | 1031 | if (!nmi_wd_enabled) { |
| @@ -879,7 +1046,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) | |||
| 879 | switch (reg) { | 1046 | switch (reg) { |
| 880 | case APIC_ID: /* Local APIC ID */ | 1047 | case APIC_ID: /* Local APIC ID */ |
| 881 | if (!apic_x2apic_mode(apic)) | 1048 | if (!apic_x2apic_mode(apic)) |
| 882 | apic_set_reg(apic, APIC_ID, val); | 1049 | kvm_apic_set_id(apic, val >> 24); |
| 883 | else | 1050 | else |
| 884 | ret = 1; | 1051 | ret = 1; |
| 885 | break; | 1052 | break; |
| @@ -895,29 +1062,30 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) | |||
| 895 | 1062 | ||
| 896 | case APIC_LDR: | 1063 | case APIC_LDR: |
| 897 | if (!apic_x2apic_mode(apic)) | 1064 | if (!apic_x2apic_mode(apic)) |
| 898 | apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK); | 1065 | kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); |
| 899 | else | 1066 | else |
| 900 | ret = 1; | 1067 | ret = 1; |
| 901 | break; | 1068 | break; |
| 902 | 1069 | ||
| 903 | case APIC_DFR: | 1070 | case APIC_DFR: |
| 904 | if (!apic_x2apic_mode(apic)) | 1071 | if (!apic_x2apic_mode(apic)) { |
| 905 | apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); | 1072 | apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); |
| 906 | else | 1073 | recalculate_apic_map(apic->vcpu->kvm); |
| 1074 | } else | ||
| 907 | ret = 1; | 1075 | ret = 1; |
| 908 | break; | 1076 | break; |
| 909 | 1077 | ||
| 910 | case APIC_SPIV: { | 1078 | case APIC_SPIV: { |
| 911 | u32 mask = 0x3ff; | 1079 | u32 mask = 0x3ff; |
| 912 | if (apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) | 1080 | if (kvm_apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) |
| 913 | mask |= APIC_SPIV_DIRECTED_EOI; | 1081 | mask |= APIC_SPIV_DIRECTED_EOI; |
| 914 | apic_set_reg(apic, APIC_SPIV, val & mask); | 1082 | apic_set_spiv(apic, val & mask); |
| 915 | if (!(val & APIC_SPIV_APIC_ENABLED)) { | 1083 | if (!(val & APIC_SPIV_APIC_ENABLED)) { |
| 916 | int i; | 1084 | int i; |
| 917 | u32 lvt_val; | 1085 | u32 lvt_val; |
| 918 | 1086 | ||
| 919 | for (i = 0; i < APIC_LVT_NUM; i++) { | 1087 | for (i = 0; i < APIC_LVT_NUM; i++) { |
| 920 | lvt_val = apic_get_reg(apic, | 1088 | lvt_val = kvm_apic_get_reg(apic, |
| 921 | APIC_LVTT + 0x10 * i); | 1089 | APIC_LVTT + 0x10 * i); |
| 922 | apic_set_reg(apic, APIC_LVTT + 0x10 * i, | 1090 | apic_set_reg(apic, APIC_LVTT + 0x10 * i, |
| 923 | lvt_val | APIC_LVT_MASKED); | 1091 | lvt_val | APIC_LVT_MASKED); |
| @@ -946,7 +1114,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) | |||
| 946 | case APIC_LVT1: | 1114 | case APIC_LVT1: |
| 947 | case APIC_LVTERR: | 1115 | case APIC_LVTERR: |
| 948 | /* TODO: Check vector */ | 1116 | /* TODO: Check vector */ |
| 949 | if (!apic_sw_enabled(apic)) | 1117 | if (!kvm_apic_sw_enabled(apic)) |
| 950 | val |= APIC_LVT_MASKED; | 1118 | val |= APIC_LVT_MASKED; |
| 951 | 1119 | ||
| 952 | val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; | 1120 | val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; |
| @@ -955,12 +1123,12 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) | |||
| 955 | break; | 1123 | break; |
| 956 | 1124 | ||
| 957 | case APIC_LVTT: | 1125 | case APIC_LVTT: |
| 958 | if ((apic_get_reg(apic, APIC_LVTT) & | 1126 | if ((kvm_apic_get_reg(apic, APIC_LVTT) & |
| 959 | apic->lapic_timer.timer_mode_mask) != | 1127 | apic->lapic_timer.timer_mode_mask) != |
| 960 | (val & apic->lapic_timer.timer_mode_mask)) | 1128 | (val & apic->lapic_timer.timer_mode_mask)) |
| 961 | hrtimer_cancel(&apic->lapic_timer.timer); | 1129 | hrtimer_cancel(&apic->lapic_timer.timer); |
| 962 | 1130 | ||
| 963 | if (!apic_sw_enabled(apic)) | 1131 | if (!kvm_apic_sw_enabled(apic)) |
| 964 | val |= APIC_LVT_MASKED; | 1132 | val |= APIC_LVT_MASKED; |
| 965 | val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); | 1133 | val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); |
| 966 | apic_set_reg(apic, APIC_LVTT, val); | 1134 | apic_set_reg(apic, APIC_LVTT, val); |
| @@ -1039,24 +1207,30 @@ static int apic_mmio_write(struct kvm_io_device *this, | |||
| 1039 | 1207 | ||
| 1040 | void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) | 1208 | void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) |
| 1041 | { | 1209 | { |
| 1042 | struct kvm_lapic *apic = vcpu->arch.apic; | 1210 | if (kvm_vcpu_has_lapic(vcpu)) |
| 1043 | |||
| 1044 | if (apic) | ||
| 1045 | apic_reg_write(vcpu->arch.apic, APIC_EOI, 0); | 1211 | apic_reg_write(vcpu->arch.apic, APIC_EOI, 0); |
| 1046 | } | 1212 | } |
| 1047 | EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); | 1213 | EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); |
| 1048 | 1214 | ||
| 1049 | void kvm_free_lapic(struct kvm_vcpu *vcpu) | 1215 | void kvm_free_lapic(struct kvm_vcpu *vcpu) |
| 1050 | { | 1216 | { |
| 1217 | struct kvm_lapic *apic = vcpu->arch.apic; | ||
| 1218 | |||
| 1051 | if (!vcpu->arch.apic) | 1219 | if (!vcpu->arch.apic) |
| 1052 | return; | 1220 | return; |
| 1053 | 1221 | ||
| 1054 | hrtimer_cancel(&vcpu->arch.apic->lapic_timer.timer); | 1222 | hrtimer_cancel(&apic->lapic_timer.timer); |
| 1223 | |||
| 1224 | if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) | ||
| 1225 | static_key_slow_dec_deferred(&apic_hw_disabled); | ||
| 1226 | |||
| 1227 | if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED)) | ||
| 1228 | static_key_slow_dec_deferred(&apic_sw_disabled); | ||
| 1055 | 1229 | ||
| 1056 | if (vcpu->arch.apic->regs) | 1230 | if (apic->regs) |
| 1057 | free_page((unsigned long)vcpu->arch.apic->regs); | 1231 | free_page((unsigned long)apic->regs); |
| 1058 | 1232 | ||
| 1059 | kfree(vcpu->arch.apic); | 1233 | kfree(apic); |
| 1060 | } | 1234 | } |
| 1061 | 1235 | ||
| 1062 | /* | 1236 | /* |
| @@ -1068,10 +1242,9 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) | |||
| 1068 | u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) | 1242 | u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) |
| 1069 | { | 1243 | { |
| 1070 | struct kvm_lapic *apic = vcpu->arch.apic; | 1244 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1071 | if (!apic) | ||
| 1072 | return 0; | ||
| 1073 | 1245 | ||
| 1074 | if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic)) | 1246 | if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) || |
| 1247 | apic_lvtt_period(apic)) | ||
| 1075 | return 0; | 1248 | return 0; |
| 1076 | 1249 | ||
| 1077 | return apic->lapic_timer.tscdeadline; | 1250 | return apic->lapic_timer.tscdeadline; |
| @@ -1080,10 +1253,9 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) | |||
| 1080 | void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) | 1253 | void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) |
| 1081 | { | 1254 | { |
| 1082 | struct kvm_lapic *apic = vcpu->arch.apic; | 1255 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1083 | if (!apic) | ||
| 1084 | return; | ||
| 1085 | 1256 | ||
| 1086 | if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic)) | 1257 | if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) || |
| 1258 | apic_lvtt_period(apic)) | ||
| 1087 | return; | 1259 | return; |
| 1088 | 1260 | ||
| 1089 | hrtimer_cancel(&apic->lapic_timer.timer); | 1261 | hrtimer_cancel(&apic->lapic_timer.timer); |
| @@ -1095,20 +1267,21 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) | |||
| 1095 | { | 1267 | { |
| 1096 | struct kvm_lapic *apic = vcpu->arch.apic; | 1268 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1097 | 1269 | ||
| 1098 | if (!apic) | 1270 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 1099 | return; | 1271 | return; |
| 1272 | |||
| 1100 | apic_set_tpr(apic, ((cr8 & 0x0f) << 4) | 1273 | apic_set_tpr(apic, ((cr8 & 0x0f) << 4) |
| 1101 | | (apic_get_reg(apic, APIC_TASKPRI) & 4)); | 1274 | | (kvm_apic_get_reg(apic, APIC_TASKPRI) & 4)); |
| 1102 | } | 1275 | } |
| 1103 | 1276 | ||
| 1104 | u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) | 1277 | u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) |
| 1105 | { | 1278 | { |
| 1106 | struct kvm_lapic *apic = vcpu->arch.apic; | ||
| 1107 | u64 tpr; | 1279 | u64 tpr; |
| 1108 | 1280 | ||
| 1109 | if (!apic) | 1281 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 1110 | return 0; | 1282 | return 0; |
| 1111 | tpr = (u64) apic_get_reg(apic, APIC_TASKPRI); | 1283 | |
| 1284 | tpr = (u64) kvm_apic_get_reg(vcpu->arch.apic, APIC_TASKPRI); | ||
| 1112 | 1285 | ||
| 1113 | return (tpr & 0xf0) >> 4; | 1286 | return (tpr & 0xf0) >> 4; |
| 1114 | } | 1287 | } |
| @@ -1123,6 +1296,15 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) | |||
| 1123 | return; | 1296 | return; |
| 1124 | } | 1297 | } |
| 1125 | 1298 | ||
| 1299 | /* update jump label if enable bit changes */ | ||
| 1300 | if ((vcpu->arch.apic_base ^ value) & MSR_IA32_APICBASE_ENABLE) { | ||
| 1301 | if (value & MSR_IA32_APICBASE_ENABLE) | ||
| 1302 | static_key_slow_dec_deferred(&apic_hw_disabled); | ||
| 1303 | else | ||
| 1304 | static_key_slow_inc(&apic_hw_disabled.key); | ||
| 1305 | recalculate_apic_map(vcpu->kvm); | ||
| 1306 | } | ||
| 1307 | |||
| 1126 | if (!kvm_vcpu_is_bsp(apic->vcpu)) | 1308 | if (!kvm_vcpu_is_bsp(apic->vcpu)) |
| 1127 | value &= ~MSR_IA32_APICBASE_BSP; | 1309 | value &= ~MSR_IA32_APICBASE_BSP; |
| 1128 | 1310 | ||
| @@ -1130,7 +1312,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) | |||
| 1130 | if (apic_x2apic_mode(apic)) { | 1312 | if (apic_x2apic_mode(apic)) { |
| 1131 | u32 id = kvm_apic_id(apic); | 1313 | u32 id = kvm_apic_id(apic); |
| 1132 | u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf)); | 1314 | u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf)); |
| 1133 | apic_set_reg(apic, APIC_LDR, ldr); | 1315 | kvm_apic_set_ldr(apic, ldr); |
| 1134 | } | 1316 | } |
| 1135 | apic->base_address = apic->vcpu->arch.apic_base & | 1317 | apic->base_address = apic->vcpu->arch.apic_base & |
| 1136 | MSR_IA32_APICBASE_BASE; | 1318 | MSR_IA32_APICBASE_BASE; |
| @@ -1155,7 +1337,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) | |||
| 1155 | /* Stop the timer in case it's a reset to an active apic */ | 1337 | /* Stop the timer in case it's a reset to an active apic */ |
| 1156 | hrtimer_cancel(&apic->lapic_timer.timer); | 1338 | hrtimer_cancel(&apic->lapic_timer.timer); |
| 1157 | 1339 | ||
| 1158 | apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24); | 1340 | kvm_apic_set_id(apic, vcpu->vcpu_id); |
| 1159 | kvm_apic_set_version(apic->vcpu); | 1341 | kvm_apic_set_version(apic->vcpu); |
| 1160 | 1342 | ||
| 1161 | for (i = 0; i < APIC_LVT_NUM; i++) | 1343 | for (i = 0; i < APIC_LVT_NUM; i++) |
| @@ -1164,9 +1346,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) | |||
| 1164 | SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); | 1346 | SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); |
| 1165 | 1347 | ||
| 1166 | apic_set_reg(apic, APIC_DFR, 0xffffffffU); | 1348 | apic_set_reg(apic, APIC_DFR, 0xffffffffU); |
| 1167 | apic_set_reg(apic, APIC_SPIV, 0xff); | 1349 | apic_set_spiv(apic, 0xff); |
| 1168 | apic_set_reg(apic, APIC_TASKPRI, 0); | 1350 | apic_set_reg(apic, APIC_TASKPRI, 0); |
| 1169 | apic_set_reg(apic, APIC_LDR, 0); | 1351 | kvm_apic_set_ldr(apic, 0); |
| 1170 | apic_set_reg(apic, APIC_ESR, 0); | 1352 | apic_set_reg(apic, APIC_ESR, 0); |
| 1171 | apic_set_reg(apic, APIC_ICR, 0); | 1353 | apic_set_reg(apic, APIC_ICR, 0); |
| 1172 | apic_set_reg(apic, APIC_ICR2, 0); | 1354 | apic_set_reg(apic, APIC_ICR2, 0); |
| @@ -1183,7 +1365,8 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) | |||
| 1183 | update_divide_count(apic); | 1365 | update_divide_count(apic); |
| 1184 | atomic_set(&apic->lapic_timer.pending, 0); | 1366 | atomic_set(&apic->lapic_timer.pending, 0); |
| 1185 | if (kvm_vcpu_is_bsp(vcpu)) | 1367 | if (kvm_vcpu_is_bsp(vcpu)) |
| 1186 | vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; | 1368 | kvm_lapic_set_base(vcpu, |
| 1369 | vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); | ||
| 1187 | vcpu->arch.pv_eoi.msr_val = 0; | 1370 | vcpu->arch.pv_eoi.msr_val = 0; |
| 1188 | apic_update_ppr(apic); | 1371 | apic_update_ppr(apic); |
| 1189 | 1372 | ||
| @@ -1196,45 +1379,34 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) | |||
| 1196 | vcpu->arch.apic_base, apic->base_address); | 1379 | vcpu->arch.apic_base, apic->base_address); |
| 1197 | } | 1380 | } |
| 1198 | 1381 | ||
| 1199 | bool kvm_apic_present(struct kvm_vcpu *vcpu) | ||
| 1200 | { | ||
| 1201 | return vcpu->arch.apic && apic_hw_enabled(vcpu->arch.apic); | ||
| 1202 | } | ||
| 1203 | |||
| 1204 | int kvm_lapic_enabled(struct kvm_vcpu *vcpu) | ||
| 1205 | { | ||
| 1206 | return kvm_apic_present(vcpu) && apic_sw_enabled(vcpu->arch.apic); | ||
| 1207 | } | ||
| 1208 | |||
| 1209 | /* | 1382 | /* |
| 1210 | *---------------------------------------------------------------------- | 1383 | *---------------------------------------------------------------------- |
| 1211 | * timer interface | 1384 | * timer interface |
| 1212 | *---------------------------------------------------------------------- | 1385 | *---------------------------------------------------------------------- |
| 1213 | */ | 1386 | */ |
| 1214 | 1387 | ||
| 1215 | static bool lapic_is_periodic(struct kvm_timer *ktimer) | 1388 | static bool lapic_is_periodic(struct kvm_lapic *apic) |
| 1216 | { | 1389 | { |
| 1217 | struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, | ||
| 1218 | lapic_timer); | ||
| 1219 | return apic_lvtt_period(apic); | 1390 | return apic_lvtt_period(apic); |
| 1220 | } | 1391 | } |
| 1221 | 1392 | ||
| 1222 | int apic_has_pending_timer(struct kvm_vcpu *vcpu) | 1393 | int apic_has_pending_timer(struct kvm_vcpu *vcpu) |
| 1223 | { | 1394 | { |
| 1224 | struct kvm_lapic *lapic = vcpu->arch.apic; | 1395 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1225 | 1396 | ||
| 1226 | if (lapic && apic_enabled(lapic) && apic_lvt_enabled(lapic, APIC_LVTT)) | 1397 | if (kvm_vcpu_has_lapic(vcpu) && apic_enabled(apic) && |
| 1227 | return atomic_read(&lapic->lapic_timer.pending); | 1398 | apic_lvt_enabled(apic, APIC_LVTT)) |
| 1399 | return atomic_read(&apic->lapic_timer.pending); | ||
| 1228 | 1400 | ||
| 1229 | return 0; | 1401 | return 0; |
| 1230 | } | 1402 | } |
| 1231 | 1403 | ||
| 1232 | int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) | 1404 | int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) |
| 1233 | { | 1405 | { |
| 1234 | u32 reg = apic_get_reg(apic, lvt_type); | 1406 | u32 reg = kvm_apic_get_reg(apic, lvt_type); |
| 1235 | int vector, mode, trig_mode; | 1407 | int vector, mode, trig_mode; |
| 1236 | 1408 | ||
| 1237 | if (apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { | 1409 | if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { |
| 1238 | vector = reg & APIC_VECTOR_MASK; | 1410 | vector = reg & APIC_VECTOR_MASK; |
| 1239 | mode = reg & APIC_MODE_MASK; | 1411 | mode = reg & APIC_MODE_MASK; |
| 1240 | trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; | 1412 | trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; |
| @@ -1251,15 +1423,40 @@ void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) | |||
| 1251 | kvm_apic_local_deliver(apic, APIC_LVT0); | 1423 | kvm_apic_local_deliver(apic, APIC_LVT0); |
| 1252 | } | 1424 | } |
| 1253 | 1425 | ||
| 1254 | static struct kvm_timer_ops lapic_timer_ops = { | ||
| 1255 | .is_periodic = lapic_is_periodic, | ||
| 1256 | }; | ||
| 1257 | |||
| 1258 | static const struct kvm_io_device_ops apic_mmio_ops = { | 1426 | static const struct kvm_io_device_ops apic_mmio_ops = { |
| 1259 | .read = apic_mmio_read, | 1427 | .read = apic_mmio_read, |
| 1260 | .write = apic_mmio_write, | 1428 | .write = apic_mmio_write, |
| 1261 | }; | 1429 | }; |
| 1262 | 1430 | ||
| 1431 | static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) | ||
| 1432 | { | ||
| 1433 | struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); | ||
| 1434 | struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); | ||
| 1435 | struct kvm_vcpu *vcpu = apic->vcpu; | ||
| 1436 | wait_queue_head_t *q = &vcpu->wq; | ||
| 1437 | |||
| 1438 | /* | ||
| 1439 | * There is a race window between reading and incrementing, but we do | ||
| 1440 | * not care about potentially losing timer events in the !reinject | ||
| 1441 | * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked | ||
| 1442 | * in vcpu_enter_guest. | ||
| 1443 | */ | ||
| 1444 | if (!atomic_read(&ktimer->pending)) { | ||
| 1445 | atomic_inc(&ktimer->pending); | ||
| 1446 | /* FIXME: this code should not know anything about vcpus */ | ||
| 1447 | kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu); | ||
| 1448 | } | ||
| 1449 | |||
| 1450 | if (waitqueue_active(q)) | ||
| 1451 | wake_up_interruptible(q); | ||
| 1452 | |||
| 1453 | if (lapic_is_periodic(apic)) { | ||
| 1454 | hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); | ||
| 1455 | return HRTIMER_RESTART; | ||
| 1456 | } else | ||
| 1457 | return HRTIMER_NORESTART; | ||
| 1458 | } | ||
| 1459 | |||
| 1263 | int kvm_create_lapic(struct kvm_vcpu *vcpu) | 1460 | int kvm_create_lapic(struct kvm_vcpu *vcpu) |
| 1264 | { | 1461 | { |
| 1265 | struct kvm_lapic *apic; | 1462 | struct kvm_lapic *apic; |
| @@ -1283,14 +1480,17 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) | |||
| 1283 | 1480 | ||
| 1284 | hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, | 1481 | hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, |
| 1285 | HRTIMER_MODE_ABS); | 1482 | HRTIMER_MODE_ABS); |
| 1286 | apic->lapic_timer.timer.function = kvm_timer_fn; | 1483 | apic->lapic_timer.timer.function = apic_timer_fn; |
| 1287 | apic->lapic_timer.t_ops = &lapic_timer_ops; | ||
| 1288 | apic->lapic_timer.kvm = vcpu->kvm; | ||
| 1289 | apic->lapic_timer.vcpu = vcpu; | ||
| 1290 | 1484 | ||
| 1291 | apic->base_address = APIC_DEFAULT_PHYS_BASE; | 1485 | /* |
| 1292 | vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE; | 1486 | * APIC is created enabled. This will prevent kvm_lapic_set_base from |
| 1487 | * thinking that APIC satet has changed. | ||
| 1488 | */ | ||
| 1489 | vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; | ||
| 1490 | kvm_lapic_set_base(vcpu, | ||
| 1491 | APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE); | ||
| 1293 | 1492 | ||
| 1493 | static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ | ||
| 1294 | kvm_lapic_reset(vcpu); | 1494 | kvm_lapic_reset(vcpu); |
| 1295 | kvm_iodevice_init(&apic->dev, &apic_mmio_ops); | 1495 | kvm_iodevice_init(&apic->dev, &apic_mmio_ops); |
| 1296 | 1496 | ||
| @@ -1306,23 +1506,23 @@ int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) | |||
| 1306 | struct kvm_lapic *apic = vcpu->arch.apic; | 1506 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1307 | int highest_irr; | 1507 | int highest_irr; |
| 1308 | 1508 | ||
| 1309 | if (!apic || !apic_enabled(apic)) | 1509 | if (!kvm_vcpu_has_lapic(vcpu) || !apic_enabled(apic)) |
| 1310 | return -1; | 1510 | return -1; |
| 1311 | 1511 | ||
| 1312 | apic_update_ppr(apic); | 1512 | apic_update_ppr(apic); |
| 1313 | highest_irr = apic_find_highest_irr(apic); | 1513 | highest_irr = apic_find_highest_irr(apic); |
| 1314 | if ((highest_irr == -1) || | 1514 | if ((highest_irr == -1) || |
| 1315 | ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI))) | 1515 | ((highest_irr & 0xF0) <= kvm_apic_get_reg(apic, APIC_PROCPRI))) |
| 1316 | return -1; | 1516 | return -1; |
| 1317 | return highest_irr; | 1517 | return highest_irr; |
| 1318 | } | 1518 | } |
| 1319 | 1519 | ||
| 1320 | int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) | 1520 | int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) |
| 1321 | { | 1521 | { |
| 1322 | u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0); | 1522 | u32 lvt0 = kvm_apic_get_reg(vcpu->arch.apic, APIC_LVT0); |
| 1323 | int r = 0; | 1523 | int r = 0; |
| 1324 | 1524 | ||
| 1325 | if (!apic_hw_enabled(vcpu->arch.apic)) | 1525 | if (!kvm_apic_hw_enabled(vcpu->arch.apic)) |
| 1326 | r = 1; | 1526 | r = 1; |
| 1327 | if ((lvt0 & APIC_LVT_MASKED) == 0 && | 1527 | if ((lvt0 & APIC_LVT_MASKED) == 0 && |
| 1328 | GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) | 1528 | GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) |
| @@ -1334,7 +1534,10 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) | |||
| 1334 | { | 1534 | { |
| 1335 | struct kvm_lapic *apic = vcpu->arch.apic; | 1535 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1336 | 1536 | ||
| 1337 | if (apic && atomic_read(&apic->lapic_timer.pending) > 0) { | 1537 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 1538 | return; | ||
| 1539 | |||
| 1540 | if (atomic_read(&apic->lapic_timer.pending) > 0) { | ||
| 1338 | if (kvm_apic_local_deliver(apic, APIC_LVTT)) | 1541 | if (kvm_apic_local_deliver(apic, APIC_LVTT)) |
| 1339 | atomic_dec(&apic->lapic_timer.pending); | 1542 | atomic_dec(&apic->lapic_timer.pending); |
| 1340 | } | 1543 | } |
| @@ -1354,12 +1557,17 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) | |||
| 1354 | return vector; | 1557 | return vector; |
| 1355 | } | 1558 | } |
| 1356 | 1559 | ||
| 1357 | void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) | 1560 | void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu, |
| 1561 | struct kvm_lapic_state *s) | ||
| 1358 | { | 1562 | { |
| 1359 | struct kvm_lapic *apic = vcpu->arch.apic; | 1563 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1360 | 1564 | ||
| 1361 | apic->base_address = vcpu->arch.apic_base & | 1565 | kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); |
| 1362 | MSR_IA32_APICBASE_BASE; | 1566 | /* set SPIV separately to get count of SW disabled APICs right */ |
| 1567 | apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); | ||
| 1568 | memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); | ||
| 1569 | /* call kvm_apic_set_id() to put apic into apic_map */ | ||
| 1570 | kvm_apic_set_id(apic, kvm_apic_id(apic)); | ||
| 1363 | kvm_apic_set_version(vcpu); | 1571 | kvm_apic_set_version(vcpu); |
| 1364 | 1572 | ||
| 1365 | apic_update_ppr(apic); | 1573 | apic_update_ppr(apic); |
| @@ -1374,13 +1582,12 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) | |||
| 1374 | 1582 | ||
| 1375 | void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) | 1583 | void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) |
| 1376 | { | 1584 | { |
| 1377 | struct kvm_lapic *apic = vcpu->arch.apic; | ||
| 1378 | struct hrtimer *timer; | 1585 | struct hrtimer *timer; |
| 1379 | 1586 | ||
| 1380 | if (!apic) | 1587 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 1381 | return; | 1588 | return; |
| 1382 | 1589 | ||
| 1383 | timer = &apic->lapic_timer.timer; | 1590 | timer = &vcpu->arch.apic->lapic_timer.timer; |
| 1384 | if (hrtimer_cancel(timer)) | 1591 | if (hrtimer_cancel(timer)) |
| 1385 | hrtimer_start_expires(timer, HRTIMER_MODE_ABS); | 1592 | hrtimer_start_expires(timer, HRTIMER_MODE_ABS); |
| 1386 | } | 1593 | } |
| @@ -1478,7 +1685,7 @@ void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) | |||
| 1478 | if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) | 1685 | if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) |
| 1479 | return; | 1686 | return; |
| 1480 | 1687 | ||
| 1481 | tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff; | 1688 | tpr = kvm_apic_get_reg(apic, APIC_TASKPRI) & 0xff; |
| 1482 | max_irr = apic_find_highest_irr(apic); | 1689 | max_irr = apic_find_highest_irr(apic); |
| 1483 | if (max_irr < 0) | 1690 | if (max_irr < 0) |
| 1484 | max_irr = 0; | 1691 | max_irr = 0; |
| @@ -1537,7 +1744,7 @@ int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) | |||
| 1537 | { | 1744 | { |
| 1538 | struct kvm_lapic *apic = vcpu->arch.apic; | 1745 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1539 | 1746 | ||
| 1540 | if (!irqchip_in_kernel(vcpu->kvm)) | 1747 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 1541 | return 1; | 1748 | return 1; |
| 1542 | 1749 | ||
| 1543 | /* if this is ICR write vector before command */ | 1750 | /* if this is ICR write vector before command */ |
| @@ -1551,7 +1758,7 @@ int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) | |||
| 1551 | struct kvm_lapic *apic = vcpu->arch.apic; | 1758 | struct kvm_lapic *apic = vcpu->arch.apic; |
| 1552 | u32 low, high = 0; | 1759 | u32 low, high = 0; |
| 1553 | 1760 | ||
| 1554 | if (!irqchip_in_kernel(vcpu->kvm)) | 1761 | if (!kvm_vcpu_has_lapic(vcpu)) |
| 1555 | return 1; | 1762 | return 1; |
| 1556 | 1763 | ||
| 1557 | if (apic_reg_read(apic, reg, 4, &low)) | 1764 | if (apic_reg_read(apic, reg, 4, &low)) |
| @@ -1576,3 +1783,10 @@ int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data) | |||
| 1576 | return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, | 1783 | return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, |
| 1577 | addr); | 1784 | addr); |
| 1578 | } | 1785 | } |
| 1786 | |||
| 1787 | void kvm_lapic_init(void) | ||
| 1788 | { | ||
| 1789 | /* do not patch jump label more than once per second */ | ||
| 1790 | jump_label_rate_limit(&apic_hw_disabled, HZ); | ||
| 1791 | jump_label_rate_limit(&apic_sw_disabled, HZ); | ||
| 1792 | } | ||
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 4af5405ae1e2..e5ebf9f3571f 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h | |||
| @@ -2,10 +2,17 @@ | |||
| 2 | #define __KVM_X86_LAPIC_H | 2 | #define __KVM_X86_LAPIC_H |
| 3 | 3 | ||
| 4 | #include "iodev.h" | 4 | #include "iodev.h" |
| 5 | #include "kvm_timer.h" | ||
| 6 | 5 | ||
| 7 | #include <linux/kvm_host.h> | 6 | #include <linux/kvm_host.h> |
| 8 | 7 | ||
| 8 | struct kvm_timer { | ||
| 9 | struct hrtimer timer; | ||
| 10 | s64 period; /* unit: ns */ | ||
| 11 | u32 timer_mode_mask; | ||
| 12 | u64 tscdeadline; | ||
| 13 | atomic_t pending; /* accumulated triggered timers */ | ||
| 14 | }; | ||
| 15 | |||
| 9 | struct kvm_lapic { | 16 | struct kvm_lapic { |
| 10 | unsigned long base_address; | 17 | unsigned long base_address; |
| 11 | struct kvm_io_device dev; | 18 | struct kvm_io_device dev; |
| @@ -45,11 +52,13 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); | |||
| 45 | int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq); | 52 | int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq); |
| 46 | int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type); | 53 | int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type); |
| 47 | 54 | ||
| 55 | bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, | ||
| 56 | struct kvm_lapic_irq *irq, int *r); | ||
| 57 | |||
| 48 | u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); | 58 | u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); |
| 49 | void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); | 59 | void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); |
| 50 | void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); | 60 | void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu, |
| 51 | int kvm_lapic_enabled(struct kvm_vcpu *vcpu); | 61 | struct kvm_lapic_state *s); |
| 52 | bool kvm_apic_present(struct kvm_vcpu *vcpu); | ||
| 53 | int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); | 62 | int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); |
| 54 | 63 | ||
| 55 | u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); | 64 | u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); |
| @@ -71,4 +80,48 @@ static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu) | |||
| 71 | } | 80 | } |
| 72 | 81 | ||
| 73 | int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data); | 82 | int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data); |
| 83 | void kvm_lapic_init(void); | ||
| 84 | |||
| 85 | static inline u32 kvm_apic_get_reg(struct kvm_lapic *apic, int reg_off) | ||
| 86 | { | ||
| 87 | return *((u32 *) (apic->regs + reg_off)); | ||
| 88 | } | ||
| 89 | |||
| 90 | extern struct static_key kvm_no_apic_vcpu; | ||
| 91 | |||
| 92 | static inline bool kvm_vcpu_has_lapic(struct kvm_vcpu *vcpu) | ||
| 93 | { | ||
| 94 | if (static_key_false(&kvm_no_apic_vcpu)) | ||
| 95 | return vcpu->arch.apic; | ||
| 96 | return true; | ||
| 97 | } | ||
| 98 | |||
| 99 | extern struct static_key_deferred apic_hw_disabled; | ||
| 100 | |||
| 101 | static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic) | ||
| 102 | { | ||
| 103 | if (static_key_false(&apic_hw_disabled.key)) | ||
| 104 | return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; | ||
| 105 | return MSR_IA32_APICBASE_ENABLE; | ||
| 106 | } | ||
| 107 | |||
| 108 | extern struct static_key_deferred apic_sw_disabled; | ||
| 109 | |||
| 110 | static inline int kvm_apic_sw_enabled(struct kvm_lapic *apic) | ||
| 111 | { | ||
| 112 | if (static_key_false(&apic_sw_disabled.key)) | ||
| 113 | return kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED; | ||
| 114 | return APIC_SPIV_APIC_ENABLED; | ||
| 115 | } | ||
| 116 | |||
| 117 | static inline bool kvm_apic_present(struct kvm_vcpu *vcpu) | ||
| 118 | { | ||
| 119 | return kvm_vcpu_has_lapic(vcpu) && kvm_apic_hw_enabled(vcpu->arch.apic); | ||
| 120 | } | ||
| 121 | |||
| 122 | static inline int kvm_lapic_enabled(struct kvm_vcpu *vcpu) | ||
| 123 | { | ||
| 124 | return kvm_apic_present(vcpu) && kvm_apic_sw_enabled(vcpu->arch.apic); | ||
| 125 | } | ||
| 126 | |||
| 74 | #endif | 127 | #endif |
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 7fbd0d273ea8..d289fee1ffb8 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c | |||
| @@ -556,6 +556,14 @@ static int mmu_spte_clear_track_bits(u64 *sptep) | |||
| 556 | return 0; | 556 | return 0; |
| 557 | 557 | ||
| 558 | pfn = spte_to_pfn(old_spte); | 558 | pfn = spte_to_pfn(old_spte); |
| 559 | |||
| 560 | /* | ||
| 561 | * KVM does not hold the refcount of the page used by | ||
| 562 | * kvm mmu, before reclaiming the page, we should | ||
| 563 | * unmap it from mmu first. | ||
| 564 | */ | ||
| 565 | WARN_ON(!kvm_is_mmio_pfn(pfn) && !page_count(pfn_to_page(pfn))); | ||
| 566 | |||
| 559 | if (!shadow_accessed_mask || old_spte & shadow_accessed_mask) | 567 | if (!shadow_accessed_mask || old_spte & shadow_accessed_mask) |
| 560 | kvm_set_pfn_accessed(pfn); | 568 | kvm_set_pfn_accessed(pfn); |
| 561 | if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask)) | 569 | if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask)) |
| @@ -960,13 +968,10 @@ static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn) | |||
| 960 | static unsigned long *__gfn_to_rmap(gfn_t gfn, int level, | 968 | static unsigned long *__gfn_to_rmap(gfn_t gfn, int level, |
| 961 | struct kvm_memory_slot *slot) | 969 | struct kvm_memory_slot *slot) |
| 962 | { | 970 | { |
| 963 | struct kvm_lpage_info *linfo; | 971 | unsigned long idx; |
| 964 | |||
| 965 | if (likely(level == PT_PAGE_TABLE_LEVEL)) | ||
| 966 | return &slot->rmap[gfn - slot->base_gfn]; | ||
| 967 | 972 | ||
| 968 | linfo = lpage_info_slot(gfn, slot, level); | 973 | idx = gfn_to_index(gfn, slot->base_gfn, level); |
| 969 | return &linfo->rmap_pde; | 974 | return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx]; |
| 970 | } | 975 | } |
| 971 | 976 | ||
| 972 | /* | 977 | /* |
| @@ -1173,7 +1178,8 @@ void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, | |||
| 1173 | unsigned long *rmapp; | 1178 | unsigned long *rmapp; |
| 1174 | 1179 | ||
| 1175 | while (mask) { | 1180 | while (mask) { |
| 1176 | rmapp = &slot->rmap[gfn_offset + __ffs(mask)]; | 1181 | rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), |
| 1182 | PT_PAGE_TABLE_LEVEL, slot); | ||
| 1177 | __rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false); | 1183 | __rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false); |
| 1178 | 1184 | ||
| 1179 | /* clear the first set bit */ | 1185 | /* clear the first set bit */ |
| @@ -1200,7 +1206,7 @@ static bool rmap_write_protect(struct kvm *kvm, u64 gfn) | |||
| 1200 | } | 1206 | } |
| 1201 | 1207 | ||
| 1202 | static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, | 1208 | static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 1203 | unsigned long data) | 1209 | struct kvm_memory_slot *slot, unsigned long data) |
| 1204 | { | 1210 | { |
| 1205 | u64 *sptep; | 1211 | u64 *sptep; |
| 1206 | struct rmap_iterator iter; | 1212 | struct rmap_iterator iter; |
| @@ -1218,7 +1224,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, | |||
| 1218 | } | 1224 | } |
| 1219 | 1225 | ||
| 1220 | static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp, | 1226 | static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 1221 | unsigned long data) | 1227 | struct kvm_memory_slot *slot, unsigned long data) |
| 1222 | { | 1228 | { |
| 1223 | u64 *sptep; | 1229 | u64 *sptep; |
| 1224 | struct rmap_iterator iter; | 1230 | struct rmap_iterator iter; |
| @@ -1259,43 +1265,67 @@ static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp, | |||
| 1259 | return 0; | 1265 | return 0; |
| 1260 | } | 1266 | } |
| 1261 | 1267 | ||
| 1262 | static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, | 1268 | static int kvm_handle_hva_range(struct kvm *kvm, |
| 1263 | unsigned long data, | 1269 | unsigned long start, |
| 1264 | int (*handler)(struct kvm *kvm, unsigned long *rmapp, | 1270 | unsigned long end, |
| 1265 | unsigned long data)) | 1271 | unsigned long data, |
| 1272 | int (*handler)(struct kvm *kvm, | ||
| 1273 | unsigned long *rmapp, | ||
| 1274 | struct kvm_memory_slot *slot, | ||
| 1275 | unsigned long data)) | ||
| 1266 | { | 1276 | { |
| 1267 | int j; | 1277 | int j; |
| 1268 | int ret; | 1278 | int ret = 0; |
| 1269 | int retval = 0; | ||
| 1270 | struct kvm_memslots *slots; | 1279 | struct kvm_memslots *slots; |
| 1271 | struct kvm_memory_slot *memslot; | 1280 | struct kvm_memory_slot *memslot; |
| 1272 | 1281 | ||
| 1273 | slots = kvm_memslots(kvm); | 1282 | slots = kvm_memslots(kvm); |
| 1274 | 1283 | ||
| 1275 | kvm_for_each_memslot(memslot, slots) { | 1284 | kvm_for_each_memslot(memslot, slots) { |
| 1276 | unsigned long start = memslot->userspace_addr; | 1285 | unsigned long hva_start, hva_end; |
| 1277 | unsigned long end; | 1286 | gfn_t gfn_start, gfn_end; |
| 1278 | 1287 | ||
| 1279 | end = start + (memslot->npages << PAGE_SHIFT); | 1288 | hva_start = max(start, memslot->userspace_addr); |
| 1280 | if (hva >= start && hva < end) { | 1289 | hva_end = min(end, memslot->userspace_addr + |
| 1281 | gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; | 1290 | (memslot->npages << PAGE_SHIFT)); |
| 1282 | gfn_t gfn = memslot->base_gfn + gfn_offset; | 1291 | if (hva_start >= hva_end) |
| 1292 | continue; | ||
| 1293 | /* | ||
| 1294 | * {gfn(page) | page intersects with [hva_start, hva_end)} = | ||
| 1295 | * {gfn_start, gfn_start+1, ..., gfn_end-1}. | ||
| 1296 | */ | ||
| 1297 | gfn_start = hva_to_gfn_memslot(hva_start, memslot); | ||
| 1298 | gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); | ||
| 1283 | 1299 | ||
| 1284 | ret = handler(kvm, &memslot->rmap[gfn_offset], data); | 1300 | for (j = PT_PAGE_TABLE_LEVEL; |
| 1301 | j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) { | ||
| 1302 | unsigned long idx, idx_end; | ||
| 1303 | unsigned long *rmapp; | ||
| 1285 | 1304 | ||
| 1286 | for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) { | 1305 | /* |
| 1287 | struct kvm_lpage_info *linfo; | 1306 | * {idx(page_j) | page_j intersects with |
| 1307 | * [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}. | ||
| 1308 | */ | ||
| 1309 | idx = gfn_to_index(gfn_start, memslot->base_gfn, j); | ||
| 1310 | idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j); | ||
| 1288 | 1311 | ||
| 1289 | linfo = lpage_info_slot(gfn, memslot, | 1312 | rmapp = __gfn_to_rmap(gfn_start, j, memslot); |
| 1290 | PT_DIRECTORY_LEVEL + j); | 1313 | |
| 1291 | ret |= handler(kvm, &linfo->rmap_pde, data); | 1314 | for (; idx <= idx_end; ++idx) |
| 1292 | } | 1315 | ret |= handler(kvm, rmapp++, memslot, data); |
| 1293 | trace_kvm_age_page(hva, memslot, ret); | ||
| 1294 | retval |= ret; | ||
| 1295 | } | 1316 | } |
| 1296 | } | 1317 | } |
| 1297 | 1318 | ||
| 1298 | return retval; | 1319 | return ret; |
| 1320 | } | ||
| 1321 | |||
| 1322 | static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, | ||
| 1323 | unsigned long data, | ||
| 1324 | int (*handler)(struct kvm *kvm, unsigned long *rmapp, | ||
| 1325 | struct kvm_memory_slot *slot, | ||
| 1326 | unsigned long data)) | ||
| 1327 | { | ||
| 1328 | return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler); | ||
| 1299 | } | 1329 | } |
| 1300 | 1330 | ||
| 1301 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) | 1331 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) |
| @@ -1303,13 +1333,18 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) | |||
| 1303 | return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp); | 1333 | return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp); |
| 1304 | } | 1334 | } |
| 1305 | 1335 | ||
| 1336 | int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end) | ||
| 1337 | { | ||
| 1338 | return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp); | ||
| 1339 | } | ||
| 1340 | |||
| 1306 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) | 1341 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) |
| 1307 | { | 1342 | { |
| 1308 | kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp); | 1343 | kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp); |
| 1309 | } | 1344 | } |
| 1310 | 1345 | ||
| 1311 | static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, | 1346 | static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 1312 | unsigned long data) | 1347 | struct kvm_memory_slot *slot, unsigned long data) |
| 1313 | { | 1348 | { |
| 1314 | u64 *sptep; | 1349 | u64 *sptep; |
| 1315 | struct rmap_iterator uninitialized_var(iter); | 1350 | struct rmap_iterator uninitialized_var(iter); |
| @@ -1323,8 +1358,10 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, | |||
| 1323 | * This has some overhead, but not as much as the cost of swapping | 1358 | * This has some overhead, but not as much as the cost of swapping |
| 1324 | * out actively used pages or breaking up actively used hugepages. | 1359 | * out actively used pages or breaking up actively used hugepages. |
| 1325 | */ | 1360 | */ |
| 1326 | if (!shadow_accessed_mask) | 1361 | if (!shadow_accessed_mask) { |
| 1327 | return kvm_unmap_rmapp(kvm, rmapp, data); | 1362 | young = kvm_unmap_rmapp(kvm, rmapp, slot, data); |
| 1363 | goto out; | ||
| 1364 | } | ||
| 1328 | 1365 | ||
| 1329 | for (sptep = rmap_get_first(*rmapp, &iter); sptep; | 1366 | for (sptep = rmap_get_first(*rmapp, &iter); sptep; |
| 1330 | sptep = rmap_get_next(&iter)) { | 1367 | sptep = rmap_get_next(&iter)) { |
| @@ -1336,12 +1373,14 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, | |||
| 1336 | (unsigned long *)sptep); | 1373 | (unsigned long *)sptep); |
| 1337 | } | 1374 | } |
| 1338 | } | 1375 | } |
| 1339 | 1376 | out: | |
| 1377 | /* @data has hva passed to kvm_age_hva(). */ | ||
| 1378 | trace_kvm_age_page(data, slot, young); | ||
| 1340 | return young; | 1379 | return young; |
| 1341 | } | 1380 | } |
| 1342 | 1381 | ||
| 1343 | static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp, | 1382 | static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 1344 | unsigned long data) | 1383 | struct kvm_memory_slot *slot, unsigned long data) |
| 1345 | { | 1384 | { |
| 1346 | u64 *sptep; | 1385 | u64 *sptep; |
| 1347 | struct rmap_iterator iter; | 1386 | struct rmap_iterator iter; |
| @@ -1379,13 +1418,13 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) | |||
| 1379 | 1418 | ||
| 1380 | rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level); | 1419 | rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level); |
| 1381 | 1420 | ||
| 1382 | kvm_unmap_rmapp(vcpu->kvm, rmapp, 0); | 1421 | kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, 0); |
| 1383 | kvm_flush_remote_tlbs(vcpu->kvm); | 1422 | kvm_flush_remote_tlbs(vcpu->kvm); |
| 1384 | } | 1423 | } |
| 1385 | 1424 | ||
| 1386 | int kvm_age_hva(struct kvm *kvm, unsigned long hva) | 1425 | int kvm_age_hva(struct kvm *kvm, unsigned long hva) |
| 1387 | { | 1426 | { |
| 1388 | return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp); | 1427 | return kvm_handle_hva(kvm, hva, hva, kvm_age_rmapp); |
| 1389 | } | 1428 | } |
| 1390 | 1429 | ||
| 1391 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) | 1430 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) |
| @@ -2457,7 +2496,9 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, | |||
| 2457 | rmap_recycle(vcpu, sptep, gfn); | 2496 | rmap_recycle(vcpu, sptep, gfn); |
| 2458 | } | 2497 | } |
| 2459 | } | 2498 | } |
| 2460 | kvm_release_pfn_clean(pfn); | 2499 | |
| 2500 | if (!is_error_pfn(pfn)) | ||
| 2501 | kvm_release_pfn_clean(pfn); | ||
| 2461 | } | 2502 | } |
| 2462 | 2503 | ||
| 2463 | static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) | 2504 | static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) |
| @@ -2469,17 +2510,12 @@ static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, | |||
| 2469 | bool no_dirty_log) | 2510 | bool no_dirty_log) |
| 2470 | { | 2511 | { |
| 2471 | struct kvm_memory_slot *slot; | 2512 | struct kvm_memory_slot *slot; |
| 2472 | unsigned long hva; | ||
| 2473 | 2513 | ||
| 2474 | slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); | 2514 | slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); |
| 2475 | if (!slot) { | 2515 | if (!slot) |
| 2476 | get_page(fault_page); | 2516 | return KVM_PFN_ERR_FAULT; |
| 2477 | return page_to_pfn(fault_page); | ||
| 2478 | } | ||
| 2479 | 2517 | ||
| 2480 | hva = gfn_to_hva_memslot(slot, gfn); | 2518 | return gfn_to_pfn_memslot_atomic(slot, gfn); |
| 2481 | |||
| 2482 | return hva_to_pfn_atomic(vcpu->kvm, hva); | ||
| 2483 | } | 2519 | } |
| 2484 | 2520 | ||
| 2485 | static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, | 2521 | static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, |
| @@ -2580,11 +2616,6 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, | |||
| 2580 | sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr, | 2616 | sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr, |
| 2581 | iterator.level - 1, | 2617 | iterator.level - 1, |
| 2582 | 1, ACC_ALL, iterator.sptep); | 2618 | 1, ACC_ALL, iterator.sptep); |
| 2583 | if (!sp) { | ||
| 2584 | pgprintk("nonpaging_map: ENOMEM\n"); | ||
| 2585 | kvm_release_pfn_clean(pfn); | ||
| 2586 | return -ENOMEM; | ||
| 2587 | } | ||
| 2588 | 2619 | ||
| 2589 | mmu_spte_set(iterator.sptep, | 2620 | mmu_spte_set(iterator.sptep, |
| 2590 | __pa(sp->spt) | 2621 | __pa(sp->spt) |
| @@ -2611,8 +2642,16 @@ static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct * | |||
| 2611 | 2642 | ||
| 2612 | static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn) | 2643 | static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn) |
| 2613 | { | 2644 | { |
| 2614 | kvm_release_pfn_clean(pfn); | 2645 | /* |
| 2615 | if (is_hwpoison_pfn(pfn)) { | 2646 | * Do not cache the mmio info caused by writing the readonly gfn |
| 2647 | * into the spte otherwise read access on readonly gfn also can | ||
| 2648 | * caused mmio page fault and treat it as mmio access. | ||
| 2649 | * Return 1 to tell kvm to emulate it. | ||
| 2650 | */ | ||
| 2651 | if (pfn == KVM_PFN_ERR_RO_FAULT) | ||
| 2652 | return 1; | ||
| 2653 | |||
| 2654 | if (pfn == KVM_PFN_ERR_HWPOISON) { | ||
| 2616 | kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current); | 2655 | kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current); |
| 2617 | return 0; | 2656 | return 0; |
| 2618 | } | 2657 | } |
| @@ -3236,8 +3275,6 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, | |||
| 3236 | if (!async) | 3275 | if (!async) |
| 3237 | return false; /* *pfn has correct page already */ | 3276 | return false; /* *pfn has correct page already */ |
| 3238 | 3277 | ||
| 3239 | put_page(pfn_to_page(*pfn)); | ||
| 3240 | |||
| 3241 | if (!prefault && can_do_async_pf(vcpu)) { | 3278 | if (!prefault && can_do_async_pf(vcpu)) { |
| 3242 | trace_kvm_try_async_get_page(gva, gfn); | 3279 | trace_kvm_try_async_get_page(gva, gfn); |
| 3243 | if (kvm_find_async_pf_gfn(vcpu, gfn)) { | 3280 | if (kvm_find_async_pf_gfn(vcpu, gfn)) { |
| @@ -3371,6 +3408,18 @@ static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level) | |||
| 3371 | return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0; | 3408 | return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0; |
| 3372 | } | 3409 | } |
| 3373 | 3410 | ||
| 3411 | static inline void protect_clean_gpte(unsigned *access, unsigned gpte) | ||
| 3412 | { | ||
| 3413 | unsigned mask; | ||
| 3414 | |||
| 3415 | BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK); | ||
| 3416 | |||
| 3417 | mask = (unsigned)~ACC_WRITE_MASK; | ||
| 3418 | /* Allow write access to dirty gptes */ | ||
| 3419 | mask |= (gpte >> (PT_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) & PT_WRITABLE_MASK; | ||
| 3420 | *access &= mask; | ||
| 3421 | } | ||
| 3422 | |||
| 3374 | static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access, | 3423 | static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access, |
| 3375 | int *nr_present) | 3424 | int *nr_present) |
| 3376 | { | 3425 | { |
| @@ -3388,6 +3437,25 @@ static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access, | |||
| 3388 | return false; | 3437 | return false; |
| 3389 | } | 3438 | } |
| 3390 | 3439 | ||
| 3440 | static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte) | ||
| 3441 | { | ||
| 3442 | unsigned access; | ||
| 3443 | |||
| 3444 | access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; | ||
| 3445 | access &= ~(gpte >> PT64_NX_SHIFT); | ||
| 3446 | |||
| 3447 | return access; | ||
| 3448 | } | ||
| 3449 | |||
| 3450 | static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte) | ||
| 3451 | { | ||
| 3452 | unsigned index; | ||
| 3453 | |||
| 3454 | index = level - 1; | ||
| 3455 | index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2); | ||
| 3456 | return mmu->last_pte_bitmap & (1 << index); | ||
| 3457 | } | ||
| 3458 | |||
| 3391 | #define PTTYPE 64 | 3459 | #define PTTYPE 64 |
| 3392 | #include "paging_tmpl.h" | 3460 | #include "paging_tmpl.h" |
| 3393 | #undef PTTYPE | 3461 | #undef PTTYPE |
| @@ -3457,6 +3525,56 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, | |||
| 3457 | } | 3525 | } |
| 3458 | } | 3526 | } |
| 3459 | 3527 | ||
| 3528 | static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) | ||
| 3529 | { | ||
| 3530 | unsigned bit, byte, pfec; | ||
| 3531 | u8 map; | ||
| 3532 | bool fault, x, w, u, wf, uf, ff, smep; | ||
| 3533 | |||
| 3534 | smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP); | ||
| 3535 | for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) { | ||
| 3536 | pfec = byte << 1; | ||
| 3537 | map = 0; | ||
| 3538 | wf = pfec & PFERR_WRITE_MASK; | ||
| 3539 | uf = pfec & PFERR_USER_MASK; | ||
| 3540 | ff = pfec & PFERR_FETCH_MASK; | ||
| 3541 | for (bit = 0; bit < 8; ++bit) { | ||
| 3542 | x = bit & ACC_EXEC_MASK; | ||
| 3543 | w = bit & ACC_WRITE_MASK; | ||
| 3544 | u = bit & ACC_USER_MASK; | ||
| 3545 | |||
| 3546 | /* Not really needed: !nx will cause pte.nx to fault */ | ||
| 3547 | x |= !mmu->nx; | ||
| 3548 | /* Allow supervisor writes if !cr0.wp */ | ||
| 3549 | w |= !is_write_protection(vcpu) && !uf; | ||
| 3550 | /* Disallow supervisor fetches of user code if cr4.smep */ | ||
| 3551 | x &= !(smep && u && !uf); | ||
| 3552 | |||
| 3553 | fault = (ff && !x) || (uf && !u) || (wf && !w); | ||
| 3554 | map |= fault << bit; | ||
| 3555 | } | ||
| 3556 | mmu->permissions[byte] = map; | ||
| 3557 | } | ||
| 3558 | } | ||
| 3559 | |||
| 3560 | static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) | ||
| 3561 | { | ||
| 3562 | u8 map; | ||
| 3563 | unsigned level, root_level = mmu->root_level; | ||
| 3564 | const unsigned ps_set_index = 1 << 2; /* bit 2 of index: ps */ | ||
| 3565 | |||
| 3566 | if (root_level == PT32E_ROOT_LEVEL) | ||
| 3567 | --root_level; | ||
| 3568 | /* PT_PAGE_TABLE_LEVEL always terminates */ | ||
| 3569 | map = 1 | (1 << ps_set_index); | ||
| 3570 | for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) { | ||
| 3571 | if (level <= PT_PDPE_LEVEL | ||
| 3572 | && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu))) | ||
| 3573 | map |= 1 << (ps_set_index | (level - 1)); | ||
| 3574 | } | ||
| 3575 | mmu->last_pte_bitmap = map; | ||
| 3576 | } | ||
| 3577 | |||
| 3460 | static int paging64_init_context_common(struct kvm_vcpu *vcpu, | 3578 | static int paging64_init_context_common(struct kvm_vcpu *vcpu, |
| 3461 | struct kvm_mmu *context, | 3579 | struct kvm_mmu *context, |
| 3462 | int level) | 3580 | int level) |
| @@ -3465,6 +3583,8 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, | |||
| 3465 | context->root_level = level; | 3583 | context->root_level = level; |
| 3466 | 3584 | ||
| 3467 | reset_rsvds_bits_mask(vcpu, context); | 3585 | reset_rsvds_bits_mask(vcpu, context); |
| 3586 | update_permission_bitmask(vcpu, context); | ||
| 3587 | update_last_pte_bitmap(vcpu, context); | ||
| 3468 | 3588 | ||
| 3469 | ASSERT(is_pae(vcpu)); | 3589 | ASSERT(is_pae(vcpu)); |
| 3470 | context->new_cr3 = paging_new_cr3; | 3590 | context->new_cr3 = paging_new_cr3; |
| @@ -3493,6 +3613,8 @@ static int paging32_init_context(struct kvm_vcpu *vcpu, | |||
| 3493 | context->root_level = PT32_ROOT_LEVEL; | 3613 | context->root_level = PT32_ROOT_LEVEL; |
| 3494 | 3614 | ||
| 3495 | reset_rsvds_bits_mask(vcpu, context); | 3615 | reset_rsvds_bits_mask(vcpu, context); |
| 3616 | update_permission_bitmask(vcpu, context); | ||
| 3617 | update_last_pte_bitmap(vcpu, context); | ||
| 3496 | 3618 | ||
| 3497 | context->new_cr3 = paging_new_cr3; | 3619 | context->new_cr3 = paging_new_cr3; |
| 3498 | context->page_fault = paging32_page_fault; | 3620 | context->page_fault = paging32_page_fault; |
| @@ -3553,6 +3675,9 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) | |||
| 3553 | context->gva_to_gpa = paging32_gva_to_gpa; | 3675 | context->gva_to_gpa = paging32_gva_to_gpa; |
| 3554 | } | 3676 | } |
| 3555 | 3677 | ||
| 3678 | update_permission_bitmask(vcpu, context); | ||
| 3679 | update_last_pte_bitmap(vcpu, context); | ||
| 3680 | |||
| 3556 | return 0; | 3681 | return 0; |
| 3557 | } | 3682 | } |
| 3558 | 3683 | ||
| @@ -3628,6 +3753,9 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu) | |||
| 3628 | g_context->gva_to_gpa = paging32_gva_to_gpa_nested; | 3753 | g_context->gva_to_gpa = paging32_gva_to_gpa_nested; |
| 3629 | } | 3754 | } |
| 3630 | 3755 | ||
| 3756 | update_permission_bitmask(vcpu, g_context); | ||
| 3757 | update_last_pte_bitmap(vcpu, g_context); | ||
| 3758 | |||
| 3631 | return 0; | 3759 | return 0; |
| 3632 | } | 3760 | } |
| 3633 | 3761 | ||
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index e374db9af021..69871080e866 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h | |||
| @@ -18,8 +18,10 @@ | |||
| 18 | #define PT_PCD_MASK (1ULL << 4) | 18 | #define PT_PCD_MASK (1ULL << 4) |
| 19 | #define PT_ACCESSED_SHIFT 5 | 19 | #define PT_ACCESSED_SHIFT 5 |
| 20 | #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT) | 20 | #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT) |
| 21 | #define PT_DIRTY_MASK (1ULL << 6) | 21 | #define PT_DIRTY_SHIFT 6 |
| 22 | #define PT_PAGE_SIZE_MASK (1ULL << 7) | 22 | #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT) |
| 23 | #define PT_PAGE_SIZE_SHIFT 7 | ||
| 24 | #define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT) | ||
| 23 | #define PT_PAT_MASK (1ULL << 7) | 25 | #define PT_PAT_MASK (1ULL << 7) |
| 24 | #define PT_GLOBAL_MASK (1ULL << 8) | 26 | #define PT_GLOBAL_MASK (1ULL << 8) |
| 25 | #define PT64_NX_SHIFT 63 | 27 | #define PT64_NX_SHIFT 63 |
| @@ -88,17 +90,14 @@ static inline bool is_write_protection(struct kvm_vcpu *vcpu) | |||
| 88 | return kvm_read_cr0_bits(vcpu, X86_CR0_WP); | 90 | return kvm_read_cr0_bits(vcpu, X86_CR0_WP); |
| 89 | } | 91 | } |
| 90 | 92 | ||
| 91 | static inline bool check_write_user_access(struct kvm_vcpu *vcpu, | 93 | /* |
| 92 | bool write_fault, bool user_fault, | 94 | * Will a fault with a given page-fault error code (pfec) cause a permission |
| 93 | unsigned long pte) | 95 | * fault with the given access (in ACC_* format)? |
| 96 | */ | ||
| 97 | static inline bool permission_fault(struct kvm_mmu *mmu, unsigned pte_access, | ||
| 98 | unsigned pfec) | ||
| 94 | { | 99 | { |
| 95 | if (unlikely(write_fault && !is_writable_pte(pte) | 100 | return (mmu->permissions[pfec >> 1] >> pte_access) & 1; |
| 96 | && (user_fault || is_write_protection(vcpu)))) | ||
| 97 | return false; | ||
| 98 | |||
| 99 | if (unlikely(user_fault && !(pte & PT_USER_MASK))) | ||
| 100 | return false; | ||
| 101 | |||
| 102 | return true; | ||
| 103 | } | 101 | } |
| 102 | |||
| 104 | #endif | 103 | #endif |
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c index 7d7d0b9e23eb..daff69e21150 100644 --- a/arch/x86/kvm/mmu_audit.c +++ b/arch/x86/kvm/mmu_audit.c | |||
| @@ -116,10 +116,8 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) | |||
| 116 | gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); | 116 | gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); |
| 117 | pfn = gfn_to_pfn_atomic(vcpu->kvm, gfn); | 117 | pfn = gfn_to_pfn_atomic(vcpu->kvm, gfn); |
| 118 | 118 | ||
| 119 | if (is_error_pfn(pfn)) { | 119 | if (is_error_pfn(pfn)) |
| 120 | kvm_release_pfn_clean(pfn); | ||
| 121 | return; | 120 | return; |
| 122 | } | ||
| 123 | 121 | ||
| 124 | hpa = pfn << PAGE_SHIFT; | 122 | hpa = pfn << PAGE_SHIFT; |
| 125 | if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) | 123 | if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) |
| @@ -190,7 +188,6 @@ static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp) | |||
| 190 | 188 | ||
| 191 | static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) | 189 | static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) |
| 192 | { | 190 | { |
| 193 | struct kvm_memory_slot *slot; | ||
| 194 | unsigned long *rmapp; | 191 | unsigned long *rmapp; |
| 195 | u64 *sptep; | 192 | u64 *sptep; |
| 196 | struct rmap_iterator iter; | 193 | struct rmap_iterator iter; |
| @@ -198,8 +195,7 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) | |||
| 198 | if (sp->role.direct || sp->unsync || sp->role.invalid) | 195 | if (sp->role.direct || sp->unsync || sp->role.invalid) |
| 199 | return; | 196 | return; |
| 200 | 197 | ||
| 201 | slot = gfn_to_memslot(kvm, sp->gfn); | 198 | rmapp = gfn_to_rmap(kvm, sp->gfn, PT_PAGE_TABLE_LEVEL); |
| 202 | rmapp = &slot->rmap[sp->gfn - slot->base_gfn]; | ||
| 203 | 199 | ||
| 204 | for (sptep = rmap_get_first(*rmapp, &iter); sptep; | 200 | for (sptep = rmap_get_first(*rmapp, &iter); sptep; |
| 205 | sptep = rmap_get_next(&iter)) { | 201 | sptep = rmap_get_next(&iter)) { |
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index bb7cf01cae76..714e2c01a6fe 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h | |||
| @@ -63,10 +63,12 @@ | |||
| 63 | */ | 63 | */ |
| 64 | struct guest_walker { | 64 | struct guest_walker { |
| 65 | int level; | 65 | int level; |
| 66 | unsigned max_level; | ||
| 66 | gfn_t table_gfn[PT_MAX_FULL_LEVELS]; | 67 | gfn_t table_gfn[PT_MAX_FULL_LEVELS]; |
| 67 | pt_element_t ptes[PT_MAX_FULL_LEVELS]; | 68 | pt_element_t ptes[PT_MAX_FULL_LEVELS]; |
| 68 | pt_element_t prefetch_ptes[PTE_PREFETCH_NUM]; | 69 | pt_element_t prefetch_ptes[PTE_PREFETCH_NUM]; |
| 69 | gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; | 70 | gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; |
| 71 | pt_element_t __user *ptep_user[PT_MAX_FULL_LEVELS]; | ||
| 70 | unsigned pt_access; | 72 | unsigned pt_access; |
| 71 | unsigned pte_access; | 73 | unsigned pte_access; |
| 72 | gfn_t gfn; | 74 | gfn_t gfn; |
| @@ -101,38 +103,41 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, | |||
| 101 | return (ret != orig_pte); | 103 | return (ret != orig_pte); |
| 102 | } | 104 | } |
| 103 | 105 | ||
| 104 | static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte, | 106 | static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu, |
| 105 | bool last) | 107 | struct kvm_mmu *mmu, |
| 108 | struct guest_walker *walker, | ||
| 109 | int write_fault) | ||
| 106 | { | 110 | { |
| 107 | unsigned access; | 111 | unsigned level, index; |
| 108 | 112 | pt_element_t pte, orig_pte; | |
| 109 | access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; | 113 | pt_element_t __user *ptep_user; |
| 110 | if (last && !is_dirty_gpte(gpte)) | 114 | gfn_t table_gfn; |
| 111 | access &= ~ACC_WRITE_MASK; | 115 | int ret; |
| 112 | 116 | ||
| 113 | #if PTTYPE == 64 | 117 | for (level = walker->max_level; level >= walker->level; --level) { |
| 114 | if (vcpu->arch.mmu.nx) | 118 | pte = orig_pte = walker->ptes[level - 1]; |
| 115 | access &= ~(gpte >> PT64_NX_SHIFT); | 119 | table_gfn = walker->table_gfn[level - 1]; |
| 116 | #endif | 120 | ptep_user = walker->ptep_user[level - 1]; |
| 117 | return access; | 121 | index = offset_in_page(ptep_user) / sizeof(pt_element_t); |
| 118 | } | 122 | if (!(pte & PT_ACCESSED_MASK)) { |
| 119 | 123 | trace_kvm_mmu_set_accessed_bit(table_gfn, index, sizeof(pte)); | |
| 120 | static bool FNAME(is_last_gpte)(struct guest_walker *walker, | 124 | pte |= PT_ACCESSED_MASK; |
| 121 | struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, | 125 | } |
| 122 | pt_element_t gpte) | 126 | if (level == walker->level && write_fault && !is_dirty_gpte(pte)) { |
| 123 | { | 127 | trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte)); |
| 124 | if (walker->level == PT_PAGE_TABLE_LEVEL) | 128 | pte |= PT_DIRTY_MASK; |
| 125 | return true; | 129 | } |
| 126 | 130 | if (pte == orig_pte) | |
| 127 | if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) && | 131 | continue; |
| 128 | (PTTYPE == 64 || is_pse(vcpu))) | ||
| 129 | return true; | ||
| 130 | 132 | ||
| 131 | if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) && | 133 | ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte); |
| 132 | (mmu->root_level == PT64_ROOT_LEVEL)) | 134 | if (ret) |
| 133 | return true; | 135 | return ret; |
| 134 | 136 | ||
| 135 | return false; | 137 | mark_page_dirty(vcpu->kvm, table_gfn); |
| 138 | walker->ptes[level] = pte; | ||
| 139 | } | ||
| 140 | return 0; | ||
| 136 | } | 141 | } |
| 137 | 142 | ||
| 138 | /* | 143 | /* |
| @@ -142,21 +147,22 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, | |||
| 142 | struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, | 147 | struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
| 143 | gva_t addr, u32 access) | 148 | gva_t addr, u32 access) |
| 144 | { | 149 | { |
| 150 | int ret; | ||
| 145 | pt_element_t pte; | 151 | pt_element_t pte; |
| 146 | pt_element_t __user *uninitialized_var(ptep_user); | 152 | pt_element_t __user *uninitialized_var(ptep_user); |
| 147 | gfn_t table_gfn; | 153 | gfn_t table_gfn; |
| 148 | unsigned index, pt_access, uninitialized_var(pte_access); | 154 | unsigned index, pt_access, pte_access, accessed_dirty, shift; |
| 149 | gpa_t pte_gpa; | 155 | gpa_t pte_gpa; |
| 150 | bool eperm, last_gpte; | ||
| 151 | int offset; | 156 | int offset; |
| 152 | const int write_fault = access & PFERR_WRITE_MASK; | 157 | const int write_fault = access & PFERR_WRITE_MASK; |
| 153 | const int user_fault = access & PFERR_USER_MASK; | 158 | const int user_fault = access & PFERR_USER_MASK; |
| 154 | const int fetch_fault = access & PFERR_FETCH_MASK; | 159 | const int fetch_fault = access & PFERR_FETCH_MASK; |
| 155 | u16 errcode = 0; | 160 | u16 errcode = 0; |
| 161 | gpa_t real_gpa; | ||
| 162 | gfn_t gfn; | ||
| 156 | 163 | ||
| 157 | trace_kvm_mmu_pagetable_walk(addr, access); | 164 | trace_kvm_mmu_pagetable_walk(addr, access); |
| 158 | retry_walk: | 165 | retry_walk: |
| 159 | eperm = false; | ||
| 160 | walker->level = mmu->root_level; | 166 | walker->level = mmu->root_level; |
| 161 | pte = mmu->get_cr3(vcpu); | 167 | pte = mmu->get_cr3(vcpu); |
| 162 | 168 | ||
| @@ -169,15 +175,21 @@ retry_walk: | |||
| 169 | --walker->level; | 175 | --walker->level; |
| 170 | } | 176 | } |
| 171 | #endif | 177 | #endif |
| 178 | walker->max_level = walker->level; | ||
| 172 | ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || | 179 | ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || |
| 173 | (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0); | 180 | (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0); |
| 174 | 181 | ||
| 175 | pt_access = ACC_ALL; | 182 | accessed_dirty = PT_ACCESSED_MASK; |
| 183 | pt_access = pte_access = ACC_ALL; | ||
| 184 | ++walker->level; | ||
| 176 | 185 | ||
| 177 | for (;;) { | 186 | do { |
| 178 | gfn_t real_gfn; | 187 | gfn_t real_gfn; |
| 179 | unsigned long host_addr; | 188 | unsigned long host_addr; |
| 180 | 189 | ||
| 190 | pt_access &= pte_access; | ||
| 191 | --walker->level; | ||
| 192 | |||
| 181 | index = PT_INDEX(addr, walker->level); | 193 | index = PT_INDEX(addr, walker->level); |
| 182 | 194 | ||
| 183 | table_gfn = gpte_to_gfn(pte); | 195 | table_gfn = gpte_to_gfn(pte); |
| @@ -199,6 +211,7 @@ retry_walk: | |||
| 199 | ptep_user = (pt_element_t __user *)((void *)host_addr + offset); | 211 | ptep_user = (pt_element_t __user *)((void *)host_addr + offset); |
| 200 | if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte)))) | 212 | if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte)))) |
| 201 | goto error; | 213 | goto error; |
| 214 | walker->ptep_user[walker->level - 1] = ptep_user; | ||
| 202 | 215 | ||
| 203 | trace_kvm_mmu_paging_element(pte, walker->level); | 216 | trace_kvm_mmu_paging_element(pte, walker->level); |
| 204 | 217 | ||
| @@ -211,92 +224,48 @@ retry_walk: | |||
| 211 | goto error; | 224 | goto error; |
| 212 | } | 225 | } |
| 213 | 226 | ||
| 214 | if (!check_write_user_access(vcpu, write_fault, user_fault, | 227 | accessed_dirty &= pte; |
| 215 | pte)) | 228 | pte_access = pt_access & gpte_access(vcpu, pte); |
| 216 | eperm = true; | ||
| 217 | |||
| 218 | #if PTTYPE == 64 | ||
| 219 | if (unlikely(fetch_fault && (pte & PT64_NX_MASK))) | ||
| 220 | eperm = true; | ||
| 221 | #endif | ||
| 222 | |||
| 223 | last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte); | ||
| 224 | if (last_gpte) { | ||
| 225 | pte_access = pt_access & | ||
| 226 | FNAME(gpte_access)(vcpu, pte, true); | ||
| 227 | /* check if the kernel is fetching from user page */ | ||
| 228 | if (unlikely(pte_access & PT_USER_MASK) && | ||
| 229 | kvm_read_cr4_bits(vcpu, X86_CR4_SMEP)) | ||
| 230 | if (fetch_fault && !user_fault) | ||
| 231 | eperm = true; | ||
| 232 | } | ||
| 233 | |||
| 234 | if (!eperm && unlikely(!(pte & PT_ACCESSED_MASK))) { | ||
| 235 | int ret; | ||
| 236 | trace_kvm_mmu_set_accessed_bit(table_gfn, index, | ||
| 237 | sizeof(pte)); | ||
| 238 | ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, | ||
| 239 | pte, pte|PT_ACCESSED_MASK); | ||
| 240 | if (unlikely(ret < 0)) | ||
| 241 | goto error; | ||
| 242 | else if (ret) | ||
| 243 | goto retry_walk; | ||
| 244 | |||
| 245 | mark_page_dirty(vcpu->kvm, table_gfn); | ||
| 246 | pte |= PT_ACCESSED_MASK; | ||
| 247 | } | ||
| 248 | 229 | ||
| 249 | walker->ptes[walker->level - 1] = pte; | 230 | walker->ptes[walker->level - 1] = pte; |
| 231 | } while (!is_last_gpte(mmu, walker->level, pte)); | ||
| 250 | 232 | ||
| 251 | if (last_gpte) { | 233 | if (unlikely(permission_fault(mmu, pte_access, access))) { |
| 252 | int lvl = walker->level; | 234 | errcode |= PFERR_PRESENT_MASK; |
| 253 | gpa_t real_gpa; | 235 | goto error; |
| 254 | gfn_t gfn; | 236 | } |
| 255 | u32 ac; | ||
| 256 | |||
| 257 | gfn = gpte_to_gfn_lvl(pte, lvl); | ||
| 258 | gfn += (addr & PT_LVL_OFFSET_MASK(lvl)) >> PAGE_SHIFT; | ||
| 259 | |||
| 260 | if (PTTYPE == 32 && | ||
| 261 | walker->level == PT_DIRECTORY_LEVEL && | ||
| 262 | is_cpuid_PSE36()) | ||
| 263 | gfn += pse36_gfn_delta(pte); | ||
| 264 | |||
| 265 | ac = write_fault | fetch_fault | user_fault; | ||
| 266 | 237 | ||
| 267 | real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), | 238 | gfn = gpte_to_gfn_lvl(pte, walker->level); |
| 268 | ac); | 239 | gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT; |
| 269 | if (real_gpa == UNMAPPED_GVA) | ||
| 270 | return 0; | ||
| 271 | 240 | ||
| 272 | walker->gfn = real_gpa >> PAGE_SHIFT; | 241 | if (PTTYPE == 32 && walker->level == PT_DIRECTORY_LEVEL && is_cpuid_PSE36()) |
| 242 | gfn += pse36_gfn_delta(pte); | ||
| 273 | 243 | ||
| 274 | break; | 244 | real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access); |
| 275 | } | 245 | if (real_gpa == UNMAPPED_GVA) |
| 246 | return 0; | ||
| 276 | 247 | ||
| 277 | pt_access &= FNAME(gpte_access)(vcpu, pte, false); | 248 | walker->gfn = real_gpa >> PAGE_SHIFT; |
| 278 | --walker->level; | ||
| 279 | } | ||
| 280 | 249 | ||
| 281 | if (unlikely(eperm)) { | 250 | if (!write_fault) |
| 282 | errcode |= PFERR_PRESENT_MASK; | 251 | protect_clean_gpte(&pte_access, pte); |
| 283 | goto error; | ||
| 284 | } | ||
| 285 | 252 | ||
| 286 | if (write_fault && unlikely(!is_dirty_gpte(pte))) { | 253 | /* |
| 287 | int ret; | 254 | * On a write fault, fold the dirty bit into accessed_dirty by shifting it one |
| 255 | * place right. | ||
| 256 | * | ||
| 257 | * On a read fault, do nothing. | ||
| 258 | */ | ||
| 259 | shift = write_fault >> ilog2(PFERR_WRITE_MASK); | ||
| 260 | shift *= PT_DIRTY_SHIFT - PT_ACCESSED_SHIFT; | ||
| 261 | accessed_dirty &= pte >> shift; | ||
| 288 | 262 | ||
| 289 | trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte)); | 263 | if (unlikely(!accessed_dirty)) { |
| 290 | ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, | 264 | ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker, write_fault); |
| 291 | pte, pte|PT_DIRTY_MASK); | ||
| 292 | if (unlikely(ret < 0)) | 265 | if (unlikely(ret < 0)) |
| 293 | goto error; | 266 | goto error; |
| 294 | else if (ret) | 267 | else if (ret) |
| 295 | goto retry_walk; | 268 | goto retry_walk; |
| 296 | |||
| 297 | mark_page_dirty(vcpu->kvm, table_gfn); | ||
| 298 | pte |= PT_DIRTY_MASK; | ||
| 299 | walker->ptes[walker->level - 1] = pte; | ||
| 300 | } | 269 | } |
| 301 | 270 | ||
| 302 | walker->pt_access = pt_access; | 271 | walker->pt_access = pt_access; |
| @@ -368,12 +337,11 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, | |||
| 368 | return; | 337 | return; |
| 369 | 338 | ||
| 370 | pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte); | 339 | pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte); |
| 371 | pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, true); | 340 | pte_access = sp->role.access & gpte_access(vcpu, gpte); |
| 341 | protect_clean_gpte(&pte_access, gpte); | ||
| 372 | pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte)); | 342 | pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte)); |
| 373 | if (mmu_invalid_pfn(pfn)) { | 343 | if (mmu_invalid_pfn(pfn)) |
| 374 | kvm_release_pfn_clean(pfn); | ||
| 375 | return; | 344 | return; |
| 376 | } | ||
| 377 | 345 | ||
| 378 | /* | 346 | /* |
| 379 | * we call mmu_set_spte() with host_writable = true because that | 347 | * we call mmu_set_spte() with host_writable = true because that |
| @@ -443,15 +411,13 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, | |||
| 443 | if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) | 411 | if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) |
| 444 | continue; | 412 | continue; |
| 445 | 413 | ||
| 446 | pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, | 414 | pte_access = sp->role.access & gpte_access(vcpu, gpte); |
| 447 | true); | 415 | protect_clean_gpte(&pte_access, gpte); |
| 448 | gfn = gpte_to_gfn(gpte); | 416 | gfn = gpte_to_gfn(gpte); |
| 449 | pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn, | 417 | pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn, |
| 450 | pte_access & ACC_WRITE_MASK); | 418 | pte_access & ACC_WRITE_MASK); |
| 451 | if (mmu_invalid_pfn(pfn)) { | 419 | if (mmu_invalid_pfn(pfn)) |
| 452 | kvm_release_pfn_clean(pfn); | ||
| 453 | break; | 420 | break; |
| 454 | } | ||
| 455 | 421 | ||
| 456 | mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0, | 422 | mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0, |
| 457 | NULL, PT_PAGE_TABLE_LEVEL, gfn, | 423 | NULL, PT_PAGE_TABLE_LEVEL, gfn, |
| @@ -798,7 +764,8 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) | |||
| 798 | 764 | ||
| 799 | gfn = gpte_to_gfn(gpte); | 765 | gfn = gpte_to_gfn(gpte); |
| 800 | pte_access = sp->role.access; | 766 | pte_access = sp->role.access; |
| 801 | pte_access &= FNAME(gpte_access)(vcpu, gpte, true); | 767 | pte_access &= gpte_access(vcpu, gpte); |
| 768 | protect_clean_gpte(&pte_access, gpte); | ||
| 802 | 769 | ||
| 803 | if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present)) | 770 | if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present)) |
| 804 | continue; | 771 | continue; |
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 9b7ec1150ab0..cfc258a6bf97 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | /* | 1 | /* |
| 2 | * Kernel-based Virtual Machine -- Performane Monitoring Unit support | 2 | * Kernel-based Virtual Machine -- Performance Monitoring Unit support |
| 3 | * | 3 | * |
| 4 | * Copyright 2011 Red Hat, Inc. and/or its affiliates. | 4 | * Copyright 2011 Red Hat, Inc. and/or its affiliates. |
| 5 | * | 5 | * |
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index baead950d6c8..d017df3899ef 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c | |||
| @@ -163,7 +163,7 @@ static DEFINE_PER_CPU(u64, current_tsc_ratio); | |||
| 163 | 163 | ||
| 164 | #define MSR_INVALID 0xffffffffU | 164 | #define MSR_INVALID 0xffffffffU |
| 165 | 165 | ||
| 166 | static struct svm_direct_access_msrs { | 166 | static const struct svm_direct_access_msrs { |
| 167 | u32 index; /* Index of the MSR */ | 167 | u32 index; /* Index of the MSR */ |
| 168 | bool always; /* True if intercept is always on */ | 168 | bool always; /* True if intercept is always on */ |
| 169 | } direct_access_msrs[] = { | 169 | } direct_access_msrs[] = { |
| @@ -400,7 +400,7 @@ struct svm_init_data { | |||
| 400 | int r; | 400 | int r; |
| 401 | }; | 401 | }; |
| 402 | 402 | ||
| 403 | static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; | 403 | static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; |
| 404 | 404 | ||
| 405 | #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) | 405 | #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) |
| 406 | #define MSRS_RANGE_SIZE 2048 | 406 | #define MSRS_RANGE_SIZE 2048 |
| @@ -1146,7 +1146,6 @@ static void init_vmcb(struct vcpu_svm *svm) | |||
| 1146 | 1146 | ||
| 1147 | svm_set_efer(&svm->vcpu, 0); | 1147 | svm_set_efer(&svm->vcpu, 0); |
| 1148 | save->dr6 = 0xffff0ff0; | 1148 | save->dr6 = 0xffff0ff0; |
| 1149 | save->dr7 = 0x400; | ||
| 1150 | kvm_set_rflags(&svm->vcpu, 2); | 1149 | kvm_set_rflags(&svm->vcpu, 2); |
| 1151 | save->rip = 0x0000fff0; | 1150 | save->rip = 0x0000fff0; |
| 1152 | svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; | 1151 | svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; |
| @@ -1643,7 +1642,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu, | |||
| 1643 | mark_dirty(svm->vmcb, VMCB_SEG); | 1642 | mark_dirty(svm->vmcb, VMCB_SEG); |
| 1644 | } | 1643 | } |
| 1645 | 1644 | ||
| 1646 | static void update_db_intercept(struct kvm_vcpu *vcpu) | 1645 | static void update_db_bp_intercept(struct kvm_vcpu *vcpu) |
| 1647 | { | 1646 | { |
| 1648 | struct vcpu_svm *svm = to_svm(vcpu); | 1647 | struct vcpu_svm *svm = to_svm(vcpu); |
| 1649 | 1648 | ||
| @@ -1663,20 +1662,6 @@ static void update_db_intercept(struct kvm_vcpu *vcpu) | |||
| 1663 | vcpu->guest_debug = 0; | 1662 | vcpu->guest_debug = 0; |
| 1664 | } | 1663 | } |
| 1665 | 1664 | ||
| 1666 | static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) | ||
| 1667 | { | ||
| 1668 | struct vcpu_svm *svm = to_svm(vcpu); | ||
| 1669 | |||
| 1670 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | ||
| 1671 | svm->vmcb->save.dr7 = dbg->arch.debugreg[7]; | ||
| 1672 | else | ||
| 1673 | svm->vmcb->save.dr7 = vcpu->arch.dr7; | ||
| 1674 | |||
| 1675 | mark_dirty(svm->vmcb, VMCB_DR); | ||
| 1676 | |||
| 1677 | update_db_intercept(vcpu); | ||
| 1678 | } | ||
| 1679 | |||
| 1680 | static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) | 1665 | static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) |
| 1681 | { | 1666 | { |
| 1682 | if (sd->next_asid > sd->max_asid) { | 1667 | if (sd->next_asid > sd->max_asid) { |
| @@ -1748,7 +1733,7 @@ static int db_interception(struct vcpu_svm *svm) | |||
| 1748 | if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) | 1733 | if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) |
| 1749 | svm->vmcb->save.rflags &= | 1734 | svm->vmcb->save.rflags &= |
| 1750 | ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | 1735 | ~(X86_EFLAGS_TF | X86_EFLAGS_RF); |
| 1751 | update_db_intercept(&svm->vcpu); | 1736 | update_db_bp_intercept(&svm->vcpu); |
| 1752 | } | 1737 | } |
| 1753 | 1738 | ||
| 1754 | if (svm->vcpu.guest_debug & | 1739 | if (svm->vcpu.guest_debug & |
| @@ -2063,7 +2048,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm) | |||
| 2063 | if (svm->nested.intercept & 1ULL) { | 2048 | if (svm->nested.intercept & 1ULL) { |
| 2064 | /* | 2049 | /* |
| 2065 | * The #vmexit can't be emulated here directly because this | 2050 | * The #vmexit can't be emulated here directly because this |
| 2066 | * code path runs with irqs and preemtion disabled. A | 2051 | * code path runs with irqs and preemption disabled. A |
| 2067 | * #vmexit emulation might sleep. Only signal request for | 2052 | * #vmexit emulation might sleep. Only signal request for |
| 2068 | * the #vmexit here. | 2053 | * the #vmexit here. |
| 2069 | */ | 2054 | */ |
| @@ -2105,7 +2090,6 @@ static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page) | |||
| 2105 | return kmap(page); | 2090 | return kmap(page); |
| 2106 | 2091 | ||
| 2107 | error: | 2092 | error: |
| 2108 | kvm_release_page_clean(page); | ||
| 2109 | kvm_inject_gp(&svm->vcpu, 0); | 2093 | kvm_inject_gp(&svm->vcpu, 0); |
| 2110 | 2094 | ||
| 2111 | return NULL; | 2095 | return NULL; |
| @@ -2409,7 +2393,7 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) | |||
| 2409 | { | 2393 | { |
| 2410 | /* | 2394 | /* |
| 2411 | * This function merges the msr permission bitmaps of kvm and the | 2395 | * This function merges the msr permission bitmaps of kvm and the |
| 2412 | * nested vmcb. It is omptimized in that it only merges the parts where | 2396 | * nested vmcb. It is optimized in that it only merges the parts where |
| 2413 | * the kvm msr permission bitmap may contain zero bits | 2397 | * the kvm msr permission bitmap may contain zero bits |
| 2414 | */ | 2398 | */ |
| 2415 | int i; | 2399 | int i; |
| @@ -3268,7 +3252,7 @@ static int pause_interception(struct vcpu_svm *svm) | |||
| 3268 | return 1; | 3252 | return 1; |
| 3269 | } | 3253 | } |
| 3270 | 3254 | ||
| 3271 | static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { | 3255 | static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = { |
| 3272 | [SVM_EXIT_READ_CR0] = cr_interception, | 3256 | [SVM_EXIT_READ_CR0] = cr_interception, |
| 3273 | [SVM_EXIT_READ_CR3] = cr_interception, | 3257 | [SVM_EXIT_READ_CR3] = cr_interception, |
| 3274 | [SVM_EXIT_READ_CR4] = cr_interception, | 3258 | [SVM_EXIT_READ_CR4] = cr_interception, |
| @@ -3660,7 +3644,7 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu) | |||
| 3660 | */ | 3644 | */ |
| 3661 | svm->nmi_singlestep = true; | 3645 | svm->nmi_singlestep = true; |
| 3662 | svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); | 3646 | svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); |
| 3663 | update_db_intercept(vcpu); | 3647 | update_db_bp_intercept(vcpu); |
| 3664 | } | 3648 | } |
| 3665 | 3649 | ||
| 3666 | static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) | 3650 | static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) |
| @@ -3783,12 +3767,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) | |||
| 3783 | svm_complete_interrupts(svm); | 3767 | svm_complete_interrupts(svm); |
| 3784 | } | 3768 | } |
| 3785 | 3769 | ||
| 3786 | #ifdef CONFIG_X86_64 | ||
| 3787 | #define R "r" | ||
| 3788 | #else | ||
| 3789 | #define R "e" | ||
| 3790 | #endif | ||
| 3791 | |||
| 3792 | static void svm_vcpu_run(struct kvm_vcpu *vcpu) | 3770 | static void svm_vcpu_run(struct kvm_vcpu *vcpu) |
| 3793 | { | 3771 | { |
| 3794 | struct vcpu_svm *svm = to_svm(vcpu); | 3772 | struct vcpu_svm *svm = to_svm(vcpu); |
| @@ -3815,13 +3793,13 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 3815 | local_irq_enable(); | 3793 | local_irq_enable(); |
| 3816 | 3794 | ||
| 3817 | asm volatile ( | 3795 | asm volatile ( |
| 3818 | "push %%"R"bp; \n\t" | 3796 | "push %%" _ASM_BP "; \n\t" |
| 3819 | "mov %c[rbx](%[svm]), %%"R"bx \n\t" | 3797 | "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t" |
| 3820 | "mov %c[rcx](%[svm]), %%"R"cx \n\t" | 3798 | "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t" |
| 3821 | "mov %c[rdx](%[svm]), %%"R"dx \n\t" | 3799 | "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t" |
| 3822 | "mov %c[rsi](%[svm]), %%"R"si \n\t" | 3800 | "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t" |
| 3823 | "mov %c[rdi](%[svm]), %%"R"di \n\t" | 3801 | "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t" |
| 3824 | "mov %c[rbp](%[svm]), %%"R"bp \n\t" | 3802 | "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t" |
| 3825 | #ifdef CONFIG_X86_64 | 3803 | #ifdef CONFIG_X86_64 |
| 3826 | "mov %c[r8](%[svm]), %%r8 \n\t" | 3804 | "mov %c[r8](%[svm]), %%r8 \n\t" |
| 3827 | "mov %c[r9](%[svm]), %%r9 \n\t" | 3805 | "mov %c[r9](%[svm]), %%r9 \n\t" |
| @@ -3834,20 +3812,20 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 3834 | #endif | 3812 | #endif |
| 3835 | 3813 | ||
| 3836 | /* Enter guest mode */ | 3814 | /* Enter guest mode */ |
| 3837 | "push %%"R"ax \n\t" | 3815 | "push %%" _ASM_AX " \n\t" |
| 3838 | "mov %c[vmcb](%[svm]), %%"R"ax \n\t" | 3816 | "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t" |
| 3839 | __ex(SVM_VMLOAD) "\n\t" | 3817 | __ex(SVM_VMLOAD) "\n\t" |
| 3840 | __ex(SVM_VMRUN) "\n\t" | 3818 | __ex(SVM_VMRUN) "\n\t" |
| 3841 | __ex(SVM_VMSAVE) "\n\t" | 3819 | __ex(SVM_VMSAVE) "\n\t" |
| 3842 | "pop %%"R"ax \n\t" | 3820 | "pop %%" _ASM_AX " \n\t" |
| 3843 | 3821 | ||
| 3844 | /* Save guest registers, load host registers */ | 3822 | /* Save guest registers, load host registers */ |
| 3845 | "mov %%"R"bx, %c[rbx](%[svm]) \n\t" | 3823 | "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t" |
| 3846 | "mov %%"R"cx, %c[rcx](%[svm]) \n\t" | 3824 | "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t" |
| 3847 | "mov %%"R"dx, %c[rdx](%[svm]) \n\t" | 3825 | "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t" |
| 3848 | "mov %%"R"si, %c[rsi](%[svm]) \n\t" | 3826 | "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t" |
| 3849 | "mov %%"R"di, %c[rdi](%[svm]) \n\t" | 3827 | "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t" |
| 3850 | "mov %%"R"bp, %c[rbp](%[svm]) \n\t" | 3828 | "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t" |
| 3851 | #ifdef CONFIG_X86_64 | 3829 | #ifdef CONFIG_X86_64 |
| 3852 | "mov %%r8, %c[r8](%[svm]) \n\t" | 3830 | "mov %%r8, %c[r8](%[svm]) \n\t" |
| 3853 | "mov %%r9, %c[r9](%[svm]) \n\t" | 3831 | "mov %%r9, %c[r9](%[svm]) \n\t" |
| @@ -3858,7 +3836,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 3858 | "mov %%r14, %c[r14](%[svm]) \n\t" | 3836 | "mov %%r14, %c[r14](%[svm]) \n\t" |
| 3859 | "mov %%r15, %c[r15](%[svm]) \n\t" | 3837 | "mov %%r15, %c[r15](%[svm]) \n\t" |
| 3860 | #endif | 3838 | #endif |
| 3861 | "pop %%"R"bp" | 3839 | "pop %%" _ASM_BP |
| 3862 | : | 3840 | : |
| 3863 | : [svm]"a"(svm), | 3841 | : [svm]"a"(svm), |
| 3864 | [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), | 3842 | [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), |
| @@ -3879,9 +3857,11 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 3879 | [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) | 3857 | [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) |
| 3880 | #endif | 3858 | #endif |
| 3881 | : "cc", "memory" | 3859 | : "cc", "memory" |
| 3882 | , R"bx", R"cx", R"dx", R"si", R"di" | ||
| 3883 | #ifdef CONFIG_X86_64 | 3860 | #ifdef CONFIG_X86_64 |
| 3861 | , "rbx", "rcx", "rdx", "rsi", "rdi" | ||
| 3884 | , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" | 3862 | , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" |
| 3863 | #else | ||
| 3864 | , "ebx", "ecx", "edx", "esi", "edi" | ||
| 3885 | #endif | 3865 | #endif |
| 3886 | ); | 3866 | ); |
| 3887 | 3867 | ||
| @@ -3941,8 +3921,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 3941 | mark_all_clean(svm->vmcb); | 3921 | mark_all_clean(svm->vmcb); |
| 3942 | } | 3922 | } |
| 3943 | 3923 | ||
| 3944 | #undef R | ||
| 3945 | |||
| 3946 | static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) | 3924 | static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) |
| 3947 | { | 3925 | { |
| 3948 | struct vcpu_svm *svm = to_svm(vcpu); | 3926 | struct vcpu_svm *svm = to_svm(vcpu); |
| @@ -4069,7 +4047,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu) | |||
| 4069 | #define POST_MEM(exit) { .exit_code = (exit), \ | 4047 | #define POST_MEM(exit) { .exit_code = (exit), \ |
| 4070 | .stage = X86_ICPT_POST_MEMACCESS, } | 4048 | .stage = X86_ICPT_POST_MEMACCESS, } |
| 4071 | 4049 | ||
| 4072 | static struct __x86_intercept { | 4050 | static const struct __x86_intercept { |
| 4073 | u32 exit_code; | 4051 | u32 exit_code; |
| 4074 | enum x86_intercept_stage stage; | 4052 | enum x86_intercept_stage stage; |
| 4075 | } x86_intercept_map[] = { | 4053 | } x86_intercept_map[] = { |
| @@ -4260,7 +4238,7 @@ static struct kvm_x86_ops svm_x86_ops = { | |||
| 4260 | .vcpu_load = svm_vcpu_load, | 4238 | .vcpu_load = svm_vcpu_load, |
| 4261 | .vcpu_put = svm_vcpu_put, | 4239 | .vcpu_put = svm_vcpu_put, |
| 4262 | 4240 | ||
| 4263 | .set_guest_debug = svm_guest_debug, | 4241 | .update_db_bp_intercept = update_db_bp_intercept, |
| 4264 | .get_msr = svm_get_msr, | 4242 | .get_msr = svm_get_msr, |
| 4265 | .set_msr = svm_set_msr, | 4243 | .set_msr = svm_set_msr, |
| 4266 | .get_segment_base = svm_get_segment_base, | 4244 | .get_segment_base = svm_get_segment_base, |
diff --git a/arch/x86/kvm/timer.c b/arch/x86/kvm/timer.c deleted file mode 100644 index 6b85cc647f34..000000000000 --- a/arch/x86/kvm/timer.c +++ /dev/null | |||
| @@ -1,47 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * Kernel-based Virtual Machine driver for Linux | ||
| 3 | * | ||
| 4 | * This module enables machines with Intel VT-x extensions to run virtual | ||
| 5 | * machines without emulation or binary translation. | ||
| 6 | * | ||
| 7 | * timer support | ||
| 8 | * | ||
| 9 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. | ||
| 10 | * | ||
| 11 | * This work is licensed under the terms of the GNU GPL, version 2. See | ||
| 12 | * the COPYING file in the top-level directory. | ||
| 13 | */ | ||
| 14 | |||
| 15 | #include <linux/kvm_host.h> | ||
| 16 | #include <linux/kvm.h> | ||
| 17 | #include <linux/hrtimer.h> | ||
| 18 | #include <linux/atomic.h> | ||
| 19 | #include "kvm_timer.h" | ||
| 20 | |||
| 21 | enum hrtimer_restart kvm_timer_fn(struct hrtimer *data) | ||
| 22 | { | ||
| 23 | struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); | ||
| 24 | struct kvm_vcpu *vcpu = ktimer->vcpu; | ||
| 25 | wait_queue_head_t *q = &vcpu->wq; | ||
| 26 | |||
| 27 | /* | ||
| 28 | * There is a race window between reading and incrementing, but we do | ||
| 29 | * not care about potentially losing timer events in the !reinject | ||
| 30 | * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked | ||
| 31 | * in vcpu_enter_guest. | ||
| 32 | */ | ||
| 33 | if (ktimer->reinject || !atomic_read(&ktimer->pending)) { | ||
| 34 | atomic_inc(&ktimer->pending); | ||
| 35 | /* FIXME: this code should not know anything about vcpus */ | ||
| 36 | kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu); | ||
| 37 | } | ||
| 38 | |||
| 39 | if (waitqueue_active(q)) | ||
| 40 | wake_up_interruptible(q); | ||
| 41 | |||
| 42 | if (ktimer->t_ops->is_periodic(ktimer)) { | ||
| 43 | hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); | ||
| 44 | return HRTIMER_RESTART; | ||
| 45 | } else | ||
| 46 | return HRTIMER_NORESTART; | ||
| 47 | } | ||
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 851aa7c3b890..ad6b1dd06f8b 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c | |||
| @@ -127,6 +127,8 @@ module_param(ple_gap, int, S_IRUGO); | |||
| 127 | static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; | 127 | static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; |
| 128 | module_param(ple_window, int, S_IRUGO); | 128 | module_param(ple_window, int, S_IRUGO); |
| 129 | 129 | ||
| 130 | extern const ulong vmx_return; | ||
| 131 | |||
| 130 | #define NR_AUTOLOAD_MSRS 8 | 132 | #define NR_AUTOLOAD_MSRS 8 |
| 131 | #define VMCS02_POOL_SIZE 1 | 133 | #define VMCS02_POOL_SIZE 1 |
| 132 | 134 | ||
| @@ -405,16 +407,16 @@ struct vcpu_vmx { | |||
| 405 | struct { | 407 | struct { |
| 406 | int vm86_active; | 408 | int vm86_active; |
| 407 | ulong save_rflags; | 409 | ulong save_rflags; |
| 410 | struct kvm_segment segs[8]; | ||
| 411 | } rmode; | ||
| 412 | struct { | ||
| 413 | u32 bitmask; /* 4 bits per segment (1 bit per field) */ | ||
| 408 | struct kvm_save_segment { | 414 | struct kvm_save_segment { |
| 409 | u16 selector; | 415 | u16 selector; |
| 410 | unsigned long base; | 416 | unsigned long base; |
| 411 | u32 limit; | 417 | u32 limit; |
| 412 | u32 ar; | 418 | u32 ar; |
| 413 | } tr, es, ds, fs, gs; | 419 | } seg[8]; |
| 414 | } rmode; | ||
| 415 | struct { | ||
| 416 | u32 bitmask; /* 4 bits per segment (1 bit per field) */ | ||
| 417 | struct kvm_save_segment seg[8]; | ||
| 418 | } segment_cache; | 420 | } segment_cache; |
| 419 | int vpid; | 421 | int vpid; |
| 420 | bool emulation_required; | 422 | bool emulation_required; |
| @@ -450,7 +452,7 @@ static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) | |||
| 450 | #define FIELD64(number, name) [number] = VMCS12_OFFSET(name), \ | 452 | #define FIELD64(number, name) [number] = VMCS12_OFFSET(name), \ |
| 451 | [number##_HIGH] = VMCS12_OFFSET(name)+4 | 453 | [number##_HIGH] = VMCS12_OFFSET(name)+4 |
| 452 | 454 | ||
| 453 | static unsigned short vmcs_field_to_offset_table[] = { | 455 | static const unsigned short vmcs_field_to_offset_table[] = { |
| 454 | FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id), | 456 | FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id), |
| 455 | FIELD(GUEST_ES_SELECTOR, guest_es_selector), | 457 | FIELD(GUEST_ES_SELECTOR, guest_es_selector), |
| 456 | FIELD(GUEST_CS_SELECTOR, guest_cs_selector), | 458 | FIELD(GUEST_CS_SELECTOR, guest_cs_selector), |
| @@ -596,10 +598,9 @@ static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu) | |||
| 596 | static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr) | 598 | static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr) |
| 597 | { | 599 | { |
| 598 | struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT); | 600 | struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT); |
| 599 | if (is_error_page(page)) { | 601 | if (is_error_page(page)) |
| 600 | kvm_release_page_clean(page); | ||
| 601 | return NULL; | 602 | return NULL; |
| 602 | } | 603 | |
| 603 | return page; | 604 | return page; |
| 604 | } | 605 | } |
| 605 | 606 | ||
| @@ -667,7 +668,7 @@ static struct vmx_capability { | |||
| 667 | .ar_bytes = GUEST_##seg##_AR_BYTES, \ | 668 | .ar_bytes = GUEST_##seg##_AR_BYTES, \ |
| 668 | } | 669 | } |
| 669 | 670 | ||
| 670 | static struct kvm_vmx_segment_field { | 671 | static const struct kvm_vmx_segment_field { |
| 671 | unsigned selector; | 672 | unsigned selector; |
| 672 | unsigned base; | 673 | unsigned base; |
| 673 | unsigned limit; | 674 | unsigned limit; |
| @@ -1343,7 +1344,7 @@ static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset) | |||
| 1343 | guest_efer = vmx->vcpu.arch.efer; | 1344 | guest_efer = vmx->vcpu.arch.efer; |
| 1344 | 1345 | ||
| 1345 | /* | 1346 | /* |
| 1346 | * NX is emulated; LMA and LME handled by hardware; SCE meaninless | 1347 | * NX is emulated; LMA and LME handled by hardware; SCE meaningless |
| 1347 | * outside long mode | 1348 | * outside long mode |
| 1348 | */ | 1349 | */ |
| 1349 | ignore_bits = EFER_NX | EFER_SCE; | 1350 | ignore_bits = EFER_NX | EFER_SCE; |
| @@ -1995,7 +1996,7 @@ static __init void nested_vmx_setup_ctls_msrs(void) | |||
| 1995 | #endif | 1996 | #endif |
| 1996 | CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | | 1997 | CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | |
| 1997 | CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING | | 1998 | CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING | |
| 1998 | CPU_BASED_RDPMC_EXITING | | 1999 | CPU_BASED_RDPMC_EXITING | CPU_BASED_RDTSC_EXITING | |
| 1999 | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; | 2000 | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; |
| 2000 | /* | 2001 | /* |
| 2001 | * We can allow some features even when not supported by the | 2002 | * We can allow some features even when not supported by the |
| @@ -2291,16 +2292,6 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) | |||
| 2291 | } | 2292 | } |
| 2292 | } | 2293 | } |
| 2293 | 2294 | ||
| 2294 | static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) | ||
| 2295 | { | ||
| 2296 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | ||
| 2297 | vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]); | ||
| 2298 | else | ||
| 2299 | vmcs_writel(GUEST_DR7, vcpu->arch.dr7); | ||
| 2300 | |||
| 2301 | update_exception_bitmap(vcpu); | ||
| 2302 | } | ||
| 2303 | |||
| 2304 | static __init int cpu_has_kvm_support(void) | 2295 | static __init int cpu_has_kvm_support(void) |
| 2305 | { | 2296 | { |
| 2306 | return cpu_has_vmx(); | 2297 | return cpu_has_vmx(); |
| @@ -2698,20 +2689,17 @@ static __exit void hardware_unsetup(void) | |||
| 2698 | free_kvm_area(); | 2689 | free_kvm_area(); |
| 2699 | } | 2690 | } |
| 2700 | 2691 | ||
| 2701 | static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save) | 2692 | static void fix_pmode_dataseg(struct kvm_vcpu *vcpu, int seg, struct kvm_segment *save) |
| 2702 | { | 2693 | { |
| 2703 | struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; | 2694 | const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; |
| 2695 | struct kvm_segment tmp = *save; | ||
| 2704 | 2696 | ||
| 2705 | if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) { | 2697 | if (!(vmcs_readl(sf->base) == tmp.base && tmp.s)) { |
| 2706 | vmcs_write16(sf->selector, save->selector); | 2698 | tmp.base = vmcs_readl(sf->base); |
| 2707 | vmcs_writel(sf->base, save->base); | 2699 | tmp.selector = vmcs_read16(sf->selector); |
| 2708 | vmcs_write32(sf->limit, save->limit); | 2700 | tmp.s = 1; |
| 2709 | vmcs_write32(sf->ar_bytes, save->ar); | ||
| 2710 | } else { | ||
| 2711 | u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK) | ||
| 2712 | << AR_DPL_SHIFT; | ||
| 2713 | vmcs_write32(sf->ar_bytes, 0x93 | dpl); | ||
| 2714 | } | 2701 | } |
| 2702 | vmx_set_segment(vcpu, &tmp, seg); | ||
| 2715 | } | 2703 | } |
| 2716 | 2704 | ||
| 2717 | static void enter_pmode(struct kvm_vcpu *vcpu) | 2705 | static void enter_pmode(struct kvm_vcpu *vcpu) |
| @@ -2724,10 +2712,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu) | |||
| 2724 | 2712 | ||
| 2725 | vmx_segment_cache_clear(vmx); | 2713 | vmx_segment_cache_clear(vmx); |
| 2726 | 2714 | ||
| 2727 | vmcs_write16(GUEST_TR_SELECTOR, vmx->rmode.tr.selector); | 2715 | vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); |
| 2728 | vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base); | ||
| 2729 | vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit); | ||
| 2730 | vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar); | ||
| 2731 | 2716 | ||
| 2732 | flags = vmcs_readl(GUEST_RFLAGS); | 2717 | flags = vmcs_readl(GUEST_RFLAGS); |
| 2733 | flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; | 2718 | flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; |
| @@ -2742,10 +2727,10 @@ static void enter_pmode(struct kvm_vcpu *vcpu) | |||
| 2742 | if (emulate_invalid_guest_state) | 2727 | if (emulate_invalid_guest_state) |
| 2743 | return; | 2728 | return; |
| 2744 | 2729 | ||
| 2745 | fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es); | 2730 | fix_pmode_dataseg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); |
| 2746 | fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds); | 2731 | fix_pmode_dataseg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); |
| 2747 | fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs); | 2732 | fix_pmode_dataseg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); |
| 2748 | fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs); | 2733 | fix_pmode_dataseg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); |
| 2749 | 2734 | ||
| 2750 | vmx_segment_cache_clear(vmx); | 2735 | vmx_segment_cache_clear(vmx); |
| 2751 | 2736 | ||
| @@ -2773,14 +2758,10 @@ static gva_t rmode_tss_base(struct kvm *kvm) | |||
| 2773 | return kvm->arch.tss_addr; | 2758 | return kvm->arch.tss_addr; |
| 2774 | } | 2759 | } |
| 2775 | 2760 | ||
| 2776 | static void fix_rmode_seg(int seg, struct kvm_save_segment *save) | 2761 | static void fix_rmode_seg(int seg, struct kvm_segment *save) |
| 2777 | { | 2762 | { |
| 2778 | struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; | 2763 | const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; |
| 2779 | 2764 | ||
| 2780 | save->selector = vmcs_read16(sf->selector); | ||
| 2781 | save->base = vmcs_readl(sf->base); | ||
| 2782 | save->limit = vmcs_read32(sf->limit); | ||
| 2783 | save->ar = vmcs_read32(sf->ar_bytes); | ||
| 2784 | vmcs_write16(sf->selector, save->base >> 4); | 2765 | vmcs_write16(sf->selector, save->base >> 4); |
| 2785 | vmcs_write32(sf->base, save->base & 0xffff0); | 2766 | vmcs_write32(sf->base, save->base & 0xffff0); |
| 2786 | vmcs_write32(sf->limit, 0xffff); | 2767 | vmcs_write32(sf->limit, 0xffff); |
| @@ -2800,9 +2781,16 @@ static void enter_rmode(struct kvm_vcpu *vcpu) | |||
| 2800 | if (enable_unrestricted_guest) | 2781 | if (enable_unrestricted_guest) |
| 2801 | return; | 2782 | return; |
| 2802 | 2783 | ||
| 2784 | vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); | ||
| 2785 | vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); | ||
| 2786 | vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); | ||
| 2787 | vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); | ||
| 2788 | vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); | ||
| 2789 | |||
| 2803 | vmx->emulation_required = 1; | 2790 | vmx->emulation_required = 1; |
| 2804 | vmx->rmode.vm86_active = 1; | 2791 | vmx->rmode.vm86_active = 1; |
| 2805 | 2792 | ||
| 2793 | |||
| 2806 | /* | 2794 | /* |
| 2807 | * Very old userspace does not call KVM_SET_TSS_ADDR before entering | 2795 | * Very old userspace does not call KVM_SET_TSS_ADDR before entering |
| 2808 | * vcpu. Call it here with phys address pointing 16M below 4G. | 2796 | * vcpu. Call it here with phys address pointing 16M below 4G. |
| @@ -2817,14 +2805,8 @@ static void enter_rmode(struct kvm_vcpu *vcpu) | |||
| 2817 | 2805 | ||
| 2818 | vmx_segment_cache_clear(vmx); | 2806 | vmx_segment_cache_clear(vmx); |
| 2819 | 2807 | ||
| 2820 | vmx->rmode.tr.selector = vmcs_read16(GUEST_TR_SELECTOR); | ||
| 2821 | vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE); | ||
| 2822 | vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); | 2808 | vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); |
| 2823 | |||
| 2824 | vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); | ||
| 2825 | vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); | 2809 | vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); |
| 2826 | |||
| 2827 | vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); | ||
| 2828 | vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); | 2810 | vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); |
| 2829 | 2811 | ||
| 2830 | flags = vmcs_readl(GUEST_RFLAGS); | 2812 | flags = vmcs_readl(GUEST_RFLAGS); |
| @@ -3117,35 +3099,24 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu, | |||
| 3117 | struct kvm_segment *var, int seg) | 3099 | struct kvm_segment *var, int seg) |
| 3118 | { | 3100 | { |
| 3119 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3101 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
| 3120 | struct kvm_save_segment *save; | ||
| 3121 | u32 ar; | 3102 | u32 ar; |
| 3122 | 3103 | ||
| 3123 | if (vmx->rmode.vm86_active | 3104 | if (vmx->rmode.vm86_active |
| 3124 | && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES | 3105 | && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES |
| 3125 | || seg == VCPU_SREG_DS || seg == VCPU_SREG_FS | 3106 | || seg == VCPU_SREG_DS || seg == VCPU_SREG_FS |
| 3126 | || seg == VCPU_SREG_GS) | 3107 | || seg == VCPU_SREG_GS)) { |
| 3127 | && !emulate_invalid_guest_state) { | 3108 | *var = vmx->rmode.segs[seg]; |
| 3128 | switch (seg) { | ||
| 3129 | case VCPU_SREG_TR: save = &vmx->rmode.tr; break; | ||
| 3130 | case VCPU_SREG_ES: save = &vmx->rmode.es; break; | ||
| 3131 | case VCPU_SREG_DS: save = &vmx->rmode.ds; break; | ||
| 3132 | case VCPU_SREG_FS: save = &vmx->rmode.fs; break; | ||
| 3133 | case VCPU_SREG_GS: save = &vmx->rmode.gs; break; | ||
| 3134 | default: BUG(); | ||
| 3135 | } | ||
| 3136 | var->selector = save->selector; | ||
| 3137 | var->base = save->base; | ||
| 3138 | var->limit = save->limit; | ||
| 3139 | ar = save->ar; | ||
| 3140 | if (seg == VCPU_SREG_TR | 3109 | if (seg == VCPU_SREG_TR |
| 3141 | || var->selector == vmx_read_guest_seg_selector(vmx, seg)) | 3110 | || var->selector == vmx_read_guest_seg_selector(vmx, seg)) |
| 3142 | goto use_saved_rmode_seg; | 3111 | return; |
| 3112 | var->base = vmx_read_guest_seg_base(vmx, seg); | ||
| 3113 | var->selector = vmx_read_guest_seg_selector(vmx, seg); | ||
| 3114 | return; | ||
| 3143 | } | 3115 | } |
| 3144 | var->base = vmx_read_guest_seg_base(vmx, seg); | 3116 | var->base = vmx_read_guest_seg_base(vmx, seg); |
| 3145 | var->limit = vmx_read_guest_seg_limit(vmx, seg); | 3117 | var->limit = vmx_read_guest_seg_limit(vmx, seg); |
| 3146 | var->selector = vmx_read_guest_seg_selector(vmx, seg); | 3118 | var->selector = vmx_read_guest_seg_selector(vmx, seg); |
| 3147 | ar = vmx_read_guest_seg_ar(vmx, seg); | 3119 | ar = vmx_read_guest_seg_ar(vmx, seg); |
| 3148 | use_saved_rmode_seg: | ||
| 3149 | if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state) | 3120 | if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state) |
| 3150 | ar = 0; | 3121 | ar = 0; |
| 3151 | var->type = ar & 15; | 3122 | var->type = ar & 15; |
| @@ -3227,23 +3198,21 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu, | |||
| 3227 | struct kvm_segment *var, int seg) | 3198 | struct kvm_segment *var, int seg) |
| 3228 | { | 3199 | { |
| 3229 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3200 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
| 3230 | struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; | 3201 | const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; |
| 3231 | u32 ar; | 3202 | u32 ar; |
| 3232 | 3203 | ||
| 3233 | vmx_segment_cache_clear(vmx); | 3204 | vmx_segment_cache_clear(vmx); |
| 3234 | 3205 | ||
| 3235 | if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) { | 3206 | if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) { |
| 3236 | vmcs_write16(sf->selector, var->selector); | 3207 | vmcs_write16(sf->selector, var->selector); |
| 3237 | vmx->rmode.tr.selector = var->selector; | 3208 | vmx->rmode.segs[VCPU_SREG_TR] = *var; |
| 3238 | vmx->rmode.tr.base = var->base; | ||
| 3239 | vmx->rmode.tr.limit = var->limit; | ||
| 3240 | vmx->rmode.tr.ar = vmx_segment_access_rights(var); | ||
| 3241 | return; | 3209 | return; |
| 3242 | } | 3210 | } |
| 3243 | vmcs_writel(sf->base, var->base); | 3211 | vmcs_writel(sf->base, var->base); |
| 3244 | vmcs_write32(sf->limit, var->limit); | 3212 | vmcs_write32(sf->limit, var->limit); |
| 3245 | vmcs_write16(sf->selector, var->selector); | 3213 | vmcs_write16(sf->selector, var->selector); |
| 3246 | if (vmx->rmode.vm86_active && var->s) { | 3214 | if (vmx->rmode.vm86_active && var->s) { |
| 3215 | vmx->rmode.segs[seg] = *var; | ||
| 3247 | /* | 3216 | /* |
| 3248 | * Hack real-mode segments into vm86 compatibility. | 3217 | * Hack real-mode segments into vm86 compatibility. |
| 3249 | */ | 3218 | */ |
| @@ -3258,7 +3227,7 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu, | |||
| 3258 | * qemu binaries. | 3227 | * qemu binaries. |
| 3259 | * IA32 arch specifies that at the time of processor reset the | 3228 | * IA32 arch specifies that at the time of processor reset the |
| 3260 | * "Accessed" bit in the AR field of segment registers is 1. And qemu | 3229 | * "Accessed" bit in the AR field of segment registers is 1. And qemu |
| 3261 | * is setting it to 0 in the usedland code. This causes invalid guest | 3230 | * is setting it to 0 in the userland code. This causes invalid guest |
| 3262 | * state vmexit when "unrestricted guest" mode is turned on. | 3231 | * state vmexit when "unrestricted guest" mode is turned on. |
| 3263 | * Fix for this setup issue in cpu_reset is being pushed in the qemu | 3232 | * Fix for this setup issue in cpu_reset is being pushed in the qemu |
| 3264 | * tree. Newer qemu binaries with that qemu fix would not need this | 3233 | * tree. Newer qemu binaries with that qemu fix would not need this |
| @@ -3288,16 +3257,10 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu, | |||
| 3288 | vmcs_readl(GUEST_CS_BASE) >> 4); | 3257 | vmcs_readl(GUEST_CS_BASE) >> 4); |
| 3289 | break; | 3258 | break; |
| 3290 | case VCPU_SREG_ES: | 3259 | case VCPU_SREG_ES: |
| 3291 | fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es); | ||
| 3292 | break; | ||
| 3293 | case VCPU_SREG_DS: | 3260 | case VCPU_SREG_DS: |
| 3294 | fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds); | ||
| 3295 | break; | ||
| 3296 | case VCPU_SREG_GS: | 3261 | case VCPU_SREG_GS: |
| 3297 | fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs); | ||
| 3298 | break; | ||
| 3299 | case VCPU_SREG_FS: | 3262 | case VCPU_SREG_FS: |
| 3300 | fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs); | 3263 | fix_rmode_seg(seg, &vmx->rmode.segs[seg]); |
| 3301 | break; | 3264 | break; |
| 3302 | case VCPU_SREG_SS: | 3265 | case VCPU_SREG_SS: |
| 3303 | vmcs_write16(GUEST_SS_SELECTOR, | 3266 | vmcs_write16(GUEST_SS_SELECTOR, |
| @@ -3351,9 +3314,9 @@ static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) | |||
| 3351 | 3314 | ||
| 3352 | if (var.base != (var.selector << 4)) | 3315 | if (var.base != (var.selector << 4)) |
| 3353 | return false; | 3316 | return false; |
| 3354 | if (var.limit != 0xffff) | 3317 | if (var.limit < 0xffff) |
| 3355 | return false; | 3318 | return false; |
| 3356 | if (ar != 0xf3) | 3319 | if (((ar | (3 << AR_DPL_SHIFT)) & ~(AR_G_MASK | AR_DB_MASK)) != 0xf3) |
| 3357 | return false; | 3320 | return false; |
| 3358 | 3321 | ||
| 3359 | return true; | 3322 | return true; |
| @@ -3605,7 +3568,7 @@ out: | |||
| 3605 | 3568 | ||
| 3606 | static void seg_setup(int seg) | 3569 | static void seg_setup(int seg) |
| 3607 | { | 3570 | { |
| 3608 | struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; | 3571 | const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; |
| 3609 | unsigned int ar; | 3572 | unsigned int ar; |
| 3610 | 3573 | ||
| 3611 | vmcs_write16(sf->selector, 0); | 3574 | vmcs_write16(sf->selector, 0); |
| @@ -3770,8 +3733,7 @@ static void vmx_set_constant_host_state(void) | |||
| 3770 | native_store_idt(&dt); | 3733 | native_store_idt(&dt); |
| 3771 | vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */ | 3734 | vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */ |
| 3772 | 3735 | ||
| 3773 | asm("mov $.Lkvm_vmx_return, %0" : "=r"(tmpl)); | 3736 | vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */ |
| 3774 | vmcs_writel(HOST_RIP, tmpl); /* 22.2.5 */ | ||
| 3775 | 3737 | ||
| 3776 | rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); | 3738 | rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); |
| 3777 | vmcs_write32(HOST_IA32_SYSENTER_CS, low32); | 3739 | vmcs_write32(HOST_IA32_SYSENTER_CS, low32); |
| @@ -4005,8 +3967,6 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) | |||
| 4005 | kvm_rip_write(vcpu, 0); | 3967 | kvm_rip_write(vcpu, 0); |
| 4006 | kvm_register_write(vcpu, VCPU_REGS_RSP, 0); | 3968 | kvm_register_write(vcpu, VCPU_REGS_RSP, 0); |
| 4007 | 3969 | ||
| 4008 | vmcs_writel(GUEST_DR7, 0x400); | ||
| 4009 | |||
| 4010 | vmcs_writel(GUEST_GDTR_BASE, 0); | 3970 | vmcs_writel(GUEST_GDTR_BASE, 0); |
| 4011 | vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); | 3971 | vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); |
| 4012 | 3972 | ||
| @@ -4456,7 +4416,7 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) | |||
| 4456 | hypercall[2] = 0xc1; | 4416 | hypercall[2] = 0xc1; |
| 4457 | } | 4417 | } |
| 4458 | 4418 | ||
| 4459 | /* called to set cr0 as approriate for a mov-to-cr0 exit. */ | 4419 | /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ |
| 4460 | static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) | 4420 | static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) |
| 4461 | { | 4421 | { |
| 4462 | if (to_vmx(vcpu)->nested.vmxon && | 4422 | if (to_vmx(vcpu)->nested.vmxon && |
| @@ -5701,7 +5661,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) | |||
| 5701 | * may resume. Otherwise they set the kvm_run parameter to indicate what needs | 5661 | * may resume. Otherwise they set the kvm_run parameter to indicate what needs |
| 5702 | * to be done to userspace and return 0. | 5662 | * to be done to userspace and return 0. |
| 5703 | */ | 5663 | */ |
| 5704 | static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { | 5664 | static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { |
| 5705 | [EXIT_REASON_EXCEPTION_NMI] = handle_exception, | 5665 | [EXIT_REASON_EXCEPTION_NMI] = handle_exception, |
| 5706 | [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, | 5666 | [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, |
| 5707 | [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, | 5667 | [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, |
| @@ -6229,17 +6189,10 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) | |||
| 6229 | msrs[i].host); | 6189 | msrs[i].host); |
| 6230 | } | 6190 | } |
| 6231 | 6191 | ||
| 6232 | #ifdef CONFIG_X86_64 | ||
| 6233 | #define R "r" | ||
| 6234 | #define Q "q" | ||
| 6235 | #else | ||
| 6236 | #define R "e" | ||
| 6237 | #define Q "l" | ||
| 6238 | #endif | ||
| 6239 | |||
| 6240 | static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | 6192 | static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) |
| 6241 | { | 6193 | { |
| 6242 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 6194 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
| 6195 | unsigned long debugctlmsr; | ||
| 6243 | 6196 | ||
| 6244 | if (is_guest_mode(vcpu) && !vmx->nested.nested_run_pending) { | 6197 | if (is_guest_mode(vcpu) && !vmx->nested.nested_run_pending) { |
| 6245 | struct vmcs12 *vmcs12 = get_vmcs12(vcpu); | 6198 | struct vmcs12 *vmcs12 = get_vmcs12(vcpu); |
| @@ -6279,34 +6232,35 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 6279 | vmx_set_interrupt_shadow(vcpu, 0); | 6232 | vmx_set_interrupt_shadow(vcpu, 0); |
| 6280 | 6233 | ||
| 6281 | atomic_switch_perf_msrs(vmx); | 6234 | atomic_switch_perf_msrs(vmx); |
| 6235 | debugctlmsr = get_debugctlmsr(); | ||
| 6282 | 6236 | ||
| 6283 | vmx->__launched = vmx->loaded_vmcs->launched; | 6237 | vmx->__launched = vmx->loaded_vmcs->launched; |
| 6284 | asm( | 6238 | asm( |
| 6285 | /* Store host registers */ | 6239 | /* Store host registers */ |
| 6286 | "push %%"R"dx; push %%"R"bp;" | 6240 | "push %%" _ASM_DX "; push %%" _ASM_BP ";" |
| 6287 | "push %%"R"cx \n\t" /* placeholder for guest rcx */ | 6241 | "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */ |
| 6288 | "push %%"R"cx \n\t" | 6242 | "push %%" _ASM_CX " \n\t" |
| 6289 | "cmp %%"R"sp, %c[host_rsp](%0) \n\t" | 6243 | "cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t" |
| 6290 | "je 1f \n\t" | 6244 | "je 1f \n\t" |
| 6291 | "mov %%"R"sp, %c[host_rsp](%0) \n\t" | 6245 | "mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t" |
| 6292 | __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t" | 6246 | __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t" |
| 6293 | "1: \n\t" | 6247 | "1: \n\t" |
| 6294 | /* Reload cr2 if changed */ | 6248 | /* Reload cr2 if changed */ |
| 6295 | "mov %c[cr2](%0), %%"R"ax \n\t" | 6249 | "mov %c[cr2](%0), %%" _ASM_AX " \n\t" |
| 6296 | "mov %%cr2, %%"R"dx \n\t" | 6250 | "mov %%cr2, %%" _ASM_DX " \n\t" |
| 6297 | "cmp %%"R"ax, %%"R"dx \n\t" | 6251 | "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t" |
| 6298 | "je 2f \n\t" | 6252 | "je 2f \n\t" |
| 6299 | "mov %%"R"ax, %%cr2 \n\t" | 6253 | "mov %%" _ASM_AX", %%cr2 \n\t" |
| 6300 | "2: \n\t" | 6254 | "2: \n\t" |
| 6301 | /* Check if vmlaunch of vmresume is needed */ | 6255 | /* Check if vmlaunch of vmresume is needed */ |
| 6302 | "cmpl $0, %c[launched](%0) \n\t" | 6256 | "cmpl $0, %c[launched](%0) \n\t" |
| 6303 | /* Load guest registers. Don't clobber flags. */ | 6257 | /* Load guest registers. Don't clobber flags. */ |
| 6304 | "mov %c[rax](%0), %%"R"ax \n\t" | 6258 | "mov %c[rax](%0), %%" _ASM_AX " \n\t" |
| 6305 | "mov %c[rbx](%0), %%"R"bx \n\t" | 6259 | "mov %c[rbx](%0), %%" _ASM_BX " \n\t" |
| 6306 | "mov %c[rdx](%0), %%"R"dx \n\t" | 6260 | "mov %c[rdx](%0), %%" _ASM_DX " \n\t" |
| 6307 | "mov %c[rsi](%0), %%"R"si \n\t" | 6261 | "mov %c[rsi](%0), %%" _ASM_SI " \n\t" |
| 6308 | "mov %c[rdi](%0), %%"R"di \n\t" | 6262 | "mov %c[rdi](%0), %%" _ASM_DI " \n\t" |
| 6309 | "mov %c[rbp](%0), %%"R"bp \n\t" | 6263 | "mov %c[rbp](%0), %%" _ASM_BP " \n\t" |
| 6310 | #ifdef CONFIG_X86_64 | 6264 | #ifdef CONFIG_X86_64 |
| 6311 | "mov %c[r8](%0), %%r8 \n\t" | 6265 | "mov %c[r8](%0), %%r8 \n\t" |
| 6312 | "mov %c[r9](%0), %%r9 \n\t" | 6266 | "mov %c[r9](%0), %%r9 \n\t" |
| @@ -6317,24 +6271,24 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 6317 | "mov %c[r14](%0), %%r14 \n\t" | 6271 | "mov %c[r14](%0), %%r14 \n\t" |
| 6318 | "mov %c[r15](%0), %%r15 \n\t" | 6272 | "mov %c[r15](%0), %%r15 \n\t" |
| 6319 | #endif | 6273 | #endif |
| 6320 | "mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */ | 6274 | "mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */ |
| 6321 | 6275 | ||
| 6322 | /* Enter guest mode */ | 6276 | /* Enter guest mode */ |
| 6323 | "jne .Llaunched \n\t" | 6277 | "jne 1f \n\t" |
| 6324 | __ex(ASM_VMX_VMLAUNCH) "\n\t" | 6278 | __ex(ASM_VMX_VMLAUNCH) "\n\t" |
| 6325 | "jmp .Lkvm_vmx_return \n\t" | 6279 | "jmp 2f \n\t" |
| 6326 | ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t" | 6280 | "1: " __ex(ASM_VMX_VMRESUME) "\n\t" |
| 6327 | ".Lkvm_vmx_return: " | 6281 | "2: " |
| 6328 | /* Save guest registers, load host registers, keep flags */ | 6282 | /* Save guest registers, load host registers, keep flags */ |
| 6329 | "mov %0, %c[wordsize](%%"R"sp) \n\t" | 6283 | "mov %0, %c[wordsize](%%" _ASM_SP ") \n\t" |
| 6330 | "pop %0 \n\t" | 6284 | "pop %0 \n\t" |
| 6331 | "mov %%"R"ax, %c[rax](%0) \n\t" | 6285 | "mov %%" _ASM_AX ", %c[rax](%0) \n\t" |
| 6332 | "mov %%"R"bx, %c[rbx](%0) \n\t" | 6286 | "mov %%" _ASM_BX ", %c[rbx](%0) \n\t" |
| 6333 | "pop"Q" %c[rcx](%0) \n\t" | 6287 | __ASM_SIZE(pop) " %c[rcx](%0) \n\t" |
| 6334 | "mov %%"R"dx, %c[rdx](%0) \n\t" | 6288 | "mov %%" _ASM_DX ", %c[rdx](%0) \n\t" |
| 6335 | "mov %%"R"si, %c[rsi](%0) \n\t" | 6289 | "mov %%" _ASM_SI ", %c[rsi](%0) \n\t" |
| 6336 | "mov %%"R"di, %c[rdi](%0) \n\t" | 6290 | "mov %%" _ASM_DI ", %c[rdi](%0) \n\t" |
| 6337 | "mov %%"R"bp, %c[rbp](%0) \n\t" | 6291 | "mov %%" _ASM_BP ", %c[rbp](%0) \n\t" |
| 6338 | #ifdef CONFIG_X86_64 | 6292 | #ifdef CONFIG_X86_64 |
| 6339 | "mov %%r8, %c[r8](%0) \n\t" | 6293 | "mov %%r8, %c[r8](%0) \n\t" |
| 6340 | "mov %%r9, %c[r9](%0) \n\t" | 6294 | "mov %%r9, %c[r9](%0) \n\t" |
| @@ -6345,11 +6299,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 6345 | "mov %%r14, %c[r14](%0) \n\t" | 6299 | "mov %%r14, %c[r14](%0) \n\t" |
| 6346 | "mov %%r15, %c[r15](%0) \n\t" | 6300 | "mov %%r15, %c[r15](%0) \n\t" |
| 6347 | #endif | 6301 | #endif |
| 6348 | "mov %%cr2, %%"R"ax \n\t" | 6302 | "mov %%cr2, %%" _ASM_AX " \n\t" |
| 6349 | "mov %%"R"ax, %c[cr2](%0) \n\t" | 6303 | "mov %%" _ASM_AX ", %c[cr2](%0) \n\t" |
| 6350 | 6304 | ||
| 6351 | "pop %%"R"bp; pop %%"R"dx \n\t" | 6305 | "pop %%" _ASM_BP "; pop %%" _ASM_DX " \n\t" |
| 6352 | "setbe %c[fail](%0) \n\t" | 6306 | "setbe %c[fail](%0) \n\t" |
| 6307 | ".pushsection .rodata \n\t" | ||
| 6308 | ".global vmx_return \n\t" | ||
| 6309 | "vmx_return: " _ASM_PTR " 2b \n\t" | ||
| 6310 | ".popsection" | ||
| 6353 | : : "c"(vmx), "d"((unsigned long)HOST_RSP), | 6311 | : : "c"(vmx), "d"((unsigned long)HOST_RSP), |
| 6354 | [launched]"i"(offsetof(struct vcpu_vmx, __launched)), | 6312 | [launched]"i"(offsetof(struct vcpu_vmx, __launched)), |
| 6355 | [fail]"i"(offsetof(struct vcpu_vmx, fail)), | 6313 | [fail]"i"(offsetof(struct vcpu_vmx, fail)), |
| @@ -6374,12 +6332,18 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 6374 | [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)), | 6332 | [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)), |
| 6375 | [wordsize]"i"(sizeof(ulong)) | 6333 | [wordsize]"i"(sizeof(ulong)) |
| 6376 | : "cc", "memory" | 6334 | : "cc", "memory" |
| 6377 | , R"ax", R"bx", R"di", R"si" | ||
| 6378 | #ifdef CONFIG_X86_64 | 6335 | #ifdef CONFIG_X86_64 |
| 6336 | , "rax", "rbx", "rdi", "rsi" | ||
| 6379 | , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" | 6337 | , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" |
| 6338 | #else | ||
| 6339 | , "eax", "ebx", "edi", "esi" | ||
| 6380 | #endif | 6340 | #endif |
| 6381 | ); | 6341 | ); |
| 6382 | 6342 | ||
| 6343 | /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ | ||
| 6344 | if (debugctlmsr) | ||
| 6345 | update_debugctlmsr(debugctlmsr); | ||
| 6346 | |||
| 6383 | #ifndef CONFIG_X86_64 | 6347 | #ifndef CONFIG_X86_64 |
| 6384 | /* | 6348 | /* |
| 6385 | * The sysexit path does not restore ds/es, so we must set them to | 6349 | * The sysexit path does not restore ds/es, so we must set them to |
| @@ -6424,9 +6388,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | |||
| 6424 | vmx_complete_interrupts(vmx); | 6388 | vmx_complete_interrupts(vmx); |
| 6425 | } | 6389 | } |
| 6426 | 6390 | ||
| 6427 | #undef R | ||
| 6428 | #undef Q | ||
| 6429 | |||
| 6430 | static void vmx_free_vcpu(struct kvm_vcpu *vcpu) | 6391 | static void vmx_free_vcpu(struct kvm_vcpu *vcpu) |
| 6431 | { | 6392 | { |
| 6432 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 6393 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
| @@ -7281,7 +7242,7 @@ static struct kvm_x86_ops vmx_x86_ops = { | |||
| 7281 | .vcpu_load = vmx_vcpu_load, | 7242 | .vcpu_load = vmx_vcpu_load, |
| 7282 | .vcpu_put = vmx_vcpu_put, | 7243 | .vcpu_put = vmx_vcpu_put, |
| 7283 | 7244 | ||
| 7284 | .set_guest_debug = set_guest_debug, | 7245 | .update_db_bp_intercept = update_exception_bitmap, |
| 7285 | .get_msr = vmx_get_msr, | 7246 | .get_msr = vmx_get_msr, |
| 7286 | .set_msr = vmx_set_msr, | 7247 | .set_msr = vmx_set_msr, |
| 7287 | .get_segment_base = vmx_get_segment_base, | 7248 | .get_segment_base = vmx_get_segment_base, |
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1f09552572fa..1eefebe5d727 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c | |||
| @@ -246,20 +246,14 @@ static void drop_user_return_notifiers(void *ignore) | |||
| 246 | 246 | ||
| 247 | u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) | 247 | u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) |
| 248 | { | 248 | { |
| 249 | if (irqchip_in_kernel(vcpu->kvm)) | 249 | return vcpu->arch.apic_base; |
| 250 | return vcpu->arch.apic_base; | ||
| 251 | else | ||
| 252 | return vcpu->arch.apic_base; | ||
| 253 | } | 250 | } |
| 254 | EXPORT_SYMBOL_GPL(kvm_get_apic_base); | 251 | EXPORT_SYMBOL_GPL(kvm_get_apic_base); |
| 255 | 252 | ||
| 256 | void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) | 253 | void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) |
| 257 | { | 254 | { |
| 258 | /* TODO: reserve bits check */ | 255 | /* TODO: reserve bits check */ |
| 259 | if (irqchip_in_kernel(vcpu->kvm)) | 256 | kvm_lapic_set_base(vcpu, data); |
| 260 | kvm_lapic_set_base(vcpu, data); | ||
| 261 | else | ||
| 262 | vcpu->arch.apic_base = data; | ||
| 263 | } | 257 | } |
| 264 | EXPORT_SYMBOL_GPL(kvm_set_apic_base); | 258 | EXPORT_SYMBOL_GPL(kvm_set_apic_base); |
| 265 | 259 | ||
| @@ -698,6 +692,18 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) | |||
| 698 | } | 692 | } |
| 699 | EXPORT_SYMBOL_GPL(kvm_get_cr8); | 693 | EXPORT_SYMBOL_GPL(kvm_get_cr8); |
| 700 | 694 | ||
| 695 | static void kvm_update_dr7(struct kvm_vcpu *vcpu) | ||
| 696 | { | ||
| 697 | unsigned long dr7; | ||
| 698 | |||
| 699 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | ||
| 700 | dr7 = vcpu->arch.guest_debug_dr7; | ||
| 701 | else | ||
| 702 | dr7 = vcpu->arch.dr7; | ||
| 703 | kvm_x86_ops->set_dr7(vcpu, dr7); | ||
| 704 | vcpu->arch.switch_db_regs = (dr7 & DR7_BP_EN_MASK); | ||
| 705 | } | ||
| 706 | |||
| 701 | static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) | 707 | static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) |
| 702 | { | 708 | { |
| 703 | switch (dr) { | 709 | switch (dr) { |
| @@ -723,10 +729,7 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) | |||
| 723 | if (val & 0xffffffff00000000ULL) | 729 | if (val & 0xffffffff00000000ULL) |
| 724 | return -1; /* #GP */ | 730 | return -1; /* #GP */ |
| 725 | vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; | 731 | vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; |
| 726 | if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { | 732 | kvm_update_dr7(vcpu); |
| 727 | kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7); | ||
| 728 | vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK); | ||
| 729 | } | ||
| 730 | break; | 733 | break; |
| 731 | } | 734 | } |
| 732 | 735 | ||
| @@ -823,7 +826,7 @@ static u32 msrs_to_save[] = { | |||
| 823 | 826 | ||
| 824 | static unsigned num_msrs_to_save; | 827 | static unsigned num_msrs_to_save; |
| 825 | 828 | ||
| 826 | static u32 emulated_msrs[] = { | 829 | static const u32 emulated_msrs[] = { |
| 827 | MSR_IA32_TSCDEADLINE, | 830 | MSR_IA32_TSCDEADLINE, |
| 828 | MSR_IA32_MISC_ENABLE, | 831 | MSR_IA32_MISC_ENABLE, |
| 829 | MSR_IA32_MCG_STATUS, | 832 | MSR_IA32_MCG_STATUS, |
| @@ -1097,7 +1100,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data) | |||
| 1097 | * For each generation, we track the original measured | 1100 | * For each generation, we track the original measured |
| 1098 | * nanosecond time, offset, and write, so if TSCs are in | 1101 | * nanosecond time, offset, and write, so if TSCs are in |
| 1099 | * sync, we can match exact offset, and if not, we can match | 1102 | * sync, we can match exact offset, and if not, we can match |
| 1100 | * exact software computaion in compute_guest_tsc() | 1103 | * exact software computation in compute_guest_tsc() |
| 1101 | * | 1104 | * |
| 1102 | * These values are tracked in kvm->arch.cur_xxx variables. | 1105 | * These values are tracked in kvm->arch.cur_xxx variables. |
| 1103 | */ | 1106 | */ |
| @@ -1140,6 +1143,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) | |||
| 1140 | unsigned long this_tsc_khz; | 1143 | unsigned long this_tsc_khz; |
| 1141 | s64 kernel_ns, max_kernel_ns; | 1144 | s64 kernel_ns, max_kernel_ns; |
| 1142 | u64 tsc_timestamp; | 1145 | u64 tsc_timestamp; |
| 1146 | u8 pvclock_flags; | ||
| 1143 | 1147 | ||
| 1144 | /* Keep irq disabled to prevent changes to the clock */ | 1148 | /* Keep irq disabled to prevent changes to the clock */ |
| 1145 | local_irq_save(flags); | 1149 | local_irq_save(flags); |
| @@ -1221,7 +1225,14 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) | |||
| 1221 | vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; | 1225 | vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; |
| 1222 | vcpu->last_kernel_ns = kernel_ns; | 1226 | vcpu->last_kernel_ns = kernel_ns; |
| 1223 | vcpu->last_guest_tsc = tsc_timestamp; | 1227 | vcpu->last_guest_tsc = tsc_timestamp; |
| 1224 | vcpu->hv_clock.flags = 0; | 1228 | |
| 1229 | pvclock_flags = 0; | ||
| 1230 | if (vcpu->pvclock_set_guest_stopped_request) { | ||
| 1231 | pvclock_flags |= PVCLOCK_GUEST_STOPPED; | ||
| 1232 | vcpu->pvclock_set_guest_stopped_request = false; | ||
| 1233 | } | ||
| 1234 | |||
| 1235 | vcpu->hv_clock.flags = pvclock_flags; | ||
| 1225 | 1236 | ||
| 1226 | /* | 1237 | /* |
| 1227 | * The interface expects us to write an even number signaling that the | 1238 | * The interface expects us to write an even number signaling that the |
| @@ -1504,7 +1515,7 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) | |||
| 1504 | { | 1515 | { |
| 1505 | gpa_t gpa = data & ~0x3f; | 1516 | gpa_t gpa = data & ~0x3f; |
| 1506 | 1517 | ||
| 1507 | /* Bits 2:5 are resrved, Should be zero */ | 1518 | /* Bits 2:5 are reserved, Should be zero */ |
| 1508 | if (data & 0x3c) | 1519 | if (data & 0x3c) |
| 1509 | return 1; | 1520 | return 1; |
| 1510 | 1521 | ||
| @@ -1639,10 +1650,9 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
| 1639 | vcpu->arch.time_page = | 1650 | vcpu->arch.time_page = |
| 1640 | gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT); | 1651 | gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT); |
| 1641 | 1652 | ||
| 1642 | if (is_error_page(vcpu->arch.time_page)) { | 1653 | if (is_error_page(vcpu->arch.time_page)) |
| 1643 | kvm_release_page_clean(vcpu->arch.time_page); | ||
| 1644 | vcpu->arch.time_page = NULL; | 1654 | vcpu->arch.time_page = NULL; |
| 1645 | } | 1655 | |
| 1646 | break; | 1656 | break; |
| 1647 | } | 1657 | } |
| 1648 | case MSR_KVM_ASYNC_PF_EN: | 1658 | case MSR_KVM_ASYNC_PF_EN: |
| @@ -1727,7 +1737,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
| 1727 | * Ignore all writes to this no longer documented MSR. | 1737 | * Ignore all writes to this no longer documented MSR. |
| 1728 | * Writes are only relevant for old K7 processors, | 1738 | * Writes are only relevant for old K7 processors, |
| 1729 | * all pre-dating SVM, but a recommended workaround from | 1739 | * all pre-dating SVM, but a recommended workaround from |
| 1730 | * AMD for these chips. It is possible to speicify the | 1740 | * AMD for these chips. It is possible to specify the |
| 1731 | * affected processor models on the command line, hence | 1741 | * affected processor models on the command line, hence |
| 1732 | * the need to ignore the workaround. | 1742 | * the need to ignore the workaround. |
| 1733 | */ | 1743 | */ |
| @@ -2177,6 +2187,8 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
| 2177 | case KVM_CAP_GET_TSC_KHZ: | 2187 | case KVM_CAP_GET_TSC_KHZ: |
| 2178 | case KVM_CAP_PCI_2_3: | 2188 | case KVM_CAP_PCI_2_3: |
| 2179 | case KVM_CAP_KVMCLOCK_CTRL: | 2189 | case KVM_CAP_KVMCLOCK_CTRL: |
| 2190 | case KVM_CAP_READONLY_MEM: | ||
| 2191 | case KVM_CAP_IRQFD_RESAMPLE: | ||
| 2180 | r = 1; | 2192 | r = 1; |
| 2181 | break; | 2193 | break; |
| 2182 | case KVM_CAP_COALESCED_MMIO: | 2194 | case KVM_CAP_COALESCED_MMIO: |
| @@ -2358,8 +2370,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, | |||
| 2358 | static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, | 2370 | static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, |
| 2359 | struct kvm_lapic_state *s) | 2371 | struct kvm_lapic_state *s) |
| 2360 | { | 2372 | { |
| 2361 | memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); | 2373 | kvm_apic_post_state_restore(vcpu, s); |
| 2362 | kvm_apic_post_state_restore(vcpu); | ||
| 2363 | update_cr8_intercept(vcpu); | 2374 | update_cr8_intercept(vcpu); |
| 2364 | 2375 | ||
| 2365 | return 0; | 2376 | return 0; |
| @@ -2368,7 +2379,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, | |||
| 2368 | static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, | 2379 | static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, |
| 2369 | struct kvm_interrupt *irq) | 2380 | struct kvm_interrupt *irq) |
| 2370 | { | 2381 | { |
| 2371 | if (irq->irq < 0 || irq->irq >= 256) | 2382 | if (irq->irq < 0 || irq->irq >= KVM_NR_INTERRUPTS) |
| 2372 | return -EINVAL; | 2383 | return -EINVAL; |
| 2373 | if (irqchip_in_kernel(vcpu->kvm)) | 2384 | if (irqchip_in_kernel(vcpu->kvm)) |
| 2374 | return -ENXIO; | 2385 | return -ENXIO; |
| @@ -2635,11 +2646,9 @@ static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, | |||
| 2635 | */ | 2646 | */ |
| 2636 | static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) | 2647 | static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) |
| 2637 | { | 2648 | { |
| 2638 | struct pvclock_vcpu_time_info *src = &vcpu->arch.hv_clock; | ||
| 2639 | if (!vcpu->arch.time_page) | 2649 | if (!vcpu->arch.time_page) |
| 2640 | return -EINVAL; | 2650 | return -EINVAL; |
| 2641 | src->flags |= PVCLOCK_GUEST_STOPPED; | 2651 | vcpu->arch.pvclock_set_guest_stopped_request = true; |
| 2642 | mark_page_dirty(vcpu->kvm, vcpu->arch.time >> PAGE_SHIFT); | ||
| 2643 | kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); | 2652 | kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); |
| 2644 | return 0; | 2653 | return 0; |
| 2645 | } | 2654 | } |
| @@ -3090,7 +3099,7 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm, | |||
| 3090 | if (!kvm->arch.vpit) | 3099 | if (!kvm->arch.vpit) |
| 3091 | return -ENXIO; | 3100 | return -ENXIO; |
| 3092 | mutex_lock(&kvm->arch.vpit->pit_state.lock); | 3101 | mutex_lock(&kvm->arch.vpit->pit_state.lock); |
| 3093 | kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject; | 3102 | kvm->arch.vpit->pit_state.reinject = control->pit_reinject; |
| 3094 | mutex_unlock(&kvm->arch.vpit->pit_state.lock); | 3103 | mutex_unlock(&kvm->arch.vpit->pit_state.lock); |
| 3095 | return 0; | 3104 | return 0; |
| 3096 | } | 3105 | } |
| @@ -3173,6 +3182,16 @@ out: | |||
| 3173 | return r; | 3182 | return r; |
| 3174 | } | 3183 | } |
| 3175 | 3184 | ||
| 3185 | int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event) | ||
| 3186 | { | ||
| 3187 | if (!irqchip_in_kernel(kvm)) | ||
| 3188 | return -ENXIO; | ||
| 3189 | |||
| 3190 | irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, | ||
| 3191 | irq_event->irq, irq_event->level); | ||
| 3192 | return 0; | ||
| 3193 | } | ||
| 3194 | |||
| 3176 | long kvm_arch_vm_ioctl(struct file *filp, | 3195 | long kvm_arch_vm_ioctl(struct file *filp, |
| 3177 | unsigned int ioctl, unsigned long arg) | 3196 | unsigned int ioctl, unsigned long arg) |
| 3178 | { | 3197 | { |
| @@ -3279,29 +3298,6 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
| 3279 | create_pit_unlock: | 3298 | create_pit_unlock: |
| 3280 | mutex_unlock(&kvm->slots_lock); | 3299 | mutex_unlock(&kvm->slots_lock); |
| 3281 | break; | 3300 | break; |
| 3282 | case KVM_IRQ_LINE_STATUS: | ||
| 3283 | case KVM_IRQ_LINE: { | ||
| 3284 | struct kvm_irq_level irq_event; | ||
| 3285 | |||
| 3286 | r = -EFAULT; | ||
| 3287 | if (copy_from_user(&irq_event, argp, sizeof irq_event)) | ||
| 3288 | goto out; | ||
| 3289 | r = -ENXIO; | ||
| 3290 | if (irqchip_in_kernel(kvm)) { | ||
| 3291 | __s32 status; | ||
| 3292 | status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, | ||
| 3293 | irq_event.irq, irq_event.level); | ||
| 3294 | if (ioctl == KVM_IRQ_LINE_STATUS) { | ||
| 3295 | r = -EFAULT; | ||
| 3296 | irq_event.status = status; | ||
| 3297 | if (copy_to_user(argp, &irq_event, | ||
| 3298 | sizeof irq_event)) | ||
| 3299 | goto out; | ||
| 3300 | } | ||
| 3301 | r = 0; | ||
| 3302 | } | ||
| 3303 | break; | ||
| 3304 | } | ||
| 3305 | case KVM_GET_IRQCHIP: { | 3301 | case KVM_GET_IRQCHIP: { |
| 3306 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ | 3302 | /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ |
| 3307 | struct kvm_irqchip *chip; | 3303 | struct kvm_irqchip *chip; |
| @@ -3689,20 +3685,17 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, | |||
| 3689 | gpa_t *gpa, struct x86_exception *exception, | 3685 | gpa_t *gpa, struct x86_exception *exception, |
| 3690 | bool write) | 3686 | bool write) |
| 3691 | { | 3687 | { |
| 3692 | u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; | 3688 | u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0) |
| 3689 | | (write ? PFERR_WRITE_MASK : 0); | ||
| 3693 | 3690 | ||
| 3694 | if (vcpu_match_mmio_gva(vcpu, gva) && | 3691 | if (vcpu_match_mmio_gva(vcpu, gva) |
| 3695 | check_write_user_access(vcpu, write, access, | 3692 | && !permission_fault(vcpu->arch.walk_mmu, vcpu->arch.access, access)) { |
| 3696 | vcpu->arch.access)) { | ||
| 3697 | *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | | 3693 | *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | |
| 3698 | (gva & (PAGE_SIZE - 1)); | 3694 | (gva & (PAGE_SIZE - 1)); |
| 3699 | trace_vcpu_match_mmio(gva, *gpa, write, false); | 3695 | trace_vcpu_match_mmio(gva, *gpa, write, false); |
| 3700 | return 1; | 3696 | return 1; |
| 3701 | } | 3697 | } |
| 3702 | 3698 | ||
| 3703 | if (write) | ||
| 3704 | access |= PFERR_WRITE_MASK; | ||
| 3705 | |||
| 3706 | *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); | 3699 | *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); |
| 3707 | 3700 | ||
| 3708 | if (*gpa == UNMAPPED_GVA) | 3701 | if (*gpa == UNMAPPED_GVA) |
| @@ -3790,14 +3783,14 @@ static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, | |||
| 3790 | return X86EMUL_CONTINUE; | 3783 | return X86EMUL_CONTINUE; |
| 3791 | } | 3784 | } |
| 3792 | 3785 | ||
| 3793 | static struct read_write_emulator_ops read_emultor = { | 3786 | static const struct read_write_emulator_ops read_emultor = { |
| 3794 | .read_write_prepare = read_prepare, | 3787 | .read_write_prepare = read_prepare, |
| 3795 | .read_write_emulate = read_emulate, | 3788 | .read_write_emulate = read_emulate, |
| 3796 | .read_write_mmio = vcpu_mmio_read, | 3789 | .read_write_mmio = vcpu_mmio_read, |
| 3797 | .read_write_exit_mmio = read_exit_mmio, | 3790 | .read_write_exit_mmio = read_exit_mmio, |
| 3798 | }; | 3791 | }; |
| 3799 | 3792 | ||
| 3800 | static struct read_write_emulator_ops write_emultor = { | 3793 | static const struct read_write_emulator_ops write_emultor = { |
| 3801 | .read_write_emulate = write_emulate, | 3794 | .read_write_emulate = write_emulate, |
| 3802 | .read_write_mmio = write_mmio, | 3795 | .read_write_mmio = write_mmio, |
| 3803 | .read_write_exit_mmio = write_exit_mmio, | 3796 | .read_write_exit_mmio = write_exit_mmio, |
| @@ -3808,7 +3801,7 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, | |||
| 3808 | unsigned int bytes, | 3801 | unsigned int bytes, |
| 3809 | struct x86_exception *exception, | 3802 | struct x86_exception *exception, |
| 3810 | struct kvm_vcpu *vcpu, | 3803 | struct kvm_vcpu *vcpu, |
| 3811 | struct read_write_emulator_ops *ops) | 3804 | const struct read_write_emulator_ops *ops) |
| 3812 | { | 3805 | { |
| 3813 | gpa_t gpa; | 3806 | gpa_t gpa; |
| 3814 | int handled, ret; | 3807 | int handled, ret; |
| @@ -3857,7 +3850,7 @@ mmio: | |||
| 3857 | int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr, | 3850 | int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr, |
| 3858 | void *val, unsigned int bytes, | 3851 | void *val, unsigned int bytes, |
| 3859 | struct x86_exception *exception, | 3852 | struct x86_exception *exception, |
| 3860 | struct read_write_emulator_ops *ops) | 3853 | const struct read_write_emulator_ops *ops) |
| 3861 | { | 3854 | { |
| 3862 | struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); | 3855 | struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); |
| 3863 | gpa_t gpa; | 3856 | gpa_t gpa; |
| @@ -3962,10 +3955,8 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, | |||
| 3962 | goto emul_write; | 3955 | goto emul_write; |
| 3963 | 3956 | ||
| 3964 | page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); | 3957 | page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); |
| 3965 | if (is_error_page(page)) { | 3958 | if (is_error_page(page)) |
| 3966 | kvm_release_page_clean(page); | ||
| 3967 | goto emul_write; | 3959 | goto emul_write; |
| 3968 | } | ||
| 3969 | 3960 | ||
| 3970 | kaddr = kmap_atomic(page); | 3961 | kaddr = kmap_atomic(page); |
| 3971 | kaddr += offset_in_page(gpa); | 3962 | kaddr += offset_in_page(gpa); |
| @@ -4332,7 +4323,19 @@ static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, | |||
| 4332 | kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx); | 4323 | kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx); |
| 4333 | } | 4324 | } |
| 4334 | 4325 | ||
| 4335 | static struct x86_emulate_ops emulate_ops = { | 4326 | static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg) |
| 4327 | { | ||
| 4328 | return kvm_register_read(emul_to_vcpu(ctxt), reg); | ||
| 4329 | } | ||
| 4330 | |||
| 4331 | static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val) | ||
| 4332 | { | ||
| 4333 | kvm_register_write(emul_to_vcpu(ctxt), reg, val); | ||
| 4334 | } | ||
| 4335 | |||
| 4336 | static const struct x86_emulate_ops emulate_ops = { | ||
| 4337 | .read_gpr = emulator_read_gpr, | ||
| 4338 | .write_gpr = emulator_write_gpr, | ||
| 4336 | .read_std = kvm_read_guest_virt_system, | 4339 | .read_std = kvm_read_guest_virt_system, |
| 4337 | .write_std = kvm_write_guest_virt_system, | 4340 | .write_std = kvm_write_guest_virt_system, |
| 4338 | .fetch = kvm_fetch_guest_virt, | 4341 | .fetch = kvm_fetch_guest_virt, |
| @@ -4367,14 +4370,6 @@ static struct x86_emulate_ops emulate_ops = { | |||
| 4367 | .get_cpuid = emulator_get_cpuid, | 4370 | .get_cpuid = emulator_get_cpuid, |
| 4368 | }; | 4371 | }; |
| 4369 | 4372 | ||
| 4370 | static void cache_all_regs(struct kvm_vcpu *vcpu) | ||
| 4371 | { | ||
| 4372 | kvm_register_read(vcpu, VCPU_REGS_RAX); | ||
| 4373 | kvm_register_read(vcpu, VCPU_REGS_RSP); | ||
| 4374 | kvm_register_read(vcpu, VCPU_REGS_RIP); | ||
| 4375 | vcpu->arch.regs_dirty = ~0; | ||
| 4376 | } | ||
| 4377 | |||
| 4378 | static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) | 4373 | static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) |
| 4379 | { | 4374 | { |
| 4380 | u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask); | 4375 | u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask); |
| @@ -4401,12 +4396,10 @@ static void inject_emulated_exception(struct kvm_vcpu *vcpu) | |||
| 4401 | kvm_queue_exception(vcpu, ctxt->exception.vector); | 4396 | kvm_queue_exception(vcpu, ctxt->exception.vector); |
| 4402 | } | 4397 | } |
| 4403 | 4398 | ||
| 4404 | static void init_decode_cache(struct x86_emulate_ctxt *ctxt, | 4399 | static void init_decode_cache(struct x86_emulate_ctxt *ctxt) |
| 4405 | const unsigned long *regs) | ||
| 4406 | { | 4400 | { |
| 4407 | memset(&ctxt->twobyte, 0, | 4401 | memset(&ctxt->twobyte, 0, |
| 4408 | (void *)&ctxt->regs - (void *)&ctxt->twobyte); | 4402 | (void *)&ctxt->_regs - (void *)&ctxt->twobyte); |
| 4409 | memcpy(ctxt->regs, regs, sizeof(ctxt->regs)); | ||
| 4410 | 4403 | ||
| 4411 | ctxt->fetch.start = 0; | 4404 | ctxt->fetch.start = 0; |
| 4412 | ctxt->fetch.end = 0; | 4405 | ctxt->fetch.end = 0; |
| @@ -4421,14 +4414,6 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) | |||
| 4421 | struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; | 4414 | struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; |
| 4422 | int cs_db, cs_l; | 4415 | int cs_db, cs_l; |
| 4423 | 4416 | ||
| 4424 | /* | ||
| 4425 | * TODO: fix emulate.c to use guest_read/write_register | ||
| 4426 | * instead of direct ->regs accesses, can save hundred cycles | ||
| 4427 | * on Intel for instructions that don't read/change RSP, for | ||
| 4428 | * for example. | ||
| 4429 | */ | ||
| 4430 | cache_all_regs(vcpu); | ||
| 4431 | |||
| 4432 | kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); | 4417 | kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); |
| 4433 | 4418 | ||
| 4434 | ctxt->eflags = kvm_get_rflags(vcpu); | 4419 | ctxt->eflags = kvm_get_rflags(vcpu); |
| @@ -4440,7 +4425,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) | |||
| 4440 | X86EMUL_MODE_PROT16; | 4425 | X86EMUL_MODE_PROT16; |
| 4441 | ctxt->guest_mode = is_guest_mode(vcpu); | 4426 | ctxt->guest_mode = is_guest_mode(vcpu); |
| 4442 | 4427 | ||
| 4443 | init_decode_cache(ctxt, vcpu->arch.regs); | 4428 | init_decode_cache(ctxt); |
| 4444 | vcpu->arch.emulate_regs_need_sync_from_vcpu = false; | 4429 | vcpu->arch.emulate_regs_need_sync_from_vcpu = false; |
| 4445 | } | 4430 | } |
| 4446 | 4431 | ||
| @@ -4460,7 +4445,6 @@ int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) | |||
| 4460 | return EMULATE_FAIL; | 4445 | return EMULATE_FAIL; |
| 4461 | 4446 | ||
| 4462 | ctxt->eip = ctxt->_eip; | 4447 | ctxt->eip = ctxt->_eip; |
| 4463 | memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); | ||
| 4464 | kvm_rip_write(vcpu, ctxt->eip); | 4448 | kvm_rip_write(vcpu, ctxt->eip); |
| 4465 | kvm_set_rflags(vcpu, ctxt->eflags); | 4449 | kvm_set_rflags(vcpu, ctxt->eflags); |
| 4466 | 4450 | ||
| @@ -4493,13 +4477,14 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu) | |||
| 4493 | static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) | 4477 | static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) |
| 4494 | { | 4478 | { |
| 4495 | gpa_t gpa; | 4479 | gpa_t gpa; |
| 4480 | pfn_t pfn; | ||
| 4496 | 4481 | ||
| 4497 | if (tdp_enabled) | 4482 | if (tdp_enabled) |
| 4498 | return false; | 4483 | return false; |
| 4499 | 4484 | ||
| 4500 | /* | 4485 | /* |
| 4501 | * if emulation was due to access to shadowed page table | 4486 | * if emulation was due to access to shadowed page table |
| 4502 | * and it failed try to unshadow page and re-entetr the | 4487 | * and it failed try to unshadow page and re-enter the |
| 4503 | * guest to let CPU execute the instruction. | 4488 | * guest to let CPU execute the instruction. |
| 4504 | */ | 4489 | */ |
| 4505 | if (kvm_mmu_unprotect_page_virt(vcpu, gva)) | 4490 | if (kvm_mmu_unprotect_page_virt(vcpu, gva)) |
| @@ -4510,8 +4495,17 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) | |||
| 4510 | if (gpa == UNMAPPED_GVA) | 4495 | if (gpa == UNMAPPED_GVA) |
| 4511 | return true; /* let cpu generate fault */ | 4496 | return true; /* let cpu generate fault */ |
| 4512 | 4497 | ||
| 4513 | if (!kvm_is_error_hva(gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT))) | 4498 | /* |
| 4499 | * Do not retry the unhandleable instruction if it faults on the | ||
| 4500 | * readonly host memory, otherwise it will goto a infinite loop: | ||
| 4501 | * retry instruction -> write #PF -> emulation fail -> retry | ||
| 4502 | * instruction -> ... | ||
| 4503 | */ | ||
| 4504 | pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa)); | ||
| 4505 | if (!is_error_pfn(pfn)) { | ||
| 4506 | kvm_release_pfn_clean(pfn); | ||
| 4514 | return true; | 4507 | return true; |
| 4508 | } | ||
| 4515 | 4509 | ||
| 4516 | return false; | 4510 | return false; |
| 4517 | } | 4511 | } |
| @@ -4560,6 +4554,9 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt, | |||
| 4560 | return true; | 4554 | return true; |
| 4561 | } | 4555 | } |
| 4562 | 4556 | ||
| 4557 | static int complete_emulated_mmio(struct kvm_vcpu *vcpu); | ||
| 4558 | static int complete_emulated_pio(struct kvm_vcpu *vcpu); | ||
| 4559 | |||
| 4563 | int x86_emulate_instruction(struct kvm_vcpu *vcpu, | 4560 | int x86_emulate_instruction(struct kvm_vcpu *vcpu, |
| 4564 | unsigned long cr2, | 4561 | unsigned long cr2, |
| 4565 | int emulation_type, | 4562 | int emulation_type, |
| @@ -4608,7 +4605,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, | |||
| 4608 | changes registers values during IO operation */ | 4605 | changes registers values during IO operation */ |
| 4609 | if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { | 4606 | if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { |
| 4610 | vcpu->arch.emulate_regs_need_sync_from_vcpu = false; | 4607 | vcpu->arch.emulate_regs_need_sync_from_vcpu = false; |
| 4611 | memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs); | 4608 | emulator_invalidate_register_cache(ctxt); |
| 4612 | } | 4609 | } |
| 4613 | 4610 | ||
| 4614 | restart: | 4611 | restart: |
| @@ -4630,13 +4627,16 @@ restart: | |||
| 4630 | } else if (vcpu->arch.pio.count) { | 4627 | } else if (vcpu->arch.pio.count) { |
| 4631 | if (!vcpu->arch.pio.in) | 4628 | if (!vcpu->arch.pio.in) |
| 4632 | vcpu->arch.pio.count = 0; | 4629 | vcpu->arch.pio.count = 0; |
| 4633 | else | 4630 | else { |
| 4634 | writeback = false; | 4631 | writeback = false; |
| 4632 | vcpu->arch.complete_userspace_io = complete_emulated_pio; | ||
| 4633 | } | ||
| 4635 | r = EMULATE_DO_MMIO; | 4634 | r = EMULATE_DO_MMIO; |
| 4636 | } else if (vcpu->mmio_needed) { | 4635 | } else if (vcpu->mmio_needed) { |
| 4637 | if (!vcpu->mmio_is_write) | 4636 | if (!vcpu->mmio_is_write) |
| 4638 | writeback = false; | 4637 | writeback = false; |
| 4639 | r = EMULATE_DO_MMIO; | 4638 | r = EMULATE_DO_MMIO; |
| 4639 | vcpu->arch.complete_userspace_io = complete_emulated_mmio; | ||
| 4640 | } else if (r == EMULATION_RESTART) | 4640 | } else if (r == EMULATION_RESTART) |
| 4641 | goto restart; | 4641 | goto restart; |
| 4642 | else | 4642 | else |
| @@ -4646,7 +4646,6 @@ restart: | |||
| 4646 | toggle_interruptibility(vcpu, ctxt->interruptibility); | 4646 | toggle_interruptibility(vcpu, ctxt->interruptibility); |
| 4647 | kvm_set_rflags(vcpu, ctxt->eflags); | 4647 | kvm_set_rflags(vcpu, ctxt->eflags); |
| 4648 | kvm_make_request(KVM_REQ_EVENT, vcpu); | 4648 | kvm_make_request(KVM_REQ_EVENT, vcpu); |
| 4649 | memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); | ||
| 4650 | vcpu->arch.emulate_regs_need_sync_to_vcpu = false; | 4649 | vcpu->arch.emulate_regs_need_sync_to_vcpu = false; |
| 4651 | kvm_rip_write(vcpu, ctxt->eip); | 4650 | kvm_rip_write(vcpu, ctxt->eip); |
| 4652 | } else | 4651 | } else |
| @@ -4929,6 +4928,7 @@ int kvm_arch_init(void *opaque) | |||
| 4929 | if (cpu_has_xsave) | 4928 | if (cpu_has_xsave) |
| 4930 | host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); | 4929 | host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); |
| 4931 | 4930 | ||
| 4931 | kvm_lapic_init(); | ||
| 4932 | return 0; | 4932 | return 0; |
| 4933 | 4933 | ||
| 4934 | out: | 4934 | out: |
| @@ -5499,6 +5499,24 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) | |||
| 5499 | return r; | 5499 | return r; |
| 5500 | } | 5500 | } |
| 5501 | 5501 | ||
| 5502 | static inline int complete_emulated_io(struct kvm_vcpu *vcpu) | ||
| 5503 | { | ||
| 5504 | int r; | ||
| 5505 | vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); | ||
| 5506 | r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); | ||
| 5507 | srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); | ||
| 5508 | if (r != EMULATE_DONE) | ||
| 5509 | return 0; | ||
| 5510 | return 1; | ||
| 5511 | } | ||
| 5512 | |||
| 5513 | static int complete_emulated_pio(struct kvm_vcpu *vcpu) | ||
| 5514 | { | ||
| 5515 | BUG_ON(!vcpu->arch.pio.count); | ||
| 5516 | |||
| 5517 | return complete_emulated_io(vcpu); | ||
| 5518 | } | ||
| 5519 | |||
| 5502 | /* | 5520 | /* |
| 5503 | * Implements the following, as a state machine: | 5521 | * Implements the following, as a state machine: |
| 5504 | * | 5522 | * |
| @@ -5515,47 +5533,37 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) | |||
| 5515 | * copy data | 5533 | * copy data |
| 5516 | * exit | 5534 | * exit |
| 5517 | */ | 5535 | */ |
| 5518 | static int complete_mmio(struct kvm_vcpu *vcpu) | 5536 | static int complete_emulated_mmio(struct kvm_vcpu *vcpu) |
| 5519 | { | 5537 | { |
| 5520 | struct kvm_run *run = vcpu->run; | 5538 | struct kvm_run *run = vcpu->run; |
| 5521 | struct kvm_mmio_fragment *frag; | 5539 | struct kvm_mmio_fragment *frag; |
| 5522 | int r; | ||
| 5523 | 5540 | ||
| 5524 | if (!(vcpu->arch.pio.count || vcpu->mmio_needed)) | 5541 | BUG_ON(!vcpu->mmio_needed); |
| 5525 | return 1; | ||
| 5526 | 5542 | ||
| 5527 | if (vcpu->mmio_needed) { | 5543 | /* Complete previous fragment */ |
| 5528 | /* Complete previous fragment */ | 5544 | frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment++]; |
| 5529 | frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment++]; | 5545 | if (!vcpu->mmio_is_write) |
| 5530 | if (!vcpu->mmio_is_write) | 5546 | memcpy(frag->data, run->mmio.data, frag->len); |
| 5531 | memcpy(frag->data, run->mmio.data, frag->len); | 5547 | if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) { |
| 5532 | if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) { | 5548 | vcpu->mmio_needed = 0; |
| 5533 | vcpu->mmio_needed = 0; | ||
| 5534 | if (vcpu->mmio_is_write) | ||
| 5535 | return 1; | ||
| 5536 | vcpu->mmio_read_completed = 1; | ||
| 5537 | goto done; | ||
| 5538 | } | ||
| 5539 | /* Initiate next fragment */ | ||
| 5540 | ++frag; | ||
| 5541 | run->exit_reason = KVM_EXIT_MMIO; | ||
| 5542 | run->mmio.phys_addr = frag->gpa; | ||
| 5543 | if (vcpu->mmio_is_write) | 5549 | if (vcpu->mmio_is_write) |
| 5544 | memcpy(run->mmio.data, frag->data, frag->len); | 5550 | return 1; |
| 5545 | run->mmio.len = frag->len; | 5551 | vcpu->mmio_read_completed = 1; |
| 5546 | run->mmio.is_write = vcpu->mmio_is_write; | 5552 | return complete_emulated_io(vcpu); |
| 5547 | return 0; | 5553 | } |
| 5548 | 5554 | /* Initiate next fragment */ | |
| 5549 | } | 5555 | ++frag; |
| 5550 | done: | 5556 | run->exit_reason = KVM_EXIT_MMIO; |
| 5551 | vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); | 5557 | run->mmio.phys_addr = frag->gpa; |
| 5552 | r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); | 5558 | if (vcpu->mmio_is_write) |
| 5553 | srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); | 5559 | memcpy(run->mmio.data, frag->data, frag->len); |
| 5554 | if (r != EMULATE_DONE) | 5560 | run->mmio.len = frag->len; |
| 5555 | return 0; | 5561 | run->mmio.is_write = vcpu->mmio_is_write; |
| 5556 | return 1; | 5562 | vcpu->arch.complete_userspace_io = complete_emulated_mmio; |
| 5563 | return 0; | ||
| 5557 | } | 5564 | } |
| 5558 | 5565 | ||
| 5566 | |||
| 5559 | int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 5567 | int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
| 5560 | { | 5568 | { |
| 5561 | int r; | 5569 | int r; |
| @@ -5582,9 +5590,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
| 5582 | } | 5590 | } |
| 5583 | } | 5591 | } |
| 5584 | 5592 | ||
| 5585 | r = complete_mmio(vcpu); | 5593 | if (unlikely(vcpu->arch.complete_userspace_io)) { |
| 5586 | if (r <= 0) | 5594 | int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; |
| 5587 | goto out; | 5595 | vcpu->arch.complete_userspace_io = NULL; |
| 5596 | r = cui(vcpu); | ||
| 5597 | if (r <= 0) | ||
| 5598 | goto out; | ||
| 5599 | } else | ||
| 5600 | WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed); | ||
| 5588 | 5601 | ||
| 5589 | r = __vcpu_run(vcpu); | 5602 | r = __vcpu_run(vcpu); |
| 5590 | 5603 | ||
| @@ -5602,12 +5615,11 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
| 5602 | /* | 5615 | /* |
| 5603 | * We are here if userspace calls get_regs() in the middle of | 5616 | * We are here if userspace calls get_regs() in the middle of |
| 5604 | * instruction emulation. Registers state needs to be copied | 5617 | * instruction emulation. Registers state needs to be copied |
| 5605 | * back from emulation context to vcpu. Usrapace shouldn't do | 5618 | * back from emulation context to vcpu. Userspace shouldn't do |
| 5606 | * that usually, but some bad designed PV devices (vmware | 5619 | * that usually, but some bad designed PV devices (vmware |
| 5607 | * backdoor interface) need this to work | 5620 | * backdoor interface) need this to work |
| 5608 | */ | 5621 | */ |
| 5609 | struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; | 5622 | emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt); |
| 5610 | memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); | ||
| 5611 | vcpu->arch.emulate_regs_need_sync_to_vcpu = false; | 5623 | vcpu->arch.emulate_regs_need_sync_to_vcpu = false; |
| 5612 | } | 5624 | } |
| 5613 | regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX); | 5625 | regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX); |
| @@ -5747,7 +5759,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, | |||
| 5747 | if (ret) | 5759 | if (ret) |
| 5748 | return EMULATE_FAIL; | 5760 | return EMULATE_FAIL; |
| 5749 | 5761 | ||
| 5750 | memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs); | ||
| 5751 | kvm_rip_write(vcpu, ctxt->eip); | 5762 | kvm_rip_write(vcpu, ctxt->eip); |
| 5752 | kvm_set_rflags(vcpu, ctxt->eflags); | 5763 | kvm_set_rflags(vcpu, ctxt->eflags); |
| 5753 | kvm_make_request(KVM_REQ_EVENT, vcpu); | 5764 | kvm_make_request(KVM_REQ_EVENT, vcpu); |
| @@ -5799,7 +5810,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |||
| 5799 | if (mmu_reset_needed) | 5810 | if (mmu_reset_needed) |
| 5800 | kvm_mmu_reset_context(vcpu); | 5811 | kvm_mmu_reset_context(vcpu); |
| 5801 | 5812 | ||
| 5802 | max_bits = (sizeof sregs->interrupt_bitmap) << 3; | 5813 | max_bits = KVM_NR_INTERRUPTS; |
| 5803 | pending_vec = find_first_bit( | 5814 | pending_vec = find_first_bit( |
| 5804 | (const unsigned long *)sregs->interrupt_bitmap, max_bits); | 5815 | (const unsigned long *)sregs->interrupt_bitmap, max_bits); |
| 5805 | if (pending_vec < max_bits) { | 5816 | if (pending_vec < max_bits) { |
| @@ -5859,13 +5870,12 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, | |||
| 5859 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { | 5870 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { |
| 5860 | for (i = 0; i < KVM_NR_DB_REGS; ++i) | 5871 | for (i = 0; i < KVM_NR_DB_REGS; ++i) |
| 5861 | vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; | 5872 | vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; |
| 5862 | vcpu->arch.switch_db_regs = | 5873 | vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7]; |
| 5863 | (dbg->arch.debugreg[7] & DR7_BP_EN_MASK); | ||
| 5864 | } else { | 5874 | } else { |
| 5865 | for (i = 0; i < KVM_NR_DB_REGS; i++) | 5875 | for (i = 0; i < KVM_NR_DB_REGS; i++) |
| 5866 | vcpu->arch.eff_db[i] = vcpu->arch.db[i]; | 5876 | vcpu->arch.eff_db[i] = vcpu->arch.db[i]; |
| 5867 | vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK); | ||
| 5868 | } | 5877 | } |
| 5878 | kvm_update_dr7(vcpu); | ||
| 5869 | 5879 | ||
| 5870 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | 5880 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) |
| 5871 | vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) + | 5881 | vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) + |
| @@ -5877,7 +5887,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, | |||
| 5877 | */ | 5887 | */ |
| 5878 | kvm_set_rflags(vcpu, rflags); | 5888 | kvm_set_rflags(vcpu, rflags); |
| 5879 | 5889 | ||
| 5880 | kvm_x86_ops->set_guest_debug(vcpu, dbg); | 5890 | kvm_x86_ops->update_db_bp_intercept(vcpu); |
| 5881 | 5891 | ||
| 5882 | r = 0; | 5892 | r = 0; |
| 5883 | 5893 | ||
| @@ -6023,7 +6033,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) | |||
| 6023 | int r; | 6033 | int r; |
| 6024 | 6034 | ||
| 6025 | vcpu->arch.mtrr_state.have_fixed = 1; | 6035 | vcpu->arch.mtrr_state.have_fixed = 1; |
| 6026 | vcpu_load(vcpu); | 6036 | r = vcpu_load(vcpu); |
| 6037 | if (r) | ||
| 6038 | return r; | ||
| 6027 | r = kvm_arch_vcpu_reset(vcpu); | 6039 | r = kvm_arch_vcpu_reset(vcpu); |
| 6028 | if (r == 0) | 6040 | if (r == 0) |
| 6029 | r = kvm_mmu_setup(vcpu); | 6041 | r = kvm_mmu_setup(vcpu); |
| @@ -6034,9 +6046,11 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) | |||
| 6034 | 6046 | ||
| 6035 | void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) | 6047 | void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) |
| 6036 | { | 6048 | { |
| 6049 | int r; | ||
| 6037 | vcpu->arch.apf.msr_val = 0; | 6050 | vcpu->arch.apf.msr_val = 0; |
| 6038 | 6051 | ||
| 6039 | vcpu_load(vcpu); | 6052 | r = vcpu_load(vcpu); |
| 6053 | BUG_ON(r); | ||
| 6040 | kvm_mmu_unload(vcpu); | 6054 | kvm_mmu_unload(vcpu); |
| 6041 | vcpu_put(vcpu); | 6055 | vcpu_put(vcpu); |
| 6042 | 6056 | ||
| @@ -6050,10 +6064,10 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) | |||
| 6050 | vcpu->arch.nmi_pending = 0; | 6064 | vcpu->arch.nmi_pending = 0; |
| 6051 | vcpu->arch.nmi_injected = false; | 6065 | vcpu->arch.nmi_injected = false; |
| 6052 | 6066 | ||
| 6053 | vcpu->arch.switch_db_regs = 0; | ||
| 6054 | memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); | 6067 | memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); |
| 6055 | vcpu->arch.dr6 = DR6_FIXED_1; | 6068 | vcpu->arch.dr6 = DR6_FIXED_1; |
| 6056 | vcpu->arch.dr7 = DR7_FIXED_1; | 6069 | vcpu->arch.dr7 = DR7_FIXED_1; |
| 6070 | kvm_update_dr7(vcpu); | ||
| 6057 | 6071 | ||
| 6058 | kvm_make_request(KVM_REQ_EVENT, vcpu); | 6072 | kvm_make_request(KVM_REQ_EVENT, vcpu); |
| 6059 | vcpu->arch.apf.msr_val = 0; | 6073 | vcpu->arch.apf.msr_val = 0; |
| @@ -6132,7 +6146,7 @@ int kvm_arch_hardware_enable(void *garbage) | |||
| 6132 | * as we reset last_host_tsc on all VCPUs to stop this from being | 6146 | * as we reset last_host_tsc on all VCPUs to stop this from being |
| 6133 | * called multiple times (one for each physical CPU bringup). | 6147 | * called multiple times (one for each physical CPU bringup). |
| 6134 | * | 6148 | * |
| 6135 | * Platforms with unnreliable TSCs don't have to deal with this, they | 6149 | * Platforms with unreliable TSCs don't have to deal with this, they |
| 6136 | * will be compensated by the logic in vcpu_load, which sets the TSC to | 6150 | * will be compensated by the logic in vcpu_load, which sets the TSC to |
| 6137 | * catchup mode. This will catchup all VCPUs to real time, but cannot | 6151 | * catchup mode. This will catchup all VCPUs to real time, but cannot |
| 6138 | * guarantee that they stay in perfect synchronization. | 6152 | * guarantee that they stay in perfect synchronization. |
| @@ -6185,6 +6199,8 @@ bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) | |||
| 6185 | return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL); | 6199 | return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL); |
| 6186 | } | 6200 | } |
| 6187 | 6201 | ||
| 6202 | struct static_key kvm_no_apic_vcpu __read_mostly; | ||
| 6203 | |||
| 6188 | int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) | 6204 | int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) |
| 6189 | { | 6205 | { |
| 6190 | struct page *page; | 6206 | struct page *page; |
| @@ -6217,7 +6233,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) | |||
| 6217 | r = kvm_create_lapic(vcpu); | 6233 | r = kvm_create_lapic(vcpu); |
| 6218 | if (r < 0) | 6234 | if (r < 0) |
| 6219 | goto fail_mmu_destroy; | 6235 | goto fail_mmu_destroy; |
| 6220 | } | 6236 | } else |
| 6237 | static_key_slow_inc(&kvm_no_apic_vcpu); | ||
| 6221 | 6238 | ||
| 6222 | vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4, | 6239 | vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4, |
| 6223 | GFP_KERNEL); | 6240 | GFP_KERNEL); |
| @@ -6257,6 +6274,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) | |||
| 6257 | kvm_mmu_destroy(vcpu); | 6274 | kvm_mmu_destroy(vcpu); |
| 6258 | srcu_read_unlock(&vcpu->kvm->srcu, idx); | 6275 | srcu_read_unlock(&vcpu->kvm->srcu, idx); |
| 6259 | free_page((unsigned long)vcpu->arch.pio_data); | 6276 | free_page((unsigned long)vcpu->arch.pio_data); |
| 6277 | if (!irqchip_in_kernel(vcpu->kvm)) | ||
| 6278 | static_key_slow_dec(&kvm_no_apic_vcpu); | ||
| 6260 | } | 6279 | } |
| 6261 | 6280 | ||
| 6262 | int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) | 6281 | int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) |
| @@ -6269,15 +6288,21 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) | |||
| 6269 | 6288 | ||
| 6270 | /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ | 6289 | /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ |
| 6271 | set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); | 6290 | set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); |
| 6291 | /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */ | ||
| 6292 | set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, | ||
| 6293 | &kvm->arch.irq_sources_bitmap); | ||
| 6272 | 6294 | ||
| 6273 | raw_spin_lock_init(&kvm->arch.tsc_write_lock); | 6295 | raw_spin_lock_init(&kvm->arch.tsc_write_lock); |
| 6296 | mutex_init(&kvm->arch.apic_map_lock); | ||
| 6274 | 6297 | ||
| 6275 | return 0; | 6298 | return 0; |
| 6276 | } | 6299 | } |
| 6277 | 6300 | ||
| 6278 | static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) | 6301 | static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) |
| 6279 | { | 6302 | { |
| 6280 | vcpu_load(vcpu); | 6303 | int r; |
| 6304 | r = vcpu_load(vcpu); | ||
| 6305 | BUG_ON(r); | ||
| 6281 | kvm_mmu_unload(vcpu); | 6306 | kvm_mmu_unload(vcpu); |
| 6282 | vcpu_put(vcpu); | 6307 | vcpu_put(vcpu); |
| 6283 | } | 6308 | } |
| @@ -6321,6 +6346,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) | |||
| 6321 | put_page(kvm->arch.apic_access_page); | 6346 | put_page(kvm->arch.apic_access_page); |
| 6322 | if (kvm->arch.ept_identity_pagetable) | 6347 | if (kvm->arch.ept_identity_pagetable) |
| 6323 | put_page(kvm->arch.ept_identity_pagetable); | 6348 | put_page(kvm->arch.ept_identity_pagetable); |
| 6349 | kfree(rcu_dereference_check(kvm->arch.apic_map, 1)); | ||
| 6324 | } | 6350 | } |
| 6325 | 6351 | ||
| 6326 | void kvm_arch_free_memslot(struct kvm_memory_slot *free, | 6352 | void kvm_arch_free_memslot(struct kvm_memory_slot *free, |
| @@ -6328,10 +6354,18 @@ void kvm_arch_free_memslot(struct kvm_memory_slot *free, | |||
| 6328 | { | 6354 | { |
| 6329 | int i; | 6355 | int i; |
| 6330 | 6356 | ||
| 6331 | for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { | 6357 | for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { |
| 6332 | if (!dont || free->arch.lpage_info[i] != dont->arch.lpage_info[i]) { | 6358 | if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) { |
| 6333 | kvm_kvfree(free->arch.lpage_info[i]); | 6359 | kvm_kvfree(free->arch.rmap[i]); |
| 6334 | free->arch.lpage_info[i] = NULL; | 6360 | free->arch.rmap[i] = NULL; |
| 6361 | } | ||
| 6362 | if (i == 0) | ||
| 6363 | continue; | ||
| 6364 | |||
| 6365 | if (!dont || free->arch.lpage_info[i - 1] != | ||
| 6366 | dont->arch.lpage_info[i - 1]) { | ||
| 6367 | kvm_kvfree(free->arch.lpage_info[i - 1]); | ||
| 6368 | free->arch.lpage_info[i - 1] = NULL; | ||
| 6335 | } | 6369 | } |
| 6336 | } | 6370 | } |
| 6337 | } | 6371 | } |
| @@ -6340,23 +6374,30 @@ int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages) | |||
| 6340 | { | 6374 | { |
| 6341 | int i; | 6375 | int i; |
| 6342 | 6376 | ||
| 6343 | for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { | 6377 | for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { |
| 6344 | unsigned long ugfn; | 6378 | unsigned long ugfn; |
| 6345 | int lpages; | 6379 | int lpages; |
| 6346 | int level = i + 2; | 6380 | int level = i + 1; |
| 6347 | 6381 | ||
| 6348 | lpages = gfn_to_index(slot->base_gfn + npages - 1, | 6382 | lpages = gfn_to_index(slot->base_gfn + npages - 1, |
| 6349 | slot->base_gfn, level) + 1; | 6383 | slot->base_gfn, level) + 1; |
| 6350 | 6384 | ||
| 6351 | slot->arch.lpage_info[i] = | 6385 | slot->arch.rmap[i] = |
| 6352 | kvm_kvzalloc(lpages * sizeof(*slot->arch.lpage_info[i])); | 6386 | kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i])); |
| 6353 | if (!slot->arch.lpage_info[i]) | 6387 | if (!slot->arch.rmap[i]) |
| 6388 | goto out_free; | ||
| 6389 | if (i == 0) | ||
| 6390 | continue; | ||
| 6391 | |||
| 6392 | slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages * | ||
| 6393 | sizeof(*slot->arch.lpage_info[i - 1])); | ||
| 6394 | if (!slot->arch.lpage_info[i - 1]) | ||
| 6354 | goto out_free; | 6395 | goto out_free; |
| 6355 | 6396 | ||
| 6356 | if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) | 6397 | if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) |
| 6357 | slot->arch.lpage_info[i][0].write_count = 1; | 6398 | slot->arch.lpage_info[i - 1][0].write_count = 1; |
| 6358 | if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) | 6399 | if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) |
| 6359 | slot->arch.lpage_info[i][lpages - 1].write_count = 1; | 6400 | slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1; |
| 6360 | ugfn = slot->userspace_addr >> PAGE_SHIFT; | 6401 | ugfn = slot->userspace_addr >> PAGE_SHIFT; |
| 6361 | /* | 6402 | /* |
| 6362 | * If the gfn and userspace address are not aligned wrt each | 6403 | * If the gfn and userspace address are not aligned wrt each |
| @@ -6368,16 +6409,21 @@ int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages) | |||
| 6368 | unsigned long j; | 6409 | unsigned long j; |
| 6369 | 6410 | ||
| 6370 | for (j = 0; j < lpages; ++j) | 6411 | for (j = 0; j < lpages; ++j) |
| 6371 | slot->arch.lpage_info[i][j].write_count = 1; | 6412 | slot->arch.lpage_info[i - 1][j].write_count = 1; |
| 6372 | } | 6413 | } |
| 6373 | } | 6414 | } |
| 6374 | 6415 | ||
| 6375 | return 0; | 6416 | return 0; |
| 6376 | 6417 | ||
| 6377 | out_free: | 6418 | out_free: |
| 6378 | for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { | 6419 | for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { |
| 6379 | kvm_kvfree(slot->arch.lpage_info[i]); | 6420 | kvm_kvfree(slot->arch.rmap[i]); |
| 6380 | slot->arch.lpage_info[i] = NULL; | 6421 | slot->arch.rmap[i] = NULL; |
| 6422 | if (i == 0) | ||
| 6423 | continue; | ||
| 6424 | |||
| 6425 | kvm_kvfree(slot->arch.lpage_info[i - 1]); | ||
| 6426 | slot->arch.lpage_info[i - 1] = NULL; | ||
| 6381 | } | 6427 | } |
| 6382 | return -ENOMEM; | 6428 | return -ENOMEM; |
| 6383 | } | 6429 | } |
| @@ -6396,10 +6442,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, | |||
| 6396 | map_flags = MAP_SHARED | MAP_ANONYMOUS; | 6442 | map_flags = MAP_SHARED | MAP_ANONYMOUS; |
| 6397 | 6443 | ||
| 6398 | /*To keep backward compatibility with older userspace, | 6444 | /*To keep backward compatibility with older userspace, |
| 6399 | *x86 needs to hanlde !user_alloc case. | 6445 | *x86 needs to handle !user_alloc case. |
| 6400 | */ | 6446 | */ |
| 6401 | if (!user_alloc) { | 6447 | if (!user_alloc) { |
| 6402 | if (npages && !old.rmap) { | 6448 | if (npages && !old.npages) { |
| 6403 | unsigned long userspace_addr; | 6449 | unsigned long userspace_addr; |
| 6404 | 6450 | ||
| 6405 | userspace_addr = vm_mmap(NULL, 0, | 6451 | userspace_addr = vm_mmap(NULL, 0, |
| @@ -6427,7 +6473,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, | |||
| 6427 | 6473 | ||
| 6428 | int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT; | 6474 | int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT; |
| 6429 | 6475 | ||
| 6430 | if (!user_alloc && !old.user_alloc && old.rmap && !npages) { | 6476 | if (!user_alloc && !old.user_alloc && old.npages && !npages) { |
| 6431 | int ret; | 6477 | int ret; |
| 6432 | 6478 | ||
| 6433 | ret = vm_munmap(old.userspace_addr, | 6479 | ret = vm_munmap(old.userspace_addr, |
| @@ -6446,14 +6492,28 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, | |||
| 6446 | kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); | 6492 | kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); |
| 6447 | kvm_mmu_slot_remove_write_access(kvm, mem->slot); | 6493 | kvm_mmu_slot_remove_write_access(kvm, mem->slot); |
| 6448 | spin_unlock(&kvm->mmu_lock); | 6494 | spin_unlock(&kvm->mmu_lock); |
| 6495 | /* | ||
| 6496 | * If memory slot is created, or moved, we need to clear all | ||
| 6497 | * mmio sptes. | ||
| 6498 | */ | ||
| 6499 | if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT) { | ||
| 6500 | kvm_mmu_zap_all(kvm); | ||
| 6501 | kvm_reload_remote_mmus(kvm); | ||
| 6502 | } | ||
| 6449 | } | 6503 | } |
| 6450 | 6504 | ||
| 6451 | void kvm_arch_flush_shadow(struct kvm *kvm) | 6505 | void kvm_arch_flush_shadow_all(struct kvm *kvm) |
| 6452 | { | 6506 | { |
| 6453 | kvm_mmu_zap_all(kvm); | 6507 | kvm_mmu_zap_all(kvm); |
| 6454 | kvm_reload_remote_mmus(kvm); | 6508 | kvm_reload_remote_mmus(kvm); |
| 6455 | } | 6509 | } |
| 6456 | 6510 | ||
| 6511 | void kvm_arch_flush_shadow_memslot(struct kvm *kvm, | ||
| 6512 | struct kvm_memory_slot *slot) | ||
| 6513 | { | ||
| 6514 | kvm_arch_flush_shadow_all(kvm); | ||
| 6515 | } | ||
| 6516 | |||
| 6457 | int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) | 6517 | int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) |
| 6458 | { | 6518 | { |
| 6459 | return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && | 6519 | return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && |
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 3d1134ddb885..2b5219c12ac8 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h | |||
| @@ -124,4 +124,5 @@ int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt, | |||
| 124 | 124 | ||
| 125 | extern u64 host_xcr0; | 125 | extern u64 host_xcr0; |
| 126 | 126 | ||
| 127 | extern struct static_key kvm_no_apic_vcpu; | ||
| 127 | #endif | 128 | #endif |
diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index b1e6c4b2e8eb..54fcffed28ed 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c | |||
| @@ -18,7 +18,11 @@ | |||
| 18 | * Copyright (C) IBM Corporation, 2002, 2004, 2009 | 18 | * Copyright (C) IBM Corporation, 2002, 2004, 2009 |
| 19 | */ | 19 | */ |
| 20 | 20 | ||
| 21 | #ifdef __KERNEL__ | ||
| 21 | #include <linux/string.h> | 22 | #include <linux/string.h> |
| 23 | #else | ||
| 24 | #include <string.h> | ||
| 25 | #endif | ||
| 22 | #include <asm/inat.h> | 26 | #include <asm/inat.h> |
| 23 | #include <asm/insn.h> | 27 | #include <asm/insn.h> |
| 24 | 28 | ||
diff --git a/arch/x86/syscalls/Makefile b/arch/x86/syscalls/Makefile index 3236aebc828d..f325af26107c 100644 --- a/arch/x86/syscalls/Makefile +++ b/arch/x86/syscalls/Makefile | |||
| @@ -1,7 +1,9 @@ | |||
| 1 | out := $(obj)/../include/generated/asm | 1 | out := $(obj)/../include/generated/asm |
| 2 | uapi := $(obj)/../include/generated/uapi/asm | ||
| 2 | 3 | ||
| 3 | # Create output directory if not already present | 4 | # Create output directory if not already present |
| 4 | _dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') | 5 | _dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') \ |
| 6 | $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') | ||
| 5 | 7 | ||
| 6 | syscall32 := $(srctree)/$(src)/syscall_32.tbl | 8 | syscall32 := $(srctree)/$(src)/syscall_32.tbl |
| 7 | syscall64 := $(srctree)/$(src)/syscall_64.tbl | 9 | syscall64 := $(srctree)/$(src)/syscall_64.tbl |
| @@ -18,7 +20,7 @@ quiet_cmd_systbl = SYSTBL $@ | |||
| 18 | cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@ | 20 | cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@ |
| 19 | 21 | ||
| 20 | syshdr_abi_unistd_32 := i386 | 22 | syshdr_abi_unistd_32 := i386 |
| 21 | $(out)/unistd_32.h: $(syscall32) $(syshdr) | 23 | $(uapi)/unistd_32.h: $(syscall32) $(syshdr) |
| 22 | $(call if_changed,syshdr) | 24 | $(call if_changed,syshdr) |
| 23 | 25 | ||
| 24 | syshdr_abi_unistd_32_ia32 := i386 | 26 | syshdr_abi_unistd_32_ia32 := i386 |
| @@ -28,11 +30,11 @@ $(out)/unistd_32_ia32.h: $(syscall32) $(syshdr) | |||
| 28 | 30 | ||
| 29 | syshdr_abi_unistd_x32 := common,x32 | 31 | syshdr_abi_unistd_x32 := common,x32 |
| 30 | syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT | 32 | syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT |
| 31 | $(out)/unistd_x32.h: $(syscall64) $(syshdr) | 33 | $(uapi)/unistd_x32.h: $(syscall64) $(syshdr) |
| 32 | $(call if_changed,syshdr) | 34 | $(call if_changed,syshdr) |
| 33 | 35 | ||
| 34 | syshdr_abi_unistd_64 := common,64 | 36 | syshdr_abi_unistd_64 := common,64 |
| 35 | $(out)/unistd_64.h: $(syscall64) $(syshdr) | 37 | $(uapi)/unistd_64.h: $(syscall64) $(syshdr) |
| 36 | $(call if_changed,syshdr) | 38 | $(call if_changed,syshdr) |
| 37 | 39 | ||
| 38 | syshdr_abi_unistd_64_x32 := x32 | 40 | syshdr_abi_unistd_64_x32 := x32 |
| @@ -45,11 +47,12 @@ $(out)/syscalls_32.h: $(syscall32) $(systbl) | |||
| 45 | $(out)/syscalls_64.h: $(syscall64) $(systbl) | 47 | $(out)/syscalls_64.h: $(syscall64) $(systbl) |
| 46 | $(call if_changed,systbl) | 48 | $(call if_changed,systbl) |
| 47 | 49 | ||
| 48 | syshdr-y += unistd_32.h unistd_64.h unistd_x32.h | 50 | uapisyshdr-y += unistd_32.h unistd_64.h unistd_x32.h |
| 49 | syshdr-y += syscalls_32.h | 51 | syshdr-y += syscalls_32.h |
| 50 | syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h | 52 | syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h |
| 51 | syshdr-$(CONFIG_X86_64) += syscalls_64.h | 53 | syshdr-$(CONFIG_X86_64) += syscalls_64.h |
| 52 | 54 | ||
| 53 | targets += $(syshdr-y) | 55 | targets += $(uapisyshdr-y) $(syshdr-y) |
| 54 | 56 | ||
| 55 | all: $(addprefix $(out)/,$(targets)) | 57 | all: $(addprefix $(uapi)/,$(uapisyshdr-y)) |
| 58 | all: $(addprefix $(out)/,$(syshdr-y)) | ||
diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile index 733057b435b0..bae601f900ef 100644 --- a/arch/x86/tools/Makefile +++ b/arch/x86/tools/Makefile | |||
| @@ -28,7 +28,7 @@ posttest: $(obj)/test_get_len vmlinux $(obj)/insn_sanity | |||
| 28 | hostprogs-y += test_get_len insn_sanity | 28 | hostprogs-y += test_get_len insn_sanity |
| 29 | 29 | ||
| 30 | # -I needed for generated C source and C source which in the kernel tree. | 30 | # -I needed for generated C source and C source which in the kernel tree. |
| 31 | HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ | 31 | HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/ |
| 32 | 32 | ||
| 33 | HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ | 33 | HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ |
| 34 | 34 | ||
diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index ec57bd3818a4..7005ced5d1ad 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c | |||
| @@ -6,8 +6,9 @@ | |||
| 6 | 6 | ||
| 7 | #include <xen/xen.h> | 7 | #include <xen/xen.h> |
| 8 | #include <xen/interface/physdev.h> | 8 | #include <xen/interface/physdev.h> |
| 9 | #include "xen-ops.h" | ||
| 9 | 10 | ||
| 10 | unsigned int xen_io_apic_read(unsigned apic, unsigned reg) | 11 | static unsigned int xen_io_apic_read(unsigned apic, unsigned reg) |
| 11 | { | 12 | { |
| 12 | struct physdev_apic apic_op; | 13 | struct physdev_apic apic_op; |
| 13 | int ret; | 14 | int ret; |
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 1fbe75a95f15..2d932c351f91 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c | |||
| @@ -80,6 +80,8 @@ | |||
| 80 | #include "smp.h" | 80 | #include "smp.h" |
| 81 | #include "multicalls.h" | 81 | #include "multicalls.h" |
| 82 | 82 | ||
| 83 | #include <xen/events.h> | ||
| 84 | |||
| 83 | EXPORT_SYMBOL_GPL(hypercall_page); | 85 | EXPORT_SYMBOL_GPL(hypercall_page); |
| 84 | 86 | ||
| 85 | DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); | 87 | DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); |
| @@ -1288,7 +1290,6 @@ asmlinkage void __init xen_start_kernel(void) | |||
| 1288 | { | 1290 | { |
| 1289 | struct physdev_set_iopl set_iopl; | 1291 | struct physdev_set_iopl set_iopl; |
| 1290 | int rc; | 1292 | int rc; |
| 1291 | pgd_t *pgd; | ||
| 1292 | 1293 | ||
| 1293 | if (!xen_start_info) | 1294 | if (!xen_start_info) |
| 1294 | return; | 1295 | return; |
| @@ -1380,8 +1381,6 @@ asmlinkage void __init xen_start_kernel(void) | |||
| 1380 | acpi_numa = -1; | 1381 | acpi_numa = -1; |
| 1381 | #endif | 1382 | #endif |
| 1382 | 1383 | ||
| 1383 | pgd = (pgd_t *)xen_start_info->pt_base; | ||
| 1384 | |||
| 1385 | /* Don't do the full vcpu_info placement stuff until we have a | 1384 | /* Don't do the full vcpu_info placement stuff until we have a |
| 1386 | possible map and a non-dummy shared_info. */ | 1385 | possible map and a non-dummy shared_info. */ |
| 1387 | per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0]; | 1386 | per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0]; |
| @@ -1390,7 +1389,7 @@ asmlinkage void __init xen_start_kernel(void) | |||
| 1390 | early_boot_irqs_disabled = true; | 1389 | early_boot_irqs_disabled = true; |
| 1391 | 1390 | ||
| 1392 | xen_raw_console_write("mapping kernel into physical memory\n"); | 1391 | xen_raw_console_write("mapping kernel into physical memory\n"); |
| 1393 | pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages); | 1392 | xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, xen_start_info->nr_pages); |
| 1394 | 1393 | ||
| 1395 | /* Allocate and initialize top and mid mfn levels for p2m structure */ | 1394 | /* Allocate and initialize top and mid mfn levels for p2m structure */ |
| 1396 | xen_build_mfn_list_list(); | 1395 | xen_build_mfn_list_list(); |
| @@ -1441,11 +1440,19 @@ asmlinkage void __init xen_start_kernel(void) | |||
| 1441 | const struct dom0_vga_console_info *info = | 1440 | const struct dom0_vga_console_info *info = |
| 1442 | (void *)((char *)xen_start_info + | 1441 | (void *)((char *)xen_start_info + |
| 1443 | xen_start_info->console.dom0.info_off); | 1442 | xen_start_info->console.dom0.info_off); |
| 1443 | struct xen_platform_op op = { | ||
| 1444 | .cmd = XENPF_firmware_info, | ||
| 1445 | .interface_version = XENPF_INTERFACE_VERSION, | ||
| 1446 | .u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS, | ||
| 1447 | }; | ||
| 1444 | 1448 | ||
| 1445 | xen_init_vga(info, xen_start_info->console.dom0.info_size); | 1449 | xen_init_vga(info, xen_start_info->console.dom0.info_size); |
| 1446 | xen_start_info->console.domU.mfn = 0; | 1450 | xen_start_info->console.domU.mfn = 0; |
| 1447 | xen_start_info->console.domU.evtchn = 0; | 1451 | xen_start_info->console.domU.evtchn = 0; |
| 1448 | 1452 | ||
| 1453 | if (HYPERVISOR_dom0_op(&op) == 0) | ||
| 1454 | boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags; | ||
| 1455 | |||
| 1449 | xen_init_apic(); | 1456 | xen_init_apic(); |
| 1450 | 1457 | ||
| 1451 | /* Make sure ACS will be enabled */ | 1458 | /* Make sure ACS will be enabled */ |
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 7a769b7526cb..5a16824cc2b3 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c | |||
| @@ -84,6 +84,7 @@ | |||
| 84 | */ | 84 | */ |
| 85 | DEFINE_SPINLOCK(xen_reservation_lock); | 85 | DEFINE_SPINLOCK(xen_reservation_lock); |
| 86 | 86 | ||
| 87 | #ifdef CONFIG_X86_32 | ||
| 87 | /* | 88 | /* |
| 88 | * Identity map, in addition to plain kernel map. This needs to be | 89 | * Identity map, in addition to plain kernel map. This needs to be |
| 89 | * large enough to allocate page table pages to allocate the rest. | 90 | * large enough to allocate page table pages to allocate the rest. |
| @@ -91,7 +92,7 @@ DEFINE_SPINLOCK(xen_reservation_lock); | |||
| 91 | */ | 92 | */ |
| 92 | #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4) | 93 | #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4) |
| 93 | static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES); | 94 | static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES); |
| 94 | 95 | #endif | |
| 95 | #ifdef CONFIG_X86_64 | 96 | #ifdef CONFIG_X86_64 |
| 96 | /* l3 pud for userspace vsyscall mapping */ | 97 | /* l3 pud for userspace vsyscall mapping */ |
| 97 | static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; | 98 | static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; |
| @@ -1176,13 +1177,6 @@ static void xen_exit_mmap(struct mm_struct *mm) | |||
| 1176 | 1177 | ||
| 1177 | static void xen_post_allocator_init(void); | 1178 | static void xen_post_allocator_init(void); |
| 1178 | 1179 | ||
| 1179 | static void __init xen_pagetable_init(void) | ||
| 1180 | { | ||
| 1181 | paging_init(); | ||
| 1182 | xen_setup_shared_info(); | ||
| 1183 | xen_post_allocator_init(); | ||
| 1184 | } | ||
| 1185 | |||
| 1186 | static __init void xen_mapping_pagetable_reserve(u64 start, u64 end) | 1180 | static __init void xen_mapping_pagetable_reserve(u64 start, u64 end) |
| 1187 | { | 1181 | { |
| 1188 | /* reserve the range used */ | 1182 | /* reserve the range used */ |
| @@ -1197,6 +1191,87 @@ static __init void xen_mapping_pagetable_reserve(u64 start, u64 end) | |||
| 1197 | } | 1191 | } |
| 1198 | } | 1192 | } |
| 1199 | 1193 | ||
| 1194 | #ifdef CONFIG_X86_64 | ||
| 1195 | static void __init xen_cleanhighmap(unsigned long vaddr, | ||
| 1196 | unsigned long vaddr_end) | ||
| 1197 | { | ||
| 1198 | unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1; | ||
| 1199 | pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr); | ||
| 1200 | |||
| 1201 | /* NOTE: The loop is more greedy than the cleanup_highmap variant. | ||
| 1202 | * We include the PMD passed in on _both_ boundaries. */ | ||
| 1203 | for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE)); | ||
| 1204 | pmd++, vaddr += PMD_SIZE) { | ||
| 1205 | if (pmd_none(*pmd)) | ||
| 1206 | continue; | ||
| 1207 | if (vaddr < (unsigned long) _text || vaddr > kernel_end) | ||
| 1208 | set_pmd(pmd, __pmd(0)); | ||
| 1209 | } | ||
| 1210 | /* In case we did something silly, we should crash in this function | ||
| 1211 | * instead of somewhere later and be confusing. */ | ||
| 1212 | xen_mc_flush(); | ||
| 1213 | } | ||
| 1214 | #endif | ||
| 1215 | static void __init xen_pagetable_init(void) | ||
| 1216 | { | ||
| 1217 | #ifdef CONFIG_X86_64 | ||
| 1218 | unsigned long size; | ||
| 1219 | unsigned long addr; | ||
| 1220 | #endif | ||
| 1221 | paging_init(); | ||
| 1222 | xen_setup_shared_info(); | ||
| 1223 | #ifdef CONFIG_X86_64 | ||
| 1224 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { | ||
| 1225 | unsigned long new_mfn_list; | ||
| 1226 | |||
| 1227 | size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); | ||
| 1228 | |||
| 1229 | /* On 32-bit, we get zero so this never gets executed. */ | ||
| 1230 | new_mfn_list = xen_revector_p2m_tree(); | ||
| 1231 | if (new_mfn_list && new_mfn_list != xen_start_info->mfn_list) { | ||
| 1232 | /* using __ka address and sticking INVALID_P2M_ENTRY! */ | ||
| 1233 | memset((void *)xen_start_info->mfn_list, 0xff, size); | ||
| 1234 | |||
| 1235 | /* We should be in __ka space. */ | ||
| 1236 | BUG_ON(xen_start_info->mfn_list < __START_KERNEL_map); | ||
| 1237 | addr = xen_start_info->mfn_list; | ||
| 1238 | /* We roundup to the PMD, which means that if anybody at this stage is | ||
| 1239 | * using the __ka address of xen_start_info or xen_start_info->shared_info | ||
| 1240 | * they are in going to crash. Fortunatly we have already revectored | ||
| 1241 | * in xen_setup_kernel_pagetable and in xen_setup_shared_info. */ | ||
| 1242 | size = roundup(size, PMD_SIZE); | ||
| 1243 | xen_cleanhighmap(addr, addr + size); | ||
| 1244 | |||
| 1245 | size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); | ||
| 1246 | memblock_free(__pa(xen_start_info->mfn_list), size); | ||
| 1247 | /* And revector! Bye bye old array */ | ||
| 1248 | xen_start_info->mfn_list = new_mfn_list; | ||
| 1249 | } else | ||
| 1250 | goto skip; | ||
| 1251 | } | ||
| 1252 | /* At this stage, cleanup_highmap has already cleaned __ka space | ||
| 1253 | * from _brk_limit way up to the max_pfn_mapped (which is the end of | ||
| 1254 | * the ramdisk). We continue on, erasing PMD entries that point to page | ||
| 1255 | * tables - do note that they are accessible at this stage via __va. | ||
| 1256 | * For good measure we also round up to the PMD - which means that if | ||
| 1257 | * anybody is using __ka address to the initial boot-stack - and try | ||
| 1258 | * to use it - they are going to crash. The xen_start_info has been | ||
| 1259 | * taken care of already in xen_setup_kernel_pagetable. */ | ||
| 1260 | addr = xen_start_info->pt_base; | ||
| 1261 | size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE); | ||
| 1262 | |||
| 1263 | xen_cleanhighmap(addr, addr + size); | ||
| 1264 | xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base)); | ||
| 1265 | #ifdef DEBUG | ||
| 1266 | /* This is superflous and is not neccessary, but you know what | ||
| 1267 | * lets do it. The MODULES_VADDR -> MODULES_END should be clear of | ||
| 1268 | * anything at this stage. */ | ||
| 1269 | xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1); | ||
| 1270 | #endif | ||
| 1271 | skip: | ||
| 1272 | #endif | ||
| 1273 | xen_post_allocator_init(); | ||
| 1274 | } | ||
| 1200 | static void xen_write_cr2(unsigned long cr2) | 1275 | static void xen_write_cr2(unsigned long cr2) |
| 1201 | { | 1276 | { |
| 1202 | this_cpu_read(xen_vcpu)->arch.cr2 = cr2; | 1277 | this_cpu_read(xen_vcpu)->arch.cr2 = cr2; |
| @@ -1652,7 +1727,7 @@ static void set_page_prot(void *addr, pgprot_t prot) | |||
| 1652 | if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0)) | 1727 | if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0)) |
| 1653 | BUG(); | 1728 | BUG(); |
| 1654 | } | 1729 | } |
| 1655 | 1730 | #ifdef CONFIG_X86_32 | |
| 1656 | static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) | 1731 | static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) |
| 1657 | { | 1732 | { |
| 1658 | unsigned pmdidx, pteidx; | 1733 | unsigned pmdidx, pteidx; |
| @@ -1703,7 +1778,7 @@ static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) | |||
| 1703 | 1778 | ||
| 1704 | set_page_prot(pmd, PAGE_KERNEL_RO); | 1779 | set_page_prot(pmd, PAGE_KERNEL_RO); |
| 1705 | } | 1780 | } |
| 1706 | 1781 | #endif | |
| 1707 | void __init xen_setup_machphys_mapping(void) | 1782 | void __init xen_setup_machphys_mapping(void) |
| 1708 | { | 1783 | { |
| 1709 | struct xen_machphys_mapping mapping; | 1784 | struct xen_machphys_mapping mapping; |
| @@ -1731,7 +1806,20 @@ static void convert_pfn_mfn(void *v) | |||
| 1731 | for (i = 0; i < PTRS_PER_PTE; i++) | 1806 | for (i = 0; i < PTRS_PER_PTE; i++) |
| 1732 | pte[i] = xen_make_pte(pte[i].pte); | 1807 | pte[i] = xen_make_pte(pte[i].pte); |
| 1733 | } | 1808 | } |
| 1734 | 1809 | static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end, | |
| 1810 | unsigned long addr) | ||
| 1811 | { | ||
| 1812 | if (*pt_base == PFN_DOWN(__pa(addr))) { | ||
| 1813 | set_page_prot((void *)addr, PAGE_KERNEL); | ||
| 1814 | clear_page((void *)addr); | ||
| 1815 | (*pt_base)++; | ||
| 1816 | } | ||
| 1817 | if (*pt_end == PFN_DOWN(__pa(addr))) { | ||
| 1818 | set_page_prot((void *)addr, PAGE_KERNEL); | ||
| 1819 | clear_page((void *)addr); | ||
| 1820 | (*pt_end)--; | ||
| 1821 | } | ||
| 1822 | } | ||
| 1735 | /* | 1823 | /* |
| 1736 | * Set up the initial kernel pagetable. | 1824 | * Set up the initial kernel pagetable. |
| 1737 | * | 1825 | * |
| @@ -1743,11 +1831,13 @@ static void convert_pfn_mfn(void *v) | |||
| 1743 | * of the physical mapping once some sort of allocator has been set | 1831 | * of the physical mapping once some sort of allocator has been set |
| 1744 | * up. | 1832 | * up. |
| 1745 | */ | 1833 | */ |
| 1746 | pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, | 1834 | void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) |
| 1747 | unsigned long max_pfn) | ||
| 1748 | { | 1835 | { |
| 1749 | pud_t *l3; | 1836 | pud_t *l3; |
| 1750 | pmd_t *l2; | 1837 | pmd_t *l2; |
| 1838 | unsigned long addr[3]; | ||
| 1839 | unsigned long pt_base, pt_end; | ||
| 1840 | unsigned i; | ||
| 1751 | 1841 | ||
| 1752 | /* max_pfn_mapped is the last pfn mapped in the initial memory | 1842 | /* max_pfn_mapped is the last pfn mapped in the initial memory |
| 1753 | * mappings. Considering that on Xen after the kernel mappings we | 1843 | * mappings. Considering that on Xen after the kernel mappings we |
| @@ -1755,32 +1845,53 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, | |||
| 1755 | * set max_pfn_mapped to the last real pfn mapped. */ | 1845 | * set max_pfn_mapped to the last real pfn mapped. */ |
| 1756 | max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list)); | 1846 | max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list)); |
| 1757 | 1847 | ||
| 1848 | pt_base = PFN_DOWN(__pa(xen_start_info->pt_base)); | ||
| 1849 | pt_end = pt_base + xen_start_info->nr_pt_frames; | ||
| 1850 | |||
| 1758 | /* Zap identity mapping */ | 1851 | /* Zap identity mapping */ |
| 1759 | init_level4_pgt[0] = __pgd(0); | 1852 | init_level4_pgt[0] = __pgd(0); |
| 1760 | 1853 | ||
| 1761 | /* Pre-constructed entries are in pfn, so convert to mfn */ | 1854 | /* Pre-constructed entries are in pfn, so convert to mfn */ |
| 1855 | /* L4[272] -> level3_ident_pgt | ||
| 1856 | * L4[511] -> level3_kernel_pgt */ | ||
| 1762 | convert_pfn_mfn(init_level4_pgt); | 1857 | convert_pfn_mfn(init_level4_pgt); |
| 1858 | |||
| 1859 | /* L3_i[0] -> level2_ident_pgt */ | ||
| 1763 | convert_pfn_mfn(level3_ident_pgt); | 1860 | convert_pfn_mfn(level3_ident_pgt); |
| 1861 | /* L3_k[510] -> level2_kernel_pgt | ||
| 1862 | * L3_i[511] -> level2_fixmap_pgt */ | ||
| 1764 | convert_pfn_mfn(level3_kernel_pgt); | 1863 | convert_pfn_mfn(level3_kernel_pgt); |
| 1765 | 1864 | ||
| 1865 | /* We get [511][511] and have Xen's version of level2_kernel_pgt */ | ||
| 1766 | l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd); | 1866 | l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd); |
| 1767 | l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud); | 1867 | l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud); |
| 1768 | 1868 | ||
| 1769 | memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); | 1869 | addr[0] = (unsigned long)pgd; |
| 1770 | memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); | 1870 | addr[1] = (unsigned long)l3; |
| 1771 | 1871 | addr[2] = (unsigned long)l2; | |
| 1872 | /* Graft it onto L4[272][0]. Note that we creating an aliasing problem: | ||
| 1873 | * Both L4[272][0] and L4[511][511] have entries that point to the same | ||
| 1874 | * L2 (PMD) tables. Meaning that if you modify it in __va space | ||
| 1875 | * it will be also modified in the __ka space! (But if you just | ||
| 1876 | * modify the PMD table to point to other PTE's or none, then you | ||
| 1877 | * are OK - which is what cleanup_highmap does) */ | ||
| 1878 | copy_page(level2_ident_pgt, l2); | ||
| 1879 | /* Graft it onto L4[511][511] */ | ||
| 1880 | copy_page(level2_kernel_pgt, l2); | ||
| 1881 | |||
| 1882 | /* Get [511][510] and graft that in level2_fixmap_pgt */ | ||
| 1772 | l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd); | 1883 | l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd); |
| 1773 | l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud); | 1884 | l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud); |
| 1774 | memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); | 1885 | copy_page(level2_fixmap_pgt, l2); |
| 1775 | 1886 | /* Note that we don't do anything with level1_fixmap_pgt which | |
| 1776 | /* Set up identity map */ | 1887 | * we don't need. */ |
| 1777 | xen_map_identity_early(level2_ident_pgt, max_pfn); | ||
| 1778 | 1888 | ||
| 1779 | /* Make pagetable pieces RO */ | 1889 | /* Make pagetable pieces RO */ |
| 1780 | set_page_prot(init_level4_pgt, PAGE_KERNEL_RO); | 1890 | set_page_prot(init_level4_pgt, PAGE_KERNEL_RO); |
| 1781 | set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO); | 1891 | set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO); |
| 1782 | set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO); | 1892 | set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO); |
| 1783 | set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); | 1893 | set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); |
| 1894 | set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO); | ||
| 1784 | set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); | 1895 | set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); |
| 1785 | set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); | 1896 | set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); |
| 1786 | 1897 | ||
| @@ -1791,22 +1902,28 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, | |||
| 1791 | /* Unpin Xen-provided one */ | 1902 | /* Unpin Xen-provided one */ |
| 1792 | pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); | 1903 | pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); |
| 1793 | 1904 | ||
| 1794 | /* Switch over */ | ||
| 1795 | pgd = init_level4_pgt; | ||
| 1796 | |||
| 1797 | /* | 1905 | /* |
| 1798 | * At this stage there can be no user pgd, and no page | 1906 | * At this stage there can be no user pgd, and no page |
| 1799 | * structure to attach it to, so make sure we just set kernel | 1907 | * structure to attach it to, so make sure we just set kernel |
| 1800 | * pgd. | 1908 | * pgd. |
| 1801 | */ | 1909 | */ |
| 1802 | xen_mc_batch(); | 1910 | xen_mc_batch(); |
| 1803 | __xen_write_cr3(true, __pa(pgd)); | 1911 | __xen_write_cr3(true, __pa(init_level4_pgt)); |
| 1804 | xen_mc_issue(PARAVIRT_LAZY_CPU); | 1912 | xen_mc_issue(PARAVIRT_LAZY_CPU); |
| 1805 | 1913 | ||
| 1806 | memblock_reserve(__pa(xen_start_info->pt_base), | 1914 | /* We can't that easily rip out L3 and L2, as the Xen pagetables are |
| 1807 | xen_start_info->nr_pt_frames * PAGE_SIZE); | 1915 | * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for |
| 1916 | * the initial domain. For guests using the toolstack, they are in: | ||
| 1917 | * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only | ||
| 1918 | * rip out the [L4] (pgd), but for guests we shave off three pages. | ||
| 1919 | */ | ||
| 1920 | for (i = 0; i < ARRAY_SIZE(addr); i++) | ||
| 1921 | check_pt_base(&pt_base, &pt_end, addr[i]); | ||
| 1808 | 1922 | ||
| 1809 | return pgd; | 1923 | /* Our (by three pages) smaller Xen pagetable that we are using */ |
| 1924 | memblock_reserve(PFN_PHYS(pt_base), (pt_end - pt_base) * PAGE_SIZE); | ||
| 1925 | /* Revector the xen_start_info */ | ||
| 1926 | xen_start_info = (struct start_info *)__va(__pa(xen_start_info)); | ||
| 1810 | } | 1927 | } |
| 1811 | #else /* !CONFIG_X86_64 */ | 1928 | #else /* !CONFIG_X86_64 */ |
| 1812 | static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD); | 1929 | static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD); |
| @@ -1831,8 +1948,7 @@ static void __init xen_write_cr3_init(unsigned long cr3) | |||
| 1831 | */ | 1948 | */ |
| 1832 | swapper_kernel_pmd = | 1949 | swapper_kernel_pmd = |
| 1833 | extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); | 1950 | extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); |
| 1834 | memcpy(swapper_kernel_pmd, initial_kernel_pmd, | 1951 | copy_page(swapper_kernel_pmd, initial_kernel_pmd); |
| 1835 | sizeof(pmd_t) * PTRS_PER_PMD); | ||
| 1836 | swapper_pg_dir[KERNEL_PGD_BOUNDARY] = | 1952 | swapper_pg_dir[KERNEL_PGD_BOUNDARY] = |
| 1837 | __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT); | 1953 | __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT); |
| 1838 | set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO); | 1954 | set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO); |
| @@ -1849,8 +1965,7 @@ static void __init xen_write_cr3_init(unsigned long cr3) | |||
| 1849 | pv_mmu_ops.write_cr3 = &xen_write_cr3; | 1965 | pv_mmu_ops.write_cr3 = &xen_write_cr3; |
| 1850 | } | 1966 | } |
| 1851 | 1967 | ||
| 1852 | pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, | 1968 | void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) |
| 1853 | unsigned long max_pfn) | ||
| 1854 | { | 1969 | { |
| 1855 | pmd_t *kernel_pmd; | 1970 | pmd_t *kernel_pmd; |
| 1856 | 1971 | ||
| @@ -1862,11 +1977,11 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, | |||
| 1862 | 512*1024); | 1977 | 512*1024); |
| 1863 | 1978 | ||
| 1864 | kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); | 1979 | kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); |
| 1865 | memcpy(initial_kernel_pmd, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD); | 1980 | copy_page(initial_kernel_pmd, kernel_pmd); |
| 1866 | 1981 | ||
| 1867 | xen_map_identity_early(initial_kernel_pmd, max_pfn); | 1982 | xen_map_identity_early(initial_kernel_pmd, max_pfn); |
| 1868 | 1983 | ||
| 1869 | memcpy(initial_page_table, pgd, sizeof(pgd_t) * PTRS_PER_PGD); | 1984 | copy_page(initial_page_table, pgd); |
| 1870 | initial_page_table[KERNEL_PGD_BOUNDARY] = | 1985 | initial_page_table[KERNEL_PGD_BOUNDARY] = |
| 1871 | __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT); | 1986 | __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT); |
| 1872 | 1987 | ||
| @@ -1882,8 +1997,6 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, | |||
| 1882 | 1997 | ||
| 1883 | memblock_reserve(__pa(xen_start_info->pt_base), | 1998 | memblock_reserve(__pa(xen_start_info->pt_base), |
| 1884 | xen_start_info->nr_pt_frames * PAGE_SIZE); | 1999 | xen_start_info->nr_pt_frames * PAGE_SIZE); |
| 1885 | |||
| 1886 | return initial_page_table; | ||
| 1887 | } | 2000 | } |
| 1888 | #endif /* CONFIG_X86_64 */ | 2001 | #endif /* CONFIG_X86_64 */ |
| 1889 | 2002 | ||
| @@ -2333,6 +2446,9 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma, | |||
| 2333 | unsigned long range; | 2446 | unsigned long range; |
| 2334 | int err = 0; | 2447 | int err = 0; |
| 2335 | 2448 | ||
| 2449 | if (xen_feature(XENFEAT_auto_translated_physmap)) | ||
| 2450 | return -EINVAL; | ||
| 2451 | |||
| 2336 | prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP); | 2452 | prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP); |
| 2337 | 2453 | ||
| 2338 | BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) == | 2454 | BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) == |
| @@ -2351,8 +2467,8 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma, | |||
| 2351 | if (err) | 2467 | if (err) |
| 2352 | goto out; | 2468 | goto out; |
| 2353 | 2469 | ||
| 2354 | err = -EFAULT; | 2470 | err = HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid); |
| 2355 | if (HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid) < 0) | 2471 | if (err < 0) |
| 2356 | goto out; | 2472 | goto out; |
| 2357 | 2473 | ||
| 2358 | nr -= batch; | 2474 | nr -= batch; |
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 72213da605f5..95fb2aa5927e 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c | |||
| @@ -22,7 +22,7 @@ | |||
| 22 | * | 22 | * |
| 23 | * P2M_PER_PAGE depends on the architecture, as a mfn is always | 23 | * P2M_PER_PAGE depends on the architecture, as a mfn is always |
| 24 | * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to | 24 | * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to |
| 25 | * 512 and 1024 entries respectively. | 25 | * 512 and 1024 entries respectively. |
| 26 | * | 26 | * |
| 27 | * In short, these structures contain the Machine Frame Number (MFN) of the PFN. | 27 | * In short, these structures contain the Machine Frame Number (MFN) of the PFN. |
| 28 | * | 28 | * |
| @@ -139,11 +139,11 @@ | |||
| 139 | * / | ~0, ~0, .... | | 139 | * / | ~0, ~0, .... | |
| 140 | * | \---------------/ | 140 | * | \---------------/ |
| 141 | * | | 141 | * | |
| 142 | * p2m_missing p2m_missing | 142 | * p2m_mid_missing p2m_missing |
| 143 | * /------------------\ /------------\ | 143 | * /-----------------\ /------------\ |
| 144 | * | [p2m_mid_missing]+---->| ~0, ~0, ~0 | | 144 | * | [p2m_missing] +---->| ~0, ~0, ~0 | |
| 145 | * | [p2m_mid_missing]+---->| ..., ~0 | | 145 | * | [p2m_missing] +---->| ..., ~0 | |
| 146 | * \------------------/ \------------/ | 146 | * \-----------------/ \------------/ |
| 147 | * | 147 | * |
| 148 | * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT) | 148 | * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT) |
| 149 | */ | 149 | */ |
| @@ -396,7 +396,85 @@ void __init xen_build_dynamic_phys_to_machine(void) | |||
| 396 | 396 | ||
| 397 | m2p_override_init(); | 397 | m2p_override_init(); |
| 398 | } | 398 | } |
| 399 | #ifdef CONFIG_X86_64 | ||
| 400 | #include <linux/bootmem.h> | ||
| 401 | unsigned long __init xen_revector_p2m_tree(void) | ||
| 402 | { | ||
| 403 | unsigned long va_start; | ||
| 404 | unsigned long va_end; | ||
| 405 | unsigned long pfn; | ||
| 406 | unsigned long pfn_free = 0; | ||
| 407 | unsigned long *mfn_list = NULL; | ||
| 408 | unsigned long size; | ||
| 409 | |||
| 410 | va_start = xen_start_info->mfn_list; | ||
| 411 | /*We copy in increments of P2M_PER_PAGE * sizeof(unsigned long), | ||
| 412 | * so make sure it is rounded up to that */ | ||
| 413 | size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); | ||
| 414 | va_end = va_start + size; | ||
| 415 | |||
| 416 | /* If we were revectored already, don't do it again. */ | ||
| 417 | if (va_start <= __START_KERNEL_map && va_start >= __PAGE_OFFSET) | ||
| 418 | return 0; | ||
| 419 | |||
| 420 | mfn_list = alloc_bootmem_align(size, PAGE_SIZE); | ||
| 421 | if (!mfn_list) { | ||
| 422 | pr_warn("Could not allocate space for a new P2M tree!\n"); | ||
| 423 | return xen_start_info->mfn_list; | ||
| 424 | } | ||
| 425 | /* Fill it out with INVALID_P2M_ENTRY value */ | ||
| 426 | memset(mfn_list, 0xFF, size); | ||
| 427 | |||
| 428 | for (pfn = 0; pfn < ALIGN(MAX_DOMAIN_PAGES, P2M_PER_PAGE); pfn += P2M_PER_PAGE) { | ||
| 429 | unsigned topidx = p2m_top_index(pfn); | ||
| 430 | unsigned mididx; | ||
| 431 | unsigned long *mid_p; | ||
| 432 | |||
| 433 | if (!p2m_top[topidx]) | ||
| 434 | continue; | ||
| 435 | |||
| 436 | if (p2m_top[topidx] == p2m_mid_missing) | ||
| 437 | continue; | ||
| 438 | |||
| 439 | mididx = p2m_mid_index(pfn); | ||
| 440 | mid_p = p2m_top[topidx][mididx]; | ||
| 441 | if (!mid_p) | ||
| 442 | continue; | ||
| 443 | if ((mid_p == p2m_missing) || (mid_p == p2m_identity)) | ||
| 444 | continue; | ||
| 445 | |||
| 446 | if ((unsigned long)mid_p == INVALID_P2M_ENTRY) | ||
| 447 | continue; | ||
| 448 | |||
| 449 | /* The old va. Rebase it on mfn_list */ | ||
| 450 | if (mid_p >= (unsigned long *)va_start && mid_p <= (unsigned long *)va_end) { | ||
| 451 | unsigned long *new; | ||
| 452 | |||
| 453 | if (pfn_free > (size / sizeof(unsigned long))) { | ||
| 454 | WARN(1, "Only allocated for %ld pages, but we want %ld!\n", | ||
| 455 | size / sizeof(unsigned long), pfn_free); | ||
| 456 | return 0; | ||
| 457 | } | ||
| 458 | new = &mfn_list[pfn_free]; | ||
| 459 | |||
| 460 | copy_page(new, mid_p); | ||
| 461 | p2m_top[topidx][mididx] = &mfn_list[pfn_free]; | ||
| 462 | p2m_top_mfn_p[topidx][mididx] = virt_to_mfn(&mfn_list[pfn_free]); | ||
| 463 | |||
| 464 | pfn_free += P2M_PER_PAGE; | ||
| 399 | 465 | ||
| 466 | } | ||
| 467 | /* This should be the leafs allocated for identity from _brk. */ | ||
| 468 | } | ||
| 469 | return (unsigned long)mfn_list; | ||
| 470 | |||
| 471 | } | ||
| 472 | #else | ||
| 473 | unsigned long __init xen_revector_p2m_tree(void) | ||
| 474 | { | ||
| 475 | return 0; | ||
| 476 | } | ||
| 477 | #endif | ||
| 400 | unsigned long get_phys_to_machine(unsigned long pfn) | 478 | unsigned long get_phys_to_machine(unsigned long pfn) |
| 401 | { | 479 | { |
| 402 | unsigned topidx, mididx, idx; | 480 | unsigned topidx, mididx, idx; |
| @@ -430,7 +508,7 @@ static void free_p2m_page(void *p) | |||
| 430 | free_page((unsigned long)p); | 508 | free_page((unsigned long)p); |
| 431 | } | 509 | } |
| 432 | 510 | ||
| 433 | /* | 511 | /* |
| 434 | * Fully allocate the p2m structure for a given pfn. We need to check | 512 | * Fully allocate the p2m structure for a given pfn. We need to check |
| 435 | * that both the top and mid levels are allocated, and make sure the | 513 | * that both the top and mid levels are allocated, and make sure the |
| 436 | * parallel mfn tree is kept in sync. We may race with other cpus, so | 514 | * parallel mfn tree is kept in sync. We may race with other cpus, so |
diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c index 967633ad98c4..969570491c39 100644 --- a/arch/x86/xen/pci-swiotlb-xen.c +++ b/arch/x86/xen/pci-swiotlb-xen.c | |||
| @@ -8,6 +8,14 @@ | |||
| 8 | #include <xen/xen.h> | 8 | #include <xen/xen.h> |
| 9 | #include <asm/iommu_table.h> | 9 | #include <asm/iommu_table.h> |
| 10 | 10 | ||
| 11 | |||
| 12 | #include <asm/xen/swiotlb-xen.h> | ||
| 13 | #ifdef CONFIG_X86_64 | ||
| 14 | #include <asm/iommu.h> | ||
| 15 | #include <asm/dma.h> | ||
| 16 | #endif | ||
| 17 | #include <linux/export.h> | ||
| 18 | |||
| 11 | int xen_swiotlb __read_mostly; | 19 | int xen_swiotlb __read_mostly; |
| 12 | 20 | ||
| 13 | static struct dma_map_ops xen_swiotlb_dma_ops = { | 21 | static struct dma_map_ops xen_swiotlb_dma_ops = { |
| @@ -34,34 +42,64 @@ static struct dma_map_ops xen_swiotlb_dma_ops = { | |||
| 34 | int __init pci_xen_swiotlb_detect(void) | 42 | int __init pci_xen_swiotlb_detect(void) |
| 35 | { | 43 | { |
| 36 | 44 | ||
| 45 | if (!xen_pv_domain()) | ||
| 46 | return 0; | ||
| 47 | |||
| 37 | /* If running as PV guest, either iommu=soft, or swiotlb=force will | 48 | /* If running as PV guest, either iommu=soft, or swiotlb=force will |
| 38 | * activate this IOMMU. If running as PV privileged, activate it | 49 | * activate this IOMMU. If running as PV privileged, activate it |
| 39 | * irregardless. | 50 | * irregardless. |
| 40 | */ | 51 | */ |
| 41 | if ((xen_initial_domain() || swiotlb || swiotlb_force) && | 52 | if ((xen_initial_domain() || swiotlb || swiotlb_force)) |
| 42 | (xen_pv_domain())) | ||
| 43 | xen_swiotlb = 1; | 53 | xen_swiotlb = 1; |
| 44 | 54 | ||
| 45 | /* If we are running under Xen, we MUST disable the native SWIOTLB. | 55 | /* If we are running under Xen, we MUST disable the native SWIOTLB. |
| 46 | * Don't worry about swiotlb_force flag activating the native, as | 56 | * Don't worry about swiotlb_force flag activating the native, as |
| 47 | * the 'swiotlb' flag is the only one turning it on. */ | 57 | * the 'swiotlb' flag is the only one turning it on. */ |
| 48 | if (xen_pv_domain()) | 58 | swiotlb = 0; |
| 49 | swiotlb = 0; | ||
| 50 | 59 | ||
| 60 | #ifdef CONFIG_X86_64 | ||
| 61 | /* pci_swiotlb_detect_4gb turns on native SWIOTLB if no_iommu == 0 | ||
| 62 | * (so no iommu=X command line over-writes). | ||
| 63 | * Considering that PV guests do not want the *native SWIOTLB* but | ||
| 64 | * only Xen SWIOTLB it is not useful to us so set no_iommu=1 here. | ||
| 65 | */ | ||
| 66 | if (max_pfn > MAX_DMA32_PFN) | ||
| 67 | no_iommu = 1; | ||
| 68 | #endif | ||
| 51 | return xen_swiotlb; | 69 | return xen_swiotlb; |
| 52 | } | 70 | } |
| 53 | 71 | ||
| 54 | void __init pci_xen_swiotlb_init(void) | 72 | void __init pci_xen_swiotlb_init(void) |
| 55 | { | 73 | { |
| 56 | if (xen_swiotlb) { | 74 | if (xen_swiotlb) { |
| 57 | xen_swiotlb_init(1); | 75 | xen_swiotlb_init(1, true /* early */); |
| 58 | dma_ops = &xen_swiotlb_dma_ops; | 76 | dma_ops = &xen_swiotlb_dma_ops; |
| 59 | 77 | ||
| 60 | /* Make sure ACS will be enabled */ | 78 | /* Make sure ACS will be enabled */ |
| 61 | pci_request_acs(); | 79 | pci_request_acs(); |
| 62 | } | 80 | } |
| 63 | } | 81 | } |
| 82 | |||
| 83 | int pci_xen_swiotlb_init_late(void) | ||
| 84 | { | ||
| 85 | int rc; | ||
| 86 | |||
| 87 | if (xen_swiotlb) | ||
| 88 | return 0; | ||
| 89 | |||
| 90 | rc = xen_swiotlb_init(1, false /* late */); | ||
| 91 | if (rc) | ||
| 92 | return rc; | ||
| 93 | |||
| 94 | dma_ops = &xen_swiotlb_dma_ops; | ||
| 95 | /* Make sure ACS will be enabled */ | ||
| 96 | pci_request_acs(); | ||
| 97 | |||
| 98 | return 0; | ||
| 99 | } | ||
| 100 | EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late); | ||
| 101 | |||
| 64 | IOMMU_INIT_FINISH(pci_xen_swiotlb_detect, | 102 | IOMMU_INIT_FINISH(pci_xen_swiotlb_detect, |
| 65 | 0, | 103 | NULL, |
| 66 | pci_xen_swiotlb_init, | 104 | pci_xen_swiotlb_init, |
| 67 | 0); | 105 | NULL); |
diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c index ffcf2615640b..0a7852483ffe 100644 --- a/arch/x86/xen/platform-pci-unplug.c +++ b/arch/x86/xen/platform-pci-unplug.c | |||
| @@ -24,6 +24,7 @@ | |||
| 24 | #include <linux/module.h> | 24 | #include <linux/module.h> |
| 25 | 25 | ||
| 26 | #include <xen/platform_pci.h> | 26 | #include <xen/platform_pci.h> |
| 27 | #include "xen-ops.h" | ||
| 27 | 28 | ||
| 28 | #define XEN_PLATFORM_ERR_MAGIC -1 | 29 | #define XEN_PLATFORM_ERR_MAGIC -1 |
| 29 | #define XEN_PLATFORM_ERR_PROTOCOL -2 | 30 | #define XEN_PLATFORM_ERR_PROTOCOL -2 |
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index e2d62d697b5d..8971a26d21ab 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c | |||
| @@ -432,6 +432,24 @@ char * __init xen_memory_setup(void) | |||
| 432 | * - mfn_list | 432 | * - mfn_list |
| 433 | * - xen_start_info | 433 | * - xen_start_info |
| 434 | * See comment above "struct start_info" in <xen/interface/xen.h> | 434 | * See comment above "struct start_info" in <xen/interface/xen.h> |
| 435 | * We tried to make the the memblock_reserve more selective so | ||
| 436 | * that it would be clear what region is reserved. Sadly we ran | ||
| 437 | * in the problem wherein on a 64-bit hypervisor with a 32-bit | ||
| 438 | * initial domain, the pt_base has the cr3 value which is not | ||
| 439 | * neccessarily where the pagetable starts! As Jan put it: " | ||
| 440 | * Actually, the adjustment turns out to be correct: The page | ||
| 441 | * tables for a 32-on-64 dom0 get allocated in the order "first L1", | ||
| 442 | * "first L2", "first L3", so the offset to the page table base is | ||
| 443 | * indeed 2. When reading xen/include/public/xen.h's comment | ||
| 444 | * very strictly, this is not a violation (since there nothing is said | ||
| 445 | * that the first thing in the page table space is pointed to by | ||
| 446 | * pt_base; I admit that this seems to be implied though, namely | ||
| 447 | * do I think that it is implied that the page table space is the | ||
| 448 | * range [pt_base, pt_base + nt_pt_frames), whereas that | ||
| 449 | * range here indeed is [pt_base - 2, pt_base - 2 + nt_pt_frames), | ||
| 450 | * which - without a priori knowledge - the kernel would have | ||
| 451 | * difficulty to figure out)." - so lets just fall back to the | ||
| 452 | * easy way and reserve the whole region. | ||
| 435 | */ | 453 | */ |
| 436 | memblock_reserve(__pa(xen_start_info->mfn_list), | 454 | memblock_reserve(__pa(xen_start_info->mfn_list), |
| 437 | xen_start_info->pt_base - xen_start_info->mfn_list); | 455 | xen_start_info->pt_base - xen_start_info->mfn_list); |
diff --git a/arch/x86/xen/vga.c b/arch/x86/xen/vga.c index 1cd7f4d11e29..6722e3733f02 100644 --- a/arch/x86/xen/vga.c +++ b/arch/x86/xen/vga.c | |||
| @@ -35,6 +35,7 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size) | |||
| 35 | info->u.text_mode_3.font_height; | 35 | info->u.text_mode_3.font_height; |
| 36 | break; | 36 | break; |
| 37 | 37 | ||
| 38 | case XEN_VGATYPE_EFI_LFB: | ||
| 38 | case XEN_VGATYPE_VESA_LFB: | 39 | case XEN_VGATYPE_VESA_LFB: |
| 39 | if (size < offsetof(struct dom0_vga_console_info, | 40 | if (size < offsetof(struct dom0_vga_console_info, |
| 40 | u.vesa_lfb.gbl_caps)) | 41 | u.vesa_lfb.gbl_caps)) |
| @@ -54,6 +55,12 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size) | |||
| 54 | screen_info->blue_pos = info->u.vesa_lfb.blue_pos; | 55 | screen_info->blue_pos = info->u.vesa_lfb.blue_pos; |
| 55 | screen_info->rsvd_size = info->u.vesa_lfb.rsvd_size; | 56 | screen_info->rsvd_size = info->u.vesa_lfb.rsvd_size; |
| 56 | screen_info->rsvd_pos = info->u.vesa_lfb.rsvd_pos; | 57 | screen_info->rsvd_pos = info->u.vesa_lfb.rsvd_pos; |
| 58 | |||
| 59 | if (info->video_type == XEN_VGATYPE_EFI_LFB) { | ||
| 60 | screen_info->orig_video_isVGA = VIDEO_TYPE_EFI; | ||
| 61 | break; | ||
| 62 | } | ||
| 63 | |||
| 57 | if (size >= offsetof(struct dom0_vga_console_info, | 64 | if (size >= offsetof(struct dom0_vga_console_info, |
| 58 | u.vesa_lfb.gbl_caps) | 65 | u.vesa_lfb.gbl_caps) |
| 59 | + sizeof(info->u.vesa_lfb.gbl_caps)) | 66 | + sizeof(info->u.vesa_lfb.gbl_caps)) |
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index aaa7291c9259..7faed5869e5b 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S | |||
| @@ -28,9 +28,61 @@ ENTRY(startup_xen) | |||
| 28 | __FINIT | 28 | __FINIT |
| 29 | 29 | ||
| 30 | .pushsection .text | 30 | .pushsection .text |
| 31 | .align PAGE_SIZE | 31 | .balign PAGE_SIZE |
| 32 | ENTRY(hypercall_page) | 32 | ENTRY(hypercall_page) |
| 33 | .skip PAGE_SIZE | 33 | #define NEXT_HYPERCALL(x) \ |
| 34 | ENTRY(xen_hypercall_##x) \ | ||
| 35 | .skip 32 | ||
| 36 | |||
| 37 | NEXT_HYPERCALL(set_trap_table) | ||
| 38 | NEXT_HYPERCALL(mmu_update) | ||
| 39 | NEXT_HYPERCALL(set_gdt) | ||
| 40 | NEXT_HYPERCALL(stack_switch) | ||
| 41 | NEXT_HYPERCALL(set_callbacks) | ||
| 42 | NEXT_HYPERCALL(fpu_taskswitch) | ||
| 43 | NEXT_HYPERCALL(sched_op_compat) | ||
| 44 | NEXT_HYPERCALL(platform_op) | ||
| 45 | NEXT_HYPERCALL(set_debugreg) | ||
| 46 | NEXT_HYPERCALL(get_debugreg) | ||
| 47 | NEXT_HYPERCALL(update_descriptor) | ||
| 48 | NEXT_HYPERCALL(ni) | ||
| 49 | NEXT_HYPERCALL(memory_op) | ||
| 50 | NEXT_HYPERCALL(multicall) | ||
| 51 | NEXT_HYPERCALL(update_va_mapping) | ||
| 52 | NEXT_HYPERCALL(set_timer_op) | ||
| 53 | NEXT_HYPERCALL(event_channel_op_compat) | ||
| 54 | NEXT_HYPERCALL(xen_version) | ||
| 55 | NEXT_HYPERCALL(console_io) | ||
| 56 | NEXT_HYPERCALL(physdev_op_compat) | ||
| 57 | NEXT_HYPERCALL(grant_table_op) | ||
| 58 | NEXT_HYPERCALL(vm_assist) | ||
| 59 | NEXT_HYPERCALL(update_va_mapping_otherdomain) | ||
| 60 | NEXT_HYPERCALL(iret) | ||
| 61 | NEXT_HYPERCALL(vcpu_op) | ||
| 62 | NEXT_HYPERCALL(set_segment_base) | ||
| 63 | NEXT_HYPERCALL(mmuext_op) | ||
| 64 | NEXT_HYPERCALL(xsm_op) | ||
| 65 | NEXT_HYPERCALL(nmi_op) | ||
| 66 | NEXT_HYPERCALL(sched_op) | ||
| 67 | NEXT_HYPERCALL(callback_op) | ||
| 68 | NEXT_HYPERCALL(xenoprof_op) | ||
| 69 | NEXT_HYPERCALL(event_channel_op) | ||
| 70 | NEXT_HYPERCALL(physdev_op) | ||
| 71 | NEXT_HYPERCALL(hvm_op) | ||
| 72 | NEXT_HYPERCALL(sysctl) | ||
| 73 | NEXT_HYPERCALL(domctl) | ||
| 74 | NEXT_HYPERCALL(kexec_op) | ||
| 75 | NEXT_HYPERCALL(tmem_op) /* 38 */ | ||
| 76 | ENTRY(xen_hypercall_rsvr) | ||
| 77 | .skip 320 | ||
| 78 | NEXT_HYPERCALL(mca) /* 48 */ | ||
| 79 | NEXT_HYPERCALL(arch_1) | ||
| 80 | NEXT_HYPERCALL(arch_2) | ||
| 81 | NEXT_HYPERCALL(arch_3) | ||
| 82 | NEXT_HYPERCALL(arch_4) | ||
| 83 | NEXT_HYPERCALL(arch_5) | ||
| 84 | NEXT_HYPERCALL(arch_6) | ||
| 85 | .balign PAGE_SIZE | ||
| 34 | .popsection | 86 | .popsection |
| 35 | 87 | ||
| 36 | ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") | 88 | ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") |
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 202d4c150154..bb5a8105ea86 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h | |||
| @@ -27,7 +27,7 @@ void xen_setup_mfn_list_list(void); | |||
| 27 | void xen_setup_shared_info(void); | 27 | void xen_setup_shared_info(void); |
| 28 | void xen_build_mfn_list_list(void); | 28 | void xen_build_mfn_list_list(void); |
| 29 | void xen_setup_machphys_mapping(void); | 29 | void xen_setup_machphys_mapping(void); |
| 30 | pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn); | 30 | void xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn); |
| 31 | void xen_reserve_top(void); | 31 | void xen_reserve_top(void); |
| 32 | extern unsigned long xen_max_p2m_pfn; | 32 | extern unsigned long xen_max_p2m_pfn; |
| 33 | 33 | ||
| @@ -45,6 +45,7 @@ void xen_hvm_init_shared_info(void); | |||
| 45 | void xen_unplug_emulated_devices(void); | 45 | void xen_unplug_emulated_devices(void); |
| 46 | 46 | ||
| 47 | void __init xen_build_dynamic_phys_to_machine(void); | 47 | void __init xen_build_dynamic_phys_to_machine(void); |
| 48 | unsigned long __init xen_revector_p2m_tree(void); | ||
| 48 | 49 | ||
| 49 | void xen_init_irq_ops(void); | 50 | void xen_init_irq_ops(void); |
| 50 | void xen_setup_timer(int cpu); | 51 | void xen_setup_timer(int cpu); |
diff --git a/arch/xtensa/include/asm/elf.h b/arch/xtensa/include/asm/elf.h index 6e65eadaae14..5293312bc6a4 100644 --- a/arch/xtensa/include/asm/elf.h +++ b/arch/xtensa/include/asm/elf.h | |||
| @@ -189,7 +189,8 @@ typedef struct { | |||
| 189 | #endif | 189 | #endif |
| 190 | } elf_xtregs_t; | 190 | } elf_xtregs_t; |
| 191 | 191 | ||
| 192 | #define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT) | 192 | #define SET_PERSONALITY(ex) \ |
| 193 | set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK))) | ||
| 193 | 194 | ||
| 194 | struct task_struct; | 195 | struct task_struct; |
| 195 | 196 | ||
diff --git a/arch/xtensa/include/uapi/asm/Kbuild b/arch/xtensa/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..baebb3da1d44 --- /dev/null +++ b/arch/xtensa/include/uapi/asm/Kbuild | |||
| @@ -0,0 +1,3 @@ | |||
| 1 | # UAPI Header export list | ||
| 2 | include include/uapi/asm-generic/Kbuild.asm | ||
| 3 | |||