summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/devicetree/bindings/crypto/artpec6-crypto.txt16
-rw-r--r--Documentation/devicetree/bindings/crypto/atmel-crypto.txt13
-rw-r--r--Documentation/devicetree/bindings/crypto/st,stm32-hash.txt30
-rw-r--r--Documentation/devicetree/bindings/rng/imx-rngc.txt21
-rw-r--r--MAINTAINERS7
-rw-r--r--arch/arm/crypto/Kconfig5
-rw-r--r--arch/arm/crypto/aes-ce-glue.c4
-rw-r--r--arch/arm/crypto/aes-cipher-core.S88
-rw-r--r--arch/arm/crypto/aes-neonbs-glue.c5
-rw-r--r--arch/arm/crypto/ghash-ce-core.S234
-rw-r--r--arch/arm/crypto/ghash-ce-glue.c24
-rw-r--r--arch/arm64/crypto/Kconfig22
-rw-r--r--arch/arm64/crypto/aes-ce-ccm-core.S30
-rw-r--r--arch/arm64/crypto/aes-ce-ccm-glue.c174
-rw-r--r--arch/arm64/crypto/aes-ce-cipher.c55
-rw-r--r--arch/arm64/crypto/aes-ce.S12
-rw-r--r--arch/arm64/crypto/aes-cipher-core.S152
-rw-r--r--arch/arm64/crypto/aes-ctr-fallback.h53
-rw-r--r--arch/arm64/crypto/aes-glue.c63
-rw-r--r--arch/arm64/crypto/aes-neonbs-glue.c53
-rw-r--r--arch/arm64/crypto/chacha20-neon-glue.c5
-rw-r--r--arch/arm64/crypto/crc32-ce-glue.c11
-rw-r--r--arch/arm64/crypto/crct10dif-ce-glue.c13
-rw-r--r--arch/arm64/crypto/ghash-ce-core.S401
-rw-r--r--arch/arm64/crypto/ghash-ce-glue.c517
-rw-r--r--arch/arm64/crypto/sha1-ce-glue.c18
-rw-r--r--arch/arm64/crypto/sha2-ce-glue.c30
-rw-r--r--arch/arm64/crypto/sha256-glue.c1
-rw-r--r--arch/sparc/crypto/aes_glue.c3
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c4
-rw-r--r--arch/x86/crypto/blowfish_glue.c3
-rw-r--r--arch/x86/crypto/cast5_avx_glue.c3
-rw-r--r--arch/x86/crypto/des3_ede_glue.c3
-rw-r--r--crypto/Kconfig2
-rw-r--r--crypto/af_alg.c691
-rw-r--r--crypto/ahash.c29
-rw-r--r--crypto/algapi.c25
-rw-r--r--crypto/algif_aead.c864
-rw-r--r--crypto/algif_skcipher.c834
-rw-r--r--crypto/ctr.c3
-rw-r--r--crypto/ecdh.c51
-rw-r--r--crypto/pcbc.c12
-rw-r--r--crypto/rng.c6
-rw-r--r--crypto/scompress.c55
-rw-r--r--crypto/serpent_generic.c77
-rw-r--r--crypto/tcrypt.c8
-rw-r--r--drivers/char/hw_random/Kconfig20
-rw-r--r--drivers/char/hw_random/Makefile1
-rw-r--r--drivers/char/hw_random/core.c42
-rw-r--r--drivers/char/hw_random/imx-rngc.c331
-rw-r--r--drivers/crypto/Kconfig49
-rw-r--r--drivers/crypto/Makefile4
-rw-r--r--drivers/crypto/atmel-ecc.c781
-rw-r--r--drivers/crypto/atmel-ecc.h128
-rw-r--r--drivers/crypto/atmel-sha.c2
-rw-r--r--drivers/crypto/atmel-tdes.c2
-rw-r--r--drivers/crypto/axis/Makefile1
-rw-r--r--drivers/crypto/axis/artpec6_crypto.c3192
-rw-r--r--drivers/crypto/bcm/cipher.c114
-rw-r--r--drivers/crypto/bcm/cipher.h13
-rw-r--r--drivers/crypto/caam/caamalg.c66
-rw-r--r--drivers/crypto/caam/caamalg_desc.c5
-rw-r--r--drivers/crypto/caam/caamalg_qi.c55
-rw-r--r--drivers/crypto/caam/caamhash.c7
-rw-r--r--drivers/crypto/caam/caamrng.c6
-rw-r--r--drivers/crypto/caam/ctrl.c127
-rw-r--r--drivers/crypto/caam/ctrl.h2
-rw-r--r--drivers/crypto/caam/error.c40
-rw-r--r--drivers/crypto/caam/error.h4
-rw-r--r--drivers/crypto/caam/intern.h11
-rw-r--r--drivers/crypto/caam/jr.c7
-rw-r--r--drivers/crypto/caam/qi.c30
-rw-r--r--drivers/crypto/caam/qi.h3
-rw-r--r--drivers/crypto/caam/regs.h1
-rw-r--r--drivers/crypto/caam/sg_sw_qm2.h81
-rw-r--r--drivers/crypto/caam/sg_sw_sec4.h43
-rw-r--r--drivers/crypto/cavium/cpt/cptpf_main.c13
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_main.c4
-rw-r--r--drivers/crypto/ccp/Kconfig22
-rw-r--r--drivers/crypto/ccp/Makefile7
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-galois.c2
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-xts.c96
-rw-r--r--drivers/crypto/ccp/ccp-crypto-des3.c2
-rw-r--r--drivers/crypto/ccp/ccp-crypto-main.c21
-rw-r--r--drivers/crypto/ccp/ccp-crypto-rsa.c299
-rw-r--r--drivers/crypto/ccp/ccp-crypto-sha.c2
-rw-r--r--drivers/crypto/ccp/ccp-crypto.h36
-rw-r--r--drivers/crypto/ccp/ccp-debugfs.c15
-rw-r--r--drivers/crypto/ccp/ccp-dev-v3.c20
-rw-r--r--drivers/crypto/ccp/ccp-dev-v5.c28
-rw-r--r--drivers/crypto/ccp/ccp-dev.c134
-rw-r--r--drivers/crypto/ccp/ccp-dev.h30
-rw-r--r--drivers/crypto/ccp/ccp-dmaengine.c2
-rw-r--r--drivers/crypto/ccp/ccp-ops.c133
-rw-r--r--drivers/crypto/ccp/ccp-pci.c356
-rw-r--r--drivers/crypto/ccp/ccp-platform.c293
-rw-r--r--drivers/crypto/ccp/sp-dev.c277
-rw-r--r--drivers/crypto/ccp/sp-dev.h133
-rw-r--r--drivers/crypto/ccp/sp-pci.c276
-rw-r--r--drivers/crypto/ccp/sp-platform.c256
-rw-r--r--drivers/crypto/geode-aes.c17
-rw-r--r--drivers/crypto/img-hash.c2
-rw-r--r--drivers/crypto/inside-secure/safexcel.c5
-rw-r--r--drivers/crypto/mediatek/mtk-platform.c2
-rw-r--r--drivers/crypto/mxc-scc.c4
-rw-r--r--drivers/crypto/mxs-dcp.c8
-rw-r--r--drivers/crypto/n2_core.c60
-rw-r--r--drivers/crypto/omap-aes.c1
-rw-r--r--drivers/crypto/omap-des.c3
-rw-r--r--drivers/crypto/omap-sham.c2
-rw-r--r--drivers/crypto/qat/qat_common/adf_aer.c2
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto.c74
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto.h15
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c103
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto_ahash.c119
-rw-r--r--drivers/crypto/sahara.c14
-rw-r--r--drivers/crypto/stm32/Kconfig19
-rw-r--r--drivers/crypto/stm32/Makefile4
-rw-r--r--drivers/crypto/stm32/stm32-hash.c1575
-rw-r--r--drivers/crypto/stm32/stm32_crc32.c17
-rw-r--r--drivers/crypto/sunxi-ss/Makefile1
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss-core.c30
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss-prng.c56
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss.h11
-rw-r--r--drivers/crypto/virtio/virtio_crypto_algs.c109
-rw-r--r--drivers/crypto/virtio/virtio_crypto_common.h22
-rw-r--r--drivers/crypto/virtio/virtio_crypto_core.c37
-rw-r--r--drivers/crypto/vmx/aes_ctr.c3
-rw-r--r--drivers/md/dm-crypt.c11
-rw-r--r--include/crypto/algapi.h23
-rw-r--r--include/crypto/if_alg.h170
-rw-r--r--include/crypto/internal/akcipher.h6
-rw-r--r--include/crypto/internal/hash.h2
-rw-r--r--include/crypto/kpp.h10
-rw-r--r--include/linux/ccp.h11
-rw-r--r--lib/mpi/longlong.h24
136 files changed, 11758 insertions, 3169 deletions
diff --git a/Documentation/devicetree/bindings/crypto/artpec6-crypto.txt b/Documentation/devicetree/bindings/crypto/artpec6-crypto.txt
new file mode 100644
index 000000000000..d9cca4875bd6
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/artpec6-crypto.txt
@@ -0,0 +1,16 @@
1Axis crypto engine with PDMA interface.
2
3Required properties:
4- compatible : Should be one of the following strings:
5 "axis,artpec6-crypto" for the version in the Axis ARTPEC-6 SoC
6 "axis,artpec7-crypto" for the version in the Axis ARTPEC-7 SoC.
7- reg: Base address and size for the PDMA register area.
8- interrupts: Interrupt handle for the PDMA interrupt line.
9
10Example:
11
12crypto@f4264000 {
13 compatible = "axis,artpec6-crypto";
14 reg = <0xf4264000 0x1000>;
15 interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
16};
diff --git a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
index f2aab3dc2b52..7de1a9674c70 100644
--- a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
+++ b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
@@ -66,3 +66,16 @@ sha@f8034000 {
66 dmas = <&dma1 2 17>; 66 dmas = <&dma1 2 17>;
67 dma-names = "tx"; 67 dma-names = "tx";
68}; 68};
69
70* Eliptic Curve Cryptography (I2C)
71
72Required properties:
73- compatible : must be "atmel,atecc508a".
74- reg: I2C bus address of the device.
75- clock-frequency: must be present in the i2c controller node.
76
77Example:
78atecc508a@C0 {
79 compatible = "atmel,atecc508a";
80 reg = <0xC0>;
81};
diff --git a/Documentation/devicetree/bindings/crypto/st,stm32-hash.txt b/Documentation/devicetree/bindings/crypto/st,stm32-hash.txt
new file mode 100644
index 000000000000..04fc246f02f7
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/st,stm32-hash.txt
@@ -0,0 +1,30 @@
1* STMicroelectronics STM32 HASH
2
3Required properties:
4- compatible: Should contain entries for this and backward compatible
5 HASH versions:
6 - "st,stm32f456-hash" for stm32 F456.
7 - "st,stm32f756-hash" for stm32 F756.
8- reg: The address and length of the peripheral registers space
9- interrupts: the interrupt specifier for the HASH
10- clocks: The input clock of the HASH instance
11
12Optional properties:
13- resets: The input reset of the HASH instance
14- dmas: DMA specifiers for the HASH. See the DMA client binding,
15 Documentation/devicetree/bindings/dma/dma.txt
16- dma-names: DMA request name. Should be "in" if a dma is present.
17- dma-maxburst: Set number of maximum dma burst supported
18
19Example:
20
21hash1: hash@50060400 {
22 compatible = "st,stm32f756-hash";
23 reg = <0x50060400 0x400>;
24 interrupts = <80>;
25 clocks = <&rcc 0 STM32F7_AHB2_CLOCK(HASH)>;
26 resets = <&rcc STM32F7_AHB2_RESET(HASH)>;
27 dmas = <&dma2 7 2 0x400 0x0>;
28 dma-names = "in";
29 dma-maxburst = <0>;
30};
diff --git a/Documentation/devicetree/bindings/rng/imx-rngc.txt b/Documentation/devicetree/bindings/rng/imx-rngc.txt
new file mode 100644
index 000000000000..93c7174a7bed
--- /dev/null
+++ b/Documentation/devicetree/bindings/rng/imx-rngc.txt
@@ -0,0 +1,21 @@
1Freescale RNGC (Random Number Generator Version C)
2
3The driver also supports version B, which is mostly compatible
4to version C.
5
6Required properties:
7- compatible : should be one of
8 "fsl,imx25-rngb"
9 "fsl,imx35-rngc"
10- reg : offset and length of the register set of this block
11- interrupts : the interrupt number for the RNGC block
12- clocks : the RNGC clk source
13
14Example:
15
16rng@53fb0000 {
17 compatible = "fsl,imx25-rngb";
18 reg = <0x53fb0000 0x4000>;
19 interrupts = <22>;
20 clocks = <&trng_clk>;
21};
diff --git a/MAINTAINERS b/MAINTAINERS
index 961423bac24c..11dde284a426 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,7 @@ L: linux-arm-kernel@axis.com
1162F: arch/arm/mach-artpec 1162F: arch/arm/mach-artpec
1163F: arch/arm/boot/dts/artpec6* 1163F: arch/arm/boot/dts/artpec6*
1164F: drivers/clk/axis 1164F: drivers/clk/axis
1165F: drivers/crypto/axis
1165F: drivers/pinctrl/pinctrl-artpec* 1166F: drivers/pinctrl/pinctrl-artpec*
1166F: Documentation/devicetree/bindings/pinctrl/axis,artpec6-pinctrl.txt 1167F: Documentation/devicetree/bindings/pinctrl/axis,artpec6-pinctrl.txt
1167 1168
@@ -8770,6 +8771,12 @@ F: drivers/dma/at_hdmac.c
8770F: drivers/dma/at_hdmac_regs.h 8771F: drivers/dma/at_hdmac_regs.h
8771F: include/linux/platform_data/dma-atmel.h 8772F: include/linux/platform_data/dma-atmel.h
8772 8773
8774MICROCHIP / ATMEL ECC DRIVER
8775M: Tudor Ambarus <tudor.ambarus@microchip.com>
8776L: linux-crypto@vger.kernel.org
8777S: Maintained
8778F: drivers/crypto/atmel-ecc.*
8779
8773MICROCHIP / ATMEL ISC DRIVER 8780MICROCHIP / ATMEL ISC DRIVER
8774M: Songjun Wu <songjun.wu@microchip.com> 8781M: Songjun Wu <songjun.wu@microchip.com>
8775L: linux-media@vger.kernel.org 8782L: linux-media@vger.kernel.org
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index b9adedcc5b2e..ec72752d5668 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -94,14 +94,15 @@ config CRYPTO_AES_ARM_CE
94 ARMv8 Crypto Extensions 94 ARMv8 Crypto Extensions
95 95
96config CRYPTO_GHASH_ARM_CE 96config CRYPTO_GHASH_ARM_CE
97 tristate "PMULL-accelerated GHASH using ARMv8 Crypto Extensions" 97 tristate "PMULL-accelerated GHASH using NEON/ARMv8 Crypto Extensions"
98 depends on KERNEL_MODE_NEON 98 depends on KERNEL_MODE_NEON
99 select CRYPTO_HASH 99 select CRYPTO_HASH
100 select CRYPTO_CRYPTD 100 select CRYPTO_CRYPTD
101 help 101 help
102 Use an implementation of GHASH (used by the GCM AEAD chaining mode) 102 Use an implementation of GHASH (used by the GCM AEAD chaining mode)
103 that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) 103 that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64)
104 that is part of the ARMv8 Crypto Extensions 104 that is part of the ARMv8 Crypto Extensions, or a slower variant that
105 uses the vmull.p8 instruction that is part of the basic NEON ISA.
105 106
106config CRYPTO_CRCT10DIF_ARM_CE 107config CRYPTO_CRCT10DIF_ARM_CE
107 tristate "CRCT10DIF digest algorithm using PMULL instructions" 108 tristate "CRCT10DIF digest algorithm using PMULL instructions"
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 0f966a8ca1ce..d0a9cec73707 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -285,9 +285,7 @@ static int ctr_encrypt(struct skcipher_request *req)
285 285
286 ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, 286 ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc,
287 num_rounds(ctx), blocks, walk.iv); 287 num_rounds(ctx), blocks, walk.iv);
288 if (tdst != tsrc) 288 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
289 memcpy(tdst, tsrc, nbytes);
290 crypto_xor(tdst, tail, nbytes);
291 err = skcipher_walk_done(&walk, 0); 289 err = skcipher_walk_done(&walk, 0);
292 } 290 }
293 kernel_neon_end(); 291 kernel_neon_end();
diff --git a/arch/arm/crypto/aes-cipher-core.S b/arch/arm/crypto/aes-cipher-core.S
index c817a86c4ca8..54b384084637 100644
--- a/arch/arm/crypto/aes-cipher-core.S
+++ b/arch/arm/crypto/aes-cipher-core.S
@@ -10,6 +10,7 @@
10 */ 10 */
11 11
12#include <linux/linkage.h> 12#include <linux/linkage.h>
13#include <asm/cache.h>
13 14
14 .text 15 .text
15 .align 5 16 .align 5
@@ -32,19 +33,19 @@
32 .endif 33 .endif
33 .endm 34 .endm
34 35
35 .macro __load, out, in, idx 36 .macro __load, out, in, idx, sz, op
36 .if __LINUX_ARM_ARCH__ < 7 && \idx > 0 37 .if __LINUX_ARM_ARCH__ < 7 && \idx > 0
37 ldr \out, [ttab, \in, lsr #(8 * \idx) - 2] 38 ldr\op \out, [ttab, \in, lsr #(8 * \idx) - \sz]
38 .else 39 .else
39 ldr \out, [ttab, \in, lsl #2] 40 ldr\op \out, [ttab, \in, lsl #\sz]
40 .endif 41 .endif
41 .endm 42 .endm
42 43
43 .macro __hround, out0, out1, in0, in1, in2, in3, t3, t4, enc 44 .macro __hround, out0, out1, in0, in1, in2, in3, t3, t4, enc, sz, op
44 __select \out0, \in0, 0 45 __select \out0, \in0, 0
45 __select t0, \in1, 1 46 __select t0, \in1, 1
46 __load \out0, \out0, 0 47 __load \out0, \out0, 0, \sz, \op
47 __load t0, t0, 1 48 __load t0, t0, 1, \sz, \op
48 49
49 .if \enc 50 .if \enc
50 __select \out1, \in1, 0 51 __select \out1, \in1, 0
@@ -53,10 +54,10 @@
53 __select \out1, \in3, 0 54 __select \out1, \in3, 0
54 __select t1, \in0, 1 55 __select t1, \in0, 1
55 .endif 56 .endif
56 __load \out1, \out1, 0 57 __load \out1, \out1, 0, \sz, \op
57 __select t2, \in2, 2 58 __select t2, \in2, 2
58 __load t1, t1, 1 59 __load t1, t1, 1, \sz, \op
59 __load t2, t2, 2 60 __load t2, t2, 2, \sz, \op
60 61
61 eor \out0, \out0, t0, ror #24 62 eor \out0, \out0, t0, ror #24
62 63
@@ -68,9 +69,9 @@
68 __select \t3, \in1, 2 69 __select \t3, \in1, 2
69 __select \t4, \in2, 3 70 __select \t4, \in2, 3
70 .endif 71 .endif
71 __load \t3, \t3, 2 72 __load \t3, \t3, 2, \sz, \op
72 __load t0, t0, 3 73 __load t0, t0, 3, \sz, \op
73 __load \t4, \t4, 3 74 __load \t4, \t4, 3, \sz, \op
74 75
75 eor \out1, \out1, t1, ror #24 76 eor \out1, \out1, t1, ror #24
76 eor \out0, \out0, t2, ror #16 77 eor \out0, \out0, t2, ror #16
@@ -82,14 +83,14 @@
82 eor \out1, \out1, t2 83 eor \out1, \out1, t2
83 .endm 84 .endm
84 85
85 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3 86 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
86 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1 87 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
87 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1 88 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
88 .endm 89 .endm
89 90
90 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3 91 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
91 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0 92 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
92 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0 93 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
93 .endm 94 .endm
94 95
95 .macro __rev, out, in 96 .macro __rev, out, in
@@ -114,7 +115,7 @@
114 .endif 115 .endif
115 .endm 116 .endm
116 117
117 .macro do_crypt, round, ttab, ltab 118 .macro do_crypt, round, ttab, ltab, bsz
118 push {r3-r11, lr} 119 push {r3-r11, lr}
119 120
120 ldr r4, [in] 121 ldr r4, [in]
@@ -146,9 +147,12 @@
146 147
1471: subs rounds, rounds, #4 1481: subs rounds, rounds, #4
148 \round r8, r9, r10, r11, r4, r5, r6, r7 149 \round r8, r9, r10, r11, r4, r5, r6, r7
149 __adrl ttab, \ltab, ls 150 bls 2f
150 \round r4, r5, r6, r7, r8, r9, r10, r11 151 \round r4, r5, r6, r7, r8, r9, r10, r11
151 bhi 0b 152 b 0b
153
1542: __adrl ttab, \ltab
155 \round r4, r5, r6, r7, r8, r9, r10, r11, \bsz, b
152 156
153#ifdef CONFIG_CPU_BIG_ENDIAN 157#ifdef CONFIG_CPU_BIG_ENDIAN
154 __rev r4, r4 158 __rev r4, r4
@@ -170,10 +174,48 @@
170 .ltorg 174 .ltorg
171 .endm 175 .endm
172 176
177 .align L1_CACHE_SHIFT
178 .type __aes_arm_inverse_sbox, %object
179__aes_arm_inverse_sbox:
180 .byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
181 .byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
182 .byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
183 .byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
184 .byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
185 .byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
186 .byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
187 .byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
188 .byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
189 .byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
190 .byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
191 .byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
192 .byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
193 .byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
194 .byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
195 .byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
196 .byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
197 .byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
198 .byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
199 .byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
200 .byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
201 .byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
202 .byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
203 .byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
204 .byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
205 .byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
206 .byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
207 .byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
208 .byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
209 .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
210 .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
211 .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
212 .size __aes_arm_inverse_sbox, . - __aes_arm_inverse_sbox
213
173ENTRY(__aes_arm_encrypt) 214ENTRY(__aes_arm_encrypt)
174 do_crypt fround, crypto_ft_tab, crypto_fl_tab 215 do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2
175ENDPROC(__aes_arm_encrypt) 216ENDPROC(__aes_arm_encrypt)
176 217
218 .align 5
177ENTRY(__aes_arm_decrypt) 219ENTRY(__aes_arm_decrypt)
178 do_crypt iround, crypto_it_tab, crypto_il_tab 220 do_crypt iround, crypto_it_tab, __aes_arm_inverse_sbox, 0
179ENDPROC(__aes_arm_decrypt) 221ENDPROC(__aes_arm_decrypt)
diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
index c76377961444..18768f330449 100644
--- a/arch/arm/crypto/aes-neonbs-glue.c
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -221,9 +221,8 @@ static int ctr_encrypt(struct skcipher_request *req)
221 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 221 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
222 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 222 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
223 223
224 if (dst != src) 224 crypto_xor_cpy(dst, src, final,
225 memcpy(dst, src, walk.total % AES_BLOCK_SIZE); 225 walk.total % AES_BLOCK_SIZE);
226 crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
227 226
228 err = skcipher_walk_done(&walk, 0); 227 err = skcipher_walk_done(&walk, 0);
229 break; 228 break;
diff --git a/arch/arm/crypto/ghash-ce-core.S b/arch/arm/crypto/ghash-ce-core.S
index f6ab8bcc9efe..2f78c10b1881 100644
--- a/arch/arm/crypto/ghash-ce-core.S
+++ b/arch/arm/crypto/ghash-ce-core.S
@@ -1,7 +1,7 @@
1/* 1/*
2 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. 2 * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions.
3 * 3 *
4 * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2015 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published 7 * under the terms of the GNU General Public License version 2 as published
@@ -12,40 +12,162 @@
12#include <asm/assembler.h> 12#include <asm/assembler.h>
13 13
14 SHASH .req q0 14 SHASH .req q0
15 SHASH2 .req q1 15 T1 .req q1
16 T1 .req q2 16 XL .req q2
17 T2 .req q3 17 XM .req q3
18 MASK .req q4 18 XH .req q4
19 XL .req q5 19 IN1 .req q4
20 XM .req q6
21 XH .req q7
22 IN1 .req q7
23 20
24 SHASH_L .req d0 21 SHASH_L .req d0
25 SHASH_H .req d1 22 SHASH_H .req d1
26 SHASH2_L .req d2 23 T1_L .req d2
27 T1_L .req d4 24 T1_H .req d3
28 MASK_L .req d8 25 XL_L .req d4
29 XL_L .req d10 26 XL_H .req d5
30 XL_H .req d11 27 XM_L .req d6
31 XM_L .req d12 28 XM_H .req d7
32 XM_H .req d13 29 XH_L .req d8
33 XH_L .req d14 30
31 t0l .req d10
32 t0h .req d11
33 t1l .req d12
34 t1h .req d13
35 t2l .req d14
36 t2h .req d15
37 t3l .req d16
38 t3h .req d17
39 t4l .req d18
40 t4h .req d19
41
42 t0q .req q5
43 t1q .req q6
44 t2q .req q7
45 t3q .req q8
46 t4q .req q9
47 T2 .req q9
48
49 s1l .req d20
50 s1h .req d21
51 s2l .req d22
52 s2h .req d23
53 s3l .req d24
54 s3h .req d25
55 s4l .req d26
56 s4h .req d27
57
58 MASK .req d28
59 SHASH2_p8 .req d28
60
61 k16 .req d29
62 k32 .req d30
63 k48 .req d31
64 SHASH2_p64 .req d31
34 65
35 .text 66 .text
36 .fpu crypto-neon-fp-armv8 67 .fpu crypto-neon-fp-armv8
37 68
69 .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4
70 vmull.p64 \rd, \rn, \rm
71 .endm
72
38 /* 73 /*
39 * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 74 * This implementation of 64x64 -> 128 bit polynomial multiplication
40 * struct ghash_key const *k, const char *head) 75 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper
76 * "Fast Software Polynomial Multiplication on ARM Processors Using
77 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and
78 * Ricardo Dahab (https://hal.inria.fr/hal-01506572)
79 *
80 * It has been slightly tweaked for in-order performance, and to allow
81 * 'rq' to overlap with 'ad' or 'bd'.
41 */ 82 */
42ENTRY(pmull_ghash_update) 83 .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l
43 vld1.64 {SHASH}, [r3] 84 vext.8 t0l, \ad, \ad, #1 @ A1
85 .ifc \b1, t4l
86 vext.8 t4l, \bd, \bd, #1 @ B1
87 .endif
88 vmull.p8 t0q, t0l, \bd @ F = A1*B
89 vext.8 t1l, \ad, \ad, #2 @ A2
90 vmull.p8 t4q, \ad, \b1 @ E = A*B1
91 .ifc \b2, t3l
92 vext.8 t3l, \bd, \bd, #2 @ B2
93 .endif
94 vmull.p8 t1q, t1l, \bd @ H = A2*B
95 vext.8 t2l, \ad, \ad, #3 @ A3
96 vmull.p8 t3q, \ad, \b2 @ G = A*B2
97 veor t0q, t0q, t4q @ L = E + F
98 .ifc \b3, t4l
99 vext.8 t4l, \bd, \bd, #3 @ B3
100 .endif
101 vmull.p8 t2q, t2l, \bd @ J = A3*B
102 veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8
103 veor t1q, t1q, t3q @ M = G + H
104 .ifc \b4, t3l
105 vext.8 t3l, \bd, \bd, #4 @ B4
106 .endif
107 vmull.p8 t4q, \ad, \b3 @ I = A*B3
108 veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16
109 vmull.p8 t3q, \ad, \b4 @ K = A*B4
110 vand t0h, t0h, k48
111 vand t1h, t1h, k32
112 veor t2q, t2q, t4q @ N = I + J
113 veor t0l, t0l, t0h
114 veor t1l, t1l, t1h
115 veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24
116 vand t2h, t2h, k16
117 veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32
118 vmov.i64 t3h, #0
119 vext.8 t0q, t0q, t0q, #15
120 veor t2l, t2l, t2h
121 vext.8 t1q, t1q, t1q, #14
122 vmull.p8 \rq, \ad, \bd @ D = A*B
123 vext.8 t2q, t2q, t2q, #13
124 vext.8 t3q, t3q, t3q, #12
125 veor t0q, t0q, t1q
126 veor t2q, t2q, t3q
127 veor \rq, \rq, t0q
128 veor \rq, \rq, t2q
129 .endm
130
131 //
132 // PMULL (64x64->128) based reduction for CPUs that can do
133 // it in a single instruction.
134 //
135 .macro __pmull_reduce_p64
136 vmull.p64 T1, XL_L, MASK
137
138 veor XH_L, XH_L, XM_H
139 vext.8 T1, T1, T1, #8
140 veor XL_H, XL_H, XM_L
141 veor T1, T1, XL
142
143 vmull.p64 XL, T1_H, MASK
144 .endm
145
146 //
147 // Alternative reduction for CPUs that lack support for the
148 // 64x64->128 PMULL instruction
149 //
150 .macro __pmull_reduce_p8
151 veor XL_H, XL_H, XM_L
152 veor XH_L, XH_L, XM_H
153
154 vshl.i64 T1, XL, #57
155 vshl.i64 T2, XL, #62
156 veor T1, T1, T2
157 vshl.i64 T2, XL, #63
158 veor T1, T1, T2
159 veor XL_H, XL_H, T1_L
160 veor XH_L, XH_L, T1_H
161
162 vshr.u64 T1, XL, #1
163 veor XH, XH, XL
164 veor XL, XL, T1
165 vshr.u64 T1, T1, #6
166 vshr.u64 XL, XL, #1
167 .endm
168
169 .macro ghash_update, pn
44 vld1.64 {XL}, [r1] 170 vld1.64 {XL}, [r1]
45 vmov.i8 MASK, #0xe1
46 vext.8 SHASH2, SHASH, SHASH, #8
47 vshl.u64 MASK, MASK, #57
48 veor SHASH2, SHASH2, SHASH
49 171
50 /* do the head block first, if supplied */ 172 /* do the head block first, if supplied */
51 ldr ip, [sp] 173 ldr ip, [sp]
@@ -62,33 +184,59 @@ ENTRY(pmull_ghash_update)
62#ifndef CONFIG_CPU_BIG_ENDIAN 184#ifndef CONFIG_CPU_BIG_ENDIAN
63 vrev64.8 T1, T1 185 vrev64.8 T1, T1
64#endif 186#endif
65 vext.8 T2, XL, XL, #8
66 vext.8 IN1, T1, T1, #8 187 vext.8 IN1, T1, T1, #8
67 veor T1, T1, T2 188 veor T1_L, T1_L, XL_H
68 veor XL, XL, IN1 189 veor XL, XL, IN1
69 190
70 vmull.p64 XH, SHASH_H, XL_H @ a1 * b1 191 __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1
71 veor T1, T1, XL 192 veor T1, T1, XL
72 vmull.p64 XL, SHASH_L, XL_L @ a0 * b0 193 __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0
73 vmull.p64 XM, SHASH2_L, T1_L @ (a1 + a0)(b1 + b0) 194 __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0)
74 195
75 vext.8 T1, XL, XH, #8 196 veor T1, XL, XH
76 veor T2, XL, XH
77 veor XM, XM, T1 197 veor XM, XM, T1
78 veor XM, XM, T2
79 vmull.p64 T2, XL_L, MASK_L
80 198
81 vmov XH_L, XM_H 199 __pmull_reduce_\pn
82 vmov XM_H, XL_L
83 200
84 veor XL, XM, T2 201 veor T1, T1, XH
85 vext.8 T2, XL, XL, #8 202 veor XL, XL, T1
86 vmull.p64 XL, XL_L, MASK_L
87 veor T2, T2, XH
88 veor XL, XL, T2
89 203
90 bne 0b 204 bne 0b
91 205
92 vst1.64 {XL}, [r1] 206 vst1.64 {XL}, [r1]
93 bx lr 207 bx lr
94ENDPROC(pmull_ghash_update) 208 .endm
209
210 /*
211 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
212 * struct ghash_key const *k, const char *head)
213 */
214ENTRY(pmull_ghash_update_p64)
215 vld1.64 {SHASH}, [r3]
216 veor SHASH2_p64, SHASH_L, SHASH_H
217
218 vmov.i8 MASK, #0xe1
219 vshl.u64 MASK, MASK, #57
220
221 ghash_update p64
222ENDPROC(pmull_ghash_update_p64)
223
224ENTRY(pmull_ghash_update_p8)
225 vld1.64 {SHASH}, [r3]
226 veor SHASH2_p8, SHASH_L, SHASH_H
227
228 vext.8 s1l, SHASH_L, SHASH_L, #1
229 vext.8 s2l, SHASH_L, SHASH_L, #2
230 vext.8 s3l, SHASH_L, SHASH_L, #3
231 vext.8 s4l, SHASH_L, SHASH_L, #4
232 vext.8 s1h, SHASH_H, SHASH_H, #1
233 vext.8 s2h, SHASH_H, SHASH_H, #2
234 vext.8 s3h, SHASH_H, SHASH_H, #3
235 vext.8 s4h, SHASH_H, SHASH_H, #4
236
237 vmov.i64 k16, #0xffff
238 vmov.i64 k32, #0xffffffff
239 vmov.i64 k48, #0xffffffffffff
240
241 ghash_update p8
242ENDPROC(pmull_ghash_update_p8)
diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index 6bac8bea9f1e..d9bb52cae2ac 100644
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -22,6 +22,7 @@
22MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions"); 22MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
23MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 23MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
24MODULE_LICENSE("GPL v2"); 24MODULE_LICENSE("GPL v2");
25MODULE_ALIAS_CRYPTO("ghash");
25 26
26#define GHASH_BLOCK_SIZE 16 27#define GHASH_BLOCK_SIZE 16
27#define GHASH_DIGEST_SIZE 16 28#define GHASH_DIGEST_SIZE 16
@@ -41,8 +42,17 @@ struct ghash_async_ctx {
41 struct cryptd_ahash *cryptd_tfm; 42 struct cryptd_ahash *cryptd_tfm;
42}; 43};
43 44
44asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src, 45asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
45 struct ghash_key const *k, const char *head); 46 struct ghash_key const *k,
47 const char *head);
48
49asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
50 struct ghash_key const *k,
51 const char *head);
52
53static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src,
54 struct ghash_key const *k,
55 const char *head);
46 56
47static int ghash_init(struct shash_desc *desc) 57static int ghash_init(struct shash_desc *desc)
48{ 58{
@@ -312,6 +322,14 @@ static int __init ghash_ce_mod_init(void)
312{ 322{
313 int err; 323 int err;
314 324
325 if (!(elf_hwcap & HWCAP_NEON))
326 return -ENODEV;
327
328 if (elf_hwcap2 & HWCAP2_PMULL)
329 pmull_ghash_update = pmull_ghash_update_p64;
330 else
331 pmull_ghash_update = pmull_ghash_update_p8;
332
315 err = crypto_register_shash(&ghash_alg); 333 err = crypto_register_shash(&ghash_alg);
316 if (err) 334 if (err)
317 return err; 335 return err;
@@ -332,5 +350,5 @@ static void __exit ghash_ce_mod_exit(void)
332 crypto_unregister_shash(&ghash_alg); 350 crypto_unregister_shash(&ghash_alg);
333} 351}
334 352
335module_cpu_feature_match(PMULL, ghash_ce_mod_init); 353module_init(ghash_ce_mod_init);
336module_exit(ghash_ce_mod_exit); 354module_exit(ghash_ce_mod_exit);
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index d92293747d63..7ca54a76f6b9 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -18,18 +18,23 @@ config CRYPTO_SHA512_ARM64
18 18
19config CRYPTO_SHA1_ARM64_CE 19config CRYPTO_SHA1_ARM64_CE
20 tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)" 20 tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)"
21 depends on ARM64 && KERNEL_MODE_NEON 21 depends on KERNEL_MODE_NEON
22 select CRYPTO_HASH 22 select CRYPTO_HASH
23 select CRYPTO_SHA1
23 24
24config CRYPTO_SHA2_ARM64_CE 25config CRYPTO_SHA2_ARM64_CE
25 tristate "SHA-224/SHA-256 digest algorithm (ARMv8 Crypto Extensions)" 26 tristate "SHA-224/SHA-256 digest algorithm (ARMv8 Crypto Extensions)"
26 depends on ARM64 && KERNEL_MODE_NEON 27 depends on KERNEL_MODE_NEON
27 select CRYPTO_HASH 28 select CRYPTO_HASH
29 select CRYPTO_SHA256_ARM64
28 30
29config CRYPTO_GHASH_ARM64_CE 31config CRYPTO_GHASH_ARM64_CE
30 tristate "GHASH (for GCM chaining mode) using ARMv8 Crypto Extensions" 32 tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions"
31 depends on ARM64 && KERNEL_MODE_NEON 33 depends on KERNEL_MODE_NEON
32 select CRYPTO_HASH 34 select CRYPTO_HASH
35 select CRYPTO_GF128MUL
36 select CRYPTO_AES
37 select CRYPTO_AES_ARM64
33 38
34config CRYPTO_CRCT10DIF_ARM64_CE 39config CRYPTO_CRCT10DIF_ARM64_CE
35 tristate "CRCT10DIF digest algorithm using PMULL instructions" 40 tristate "CRCT10DIF digest algorithm using PMULL instructions"
@@ -49,25 +54,29 @@ config CRYPTO_AES_ARM64_CE
49 tristate "AES core cipher using ARMv8 Crypto Extensions" 54 tristate "AES core cipher using ARMv8 Crypto Extensions"
50 depends on ARM64 && KERNEL_MODE_NEON 55 depends on ARM64 && KERNEL_MODE_NEON
51 select CRYPTO_ALGAPI 56 select CRYPTO_ALGAPI
57 select CRYPTO_AES_ARM64
52 58
53config CRYPTO_AES_ARM64_CE_CCM 59config CRYPTO_AES_ARM64_CE_CCM
54 tristate "AES in CCM mode using ARMv8 Crypto Extensions" 60 tristate "AES in CCM mode using ARMv8 Crypto Extensions"
55 depends on ARM64 && KERNEL_MODE_NEON 61 depends on ARM64 && KERNEL_MODE_NEON
56 select CRYPTO_ALGAPI 62 select CRYPTO_ALGAPI
57 select CRYPTO_AES_ARM64_CE 63 select CRYPTO_AES_ARM64_CE
64 select CRYPTO_AES_ARM64
58 select CRYPTO_AEAD 65 select CRYPTO_AEAD
59 66
60config CRYPTO_AES_ARM64_CE_BLK 67config CRYPTO_AES_ARM64_CE_BLK
61 tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions" 68 tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions"
62 depends on ARM64 && KERNEL_MODE_NEON 69 depends on KERNEL_MODE_NEON
63 select CRYPTO_BLKCIPHER 70 select CRYPTO_BLKCIPHER
64 select CRYPTO_AES_ARM64_CE 71 select CRYPTO_AES_ARM64_CE
72 select CRYPTO_AES_ARM64
65 select CRYPTO_SIMD 73 select CRYPTO_SIMD
66 74
67config CRYPTO_AES_ARM64_NEON_BLK 75config CRYPTO_AES_ARM64_NEON_BLK
68 tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions" 76 tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions"
69 depends on ARM64 && KERNEL_MODE_NEON 77 depends on KERNEL_MODE_NEON
70 select CRYPTO_BLKCIPHER 78 select CRYPTO_BLKCIPHER
79 select CRYPTO_AES_ARM64
71 select CRYPTO_AES 80 select CRYPTO_AES
72 select CRYPTO_SIMD 81 select CRYPTO_SIMD
73 82
@@ -82,6 +91,7 @@ config CRYPTO_AES_ARM64_BS
82 depends on KERNEL_MODE_NEON 91 depends on KERNEL_MODE_NEON
83 select CRYPTO_BLKCIPHER 92 select CRYPTO_BLKCIPHER
84 select CRYPTO_AES_ARM64_NEON_BLK 93 select CRYPTO_AES_ARM64_NEON_BLK
94 select CRYPTO_AES_ARM64
85 select CRYPTO_SIMD 95 select CRYPTO_SIMD
86 96
87endif 97endif
diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S
index 3363560c79b7..e3a375c4cb83 100644
--- a/arch/arm64/crypto/aes-ce-ccm-core.S
+++ b/arch/arm64/crypto/aes-ce-ccm-core.S
@@ -1,7 +1,7 @@
1/* 1/*
2 * aesce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions 2 * aesce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions
3 * 3 *
4 * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -32,7 +32,7 @@ ENTRY(ce_aes_ccm_auth_data)
32 beq 8f /* out of input? */ 32 beq 8f /* out of input? */
33 cbnz w8, 0b 33 cbnz w8, 0b
34 eor v0.16b, v0.16b, v1.16b 34 eor v0.16b, v0.16b, v1.16b
351: ld1 {v3.16b}, [x4] /* load first round key */ 351: ld1 {v3.4s}, [x4] /* load first round key */
36 prfm pldl1strm, [x1] 36 prfm pldl1strm, [x1]
37 cmp w5, #12 /* which key size? */ 37 cmp w5, #12 /* which key size? */
38 add x6, x4, #16 38 add x6, x4, #16
@@ -42,17 +42,17 @@ ENTRY(ce_aes_ccm_auth_data)
42 mov v5.16b, v3.16b 42 mov v5.16b, v3.16b
43 b 4f 43 b 4f
442: mov v4.16b, v3.16b 442: mov v4.16b, v3.16b
45 ld1 {v5.16b}, [x6], #16 /* load 2nd round key */ 45 ld1 {v5.4s}, [x6], #16 /* load 2nd round key */
463: aese v0.16b, v4.16b 463: aese v0.16b, v4.16b
47 aesmc v0.16b, v0.16b 47 aesmc v0.16b, v0.16b
484: ld1 {v3.16b}, [x6], #16 /* load next round key */ 484: ld1 {v3.4s}, [x6], #16 /* load next round key */
49 aese v0.16b, v5.16b 49 aese v0.16b, v5.16b
50 aesmc v0.16b, v0.16b 50 aesmc v0.16b, v0.16b
515: ld1 {v4.16b}, [x6], #16 /* load next round key */ 515: ld1 {v4.4s}, [x6], #16 /* load next round key */
52 subs w7, w7, #3 52 subs w7, w7, #3
53 aese v0.16b, v3.16b 53 aese v0.16b, v3.16b
54 aesmc v0.16b, v0.16b 54 aesmc v0.16b, v0.16b
55 ld1 {v5.16b}, [x6], #16 /* load next round key */ 55 ld1 {v5.4s}, [x6], #16 /* load next round key */
56 bpl 3b 56 bpl 3b
57 aese v0.16b, v4.16b 57 aese v0.16b, v4.16b
58 subs w2, w2, #16 /* last data? */ 58 subs w2, w2, #16 /* last data? */
@@ -90,7 +90,7 @@ ENDPROC(ce_aes_ccm_auth_data)
90 * u32 rounds); 90 * u32 rounds);
91 */ 91 */
92ENTRY(ce_aes_ccm_final) 92ENTRY(ce_aes_ccm_final)
93 ld1 {v3.16b}, [x2], #16 /* load first round key */ 93 ld1 {v3.4s}, [x2], #16 /* load first round key */
94 ld1 {v0.16b}, [x0] /* load mac */ 94 ld1 {v0.16b}, [x0] /* load mac */
95 cmp w3, #12 /* which key size? */ 95 cmp w3, #12 /* which key size? */
96 sub w3, w3, #2 /* modified # of rounds */ 96 sub w3, w3, #2 /* modified # of rounds */
@@ -100,17 +100,17 @@ ENTRY(ce_aes_ccm_final)
100 mov v5.16b, v3.16b 100 mov v5.16b, v3.16b
101 b 2f 101 b 2f
1020: mov v4.16b, v3.16b 1020: mov v4.16b, v3.16b
1031: ld1 {v5.16b}, [x2], #16 /* load next round key */ 1031: ld1 {v5.4s}, [x2], #16 /* load next round key */
104 aese v0.16b, v4.16b 104 aese v0.16b, v4.16b
105 aesmc v0.16b, v0.16b 105 aesmc v0.16b, v0.16b
106 aese v1.16b, v4.16b 106 aese v1.16b, v4.16b
107 aesmc v1.16b, v1.16b 107 aesmc v1.16b, v1.16b
1082: ld1 {v3.16b}, [x2], #16 /* load next round key */ 1082: ld1 {v3.4s}, [x2], #16 /* load next round key */
109 aese v0.16b, v5.16b 109 aese v0.16b, v5.16b
110 aesmc v0.16b, v0.16b 110 aesmc v0.16b, v0.16b
111 aese v1.16b, v5.16b 111 aese v1.16b, v5.16b
112 aesmc v1.16b, v1.16b 112 aesmc v1.16b, v1.16b
1133: ld1 {v4.16b}, [x2], #16 /* load next round key */ 1133: ld1 {v4.4s}, [x2], #16 /* load next round key */
114 subs w3, w3, #3 114 subs w3, w3, #3
115 aese v0.16b, v3.16b 115 aese v0.16b, v3.16b
116 aesmc v0.16b, v0.16b 116 aesmc v0.16b, v0.16b
@@ -137,31 +137,31 @@ CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */
137 cmp w4, #12 /* which key size? */ 137 cmp w4, #12 /* which key size? */
138 sub w7, w4, #2 /* get modified # of rounds */ 138 sub w7, w4, #2 /* get modified # of rounds */
139 ins v1.d[1], x9 /* no carry in lower ctr */ 139 ins v1.d[1], x9 /* no carry in lower ctr */
140 ld1 {v3.16b}, [x3] /* load first round key */ 140 ld1 {v3.4s}, [x3] /* load first round key */
141 add x10, x3, #16 141 add x10, x3, #16
142 bmi 1f 142 bmi 1f
143 bne 4f 143 bne 4f
144 mov v5.16b, v3.16b 144 mov v5.16b, v3.16b
145 b 3f 145 b 3f
1461: mov v4.16b, v3.16b 1461: mov v4.16b, v3.16b
147 ld1 {v5.16b}, [x10], #16 /* load 2nd round key */ 147 ld1 {v5.4s}, [x10], #16 /* load 2nd round key */
1482: /* inner loop: 3 rounds, 2x interleaved */ 1482: /* inner loop: 3 rounds, 2x interleaved */
149 aese v0.16b, v4.16b 149 aese v0.16b, v4.16b
150 aesmc v0.16b, v0.16b 150 aesmc v0.16b, v0.16b
151 aese v1.16b, v4.16b 151 aese v1.16b, v4.16b
152 aesmc v1.16b, v1.16b 152 aesmc v1.16b, v1.16b
1533: ld1 {v3.16b}, [x10], #16 /* load next round key */ 1533: ld1 {v3.4s}, [x10], #16 /* load next round key */
154 aese v0.16b, v5.16b 154 aese v0.16b, v5.16b
155 aesmc v0.16b, v0.16b 155 aesmc v0.16b, v0.16b
156 aese v1.16b, v5.16b 156 aese v1.16b, v5.16b
157 aesmc v1.16b, v1.16b 157 aesmc v1.16b, v1.16b
1584: ld1 {v4.16b}, [x10], #16 /* load next round key */ 1584: ld1 {v4.4s}, [x10], #16 /* load next round key */
159 subs w7, w7, #3 159 subs w7, w7, #3
160 aese v0.16b, v3.16b 160 aese v0.16b, v3.16b
161 aesmc v0.16b, v0.16b 161 aesmc v0.16b, v0.16b
162 aese v1.16b, v3.16b 162 aese v1.16b, v3.16b
163 aesmc v1.16b, v1.16b 163 aesmc v1.16b, v1.16b
164 ld1 {v5.16b}, [x10], #16 /* load next round key */ 164 ld1 {v5.4s}, [x10], #16 /* load next round key */
165 bpl 2b 165 bpl 2b
166 aese v0.16b, v4.16b 166 aese v0.16b, v4.16b
167 aese v1.16b, v4.16b 167 aese v1.16b, v4.16b
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
index 6a7dbc7c83a6..a1254036f2b1 100644
--- a/arch/arm64/crypto/aes-ce-ccm-glue.c
+++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * aes-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions 2 * aes-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions
3 * 3 *
4 * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -9,6 +9,7 @@
9 */ 9 */
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/simd.h>
12#include <asm/unaligned.h> 13#include <asm/unaligned.h>
13#include <crypto/aes.h> 14#include <crypto/aes.h>
14#include <crypto/scatterwalk.h> 15#include <crypto/scatterwalk.h>
@@ -44,6 +45,8 @@ asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
44asmlinkage void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u32 const rk[], 45asmlinkage void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u32 const rk[],
45 u32 rounds); 46 u32 rounds);
46 47
48asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
49
47static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key, 50static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
48 unsigned int key_len) 51 unsigned int key_len)
49{ 52{
@@ -103,7 +106,45 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
103 return 0; 106 return 0;
104} 107}
105 108
106static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) 109static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[],
110 u32 abytes, u32 *macp, bool use_neon)
111{
112 if (likely(use_neon)) {
113 ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc,
114 num_rounds(key));
115 } else {
116 if (*macp > 0 && *macp < AES_BLOCK_SIZE) {
117 int added = min(abytes, AES_BLOCK_SIZE - *macp);
118
119 crypto_xor(&mac[*macp], in, added);
120
121 *macp += added;
122 in += added;
123 abytes -= added;
124 }
125
126 while (abytes > AES_BLOCK_SIZE) {
127 __aes_arm64_encrypt(key->key_enc, mac, mac,
128 num_rounds(key));
129 crypto_xor(mac, in, AES_BLOCK_SIZE);
130
131 in += AES_BLOCK_SIZE;
132 abytes -= AES_BLOCK_SIZE;
133 }
134
135 if (abytes > 0) {
136 __aes_arm64_encrypt(key->key_enc, mac, mac,
137 num_rounds(key));
138 crypto_xor(mac, in, abytes);
139 *macp = abytes;
140 } else {
141 *macp = 0;
142 }
143 }
144}
145
146static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[],
147 bool use_neon)
107{ 148{
108 struct crypto_aead *aead = crypto_aead_reqtfm(req); 149 struct crypto_aead *aead = crypto_aead_reqtfm(req);
109 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); 150 struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
@@ -122,8 +163,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
122 ltag.len = 6; 163 ltag.len = 6;
123 } 164 }
124 165
125 ce_aes_ccm_auth_data(mac, (u8 *)&ltag, ltag.len, &macp, ctx->key_enc, 166 ccm_update_mac(ctx, mac, (u8 *)&ltag, ltag.len, &macp, use_neon);
126 num_rounds(ctx));
127 scatterwalk_start(&walk, req->src); 167 scatterwalk_start(&walk, req->src);
128 168
129 do { 169 do {
@@ -135,8 +175,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
135 n = scatterwalk_clamp(&walk, len); 175 n = scatterwalk_clamp(&walk, len);
136 } 176 }
137 p = scatterwalk_map(&walk); 177 p = scatterwalk_map(&walk);
138 ce_aes_ccm_auth_data(mac, p, n, &macp, ctx->key_enc, 178 ccm_update_mac(ctx, mac, p, n, &macp, use_neon);
139 num_rounds(ctx));
140 len -= n; 179 len -= n;
141 180
142 scatterwalk_unmap(p); 181 scatterwalk_unmap(p);
@@ -145,6 +184,56 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
145 } while (len); 184 } while (len);
146} 185}
147 186
187static int ccm_crypt_fallback(struct skcipher_walk *walk, u8 mac[], u8 iv0[],
188 struct crypto_aes_ctx *ctx, bool enc)
189{
190 u8 buf[AES_BLOCK_SIZE];
191 int err = 0;
192
193 while (walk->nbytes) {
194 int blocks = walk->nbytes / AES_BLOCK_SIZE;
195 u32 tail = walk->nbytes % AES_BLOCK_SIZE;
196 u8 *dst = walk->dst.virt.addr;
197 u8 *src = walk->src.virt.addr;
198 u32 nbytes = walk->nbytes;
199
200 if (nbytes == walk->total && tail > 0) {
201 blocks++;
202 tail = 0;
203 }
204
205 do {
206 u32 bsize = AES_BLOCK_SIZE;
207
208 if (nbytes < AES_BLOCK_SIZE)
209 bsize = nbytes;
210
211 crypto_inc(walk->iv, AES_BLOCK_SIZE);
212 __aes_arm64_encrypt(ctx->key_enc, buf, walk->iv,
213 num_rounds(ctx));
214 __aes_arm64_encrypt(ctx->key_enc, mac, mac,
215 num_rounds(ctx));
216 if (enc)
217 crypto_xor(mac, src, bsize);
218 crypto_xor_cpy(dst, src, buf, bsize);
219 if (!enc)
220 crypto_xor(mac, dst, bsize);
221 dst += bsize;
222 src += bsize;
223 nbytes -= bsize;
224 } while (--blocks);
225
226 err = skcipher_walk_done(walk, tail);
227 }
228
229 if (!err) {
230 __aes_arm64_encrypt(ctx->key_enc, buf, iv0, num_rounds(ctx));
231 __aes_arm64_encrypt(ctx->key_enc, mac, mac, num_rounds(ctx));
232 crypto_xor(mac, buf, AES_BLOCK_SIZE);
233 }
234 return err;
235}
236
148static int ccm_encrypt(struct aead_request *req) 237static int ccm_encrypt(struct aead_request *req)
149{ 238{
150 struct crypto_aead *aead = crypto_aead_reqtfm(req); 239 struct crypto_aead *aead = crypto_aead_reqtfm(req);
@@ -153,39 +242,46 @@ static int ccm_encrypt(struct aead_request *req)
153 u8 __aligned(8) mac[AES_BLOCK_SIZE]; 242 u8 __aligned(8) mac[AES_BLOCK_SIZE];
154 u8 buf[AES_BLOCK_SIZE]; 243 u8 buf[AES_BLOCK_SIZE];
155 u32 len = req->cryptlen; 244 u32 len = req->cryptlen;
245 bool use_neon = may_use_simd();
156 int err; 246 int err;
157 247
158 err = ccm_init_mac(req, mac, len); 248 err = ccm_init_mac(req, mac, len);
159 if (err) 249 if (err)
160 return err; 250 return err;
161 251
162 kernel_neon_begin_partial(6); 252 if (likely(use_neon))
253 kernel_neon_begin();
163 254
164 if (req->assoclen) 255 if (req->assoclen)
165 ccm_calculate_auth_mac(req, mac); 256 ccm_calculate_auth_mac(req, mac, use_neon);
166 257
167 /* preserve the original iv for the final round */ 258 /* preserve the original iv for the final round */
168 memcpy(buf, req->iv, AES_BLOCK_SIZE); 259 memcpy(buf, req->iv, AES_BLOCK_SIZE);
169 260
170 err = skcipher_walk_aead_encrypt(&walk, req, true); 261 err = skcipher_walk_aead_encrypt(&walk, req, true);
171 262
172 while (walk.nbytes) { 263 if (likely(use_neon)) {
173 u32 tail = walk.nbytes % AES_BLOCK_SIZE; 264 while (walk.nbytes) {
174 265 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
175 if (walk.nbytes == walk.total)
176 tail = 0;
177 266
178 ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 267 if (walk.nbytes == walk.total)
179 walk.nbytes - tail, ctx->key_enc, 268 tail = 0;
180 num_rounds(ctx), mac, walk.iv);
181 269
182 err = skcipher_walk_done(&walk, tail); 270 ce_aes_ccm_encrypt(walk.dst.virt.addr,
183 } 271 walk.src.virt.addr,
184 if (!err) 272 walk.nbytes - tail, ctx->key_enc,
185 ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); 273 num_rounds(ctx), mac, walk.iv);
186 274
187 kernel_neon_end(); 275 err = skcipher_walk_done(&walk, tail);
276 }
277 if (!err)
278 ce_aes_ccm_final(mac, buf, ctx->key_enc,
279 num_rounds(ctx));
188 280
281 kernel_neon_end();
282 } else {
283 err = ccm_crypt_fallback(&walk, mac, buf, ctx, true);
284 }
189 if (err) 285 if (err)
190 return err; 286 return err;
191 287
@@ -205,38 +301,46 @@ static int ccm_decrypt(struct aead_request *req)
205 u8 __aligned(8) mac[AES_BLOCK_SIZE]; 301 u8 __aligned(8) mac[AES_BLOCK_SIZE];
206 u8 buf[AES_BLOCK_SIZE]; 302 u8 buf[AES_BLOCK_SIZE];
207 u32 len = req->cryptlen - authsize; 303 u32 len = req->cryptlen - authsize;
304 bool use_neon = may_use_simd();
208 int err; 305 int err;
209 306
210 err = ccm_init_mac(req, mac, len); 307 err = ccm_init_mac(req, mac, len);
211 if (err) 308 if (err)
212 return err; 309 return err;
213 310
214 kernel_neon_begin_partial(6); 311 if (likely(use_neon))
312 kernel_neon_begin();
215 313
216 if (req->assoclen) 314 if (req->assoclen)
217 ccm_calculate_auth_mac(req, mac); 315 ccm_calculate_auth_mac(req, mac, use_neon);
218 316
219 /* preserve the original iv for the final round */ 317 /* preserve the original iv for the final round */
220 memcpy(buf, req->iv, AES_BLOCK_SIZE); 318 memcpy(buf, req->iv, AES_BLOCK_SIZE);
221 319
222 err = skcipher_walk_aead_decrypt(&walk, req, true); 320 err = skcipher_walk_aead_decrypt(&walk, req, true);
223 321
224 while (walk.nbytes) { 322 if (likely(use_neon)) {
225 u32 tail = walk.nbytes % AES_BLOCK_SIZE; 323 while (walk.nbytes) {
324 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
226 325
227 if (walk.nbytes == walk.total) 326 if (walk.nbytes == walk.total)
228 tail = 0; 327 tail = 0;
229 328
230 ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 329 ce_aes_ccm_decrypt(walk.dst.virt.addr,
231 walk.nbytes - tail, ctx->key_enc, 330 walk.src.virt.addr,
232 num_rounds(ctx), mac, walk.iv); 331 walk.nbytes - tail, ctx->key_enc,
332 num_rounds(ctx), mac, walk.iv);
233 333
234 err = skcipher_walk_done(&walk, tail); 334 err = skcipher_walk_done(&walk, tail);
235 } 335 }
236 if (!err) 336 if (!err)
237 ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); 337 ce_aes_ccm_final(mac, buf, ctx->key_enc,
338 num_rounds(ctx));
238 339
239 kernel_neon_end(); 340 kernel_neon_end();
341 } else {
342 err = ccm_crypt_fallback(&walk, mac, buf, ctx, false);
343 }
240 344
241 if (err) 345 if (err)
242 return err; 346 return err;
diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
index 50d9fe11d0c8..6a75cd75ed11 100644
--- a/arch/arm64/crypto/aes-ce-cipher.c
+++ b/arch/arm64/crypto/aes-ce-cipher.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions 2 * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
3 * 3 *
4 * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -9,6 +9,8 @@
9 */ 9 */
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/simd.h>
13#include <asm/unaligned.h>
12#include <crypto/aes.h> 14#include <crypto/aes.h>
13#include <linux/cpufeature.h> 15#include <linux/cpufeature.h>
14#include <linux/crypto.h> 16#include <linux/crypto.h>
@@ -20,6 +22,9 @@ MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
20MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 22MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
21MODULE_LICENSE("GPL v2"); 23MODULE_LICENSE("GPL v2");
22 24
25asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
26asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
27
23struct aes_block { 28struct aes_block {
24 u8 b[AES_BLOCK_SIZE]; 29 u8 b[AES_BLOCK_SIZE];
25}; 30};
@@ -44,27 +49,32 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
44 void *dummy0; 49 void *dummy0;
45 int dummy1; 50 int dummy1;
46 51
47 kernel_neon_begin_partial(4); 52 if (!may_use_simd()) {
53 __aes_arm64_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
54 return;
55 }
56
57 kernel_neon_begin();
48 58
49 __asm__(" ld1 {v0.16b}, %[in] ;" 59 __asm__(" ld1 {v0.16b}, %[in] ;"
50 " ld1 {v1.16b}, [%[key]], #16 ;" 60 " ld1 {v1.4s}, [%[key]], #16 ;"
51 " cmp %w[rounds], #10 ;" 61 " cmp %w[rounds], #10 ;"
52 " bmi 0f ;" 62 " bmi 0f ;"
53 " bne 3f ;" 63 " bne 3f ;"
54 " mov v3.16b, v1.16b ;" 64 " mov v3.16b, v1.16b ;"
55 " b 2f ;" 65 " b 2f ;"
56 "0: mov v2.16b, v1.16b ;" 66 "0: mov v2.16b, v1.16b ;"
57 " ld1 {v3.16b}, [%[key]], #16 ;" 67 " ld1 {v3.4s}, [%[key]], #16 ;"
58 "1: aese v0.16b, v2.16b ;" 68 "1: aese v0.16b, v2.16b ;"
59 " aesmc v0.16b, v0.16b ;" 69 " aesmc v0.16b, v0.16b ;"
60 "2: ld1 {v1.16b}, [%[key]], #16 ;" 70 "2: ld1 {v1.4s}, [%[key]], #16 ;"
61 " aese v0.16b, v3.16b ;" 71 " aese v0.16b, v3.16b ;"
62 " aesmc v0.16b, v0.16b ;" 72 " aesmc v0.16b, v0.16b ;"
63 "3: ld1 {v2.16b}, [%[key]], #16 ;" 73 "3: ld1 {v2.4s}, [%[key]], #16 ;"
64 " subs %w[rounds], %w[rounds], #3 ;" 74 " subs %w[rounds], %w[rounds], #3 ;"
65 " aese v0.16b, v1.16b ;" 75 " aese v0.16b, v1.16b ;"
66 " aesmc v0.16b, v0.16b ;" 76 " aesmc v0.16b, v0.16b ;"
67 " ld1 {v3.16b}, [%[key]], #16 ;" 77 " ld1 {v3.4s}, [%[key]], #16 ;"
68 " bpl 1b ;" 78 " bpl 1b ;"
69 " aese v0.16b, v2.16b ;" 79 " aese v0.16b, v2.16b ;"
70 " eor v0.16b, v0.16b, v3.16b ;" 80 " eor v0.16b, v0.16b, v3.16b ;"
@@ -89,27 +99,32 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
89 void *dummy0; 99 void *dummy0;
90 int dummy1; 100 int dummy1;
91 101
92 kernel_neon_begin_partial(4); 102 if (!may_use_simd()) {
103 __aes_arm64_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
104 return;
105 }
106
107 kernel_neon_begin();
93 108
94 __asm__(" ld1 {v0.16b}, %[in] ;" 109 __asm__(" ld1 {v0.16b}, %[in] ;"
95 " ld1 {v1.16b}, [%[key]], #16 ;" 110 " ld1 {v1.4s}, [%[key]], #16 ;"
96 " cmp %w[rounds], #10 ;" 111 " cmp %w[rounds], #10 ;"
97 " bmi 0f ;" 112 " bmi 0f ;"
98 " bne 3f ;" 113 " bne 3f ;"
99 " mov v3.16b, v1.16b ;" 114 " mov v3.16b, v1.16b ;"
100 " b 2f ;" 115 " b 2f ;"
101 "0: mov v2.16b, v1.16b ;" 116 "0: mov v2.16b, v1.16b ;"
102 " ld1 {v3.16b}, [%[key]], #16 ;" 117 " ld1 {v3.4s}, [%[key]], #16 ;"
103 "1: aesd v0.16b, v2.16b ;" 118 "1: aesd v0.16b, v2.16b ;"
104 " aesimc v0.16b, v0.16b ;" 119 " aesimc v0.16b, v0.16b ;"
105 "2: ld1 {v1.16b}, [%[key]], #16 ;" 120 "2: ld1 {v1.4s}, [%[key]], #16 ;"
106 " aesd v0.16b, v3.16b ;" 121 " aesd v0.16b, v3.16b ;"
107 " aesimc v0.16b, v0.16b ;" 122 " aesimc v0.16b, v0.16b ;"
108 "3: ld1 {v2.16b}, [%[key]], #16 ;" 123 "3: ld1 {v2.4s}, [%[key]], #16 ;"
109 " subs %w[rounds], %w[rounds], #3 ;" 124 " subs %w[rounds], %w[rounds], #3 ;"
110 " aesd v0.16b, v1.16b ;" 125 " aesd v0.16b, v1.16b ;"
111 " aesimc v0.16b, v0.16b ;" 126 " aesimc v0.16b, v0.16b ;"
112 " ld1 {v3.16b}, [%[key]], #16 ;" 127 " ld1 {v3.4s}, [%[key]], #16 ;"
113 " bpl 1b ;" 128 " bpl 1b ;"
114 " aesd v0.16b, v2.16b ;" 129 " aesd v0.16b, v2.16b ;"
115 " eor v0.16b, v0.16b, v3.16b ;" 130 " eor v0.16b, v0.16b, v3.16b ;"
@@ -165,20 +180,16 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
165 key_len != AES_KEYSIZE_256) 180 key_len != AES_KEYSIZE_256)
166 return -EINVAL; 181 return -EINVAL;
167 182
168 memcpy(ctx->key_enc, in_key, key_len);
169 ctx->key_length = key_len; 183 ctx->key_length = key_len;
184 for (i = 0; i < kwords; i++)
185 ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
170 186
171 kernel_neon_begin_partial(2); 187 kernel_neon_begin();
172 for (i = 0; i < sizeof(rcon); i++) { 188 for (i = 0; i < sizeof(rcon); i++) {
173 u32 *rki = ctx->key_enc + (i * kwords); 189 u32 *rki = ctx->key_enc + (i * kwords);
174 u32 *rko = rki + kwords; 190 u32 *rko = rki + kwords;
175 191
176#ifndef CONFIG_CPU_BIG_ENDIAN
177 rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; 192 rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
178#else
179 rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^
180 rki[0];
181#endif
182 rko[1] = rko[0] ^ rki[1]; 193 rko[1] = rko[0] ^ rki[1];
183 rko[2] = rko[1] ^ rki[2]; 194 rko[2] = rko[1] ^ rki[2];
184 rko[3] = rko[2] ^ rki[3]; 195 rko[3] = rko[2] ^ rki[3];
@@ -210,9 +221,9 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
210 221
211 key_dec[0] = key_enc[j]; 222 key_dec[0] = key_enc[j];
212 for (i = 1, j--; j > 0; i++, j--) 223 for (i = 1, j--; j > 0; i++, j--)
213 __asm__("ld1 {v0.16b}, %[in] ;" 224 __asm__("ld1 {v0.4s}, %[in] ;"
214 "aesimc v1.16b, v0.16b ;" 225 "aesimc v1.16b, v0.16b ;"
215 "st1 {v1.16b}, %[out] ;" 226 "st1 {v1.4s}, %[out] ;"
216 227
217 : [out] "=Q"(key_dec[i]) 228 : [out] "=Q"(key_dec[i])
218 : [in] "Q"(key_enc[j]) 229 : [in] "Q"(key_enc[j])
diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S
index b46093d567e5..50330f5c3adc 100644
--- a/arch/arm64/crypto/aes-ce.S
+++ b/arch/arm64/crypto/aes-ce.S
@@ -2,7 +2,7 @@
2 * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with 2 * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
3 * Crypto Extensions 3 * Crypto Extensions
4 * 4 *
5 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 * 6 *
7 * This program is free software; you can redistribute it and/or modify 7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as 8 * it under the terms of the GNU General Public License version 2 as
@@ -22,11 +22,11 @@
22 cmp \rounds, #12 22 cmp \rounds, #12
23 blo 2222f /* 128 bits */ 23 blo 2222f /* 128 bits */
24 beq 1111f /* 192 bits */ 24 beq 1111f /* 192 bits */
25 ld1 {v17.16b-v18.16b}, [\rk], #32 25 ld1 {v17.4s-v18.4s}, [\rk], #32
261111: ld1 {v19.16b-v20.16b}, [\rk], #32 261111: ld1 {v19.4s-v20.4s}, [\rk], #32
272222: ld1 {v21.16b-v24.16b}, [\rk], #64 272222: ld1 {v21.4s-v24.4s}, [\rk], #64
28 ld1 {v25.16b-v28.16b}, [\rk], #64 28 ld1 {v25.4s-v28.4s}, [\rk], #64
29 ld1 {v29.16b-v31.16b}, [\rk] 29 ld1 {v29.4s-v31.4s}, [\rk]
30 .endm 30 .endm
31 31
32 /* prepare for encryption with key in rk[] */ 32 /* prepare for encryption with key in rk[] */
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
index f2f9cc519309..6d2445d603cc 100644
--- a/arch/arm64/crypto/aes-cipher-core.S
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -10,6 +10,7 @@
10 10
11#include <linux/linkage.h> 11#include <linux/linkage.h>
12#include <asm/assembler.h> 12#include <asm/assembler.h>
13#include <asm/cache.h>
13 14
14 .text 15 .text
15 16
@@ -17,94 +18,155 @@
17 out .req x1 18 out .req x1
18 in .req x2 19 in .req x2
19 rounds .req x3 20 rounds .req x3
20 tt .req x4 21 tt .req x2
21 lt .req x2
22 22
23 .macro __pair, enc, reg0, reg1, in0, in1e, in1d, shift 23 .macro __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
24 .ifc \op\shift, b0
25 ubfiz \reg0, \in0, #2, #8
26 ubfiz \reg1, \in1e, #2, #8
27 .else
24 ubfx \reg0, \in0, #\shift, #8 28 ubfx \reg0, \in0, #\shift, #8
25 .if \enc
26 ubfx \reg1, \in1e, #\shift, #8 29 ubfx \reg1, \in1e, #\shift, #8
27 .else
28 ubfx \reg1, \in1d, #\shift, #8
29 .endif 30 .endif
31
32 /*
33 * AArch64 cannot do byte size indexed loads from a table containing
34 * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
35 * valid instruction. So perform the shift explicitly first for the
36 * high bytes (the low byte is shifted implicitly by using ubfiz rather
37 * than ubfx above)
38 */
39 .ifnc \op, b
30 ldr \reg0, [tt, \reg0, uxtw #2] 40 ldr \reg0, [tt, \reg0, uxtw #2]
31 ldr \reg1, [tt, \reg1, uxtw #2] 41 ldr \reg1, [tt, \reg1, uxtw #2]
42 .else
43 .if \shift > 0
44 lsl \reg0, \reg0, #2
45 lsl \reg1, \reg1, #2
46 .endif
47 ldrb \reg0, [tt, \reg0, uxtw]
48 ldrb \reg1, [tt, \reg1, uxtw]
49 .endif
32 .endm 50 .endm
33 51
34 .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc 52 .macro __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
53 ubfx \reg0, \in0, #\shift, #8
54 ubfx \reg1, \in1d, #\shift, #8
55 ldr\op \reg0, [tt, \reg0, uxtw #\sz]
56 ldr\op \reg1, [tt, \reg1, uxtw #\sz]
57 .endm
58
59 .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
35 ldp \out0, \out1, [rk], #8 60 ldp \out0, \out1, [rk], #8
36 61
37 __pair \enc, w13, w14, \in0, \in1, \in3, 0 62 __pair\enc \sz, \op, w12, w13, \in0, \in1, \in3, 0
38 __pair \enc, w15, w16, \in1, \in2, \in0, 8 63 __pair\enc \sz, \op, w14, w15, \in1, \in2, \in0, 8
39 __pair \enc, w17, w18, \in2, \in3, \in1, 16 64 __pair\enc \sz, \op, w16, w17, \in2, \in3, \in1, 16
40 __pair \enc, \t0, \t1, \in3, \in0, \in2, 24 65 __pair\enc \sz, \op, \t0, \t1, \in3, \in0, \in2, 24
41 66
42 eor \out0, \out0, w13 67 eor \out0, \out0, w12
43 eor \out1, \out1, w14 68 eor \out1, \out1, w13
44 eor \out0, \out0, w15, ror #24 69 eor \out0, \out0, w14, ror #24
45 eor \out1, \out1, w16, ror #24 70 eor \out1, \out1, w15, ror #24
46 eor \out0, \out0, w17, ror #16 71 eor \out0, \out0, w16, ror #16
47 eor \out1, \out1, w18, ror #16 72 eor \out1, \out1, w17, ror #16
48 eor \out0, \out0, \t0, ror #8 73 eor \out0, \out0, \t0, ror #8
49 eor \out1, \out1, \t1, ror #8 74 eor \out1, \out1, \t1, ror #8
50 .endm 75 .endm
51 76
52 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3 77 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
53 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1 78 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
54 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1 79 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
55 .endm 80 .endm
56 81
57 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3 82 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
58 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0 83 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
59 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0 84 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
60 .endm 85 .endm
61 86
62 .macro do_crypt, round, ttab, ltab 87 .macro do_crypt, round, ttab, ltab, bsz
63 ldp w5, w6, [in] 88 ldp w4, w5, [in]
64 ldp w7, w8, [in, #8] 89 ldp w6, w7, [in, #8]
65 ldp w9, w10, [rk], #16 90 ldp w8, w9, [rk], #16
66 ldp w11, w12, [rk, #-8] 91 ldp w10, w11, [rk, #-8]
67 92
93CPU_BE( rev w4, w4 )
68CPU_BE( rev w5, w5 ) 94CPU_BE( rev w5, w5 )
69CPU_BE( rev w6, w6 ) 95CPU_BE( rev w6, w6 )
70CPU_BE( rev w7, w7 ) 96CPU_BE( rev w7, w7 )
71CPU_BE( rev w8, w8 )
72 97
98 eor w4, w4, w8
73 eor w5, w5, w9 99 eor w5, w5, w9
74 eor w6, w6, w10 100 eor w6, w6, w10
75 eor w7, w7, w11 101 eor w7, w7, w11
76 eor w8, w8, w12
77 102
78 adr_l tt, \ttab 103 adr_l tt, \ttab
79 adr_l lt, \ltab
80 104
81 tbnz rounds, #1, 1f 105 tbnz rounds, #1, 1f
82 106
830: \round w9, w10, w11, w12, w5, w6, w7, w8 1070: \round w8, w9, w10, w11, w4, w5, w6, w7
84 \round w5, w6, w7, w8, w9, w10, w11, w12 108 \round w4, w5, w6, w7, w8, w9, w10, w11
85 109
861: subs rounds, rounds, #4 1101: subs rounds, rounds, #4
87 \round w9, w10, w11, w12, w5, w6, w7, w8 111 \round w8, w9, w10, w11, w4, w5, w6, w7
88 csel tt, tt, lt, hi 112 b.ls 3f
89 \round w5, w6, w7, w8, w9, w10, w11, w12 1132: \round w4, w5, w6, w7, w8, w9, w10, w11
90 b.hi 0b 114 b 0b
91 1153: adr_l tt, \ltab
116 \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
117
118CPU_BE( rev w4, w4 )
92CPU_BE( rev w5, w5 ) 119CPU_BE( rev w5, w5 )
93CPU_BE( rev w6, w6 ) 120CPU_BE( rev w6, w6 )
94CPU_BE( rev w7, w7 ) 121CPU_BE( rev w7, w7 )
95CPU_BE( rev w8, w8 )
96 122
97 stp w5, w6, [out] 123 stp w4, w5, [out]
98 stp w7, w8, [out, #8] 124 stp w6, w7, [out, #8]
99 ret 125 ret
100 .endm 126 .endm
101 127
102 .align 5 128 .align L1_CACHE_SHIFT
129 .type __aes_arm64_inverse_sbox, %object
130__aes_arm64_inverse_sbox:
131 .byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
132 .byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
133 .byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
134 .byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
135 .byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
136 .byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
137 .byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
138 .byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
139 .byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
140 .byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
141 .byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
142 .byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
143 .byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
144 .byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
145 .byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
146 .byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
147 .byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
148 .byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
149 .byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
150 .byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
151 .byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
152 .byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
153 .byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
154 .byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
155 .byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
156 .byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
157 .byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
158 .byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
159 .byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
160 .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
161 .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
162 .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
163 .size __aes_arm64_inverse_sbox, . - __aes_arm64_inverse_sbox
164
103ENTRY(__aes_arm64_encrypt) 165ENTRY(__aes_arm64_encrypt)
104 do_crypt fround, crypto_ft_tab, crypto_fl_tab 166 do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2
105ENDPROC(__aes_arm64_encrypt) 167ENDPROC(__aes_arm64_encrypt)
106 168
107 .align 5 169 .align 5
108ENTRY(__aes_arm64_decrypt) 170ENTRY(__aes_arm64_decrypt)
109 do_crypt iround, crypto_it_tab, crypto_il_tab 171 do_crypt iround, crypto_it_tab, __aes_arm64_inverse_sbox, 0
110ENDPROC(__aes_arm64_decrypt) 172ENDPROC(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-ctr-fallback.h b/arch/arm64/crypto/aes-ctr-fallback.h
new file mode 100644
index 000000000000..c9285717b6b5
--- /dev/null
+++ b/arch/arm64/crypto/aes-ctr-fallback.h
@@ -0,0 +1,53 @@
1/*
2 * Fallback for sync aes(ctr) in contexts where kernel mode NEON
3 * is not allowed
4 *
5 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <crypto/aes.h>
13#include <crypto/internal/skcipher.h>
14
15asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
16
17static inline int aes_ctr_encrypt_fallback(struct crypto_aes_ctx *ctx,
18 struct skcipher_request *req)
19{
20 struct skcipher_walk walk;
21 u8 buf[AES_BLOCK_SIZE];
22 int err;
23
24 err = skcipher_walk_virt(&walk, req, true);
25
26 while (walk.nbytes > 0) {
27 u8 *dst = walk.dst.virt.addr;
28 u8 *src = walk.src.virt.addr;
29 int nbytes = walk.nbytes;
30 int tail = 0;
31
32 if (nbytes < walk.total) {
33 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
34 tail = walk.nbytes % AES_BLOCK_SIZE;
35 }
36
37 do {
38 int bsize = min(nbytes, AES_BLOCK_SIZE);
39
40 __aes_arm64_encrypt(ctx->key_enc, buf, walk.iv,
41 6 + ctx->key_length / 4);
42 crypto_xor_cpy(dst, src, buf, bsize);
43 crypto_inc(walk.iv, AES_BLOCK_SIZE);
44
45 dst += AES_BLOCK_SIZE;
46 src += AES_BLOCK_SIZE;
47 nbytes -= AES_BLOCK_SIZE;
48 } while (nbytes > 0);
49
50 err = skcipher_walk_done(&walk, tail);
51 }
52 return err;
53}
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index bcf596b0197e..998ba519a026 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -10,6 +10,7 @@
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/hwcap.h> 12#include <asm/hwcap.h>
13#include <asm/simd.h>
13#include <crypto/aes.h> 14#include <crypto/aes.h>
14#include <crypto/internal/hash.h> 15#include <crypto/internal/hash.h>
15#include <crypto/internal/simd.h> 16#include <crypto/internal/simd.h>
@@ -19,6 +20,7 @@
19#include <crypto/xts.h> 20#include <crypto/xts.h>
20 21
21#include "aes-ce-setkey.h" 22#include "aes-ce-setkey.h"
23#include "aes-ctr-fallback.h"
22 24
23#ifdef USE_V8_CRYPTO_EXTENSIONS 25#ifdef USE_V8_CRYPTO_EXTENSIONS
24#define MODE "ce" 26#define MODE "ce"
@@ -241,9 +243,7 @@ static int ctr_encrypt(struct skcipher_request *req)
241 243
242 aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds, 244 aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
243 blocks, walk.iv, first); 245 blocks, walk.iv, first);
244 if (tdst != tsrc) 246 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
245 memcpy(tdst, tsrc, nbytes);
246 crypto_xor(tdst, tail, nbytes);
247 err = skcipher_walk_done(&walk, 0); 247 err = skcipher_walk_done(&walk, 0);
248 } 248 }
249 kernel_neon_end(); 249 kernel_neon_end();
@@ -251,6 +251,17 @@ static int ctr_encrypt(struct skcipher_request *req)
251 return err; 251 return err;
252} 252}
253 253
254static int ctr_encrypt_sync(struct skcipher_request *req)
255{
256 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
257 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
258
259 if (!may_use_simd())
260 return aes_ctr_encrypt_fallback(ctx, req);
261
262 return ctr_encrypt(req);
263}
264
254static int xts_encrypt(struct skcipher_request *req) 265static int xts_encrypt(struct skcipher_request *req)
255{ 266{
256 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 267 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -357,8 +368,8 @@ static struct skcipher_alg aes_algs[] = { {
357 .ivsize = AES_BLOCK_SIZE, 368 .ivsize = AES_BLOCK_SIZE,
358 .chunksize = AES_BLOCK_SIZE, 369 .chunksize = AES_BLOCK_SIZE,
359 .setkey = skcipher_aes_setkey, 370 .setkey = skcipher_aes_setkey,
360 .encrypt = ctr_encrypt, 371 .encrypt = ctr_encrypt_sync,
361 .decrypt = ctr_encrypt, 372 .decrypt = ctr_encrypt_sync,
362}, { 373}, {
363 .base = { 374 .base = {
364 .cra_name = "__xts(aes)", 375 .cra_name = "__xts(aes)",
@@ -460,11 +471,35 @@ static int mac_init(struct shash_desc *desc)
460 return 0; 471 return 0;
461} 472}
462 473
474static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
475 u8 dg[], int enc_before, int enc_after)
476{
477 int rounds = 6 + ctx->key_length / 4;
478
479 if (may_use_simd()) {
480 kernel_neon_begin();
481 aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before,
482 enc_after);
483 kernel_neon_end();
484 } else {
485 if (enc_before)
486 __aes_arm64_encrypt(ctx->key_enc, dg, dg, rounds);
487
488 while (blocks--) {
489 crypto_xor(dg, in, AES_BLOCK_SIZE);
490 in += AES_BLOCK_SIZE;
491
492 if (blocks || enc_after)
493 __aes_arm64_encrypt(ctx->key_enc, dg, dg,
494 rounds);
495 }
496 }
497}
498
463static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) 499static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
464{ 500{
465 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 501 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
466 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 502 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
467 int rounds = 6 + tctx->key.key_length / 4;
468 503
469 while (len > 0) { 504 while (len > 0) {
470 unsigned int l; 505 unsigned int l;
@@ -476,10 +511,8 @@ static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
476 511
477 len %= AES_BLOCK_SIZE; 512 len %= AES_BLOCK_SIZE;
478 513
479 kernel_neon_begin(); 514 mac_do_update(&tctx->key, p, blocks, ctx->dg,
480 aes_mac_update(p, tctx->key.key_enc, rounds, blocks, 515 (ctx->len != 0), (len != 0));
481 ctx->dg, (ctx->len != 0), (len != 0));
482 kernel_neon_end();
483 516
484 p += blocks * AES_BLOCK_SIZE; 517 p += blocks * AES_BLOCK_SIZE;
485 518
@@ -507,11 +540,8 @@ static int cbcmac_final(struct shash_desc *desc, u8 *out)
507{ 540{
508 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 541 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
509 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 542 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
510 int rounds = 6 + tctx->key.key_length / 4;
511 543
512 kernel_neon_begin(); 544 mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1, 0);
513 aes_mac_update(NULL, tctx->key.key_enc, rounds, 0, ctx->dg, 1, 0);
514 kernel_neon_end();
515 545
516 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 546 memcpy(out, ctx->dg, AES_BLOCK_SIZE);
517 547
@@ -522,7 +552,6 @@ static int cmac_final(struct shash_desc *desc, u8 *out)
522{ 552{
523 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 553 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
524 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 554 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
525 int rounds = 6 + tctx->key.key_length / 4;
526 u8 *consts = tctx->consts; 555 u8 *consts = tctx->consts;
527 556
528 if (ctx->len != AES_BLOCK_SIZE) { 557 if (ctx->len != AES_BLOCK_SIZE) {
@@ -530,9 +559,7 @@ static int cmac_final(struct shash_desc *desc, u8 *out)
530 consts += AES_BLOCK_SIZE; 559 consts += AES_BLOCK_SIZE;
531 } 560 }
532 561
533 kernel_neon_begin(); 562 mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1);
534 aes_mac_update(consts, tctx->key.key_enc, rounds, 1, ctx->dg, 0, 1);
535 kernel_neon_end();
536 563
537 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 564 memcpy(out, ctx->dg, AES_BLOCK_SIZE);
538 565
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
index db2501d93550..c55d68ccb89f 100644
--- a/arch/arm64/crypto/aes-neonbs-glue.c
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * Bit sliced AES using NEON instructions 2 * Bit sliced AES using NEON instructions
3 * 3 *
4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -9,12 +9,15 @@
9 */ 9 */
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/simd.h>
12#include <crypto/aes.h> 13#include <crypto/aes.h>
13#include <crypto/internal/simd.h> 14#include <crypto/internal/simd.h>
14#include <crypto/internal/skcipher.h> 15#include <crypto/internal/skcipher.h>
15#include <crypto/xts.h> 16#include <crypto/xts.h>
16#include <linux/module.h> 17#include <linux/module.h>
17 18
19#include "aes-ctr-fallback.h"
20
18MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 21MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
19MODULE_LICENSE("GPL v2"); 22MODULE_LICENSE("GPL v2");
20 23
@@ -58,6 +61,11 @@ struct aesbs_cbc_ctx {
58 u32 enc[AES_MAX_KEYLENGTH_U32]; 61 u32 enc[AES_MAX_KEYLENGTH_U32];
59}; 62};
60 63
64struct aesbs_ctr_ctx {
65 struct aesbs_ctx key; /* must be first member */
66 struct crypto_aes_ctx fallback;
67};
68
61struct aesbs_xts_ctx { 69struct aesbs_xts_ctx {
62 struct aesbs_ctx key; 70 struct aesbs_ctx key;
63 u32 twkey[AES_MAX_KEYLENGTH_U32]; 71 u32 twkey[AES_MAX_KEYLENGTH_U32];
@@ -196,6 +204,25 @@ static int cbc_decrypt(struct skcipher_request *req)
196 return err; 204 return err;
197} 205}
198 206
207static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key,
208 unsigned int key_len)
209{
210 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
211 int err;
212
213 err = crypto_aes_expand_key(&ctx->fallback, in_key, key_len);
214 if (err)
215 return err;
216
217 ctx->key.rounds = 6 + key_len / 4;
218
219 kernel_neon_begin();
220 aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds);
221 kernel_neon_end();
222
223 return 0;
224}
225
199static int ctr_encrypt(struct skcipher_request *req) 226static int ctr_encrypt(struct skcipher_request *req)
200{ 227{
201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 228 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -224,9 +251,8 @@ static int ctr_encrypt(struct skcipher_request *req)
224 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 251 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
225 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 252 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
226 253
227 if (dst != src) 254 crypto_xor_cpy(dst, src, final,
228 memcpy(dst, src, walk.total % AES_BLOCK_SIZE); 255 walk.total % AES_BLOCK_SIZE);
229 crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
230 256
231 err = skcipher_walk_done(&walk, 0); 257 err = skcipher_walk_done(&walk, 0);
232 break; 258 break;
@@ -260,6 +286,17 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
260 return aesbs_setkey(tfm, in_key, key_len); 286 return aesbs_setkey(tfm, in_key, key_len);
261} 287}
262 288
289static int ctr_encrypt_sync(struct skcipher_request *req)
290{
291 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
292 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
293
294 if (!may_use_simd())
295 return aes_ctr_encrypt_fallback(&ctx->fallback, req);
296
297 return ctr_encrypt(req);
298}
299
263static int __xts_crypt(struct skcipher_request *req, 300static int __xts_crypt(struct skcipher_request *req,
264 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 301 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
265 int rounds, int blocks, u8 iv[])) 302 int rounds, int blocks, u8 iv[]))
@@ -356,7 +393,7 @@ static struct skcipher_alg aes_algs[] = { {
356 .base.cra_driver_name = "ctr-aes-neonbs", 393 .base.cra_driver_name = "ctr-aes-neonbs",
357 .base.cra_priority = 250 - 1, 394 .base.cra_priority = 250 - 1,
358 .base.cra_blocksize = 1, 395 .base.cra_blocksize = 1,
359 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 396 .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
360 .base.cra_module = THIS_MODULE, 397 .base.cra_module = THIS_MODULE,
361 398
362 .min_keysize = AES_MIN_KEY_SIZE, 399 .min_keysize = AES_MIN_KEY_SIZE,
@@ -364,9 +401,9 @@ static struct skcipher_alg aes_algs[] = { {
364 .chunksize = AES_BLOCK_SIZE, 401 .chunksize = AES_BLOCK_SIZE,
365 .walksize = 8 * AES_BLOCK_SIZE, 402 .walksize = 8 * AES_BLOCK_SIZE,
366 .ivsize = AES_BLOCK_SIZE, 403 .ivsize = AES_BLOCK_SIZE,
367 .setkey = aesbs_setkey, 404 .setkey = aesbs_ctr_setkey_sync,
368 .encrypt = ctr_encrypt, 405 .encrypt = ctr_encrypt_sync,
369 .decrypt = ctr_encrypt, 406 .decrypt = ctr_encrypt_sync,
370}, { 407}, {
371 .base.cra_name = "__xts(aes)", 408 .base.cra_name = "__xts(aes)",
372 .base.cra_driver_name = "__xts-aes-neonbs", 409 .base.cra_driver_name = "__xts-aes-neonbs",
diff --git a/arch/arm64/crypto/chacha20-neon-glue.c b/arch/arm64/crypto/chacha20-neon-glue.c
index a7cd575ea223..cbdb75d15cd0 100644
--- a/arch/arm64/crypto/chacha20-neon-glue.c
+++ b/arch/arm64/crypto/chacha20-neon-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions 2 * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
3 * 3 *
4 * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -26,6 +26,7 @@
26 26
27#include <asm/hwcap.h> 27#include <asm/hwcap.h>
28#include <asm/neon.h> 28#include <asm/neon.h>
29#include <asm/simd.h>
29 30
30asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src); 31asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
31asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src); 32asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
@@ -64,7 +65,7 @@ static int chacha20_neon(struct skcipher_request *req)
64 u32 state[16]; 65 u32 state[16];
65 int err; 66 int err;
66 67
67 if (req->cryptlen <= CHACHA20_BLOCK_SIZE) 68 if (!may_use_simd() || req->cryptlen <= CHACHA20_BLOCK_SIZE)
68 return crypto_chacha20_crypt(req); 69 return crypto_chacha20_crypt(req);
69 70
70 err = skcipher_walk_virt(&walk, req, true); 71 err = skcipher_walk_virt(&walk, req, true);
diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
index eccb1ae90064..624f4137918c 100644
--- a/arch/arm64/crypto/crc32-ce-glue.c
+++ b/arch/arm64/crypto/crc32-ce-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions 2 * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
3 * 3 *
4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -19,6 +19,7 @@
19 19
20#include <asm/hwcap.h> 20#include <asm/hwcap.h>
21#include <asm/neon.h> 21#include <asm/neon.h>
22#include <asm/simd.h>
22#include <asm/unaligned.h> 23#include <asm/unaligned.h>
23 24
24#define PMULL_MIN_LEN 64L /* minimum size of buffer 25#define PMULL_MIN_LEN 64L /* minimum size of buffer
@@ -105,10 +106,10 @@ static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
105 length -= l; 106 length -= l;
106 } 107 }
107 108
108 if (length >= PMULL_MIN_LEN) { 109 if (length >= PMULL_MIN_LEN && may_use_simd()) {
109 l = round_down(length, SCALE_F); 110 l = round_down(length, SCALE_F);
110 111
111 kernel_neon_begin_partial(10); 112 kernel_neon_begin();
112 *crc = crc32_pmull_le(data, l, *crc); 113 *crc = crc32_pmull_le(data, l, *crc);
113 kernel_neon_end(); 114 kernel_neon_end();
114 115
@@ -137,10 +138,10 @@ static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
137 length -= l; 138 length -= l;
138 } 139 }
139 140
140 if (length >= PMULL_MIN_LEN) { 141 if (length >= PMULL_MIN_LEN && may_use_simd()) {
141 l = round_down(length, SCALE_F); 142 l = round_down(length, SCALE_F);
142 143
143 kernel_neon_begin_partial(10); 144 kernel_neon_begin();
144 *crc = crc32c_pmull_le(data, l, *crc); 145 *crc = crc32c_pmull_le(data, l, *crc);
145 kernel_neon_end(); 146 kernel_neon_end();
146 147
diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c
index 60cb590c2590..96f0cae4a022 100644
--- a/arch/arm64/crypto/crct10dif-ce-glue.c
+++ b/arch/arm64/crypto/crct10dif-ce-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 2 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
3 * 3 *
4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -18,6 +18,7 @@
18#include <crypto/internal/hash.h> 18#include <crypto/internal/hash.h>
19 19
20#include <asm/neon.h> 20#include <asm/neon.h>
21#include <asm/simd.h>
21 22
22#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 23#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
23 24
@@ -48,9 +49,13 @@ static int crct10dif_update(struct shash_desc *desc, const u8 *data,
48 } 49 }
49 50
50 if (length > 0) { 51 if (length > 0) {
51 kernel_neon_begin_partial(14); 52 if (may_use_simd()) {
52 *crc = crc_t10dif_pmull(*crc, data, length); 53 kernel_neon_begin();
53 kernel_neon_end(); 54 *crc = crc_t10dif_pmull(*crc, data, length);
55 kernel_neon_end();
56 } else {
57 *crc = crc_t10dif_generic(*crc, data, length);
58 }
54 } 59 }
55 60
56 return 0; 61 return 0;
diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S
index f0bb9f0b524f..11ebf1ae248a 100644
--- a/arch/arm64/crypto/ghash-ce-core.S
+++ b/arch/arm64/crypto/ghash-ce-core.S
@@ -1,7 +1,7 @@
1/* 1/*
2 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 2 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
3 * 3 *
4 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2014 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published 7 * under the terms of the GNU General Public License version 2 as published
@@ -11,31 +11,215 @@
11#include <linux/linkage.h> 11#include <linux/linkage.h>
12#include <asm/assembler.h> 12#include <asm/assembler.h>
13 13
14 SHASH .req v0 14 SHASH .req v0
15 SHASH2 .req v1 15 SHASH2 .req v1
16 T1 .req v2 16 T1 .req v2
17 T2 .req v3 17 T2 .req v3
18 MASK .req v4 18 MASK .req v4
19 XL .req v5 19 XL .req v5
20 XM .req v6 20 XM .req v6
21 XH .req v7 21 XH .req v7
22 IN1 .req v7 22 IN1 .req v7
23
24 k00_16 .req v8
25 k32_48 .req v9
26
27 t3 .req v10
28 t4 .req v11
29 t5 .req v12
30 t6 .req v13
31 t7 .req v14
32 t8 .req v15
33 t9 .req v16
34
35 perm1 .req v17
36 perm2 .req v18
37 perm3 .req v19
38
39 sh1 .req v20
40 sh2 .req v21
41 sh3 .req v22
42 sh4 .req v23
43
44 ss1 .req v24
45 ss2 .req v25
46 ss3 .req v26
47 ss4 .req v27
23 48
24 .text 49 .text
25 .arch armv8-a+crypto 50 .arch armv8-a+crypto
26 51
27 /* 52 .macro __pmull_p64, rd, rn, rm
28 * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 53 pmull \rd\().1q, \rn\().1d, \rm\().1d
29 * struct ghash_key const *k, const char *head) 54 .endm
30 */ 55
31ENTRY(pmull_ghash_update) 56 .macro __pmull2_p64, rd, rn, rm
57 pmull2 \rd\().1q, \rn\().2d, \rm\().2d
58 .endm
59
60 .macro __pmull_p8, rq, ad, bd
61 ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1
62 ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2
63 ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3
64
65 __pmull_p8_\bd \rq, \ad
66 .endm
67
68 .macro __pmull2_p8, rq, ad, bd
69 tbl t3.16b, {\ad\().16b}, perm1.16b // A1
70 tbl t5.16b, {\ad\().16b}, perm2.16b // A2
71 tbl t7.16b, {\ad\().16b}, perm3.16b // A3
72
73 __pmull2_p8_\bd \rq, \ad
74 .endm
75
76 .macro __pmull_p8_SHASH, rq, ad
77 __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4
78 .endm
79
80 .macro __pmull_p8_SHASH2, rq, ad
81 __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4
82 .endm
83
84 .macro __pmull2_p8_SHASH, rq, ad
85 __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4
86 .endm
87
88 .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4
89 pmull\t t3.8h, t3.\nb, \bd // F = A1*B
90 pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1
91 pmull\t t5.8h, t5.\nb, \bd // H = A2*B
92 pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2
93 pmull\t t7.8h, t7.\nb, \bd // J = A3*B
94 pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3
95 pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4
96 pmull\t \rq\().8h, \ad, \bd // D = A*B
97
98 eor t3.16b, t3.16b, t4.16b // L = E + F
99 eor t5.16b, t5.16b, t6.16b // M = G + H
100 eor t7.16b, t7.16b, t8.16b // N = I + J
101
102 uzp1 t4.2d, t3.2d, t5.2d
103 uzp2 t3.2d, t3.2d, t5.2d
104 uzp1 t6.2d, t7.2d, t9.2d
105 uzp2 t7.2d, t7.2d, t9.2d
106
107 // t3 = (L) (P0 + P1) << 8
108 // t5 = (M) (P2 + P3) << 16
109 eor t4.16b, t4.16b, t3.16b
110 and t3.16b, t3.16b, k32_48.16b
111
112 // t7 = (N) (P4 + P5) << 24
113 // t9 = (K) (P6 + P7) << 32
114 eor t6.16b, t6.16b, t7.16b
115 and t7.16b, t7.16b, k00_16.16b
116
117 eor t4.16b, t4.16b, t3.16b
118 eor t6.16b, t6.16b, t7.16b
119
120 zip2 t5.2d, t4.2d, t3.2d
121 zip1 t3.2d, t4.2d, t3.2d
122 zip2 t9.2d, t6.2d, t7.2d
123 zip1 t7.2d, t6.2d, t7.2d
124
125 ext t3.16b, t3.16b, t3.16b, #15
126 ext t5.16b, t5.16b, t5.16b, #14
127 ext t7.16b, t7.16b, t7.16b, #13
128 ext t9.16b, t9.16b, t9.16b, #12
129
130 eor t3.16b, t3.16b, t5.16b
131 eor t7.16b, t7.16b, t9.16b
132 eor \rq\().16b, \rq\().16b, t3.16b
133 eor \rq\().16b, \rq\().16b, t7.16b
134 .endm
135
136 .macro __pmull_pre_p64
137 movi MASK.16b, #0xe1
138 shl MASK.2d, MASK.2d, #57
139 .endm
140
141 .macro __pmull_pre_p8
142 // k00_16 := 0x0000000000000000_000000000000ffff
143 // k32_48 := 0x00000000ffffffff_0000ffffffffffff
144 movi k32_48.2d, #0xffffffff
145 mov k32_48.h[2], k32_48.h[0]
146 ushr k00_16.2d, k32_48.2d, #32
147
148 // prepare the permutation vectors
149 mov_q x5, 0x080f0e0d0c0b0a09
150 movi T1.8b, #8
151 dup perm1.2d, x5
152 eor perm1.16b, perm1.16b, T1.16b
153 ushr perm2.2d, perm1.2d, #8
154 ushr perm3.2d, perm1.2d, #16
155 ushr T1.2d, perm1.2d, #24
156 sli perm2.2d, perm1.2d, #56
157 sli perm3.2d, perm1.2d, #48
158 sli T1.2d, perm1.2d, #40
159
160 // precompute loop invariants
161 tbl sh1.16b, {SHASH.16b}, perm1.16b
162 tbl sh2.16b, {SHASH.16b}, perm2.16b
163 tbl sh3.16b, {SHASH.16b}, perm3.16b
164 tbl sh4.16b, {SHASH.16b}, T1.16b
165 ext ss1.8b, SHASH2.8b, SHASH2.8b, #1
166 ext ss2.8b, SHASH2.8b, SHASH2.8b, #2
167 ext ss3.8b, SHASH2.8b, SHASH2.8b, #3
168 ext ss4.8b, SHASH2.8b, SHASH2.8b, #4
169 .endm
170
171 //
172 // PMULL (64x64->128) based reduction for CPUs that can do
173 // it in a single instruction.
174 //
175 .macro __pmull_reduce_p64
176 pmull T2.1q, XL.1d, MASK.1d
177 eor XM.16b, XM.16b, T1.16b
178
179 mov XH.d[0], XM.d[1]
180 mov XM.d[1], XL.d[0]
181
182 eor XL.16b, XM.16b, T2.16b
183 ext T2.16b, XL.16b, XL.16b, #8
184 pmull XL.1q, XL.1d, MASK.1d
185 .endm
186
187 //
188 // Alternative reduction for CPUs that lack support for the
189 // 64x64->128 PMULL instruction
190 //
191 .macro __pmull_reduce_p8
192 eor XM.16b, XM.16b, T1.16b
193
194 mov XL.d[1], XM.d[0]
195 mov XH.d[0], XM.d[1]
196
197 shl T1.2d, XL.2d, #57
198 shl T2.2d, XL.2d, #62
199 eor T2.16b, T2.16b, T1.16b
200 shl T1.2d, XL.2d, #63
201 eor T2.16b, T2.16b, T1.16b
202 ext T1.16b, XL.16b, XH.16b, #8
203 eor T2.16b, T2.16b, T1.16b
204
205 mov XL.d[1], T2.d[0]
206 mov XH.d[0], T2.d[1]
207
208 ushr T2.2d, XL.2d, #1
209 eor XH.16b, XH.16b, XL.16b
210 eor XL.16b, XL.16b, T2.16b
211 ushr T2.2d, T2.2d, #6
212 ushr XL.2d, XL.2d, #1
213 .endm
214
215 .macro __pmull_ghash, pn
32 ld1 {SHASH.2d}, [x3] 216 ld1 {SHASH.2d}, [x3]
33 ld1 {XL.2d}, [x1] 217 ld1 {XL.2d}, [x1]
34 movi MASK.16b, #0xe1
35 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 218 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8
36 shl MASK.2d, MASK.2d, #57
37 eor SHASH2.16b, SHASH2.16b, SHASH.16b 219 eor SHASH2.16b, SHASH2.16b, SHASH.16b
38 220
221 __pmull_pre_\pn
222
39 /* do the head block first, if supplied */ 223 /* do the head block first, if supplied */
40 cbz x4, 0f 224 cbz x4, 0f
41 ld1 {T1.2d}, [x4] 225 ld1 {T1.2d}, [x4]
@@ -52,28 +236,209 @@ CPU_LE( rev64 T1.16b, T1.16b )
52 eor T1.16b, T1.16b, T2.16b 236 eor T1.16b, T1.16b, T2.16b
53 eor XL.16b, XL.16b, IN1.16b 237 eor XL.16b, XL.16b, IN1.16b
54 238
239 __pmull2_\pn XH, XL, SHASH // a1 * b1
240 eor T1.16b, T1.16b, XL.16b
241 __pmull_\pn XL, XL, SHASH // a0 * b0
242 __pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0)
243
244 eor T2.16b, XL.16b, XH.16b
245 ext T1.16b, XL.16b, XH.16b, #8
246 eor XM.16b, XM.16b, T2.16b
247
248 __pmull_reduce_\pn
249
250 eor T2.16b, T2.16b, XH.16b
251 eor XL.16b, XL.16b, T2.16b
252
253 cbnz w0, 0b
254
255 st1 {XL.2d}, [x1]
256 ret
257 .endm
258
259 /*
260 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
261 * struct ghash_key const *k, const char *head)
262 */
263ENTRY(pmull_ghash_update_p64)
264 __pmull_ghash p64
265ENDPROC(pmull_ghash_update_p64)
266
267ENTRY(pmull_ghash_update_p8)
268 __pmull_ghash p8
269ENDPROC(pmull_ghash_update_p8)
270
271 KS .req v8
272 CTR .req v9
273 INP .req v10
274
275 .macro load_round_keys, rounds, rk
276 cmp \rounds, #12
277 blo 2222f /* 128 bits */
278 beq 1111f /* 192 bits */
279 ld1 {v17.4s-v18.4s}, [\rk], #32
2801111: ld1 {v19.4s-v20.4s}, [\rk], #32
2812222: ld1 {v21.4s-v24.4s}, [\rk], #64
282 ld1 {v25.4s-v28.4s}, [\rk], #64
283 ld1 {v29.4s-v31.4s}, [\rk]
284 .endm
285
286 .macro enc_round, state, key
287 aese \state\().16b, \key\().16b
288 aesmc \state\().16b, \state\().16b
289 .endm
290
291 .macro enc_block, state, rounds
292 cmp \rounds, #12
293 b.lo 2222f /* 128 bits */
294 b.eq 1111f /* 192 bits */
295 enc_round \state, v17
296 enc_round \state, v18
2971111: enc_round \state, v19
298 enc_round \state, v20
2992222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29
300 enc_round \state, \key
301 .endr
302 aese \state\().16b, v30.16b
303 eor \state\().16b, \state\().16b, v31.16b
304 .endm
305
306 .macro pmull_gcm_do_crypt, enc
307 ld1 {SHASH.2d}, [x4]
308 ld1 {XL.2d}, [x1]
309 ldr x8, [x5, #8] // load lower counter
310
311 movi MASK.16b, #0xe1
312 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8
313CPU_LE( rev x8, x8 )
314 shl MASK.2d, MASK.2d, #57
315 eor SHASH2.16b, SHASH2.16b, SHASH.16b
316
317 .if \enc == 1
318 ld1 {KS.16b}, [x7]
319 .endif
320
3210: ld1 {CTR.8b}, [x5] // load upper counter
322 ld1 {INP.16b}, [x3], #16
323 rev x9, x8
324 add x8, x8, #1
325 sub w0, w0, #1
326 ins CTR.d[1], x9 // set lower counter
327
328 .if \enc == 1
329 eor INP.16b, INP.16b, KS.16b // encrypt input
330 st1 {INP.16b}, [x2], #16
331 .endif
332
333 rev64 T1.16b, INP.16b
334
335 cmp w6, #12
336 b.ge 2f // AES-192/256?
337
3381: enc_round CTR, v21
339
340 ext T2.16b, XL.16b, XL.16b, #8
341 ext IN1.16b, T1.16b, T1.16b, #8
342
343 enc_round CTR, v22
344
345 eor T1.16b, T1.16b, T2.16b
346 eor XL.16b, XL.16b, IN1.16b
347
348 enc_round CTR, v23
349
55 pmull2 XH.1q, SHASH.2d, XL.2d // a1 * b1 350 pmull2 XH.1q, SHASH.2d, XL.2d // a1 * b1
56 eor T1.16b, T1.16b, XL.16b 351 eor T1.16b, T1.16b, XL.16b
352
353 enc_round CTR, v24
354
57 pmull XL.1q, SHASH.1d, XL.1d // a0 * b0 355 pmull XL.1q, SHASH.1d, XL.1d // a0 * b0
58 pmull XM.1q, SHASH2.1d, T1.1d // (a1 + a0)(b1 + b0) 356 pmull XM.1q, SHASH2.1d, T1.1d // (a1 + a0)(b1 + b0)
59 357
358 enc_round CTR, v25
359
60 ext T1.16b, XL.16b, XH.16b, #8 360 ext T1.16b, XL.16b, XH.16b, #8
61 eor T2.16b, XL.16b, XH.16b 361 eor T2.16b, XL.16b, XH.16b
62 eor XM.16b, XM.16b, T1.16b 362 eor XM.16b, XM.16b, T1.16b
363
364 enc_round CTR, v26
365
63 eor XM.16b, XM.16b, T2.16b 366 eor XM.16b, XM.16b, T2.16b
64 pmull T2.1q, XL.1d, MASK.1d 367 pmull T2.1q, XL.1d, MASK.1d
65 368
369 enc_round CTR, v27
370
66 mov XH.d[0], XM.d[1] 371 mov XH.d[0], XM.d[1]
67 mov XM.d[1], XL.d[0] 372 mov XM.d[1], XL.d[0]
68 373
374 enc_round CTR, v28
375
69 eor XL.16b, XM.16b, T2.16b 376 eor XL.16b, XM.16b, T2.16b
377
378 enc_round CTR, v29
379
70 ext T2.16b, XL.16b, XL.16b, #8 380 ext T2.16b, XL.16b, XL.16b, #8
381
382 aese CTR.16b, v30.16b
383
71 pmull XL.1q, XL.1d, MASK.1d 384 pmull XL.1q, XL.1d, MASK.1d
72 eor T2.16b, T2.16b, XH.16b 385 eor T2.16b, T2.16b, XH.16b
386
387 eor KS.16b, CTR.16b, v31.16b
388
73 eor XL.16b, XL.16b, T2.16b 389 eor XL.16b, XL.16b, T2.16b
74 390
391 .if \enc == 0
392 eor INP.16b, INP.16b, KS.16b
393 st1 {INP.16b}, [x2], #16
394 .endif
395
75 cbnz w0, 0b 396 cbnz w0, 0b
76 397
398CPU_LE( rev x8, x8 )
77 st1 {XL.2d}, [x1] 399 st1 {XL.2d}, [x1]
400 str x8, [x5, #8] // store lower counter
401
402 .if \enc == 1
403 st1 {KS.16b}, [x7]
404 .endif
405
406 ret
407
4082: b.eq 3f // AES-192?
409 enc_round CTR, v17
410 enc_round CTR, v18
4113: enc_round CTR, v19
412 enc_round CTR, v20
413 b 1b
414 .endm
415
416 /*
417 * void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[], const u8 src[],
418 * struct ghash_key const *k, u8 ctr[],
419 * int rounds, u8 ks[])
420 */
421ENTRY(pmull_gcm_encrypt)
422 pmull_gcm_do_crypt 1
423ENDPROC(pmull_gcm_encrypt)
424
425 /*
426 * void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[], const u8 src[],
427 * struct ghash_key const *k, u8 ctr[],
428 * int rounds)
429 */
430ENTRY(pmull_gcm_decrypt)
431 pmull_gcm_do_crypt 0
432ENDPROC(pmull_gcm_decrypt)
433
434 /*
435 * void pmull_gcm_encrypt_block(u8 dst[], u8 src[], u8 rk[], int rounds)
436 */
437ENTRY(pmull_gcm_encrypt_block)
438 cbz x2, 0f
439 load_round_keys w3, x2
4400: ld1 {v0.16b}, [x1]
441 enc_block v0, w3
442 st1 {v0.16b}, [x0]
78 ret 443 ret
79ENDPROC(pmull_ghash_update) 444ENDPROC(pmull_gcm_encrypt_block)
diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c
index 833ec1e3f3e9..cfc9c92814fd 100644
--- a/arch/arm64/crypto/ghash-ce-glue.c
+++ b/arch/arm64/crypto/ghash-ce-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 2 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
3 * 3 *
4 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2014 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published 7 * under the terms of the GNU General Public License version 2 as published
@@ -9,22 +9,33 @@
9 */ 9 */
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/simd.h>
12#include <asm/unaligned.h> 13#include <asm/unaligned.h>
14#include <crypto/aes.h>
15#include <crypto/algapi.h>
16#include <crypto/b128ops.h>
17#include <crypto/gf128mul.h>
18#include <crypto/internal/aead.h>
13#include <crypto/internal/hash.h> 19#include <crypto/internal/hash.h>
20#include <crypto/internal/skcipher.h>
21#include <crypto/scatterwalk.h>
14#include <linux/cpufeature.h> 22#include <linux/cpufeature.h>
15#include <linux/crypto.h> 23#include <linux/crypto.h>
16#include <linux/module.h> 24#include <linux/module.h>
17 25
18MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions"); 26MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
19MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 27MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
20MODULE_LICENSE("GPL v2"); 28MODULE_LICENSE("GPL v2");
29MODULE_ALIAS_CRYPTO("ghash");
21 30
22#define GHASH_BLOCK_SIZE 16 31#define GHASH_BLOCK_SIZE 16
23#define GHASH_DIGEST_SIZE 16 32#define GHASH_DIGEST_SIZE 16
33#define GCM_IV_SIZE 12
24 34
25struct ghash_key { 35struct ghash_key {
26 u64 a; 36 u64 a;
27 u64 b; 37 u64 b;
38 be128 k;
28}; 39};
29 40
30struct ghash_desc_ctx { 41struct ghash_desc_ctx {
@@ -33,8 +44,35 @@ struct ghash_desc_ctx {
33 u32 count; 44 u32 count;
34}; 45};
35 46
36asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src, 47struct gcm_aes_ctx {
37 struct ghash_key const *k, const char *head); 48 struct crypto_aes_ctx aes_key;
49 struct ghash_key ghash_key;
50};
51
52asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
53 struct ghash_key const *k,
54 const char *head);
55
56asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
57 struct ghash_key const *k,
58 const char *head);
59
60static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src,
61 struct ghash_key const *k,
62 const char *head);
63
64asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[],
65 const u8 src[], struct ghash_key const *k,
66 u8 ctr[], int rounds, u8 ks[]);
67
68asmlinkage void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[],
69 const u8 src[], struct ghash_key const *k,
70 u8 ctr[], int rounds);
71
72asmlinkage void pmull_gcm_encrypt_block(u8 dst[], u8 const src[],
73 u32 const rk[], int rounds);
74
75asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
38 76
39static int ghash_init(struct shash_desc *desc) 77static int ghash_init(struct shash_desc *desc)
40{ 78{
@@ -44,6 +82,36 @@ static int ghash_init(struct shash_desc *desc)
44 return 0; 82 return 0;
45} 83}
46 84
85static void ghash_do_update(int blocks, u64 dg[], const char *src,
86 struct ghash_key *key, const char *head)
87{
88 if (likely(may_use_simd())) {
89 kernel_neon_begin();
90 pmull_ghash_update(blocks, dg, src, key, head);
91 kernel_neon_end();
92 } else {
93 be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
94
95 do {
96 const u8 *in = src;
97
98 if (head) {
99 in = head;
100 blocks++;
101 head = NULL;
102 } else {
103 src += GHASH_BLOCK_SIZE;
104 }
105
106 crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
107 gf128mul_lle(&dst, &key->k);
108 } while (--blocks);
109
110 dg[0] = be64_to_cpu(dst.b);
111 dg[1] = be64_to_cpu(dst.a);
112 }
113}
114
47static int ghash_update(struct shash_desc *desc, const u8 *src, 115static int ghash_update(struct shash_desc *desc, const u8 *src,
48 unsigned int len) 116 unsigned int len)
49{ 117{
@@ -67,10 +135,9 @@ static int ghash_update(struct shash_desc *desc, const u8 *src,
67 blocks = len / GHASH_BLOCK_SIZE; 135 blocks = len / GHASH_BLOCK_SIZE;
68 len %= GHASH_BLOCK_SIZE; 136 len %= GHASH_BLOCK_SIZE;
69 137
70 kernel_neon_begin_partial(8); 138 ghash_do_update(blocks, ctx->digest, src, key,
71 pmull_ghash_update(blocks, ctx->digest, src, key, 139 partial ? ctx->buf : NULL);
72 partial ? ctx->buf : NULL); 140
73 kernel_neon_end();
74 src += blocks * GHASH_BLOCK_SIZE; 141 src += blocks * GHASH_BLOCK_SIZE;
75 partial = 0; 142 partial = 0;
76 } 143 }
@@ -89,9 +156,7 @@ static int ghash_final(struct shash_desc *desc, u8 *dst)
89 156
90 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 157 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
91 158
92 kernel_neon_begin_partial(8); 159 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
93 pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
94 kernel_neon_end();
95 } 160 }
96 put_unaligned_be64(ctx->digest[1], dst); 161 put_unaligned_be64(ctx->digest[1], dst);
97 put_unaligned_be64(ctx->digest[0], dst + 8); 162 put_unaligned_be64(ctx->digest[0], dst + 8);
@@ -100,16 +165,13 @@ static int ghash_final(struct shash_desc *desc, u8 *dst)
100 return 0; 165 return 0;
101} 166}
102 167
103static int ghash_setkey(struct crypto_shash *tfm, 168static int __ghash_setkey(struct ghash_key *key,
104 const u8 *inkey, unsigned int keylen) 169 const u8 *inkey, unsigned int keylen)
105{ 170{
106 struct ghash_key *key = crypto_shash_ctx(tfm);
107 u64 a, b; 171 u64 a, b;
108 172
109 if (keylen != GHASH_BLOCK_SIZE) { 173 /* needed for the fallback */
110 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 174 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
111 return -EINVAL;
112 }
113 175
114 /* perform multiplication by 'x' in GF(2^128) */ 176 /* perform multiplication by 'x' in GF(2^128) */
115 b = get_unaligned_be64(inkey); 177 b = get_unaligned_be64(inkey);
@@ -124,33 +186,418 @@ static int ghash_setkey(struct crypto_shash *tfm,
124 return 0; 186 return 0;
125} 187}
126 188
189static int ghash_setkey(struct crypto_shash *tfm,
190 const u8 *inkey, unsigned int keylen)
191{
192 struct ghash_key *key = crypto_shash_ctx(tfm);
193
194 if (keylen != GHASH_BLOCK_SIZE) {
195 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
196 return -EINVAL;
197 }
198
199 return __ghash_setkey(key, inkey, keylen);
200}
201
127static struct shash_alg ghash_alg = { 202static struct shash_alg ghash_alg = {
128 .digestsize = GHASH_DIGEST_SIZE, 203 .base.cra_name = "ghash",
129 .init = ghash_init, 204 .base.cra_driver_name = "ghash-ce",
130 .update = ghash_update, 205 .base.cra_priority = 200,
131 .final = ghash_final, 206 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
132 .setkey = ghash_setkey, 207 .base.cra_blocksize = GHASH_BLOCK_SIZE,
133 .descsize = sizeof(struct ghash_desc_ctx), 208 .base.cra_ctxsize = sizeof(struct ghash_key),
134 .base = { 209 .base.cra_module = THIS_MODULE,
135 .cra_name = "ghash", 210
136 .cra_driver_name = "ghash-ce", 211 .digestsize = GHASH_DIGEST_SIZE,
137 .cra_priority = 200, 212 .init = ghash_init,
138 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 213 .update = ghash_update,
139 .cra_blocksize = GHASH_BLOCK_SIZE, 214 .final = ghash_final,
140 .cra_ctxsize = sizeof(struct ghash_key), 215 .setkey = ghash_setkey,
141 .cra_module = THIS_MODULE, 216 .descsize = sizeof(struct ghash_desc_ctx),
142 }, 217};
218
219static int num_rounds(struct crypto_aes_ctx *ctx)
220{
221 /*
222 * # of rounds specified by AES:
223 * 128 bit key 10 rounds
224 * 192 bit key 12 rounds
225 * 256 bit key 14 rounds
226 * => n byte key => 6 + (n/4) rounds
227 */
228 return 6 + ctx->key_length / 4;
229}
230
231static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey,
232 unsigned int keylen)
233{
234 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
235 u8 key[GHASH_BLOCK_SIZE];
236 int ret;
237
238 ret = crypto_aes_expand_key(&ctx->aes_key, inkey, keylen);
239 if (ret) {
240 tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
241 return -EINVAL;
242 }
243
244 __aes_arm64_encrypt(ctx->aes_key.key_enc, key, (u8[AES_BLOCK_SIZE]){},
245 num_rounds(&ctx->aes_key));
246
247 return __ghash_setkey(&ctx->ghash_key, key, sizeof(key));
248}
249
250static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
251{
252 switch (authsize) {
253 case 4:
254 case 8:
255 case 12 ... 16:
256 break;
257 default:
258 return -EINVAL;
259 }
260 return 0;
261}
262
263static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
264 int *buf_count, struct gcm_aes_ctx *ctx)
265{
266 if (*buf_count > 0) {
267 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
268
269 memcpy(&buf[*buf_count], src, buf_added);
270
271 *buf_count += buf_added;
272 src += buf_added;
273 count -= buf_added;
274 }
275
276 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
277 int blocks = count / GHASH_BLOCK_SIZE;
278
279 ghash_do_update(blocks, dg, src, &ctx->ghash_key,
280 *buf_count ? buf : NULL);
281
282 src += blocks * GHASH_BLOCK_SIZE;
283 count %= GHASH_BLOCK_SIZE;
284 *buf_count = 0;
285 }
286
287 if (count > 0) {
288 memcpy(buf, src, count);
289 *buf_count = count;
290 }
291}
292
293static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])
294{
295 struct crypto_aead *aead = crypto_aead_reqtfm(req);
296 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
297 u8 buf[GHASH_BLOCK_SIZE];
298 struct scatter_walk walk;
299 u32 len = req->assoclen;
300 int buf_count = 0;
301
302 scatterwalk_start(&walk, req->src);
303
304 do {
305 u32 n = scatterwalk_clamp(&walk, len);
306 u8 *p;
307
308 if (!n) {
309 scatterwalk_start(&walk, sg_next(walk.sg));
310 n = scatterwalk_clamp(&walk, len);
311 }
312 p = scatterwalk_map(&walk);
313
314 gcm_update_mac(dg, p, n, buf, &buf_count, ctx);
315 len -= n;
316
317 scatterwalk_unmap(p);
318 scatterwalk_advance(&walk, n);
319 scatterwalk_done(&walk, 0, len);
320 } while (len);
321
322 if (buf_count) {
323 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
324 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL);
325 }
326}
327
328static void gcm_final(struct aead_request *req, struct gcm_aes_ctx *ctx,
329 u64 dg[], u8 tag[], int cryptlen)
330{
331 u8 mac[AES_BLOCK_SIZE];
332 u128 lengths;
333
334 lengths.a = cpu_to_be64(req->assoclen * 8);
335 lengths.b = cpu_to_be64(cryptlen * 8);
336
337 ghash_do_update(1, dg, (void *)&lengths, &ctx->ghash_key, NULL);
338
339 put_unaligned_be64(dg[1], mac);
340 put_unaligned_be64(dg[0], mac + 8);
341
342 crypto_xor(tag, mac, AES_BLOCK_SIZE);
343}
344
345static int gcm_encrypt(struct aead_request *req)
346{
347 struct crypto_aead *aead = crypto_aead_reqtfm(req);
348 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
349 struct skcipher_walk walk;
350 u8 iv[AES_BLOCK_SIZE];
351 u8 ks[AES_BLOCK_SIZE];
352 u8 tag[AES_BLOCK_SIZE];
353 u64 dg[2] = {};
354 int err;
355
356 if (req->assoclen)
357 gcm_calculate_auth_mac(req, dg);
358
359 memcpy(iv, req->iv, GCM_IV_SIZE);
360 put_unaligned_be32(1, iv + GCM_IV_SIZE);
361
362 if (likely(may_use_simd())) {
363 kernel_neon_begin();
364
365 pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc,
366 num_rounds(&ctx->aes_key));
367 put_unaligned_be32(2, iv + GCM_IV_SIZE);
368 pmull_gcm_encrypt_block(ks, iv, NULL,
369 num_rounds(&ctx->aes_key));
370 put_unaligned_be32(3, iv + GCM_IV_SIZE);
371
372 err = skcipher_walk_aead_encrypt(&walk, req, true);
373
374 while (walk.nbytes >= AES_BLOCK_SIZE) {
375 int blocks = walk.nbytes / AES_BLOCK_SIZE;
376
377 pmull_gcm_encrypt(blocks, dg, walk.dst.virt.addr,
378 walk.src.virt.addr, &ctx->ghash_key,
379 iv, num_rounds(&ctx->aes_key), ks);
380
381 err = skcipher_walk_done(&walk,
382 walk.nbytes % AES_BLOCK_SIZE);
383 }
384 kernel_neon_end();
385 } else {
386 __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv,
387 num_rounds(&ctx->aes_key));
388 put_unaligned_be32(2, iv + GCM_IV_SIZE);
389
390 err = skcipher_walk_aead_encrypt(&walk, req, true);
391
392 while (walk.nbytes >= AES_BLOCK_SIZE) {
393 int blocks = walk.nbytes / AES_BLOCK_SIZE;
394 u8 *dst = walk.dst.virt.addr;
395 u8 *src = walk.src.virt.addr;
396
397 do {
398 __aes_arm64_encrypt(ctx->aes_key.key_enc,
399 ks, iv,
400 num_rounds(&ctx->aes_key));
401 crypto_xor_cpy(dst, src, ks, AES_BLOCK_SIZE);
402 crypto_inc(iv, AES_BLOCK_SIZE);
403
404 dst += AES_BLOCK_SIZE;
405 src += AES_BLOCK_SIZE;
406 } while (--blocks > 0);
407
408 ghash_do_update(walk.nbytes / AES_BLOCK_SIZE, dg,
409 walk.dst.virt.addr, &ctx->ghash_key,
410 NULL);
411
412 err = skcipher_walk_done(&walk,
413 walk.nbytes % AES_BLOCK_SIZE);
414 }
415 if (walk.nbytes)
416 __aes_arm64_encrypt(ctx->aes_key.key_enc, ks, iv,
417 num_rounds(&ctx->aes_key));
418 }
419
420 /* handle the tail */
421 if (walk.nbytes) {
422 u8 buf[GHASH_BLOCK_SIZE];
423
424 crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, ks,
425 walk.nbytes);
426
427 memcpy(buf, walk.dst.virt.addr, walk.nbytes);
428 memset(buf + walk.nbytes, 0, GHASH_BLOCK_SIZE - walk.nbytes);
429 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL);
430
431 err = skcipher_walk_done(&walk, 0);
432 }
433
434 if (err)
435 return err;
436
437 gcm_final(req, ctx, dg, tag, req->cryptlen);
438
439 /* copy authtag to end of dst */
440 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
441 crypto_aead_authsize(aead), 1);
442
443 return 0;
444}
445
446static int gcm_decrypt(struct aead_request *req)
447{
448 struct crypto_aead *aead = crypto_aead_reqtfm(req);
449 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
450 unsigned int authsize = crypto_aead_authsize(aead);
451 struct skcipher_walk walk;
452 u8 iv[AES_BLOCK_SIZE];
453 u8 tag[AES_BLOCK_SIZE];
454 u8 buf[GHASH_BLOCK_SIZE];
455 u64 dg[2] = {};
456 int err;
457
458 if (req->assoclen)
459 gcm_calculate_auth_mac(req, dg);
460
461 memcpy(iv, req->iv, GCM_IV_SIZE);
462 put_unaligned_be32(1, iv + GCM_IV_SIZE);
463
464 if (likely(may_use_simd())) {
465 kernel_neon_begin();
466
467 pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc,
468 num_rounds(&ctx->aes_key));
469 put_unaligned_be32(2, iv + GCM_IV_SIZE);
470
471 err = skcipher_walk_aead_decrypt(&walk, req, true);
472
473 while (walk.nbytes >= AES_BLOCK_SIZE) {
474 int blocks = walk.nbytes / AES_BLOCK_SIZE;
475
476 pmull_gcm_decrypt(blocks, dg, walk.dst.virt.addr,
477 walk.src.virt.addr, &ctx->ghash_key,
478 iv, num_rounds(&ctx->aes_key));
479
480 err = skcipher_walk_done(&walk,
481 walk.nbytes % AES_BLOCK_SIZE);
482 }
483 if (walk.nbytes)
484 pmull_gcm_encrypt_block(iv, iv, NULL,
485 num_rounds(&ctx->aes_key));
486
487 kernel_neon_end();
488 } else {
489 __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv,
490 num_rounds(&ctx->aes_key));
491 put_unaligned_be32(2, iv + GCM_IV_SIZE);
492
493 err = skcipher_walk_aead_decrypt(&walk, req, true);
494
495 while (walk.nbytes >= AES_BLOCK_SIZE) {
496 int blocks = walk.nbytes / AES_BLOCK_SIZE;
497 u8 *dst = walk.dst.virt.addr;
498 u8 *src = walk.src.virt.addr;
499
500 ghash_do_update(blocks, dg, walk.src.virt.addr,
501 &ctx->ghash_key, NULL);
502
503 do {
504 __aes_arm64_encrypt(ctx->aes_key.key_enc,
505 buf, iv,
506 num_rounds(&ctx->aes_key));
507 crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
508 crypto_inc(iv, AES_BLOCK_SIZE);
509
510 dst += AES_BLOCK_SIZE;
511 src += AES_BLOCK_SIZE;
512 } while (--blocks > 0);
513
514 err = skcipher_walk_done(&walk,
515 walk.nbytes % AES_BLOCK_SIZE);
516 }
517 if (walk.nbytes)
518 __aes_arm64_encrypt(ctx->aes_key.key_enc, iv, iv,
519 num_rounds(&ctx->aes_key));
520 }
521
522 /* handle the tail */
523 if (walk.nbytes) {
524 memcpy(buf, walk.src.virt.addr, walk.nbytes);
525 memset(buf + walk.nbytes, 0, GHASH_BLOCK_SIZE - walk.nbytes);
526 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL);
527
528 crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, iv,
529 walk.nbytes);
530
531 err = skcipher_walk_done(&walk, 0);
532 }
533
534 if (err)
535 return err;
536
537 gcm_final(req, ctx, dg, tag, req->cryptlen - authsize);
538
539 /* compare calculated auth tag with the stored one */
540 scatterwalk_map_and_copy(buf, req->src,
541 req->assoclen + req->cryptlen - authsize,
542 authsize, 0);
543
544 if (crypto_memneq(tag, buf, authsize))
545 return -EBADMSG;
546 return 0;
547}
548
549static struct aead_alg gcm_aes_alg = {
550 .ivsize = GCM_IV_SIZE,
551 .chunksize = AES_BLOCK_SIZE,
552 .maxauthsize = AES_BLOCK_SIZE,
553 .setkey = gcm_setkey,
554 .setauthsize = gcm_setauthsize,
555 .encrypt = gcm_encrypt,
556 .decrypt = gcm_decrypt,
557
558 .base.cra_name = "gcm(aes)",
559 .base.cra_driver_name = "gcm-aes-ce",
560 .base.cra_priority = 300,
561 .base.cra_blocksize = 1,
562 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx),
563 .base.cra_module = THIS_MODULE,
143}; 564};
144 565
145static int __init ghash_ce_mod_init(void) 566static int __init ghash_ce_mod_init(void)
146{ 567{
147 return crypto_register_shash(&ghash_alg); 568 int ret;
569
570 if (!(elf_hwcap & HWCAP_ASIMD))
571 return -ENODEV;
572
573 if (elf_hwcap & HWCAP_PMULL)
574 pmull_ghash_update = pmull_ghash_update_p64;
575
576 else
577 pmull_ghash_update = pmull_ghash_update_p8;
578
579 ret = crypto_register_shash(&ghash_alg);
580 if (ret)
581 return ret;
582
583 if (elf_hwcap & HWCAP_PMULL) {
584 ret = crypto_register_aead(&gcm_aes_alg);
585 if (ret)
586 crypto_unregister_shash(&ghash_alg);
587 }
588 return ret;
148} 589}
149 590
150static void __exit ghash_ce_mod_exit(void) 591static void __exit ghash_ce_mod_exit(void)
151{ 592{
152 crypto_unregister_shash(&ghash_alg); 593 crypto_unregister_shash(&ghash_alg);
594 crypto_unregister_aead(&gcm_aes_alg);
153} 595}
154 596
155module_cpu_feature_match(PMULL, ghash_ce_mod_init); 597static const struct cpu_feature ghash_cpu_feature[] = {
598 { cpu_feature(PMULL) }, { }
599};
600MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);
601
602module_init(ghash_ce_mod_init);
156module_exit(ghash_ce_mod_exit); 603module_exit(ghash_ce_mod_exit);
diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c
index ea319c055f5d..efbeb3e0dcfb 100644
--- a/arch/arm64/crypto/sha1-ce-glue.c
+++ b/arch/arm64/crypto/sha1-ce-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions 2 * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions
3 * 3 *
4 * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -9,6 +9,7 @@
9 */ 9 */
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/simd.h>
12#include <asm/unaligned.h> 13#include <asm/unaligned.h>
13#include <crypto/internal/hash.h> 14#include <crypto/internal/hash.h>
14#include <crypto/sha.h> 15#include <crypto/sha.h>
@@ -37,8 +38,11 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data,
37{ 38{
38 struct sha1_ce_state *sctx = shash_desc_ctx(desc); 39 struct sha1_ce_state *sctx = shash_desc_ctx(desc);
39 40
41 if (!may_use_simd())
42 return crypto_sha1_update(desc, data, len);
43
40 sctx->finalize = 0; 44 sctx->finalize = 0;
41 kernel_neon_begin_partial(16); 45 kernel_neon_begin();
42 sha1_base_do_update(desc, data, len, 46 sha1_base_do_update(desc, data, len,
43 (sha1_block_fn *)sha1_ce_transform); 47 (sha1_block_fn *)sha1_ce_transform);
44 kernel_neon_end(); 48 kernel_neon_end();
@@ -52,13 +56,16 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
52 struct sha1_ce_state *sctx = shash_desc_ctx(desc); 56 struct sha1_ce_state *sctx = shash_desc_ctx(desc);
53 bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); 57 bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE);
54 58
59 if (!may_use_simd())
60 return crypto_sha1_finup(desc, data, len, out);
61
55 /* 62 /*
56 * Allow the asm code to perform the finalization if there is no 63 * Allow the asm code to perform the finalization if there is no
57 * partial data and the input is a round multiple of the block size. 64 * partial data and the input is a round multiple of the block size.
58 */ 65 */
59 sctx->finalize = finalize; 66 sctx->finalize = finalize;
60 67
61 kernel_neon_begin_partial(16); 68 kernel_neon_begin();
62 sha1_base_do_update(desc, data, len, 69 sha1_base_do_update(desc, data, len,
63 (sha1_block_fn *)sha1_ce_transform); 70 (sha1_block_fn *)sha1_ce_transform);
64 if (!finalize) 71 if (!finalize)
@@ -71,8 +78,11 @@ static int sha1_ce_final(struct shash_desc *desc, u8 *out)
71{ 78{
72 struct sha1_ce_state *sctx = shash_desc_ctx(desc); 79 struct sha1_ce_state *sctx = shash_desc_ctx(desc);
73 80
81 if (!may_use_simd())
82 return crypto_sha1_finup(desc, NULL, 0, out);
83
74 sctx->finalize = 0; 84 sctx->finalize = 0;
75 kernel_neon_begin_partial(16); 85 kernel_neon_begin();
76 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); 86 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform);
77 kernel_neon_end(); 87 kernel_neon_end();
78 return sha1_base_finish(desc, out); 88 return sha1_base_finish(desc, out);
diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c
index 0ed9486f75dd..fd1ff2b13dfa 100644
--- a/arch/arm64/crypto/sha2-ce-glue.c
+++ b/arch/arm64/crypto/sha2-ce-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions 2 * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions
3 * 3 *
4 * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -9,6 +9,7 @@
9 */ 9 */
10 10
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/simd.h>
12#include <asm/unaligned.h> 13#include <asm/unaligned.h>
13#include <crypto/internal/hash.h> 14#include <crypto/internal/hash.h>
14#include <crypto/sha.h> 15#include <crypto/sha.h>
@@ -34,13 +35,19 @@ const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state,
34const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, 35const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state,
35 finalize); 36 finalize);
36 37
38asmlinkage void sha256_block_data_order(u32 *digest, u8 const *src, int blocks);
39
37static int sha256_ce_update(struct shash_desc *desc, const u8 *data, 40static int sha256_ce_update(struct shash_desc *desc, const u8 *data,
38 unsigned int len) 41 unsigned int len)
39{ 42{
40 struct sha256_ce_state *sctx = shash_desc_ctx(desc); 43 struct sha256_ce_state *sctx = shash_desc_ctx(desc);
41 44
45 if (!may_use_simd())
46 return sha256_base_do_update(desc, data, len,
47 (sha256_block_fn *)sha256_block_data_order);
48
42 sctx->finalize = 0; 49 sctx->finalize = 0;
43 kernel_neon_begin_partial(28); 50 kernel_neon_begin();
44 sha256_base_do_update(desc, data, len, 51 sha256_base_do_update(desc, data, len,
45 (sha256_block_fn *)sha2_ce_transform); 52 (sha256_block_fn *)sha2_ce_transform);
46 kernel_neon_end(); 53 kernel_neon_end();
@@ -54,13 +61,22 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data,
54 struct sha256_ce_state *sctx = shash_desc_ctx(desc); 61 struct sha256_ce_state *sctx = shash_desc_ctx(desc);
55 bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); 62 bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE);
56 63
64 if (!may_use_simd()) {
65 if (len)
66 sha256_base_do_update(desc, data, len,
67 (sha256_block_fn *)sha256_block_data_order);
68 sha256_base_do_finalize(desc,
69 (sha256_block_fn *)sha256_block_data_order);
70 return sha256_base_finish(desc, out);
71 }
72
57 /* 73 /*
58 * Allow the asm code to perform the finalization if there is no 74 * Allow the asm code to perform the finalization if there is no
59 * partial data and the input is a round multiple of the block size. 75 * partial data and the input is a round multiple of the block size.
60 */ 76 */
61 sctx->finalize = finalize; 77 sctx->finalize = finalize;
62 78
63 kernel_neon_begin_partial(28); 79 kernel_neon_begin();
64 sha256_base_do_update(desc, data, len, 80 sha256_base_do_update(desc, data, len,
65 (sha256_block_fn *)sha2_ce_transform); 81 (sha256_block_fn *)sha2_ce_transform);
66 if (!finalize) 82 if (!finalize)
@@ -74,8 +90,14 @@ static int sha256_ce_final(struct shash_desc *desc, u8 *out)
74{ 90{
75 struct sha256_ce_state *sctx = shash_desc_ctx(desc); 91 struct sha256_ce_state *sctx = shash_desc_ctx(desc);
76 92
93 if (!may_use_simd()) {
94 sha256_base_do_finalize(desc,
95 (sha256_block_fn *)sha256_block_data_order);
96 return sha256_base_finish(desc, out);
97 }
98
77 sctx->finalize = 0; 99 sctx->finalize = 0;
78 kernel_neon_begin_partial(28); 100 kernel_neon_begin();
79 sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); 101 sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform);
80 kernel_neon_end(); 102 kernel_neon_end();
81 return sha256_base_finish(desc, out); 103 return sha256_base_finish(desc, out);
diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c
index a2226f841960..b064d925fe2a 100644
--- a/arch/arm64/crypto/sha256-glue.c
+++ b/arch/arm64/crypto/sha256-glue.c
@@ -29,6 +29,7 @@ MODULE_ALIAS_CRYPTO("sha256");
29 29
30asmlinkage void sha256_block_data_order(u32 *digest, const void *data, 30asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
31 unsigned int num_blks); 31 unsigned int num_blks);
32EXPORT_SYMBOL(sha256_block_data_order);
32 33
33asmlinkage void sha256_block_neon(u32 *digest, const void *data, 34asmlinkage void sha256_block_neon(u32 *digest, const void *data,
34 unsigned int num_blks); 35 unsigned int num_blks);
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index c90930de76ba..3cd4f6b198b6 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -344,8 +344,7 @@ static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx,
344 344
345 ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk, 345 ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
346 keystream, AES_BLOCK_SIZE); 346 keystream, AES_BLOCK_SIZE);
347 crypto_xor((u8 *) keystream, src, nbytes); 347 crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
348 memcpy(dst, keystream, nbytes);
349 crypto_inc(ctrblk, AES_BLOCK_SIZE); 348 crypto_inc(ctrblk, AES_BLOCK_SIZE);
350} 349}
351 350
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 4a55cdcdc008..5c15d6b57329 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -475,8 +475,8 @@ static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
475 unsigned int nbytes = walk->nbytes; 475 unsigned int nbytes = walk->nbytes;
476 476
477 aesni_enc(ctx, keystream, ctrblk); 477 aesni_enc(ctx, keystream, ctrblk);
478 crypto_xor(keystream, src, nbytes); 478 crypto_xor_cpy(dst, keystream, src, nbytes);
479 memcpy(dst, keystream, nbytes); 479
480 crypto_inc(ctrblk, AES_BLOCK_SIZE); 480 crypto_inc(ctrblk, AES_BLOCK_SIZE);
481} 481}
482 482
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
index 17c05531dfd1..f9eca34301e2 100644
--- a/arch/x86/crypto/blowfish_glue.c
+++ b/arch/x86/crypto/blowfish_glue.c
@@ -271,8 +271,7 @@ static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
271 unsigned int nbytes = walk->nbytes; 271 unsigned int nbytes = walk->nbytes;
272 272
273 blowfish_enc_blk(ctx, keystream, ctrblk); 273 blowfish_enc_blk(ctx, keystream, ctrblk);
274 crypto_xor(keystream, src, nbytes); 274 crypto_xor_cpy(dst, keystream, src, nbytes);
275 memcpy(dst, keystream, nbytes);
276 275
277 crypto_inc(ctrblk, BF_BLOCK_SIZE); 276 crypto_inc(ctrblk, BF_BLOCK_SIZE);
278} 277}
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
index 8648158f3916..dbea6020ffe7 100644
--- a/arch/x86/crypto/cast5_avx_glue.c
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -256,8 +256,7 @@ static void ctr_crypt_final(struct blkcipher_desc *desc,
256 unsigned int nbytes = walk->nbytes; 256 unsigned int nbytes = walk->nbytes;
257 257
258 __cast5_encrypt(ctx, keystream, ctrblk); 258 __cast5_encrypt(ctx, keystream, ctrblk);
259 crypto_xor(keystream, src, nbytes); 259 crypto_xor_cpy(dst, keystream, src, nbytes);
260 memcpy(dst, keystream, nbytes);
261 260
262 crypto_inc(ctrblk, CAST5_BLOCK_SIZE); 261 crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
263} 262}
diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c
index d6fc59aaaadf..30c0a37f4882 100644
--- a/arch/x86/crypto/des3_ede_glue.c
+++ b/arch/x86/crypto/des3_ede_glue.c
@@ -277,8 +277,7 @@ static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx,
277 unsigned int nbytes = walk->nbytes; 277 unsigned int nbytes = walk->nbytes;
278 278
279 des3_ede_enc_blk(ctx, keystream, ctrblk); 279 des3_ede_enc_blk(ctx, keystream, ctrblk);
280 crypto_xor(keystream, src, nbytes); 280 crypto_xor_cpy(dst, keystream, src, nbytes);
281 memcpy(dst, keystream, nbytes);
282 281
283 crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE); 282 crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE);
284} 283}
diff --git a/crypto/Kconfig b/crypto/Kconfig
index caa770e535a2..0a121f9ddf8e 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -1753,6 +1753,8 @@ config CRYPTO_USER_API_AEAD
1753 tristate "User-space interface for AEAD cipher algorithms" 1753 tristate "User-space interface for AEAD cipher algorithms"
1754 depends on NET 1754 depends on NET
1755 select CRYPTO_AEAD 1755 select CRYPTO_AEAD
1756 select CRYPTO_BLKCIPHER
1757 select CRYPTO_NULL
1756 select CRYPTO_USER_API 1758 select CRYPTO_USER_API
1757 help 1759 help
1758 This option enables the user-spaces interface for AEAD 1760 This option enables the user-spaces interface for AEAD
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 92a3d540d920..ffa9f4ccd9b4 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -21,6 +21,7 @@
21#include <linux/module.h> 21#include <linux/module.h>
22#include <linux/net.h> 22#include <linux/net.h>
23#include <linux/rwsem.h> 23#include <linux/rwsem.h>
24#include <linux/sched/signal.h>
24#include <linux/security.h> 25#include <linux/security.h>
25 26
26struct alg_type_list { 27struct alg_type_list {
@@ -507,6 +508,696 @@ void af_alg_complete(struct crypto_async_request *req, int err)
507} 508}
508EXPORT_SYMBOL_GPL(af_alg_complete); 509EXPORT_SYMBOL_GPL(af_alg_complete);
509 510
511/**
512 * af_alg_alloc_tsgl - allocate the TX SGL
513 *
514 * @sk socket of connection to user space
515 * @return: 0 upon success, < 0 upon error
516 */
517int af_alg_alloc_tsgl(struct sock *sk)
518{
519 struct alg_sock *ask = alg_sk(sk);
520 struct af_alg_ctx *ctx = ask->private;
521 struct af_alg_tsgl *sgl;
522 struct scatterlist *sg = NULL;
523
524 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
525 if (!list_empty(&ctx->tsgl_list))
526 sg = sgl->sg;
527
528 if (!sg || sgl->cur >= MAX_SGL_ENTS) {
529 sgl = sock_kmalloc(sk, sizeof(*sgl) +
530 sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
531 GFP_KERNEL);
532 if (!sgl)
533 return -ENOMEM;
534
535 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
536 sgl->cur = 0;
537
538 if (sg)
539 sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
540
541 list_add_tail(&sgl->list, &ctx->tsgl_list);
542 }
543
544 return 0;
545}
546EXPORT_SYMBOL_GPL(af_alg_alloc_tsgl);
547
548/**
549 * aead_count_tsgl - Count number of TX SG entries
550 *
551 * The counting starts from the beginning of the SGL to @bytes. If
552 * an offset is provided, the counting of the SG entries starts at the offset.
553 *
554 * @sk socket of connection to user space
555 * @bytes Count the number of SG entries holding given number of bytes.
556 * @offset Start the counting of SG entries from the given offset.
557 * @return Number of TX SG entries found given the constraints
558 */
559unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset)
560{
561 struct alg_sock *ask = alg_sk(sk);
562 struct af_alg_ctx *ctx = ask->private;
563 struct af_alg_tsgl *sgl, *tmp;
564 unsigned int i;
565 unsigned int sgl_count = 0;
566
567 if (!bytes)
568 return 0;
569
570 list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) {
571 struct scatterlist *sg = sgl->sg;
572
573 for (i = 0; i < sgl->cur; i++) {
574 size_t bytes_count;
575
576 /* Skip offset */
577 if (offset >= sg[i].length) {
578 offset -= sg[i].length;
579 bytes -= sg[i].length;
580 continue;
581 }
582
583 bytes_count = sg[i].length - offset;
584
585 offset = 0;
586 sgl_count++;
587
588 /* If we have seen requested number of bytes, stop */
589 if (bytes_count >= bytes)
590 return sgl_count;
591
592 bytes -= bytes_count;
593 }
594 }
595
596 return sgl_count;
597}
598EXPORT_SYMBOL_GPL(af_alg_count_tsgl);
599
600/**
601 * aead_pull_tsgl - Release the specified buffers from TX SGL
602 *
603 * If @dst is non-null, reassign the pages to dst. The caller must release
604 * the pages. If @dst_offset is given only reassign the pages to @dst starting
605 * at the @dst_offset (byte). The caller must ensure that @dst is large
606 * enough (e.g. by using af_alg_count_tsgl with the same offset).
607 *
608 * @sk socket of connection to user space
609 * @used Number of bytes to pull from TX SGL
610 * @dst If non-NULL, buffer is reassigned to dst SGL instead of releasing. The
611 * caller must release the buffers in dst.
612 * @dst_offset Reassign the TX SGL from given offset. All buffers before
613 * reaching the offset is released.
614 */
615void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst,
616 size_t dst_offset)
617{
618 struct alg_sock *ask = alg_sk(sk);
619 struct af_alg_ctx *ctx = ask->private;
620 struct af_alg_tsgl *sgl;
621 struct scatterlist *sg;
622 unsigned int i, j;
623
624 while (!list_empty(&ctx->tsgl_list)) {
625 sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl,
626 list);
627 sg = sgl->sg;
628
629 for (i = 0, j = 0; i < sgl->cur; i++) {
630 size_t plen = min_t(size_t, used, sg[i].length);
631 struct page *page = sg_page(sg + i);
632
633 if (!page)
634 continue;
635
636 /*
637 * Assumption: caller created af_alg_count_tsgl(len)
638 * SG entries in dst.
639 */
640 if (dst) {
641 if (dst_offset >= plen) {
642 /* discard page before offset */
643 dst_offset -= plen;
644 } else {
645 /* reassign page to dst after offset */
646 get_page(page);
647 sg_set_page(dst + j, page,
648 plen - dst_offset,
649 sg[i].offset + dst_offset);
650 dst_offset = 0;
651 j++;
652 }
653 }
654
655 sg[i].length -= plen;
656 sg[i].offset += plen;
657
658 used -= plen;
659 ctx->used -= plen;
660
661 if (sg[i].length)
662 return;
663
664 put_page(page);
665 sg_assign_page(sg + i, NULL);
666 }
667
668 list_del(&sgl->list);
669 sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) *
670 (MAX_SGL_ENTS + 1));
671 }
672
673 if (!ctx->used)
674 ctx->merge = 0;
675}
676EXPORT_SYMBOL_GPL(af_alg_pull_tsgl);
677
678/**
679 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request
680 *
681 * @areq Request holding the TX and RX SGL
682 */
683void af_alg_free_areq_sgls(struct af_alg_async_req *areq)
684{
685 struct sock *sk = areq->sk;
686 struct alg_sock *ask = alg_sk(sk);
687 struct af_alg_ctx *ctx = ask->private;
688 struct af_alg_rsgl *rsgl, *tmp;
689 struct scatterlist *tsgl;
690 struct scatterlist *sg;
691 unsigned int i;
692
693 list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
694 ctx->rcvused -= rsgl->sg_num_bytes;
695 af_alg_free_sg(&rsgl->sgl);
696 list_del(&rsgl->list);
697 if (rsgl != &areq->first_rsgl)
698 sock_kfree_s(sk, rsgl, sizeof(*rsgl));
699 }
700
701 tsgl = areq->tsgl;
702 for_each_sg(tsgl, sg, areq->tsgl_entries, i) {
703 if (!sg_page(sg))
704 continue;
705 put_page(sg_page(sg));
706 }
707
708 if (areq->tsgl && areq->tsgl_entries)
709 sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl));
710}
711EXPORT_SYMBOL_GPL(af_alg_free_areq_sgls);
712
713/**
714 * af_alg_wait_for_wmem - wait for availability of writable memory
715 *
716 * @sk socket of connection to user space
717 * @flags If MSG_DONTWAIT is set, then only report if function would sleep
718 * @return 0 when writable memory is available, < 0 upon error
719 */
720int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags)
721{
722 DEFINE_WAIT_FUNC(wait, woken_wake_function);
723 int err = -ERESTARTSYS;
724 long timeout;
725
726 if (flags & MSG_DONTWAIT)
727 return -EAGAIN;
728
729 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
730
731 add_wait_queue(sk_sleep(sk), &wait);
732 for (;;) {
733 if (signal_pending(current))
734 break;
735 timeout = MAX_SCHEDULE_TIMEOUT;
736 if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) {
737 err = 0;
738 break;
739 }
740 }
741 remove_wait_queue(sk_sleep(sk), &wait);
742
743 return err;
744}
745EXPORT_SYMBOL_GPL(af_alg_wait_for_wmem);
746
747/**
748 * af_alg_wmem_wakeup - wakeup caller when writable memory is available
749 *
750 * @sk socket of connection to user space
751 */
752void af_alg_wmem_wakeup(struct sock *sk)
753{
754 struct socket_wq *wq;
755
756 if (!af_alg_writable(sk))
757 return;
758
759 rcu_read_lock();
760 wq = rcu_dereference(sk->sk_wq);
761 if (skwq_has_sleeper(wq))
762 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
763 POLLRDNORM |
764 POLLRDBAND);
765 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
766 rcu_read_unlock();
767}
768EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup);
769
770/**
771 * af_alg_wait_for_data - wait for availability of TX data
772 *
773 * @sk socket of connection to user space
774 * @flags If MSG_DONTWAIT is set, then only report if function would sleep
775 * @return 0 when writable memory is available, < 0 upon error
776 */
777int af_alg_wait_for_data(struct sock *sk, unsigned flags)
778{
779 DEFINE_WAIT_FUNC(wait, woken_wake_function);
780 struct alg_sock *ask = alg_sk(sk);
781 struct af_alg_ctx *ctx = ask->private;
782 long timeout;
783 int err = -ERESTARTSYS;
784
785 if (flags & MSG_DONTWAIT)
786 return -EAGAIN;
787
788 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
789
790 add_wait_queue(sk_sleep(sk), &wait);
791 for (;;) {
792 if (signal_pending(current))
793 break;
794 timeout = MAX_SCHEDULE_TIMEOUT;
795 if (sk_wait_event(sk, &timeout, (ctx->used || !ctx->more),
796 &wait)) {
797 err = 0;
798 break;
799 }
800 }
801 remove_wait_queue(sk_sleep(sk), &wait);
802
803 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
804
805 return err;
806}
807EXPORT_SYMBOL_GPL(af_alg_wait_for_data);
808
809/**
810 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel
811 *
812 * @sk socket of connection to user space
813 */
814
815void af_alg_data_wakeup(struct sock *sk)
816{
817 struct alg_sock *ask = alg_sk(sk);
818 struct af_alg_ctx *ctx = ask->private;
819 struct socket_wq *wq;
820
821 if (!ctx->used)
822 return;
823
824 rcu_read_lock();
825 wq = rcu_dereference(sk->sk_wq);
826 if (skwq_has_sleeper(wq))
827 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
828 POLLRDNORM |
829 POLLRDBAND);
830 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
831 rcu_read_unlock();
832}
833EXPORT_SYMBOL_GPL(af_alg_data_wakeup);
834
835/**
836 * af_alg_sendmsg - implementation of sendmsg system call handler
837 *
838 * The sendmsg system call handler obtains the user data and stores it
839 * in ctx->tsgl_list. This implies allocation of the required numbers of
840 * struct af_alg_tsgl.
841 *
842 * In addition, the ctx is filled with the information sent via CMSG.
843 *
844 * @sock socket of connection to user space
845 * @msg message from user space
846 * @size size of message from user space
847 * @ivsize the size of the IV for the cipher operation to verify that the
848 * user-space-provided IV has the right size
849 * @return the number of copied data upon success, < 0 upon error
850 */
851int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
852 unsigned int ivsize)
853{
854 struct sock *sk = sock->sk;
855 struct alg_sock *ask = alg_sk(sk);
856 struct af_alg_ctx *ctx = ask->private;
857 struct af_alg_tsgl *sgl;
858 struct af_alg_control con = {};
859 long copied = 0;
860 bool enc = 0;
861 bool init = 0;
862 int err = 0;
863
864 if (msg->msg_controllen) {
865 err = af_alg_cmsg_send(msg, &con);
866 if (err)
867 return err;
868
869 init = 1;
870 switch (con.op) {
871 case ALG_OP_ENCRYPT:
872 enc = 1;
873 break;
874 case ALG_OP_DECRYPT:
875 enc = 0;
876 break;
877 default:
878 return -EINVAL;
879 }
880
881 if (con.iv && con.iv->ivlen != ivsize)
882 return -EINVAL;
883 }
884
885 lock_sock(sk);
886 if (!ctx->more && ctx->used) {
887 err = -EINVAL;
888 goto unlock;
889 }
890
891 if (init) {
892 ctx->enc = enc;
893 if (con.iv)
894 memcpy(ctx->iv, con.iv->iv, ivsize);
895
896 ctx->aead_assoclen = con.aead_assoclen;
897 }
898
899 while (size) {
900 struct scatterlist *sg;
901 size_t len = size;
902 size_t plen;
903
904 /* use the existing memory in an allocated page */
905 if (ctx->merge) {
906 sgl = list_entry(ctx->tsgl_list.prev,
907 struct af_alg_tsgl, list);
908 sg = sgl->sg + sgl->cur - 1;
909 len = min_t(size_t, len,
910 PAGE_SIZE - sg->offset - sg->length);
911
912 err = memcpy_from_msg(page_address(sg_page(sg)) +
913 sg->offset + sg->length,
914 msg, len);
915 if (err)
916 goto unlock;
917
918 sg->length += len;
919 ctx->merge = (sg->offset + sg->length) &
920 (PAGE_SIZE - 1);
921
922 ctx->used += len;
923 copied += len;
924 size -= len;
925 continue;
926 }
927
928 if (!af_alg_writable(sk)) {
929 err = af_alg_wait_for_wmem(sk, msg->msg_flags);
930 if (err)
931 goto unlock;
932 }
933
934 /* allocate a new page */
935 len = min_t(unsigned long, len, af_alg_sndbuf(sk));
936
937 err = af_alg_alloc_tsgl(sk);
938 if (err)
939 goto unlock;
940
941 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl,
942 list);
943 sg = sgl->sg;
944 if (sgl->cur)
945 sg_unmark_end(sg + sgl->cur - 1);
946
947 do {
948 unsigned int i = sgl->cur;
949
950 plen = min_t(size_t, len, PAGE_SIZE);
951
952 sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
953 if (!sg_page(sg + i)) {
954 err = -ENOMEM;
955 goto unlock;
956 }
957
958 err = memcpy_from_msg(page_address(sg_page(sg + i)),
959 msg, plen);
960 if (err) {
961 __free_page(sg_page(sg + i));
962 sg_assign_page(sg + i, NULL);
963 goto unlock;
964 }
965
966 sg[i].length = plen;
967 len -= plen;
968 ctx->used += plen;
969 copied += plen;
970 size -= plen;
971 sgl->cur++;
972 } while (len && sgl->cur < MAX_SGL_ENTS);
973
974 if (!size)
975 sg_mark_end(sg + sgl->cur - 1);
976
977 ctx->merge = plen & (PAGE_SIZE - 1);
978 }
979
980 err = 0;
981
982 ctx->more = msg->msg_flags & MSG_MORE;
983
984unlock:
985 af_alg_data_wakeup(sk);
986 release_sock(sk);
987
988 return copied ?: err;
989}
990EXPORT_SYMBOL_GPL(af_alg_sendmsg);
991
992/**
993 * af_alg_sendpage - sendpage system call handler
994 *
995 * This is a generic implementation of sendpage to fill ctx->tsgl_list.
996 */
997ssize_t af_alg_sendpage(struct socket *sock, struct page *page,
998 int offset, size_t size, int flags)
999{
1000 struct sock *sk = sock->sk;
1001 struct alg_sock *ask = alg_sk(sk);
1002 struct af_alg_ctx *ctx = ask->private;
1003 struct af_alg_tsgl *sgl;
1004 int err = -EINVAL;
1005
1006 if (flags & MSG_SENDPAGE_NOTLAST)
1007 flags |= MSG_MORE;
1008
1009 lock_sock(sk);
1010 if (!ctx->more && ctx->used)
1011 goto unlock;
1012
1013 if (!size)
1014 goto done;
1015
1016 if (!af_alg_writable(sk)) {
1017 err = af_alg_wait_for_wmem(sk, flags);
1018 if (err)
1019 goto unlock;
1020 }
1021
1022 err = af_alg_alloc_tsgl(sk);
1023 if (err)
1024 goto unlock;
1025
1026 ctx->merge = 0;
1027 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
1028
1029 if (sgl->cur)
1030 sg_unmark_end(sgl->sg + sgl->cur - 1);
1031
1032 sg_mark_end(sgl->sg + sgl->cur);
1033
1034 get_page(page);
1035 sg_set_page(sgl->sg + sgl->cur, page, size, offset);
1036 sgl->cur++;
1037 ctx->used += size;
1038
1039done:
1040 ctx->more = flags & MSG_MORE;
1041
1042unlock:
1043 af_alg_data_wakeup(sk);
1044 release_sock(sk);
1045
1046 return err ?: size;
1047}
1048EXPORT_SYMBOL_GPL(af_alg_sendpage);
1049
1050/**
1051 * af_alg_async_cb - AIO callback handler
1052 *
1053 * This handler cleans up the struct af_alg_async_req upon completion of the
1054 * AIO operation.
1055 *
1056 * The number of bytes to be generated with the AIO operation must be set
1057 * in areq->outlen before the AIO callback handler is invoked.
1058 */
1059void af_alg_async_cb(struct crypto_async_request *_req, int err)
1060{
1061 struct af_alg_async_req *areq = _req->data;
1062 struct sock *sk = areq->sk;
1063 struct kiocb *iocb = areq->iocb;
1064 unsigned int resultlen;
1065
1066 lock_sock(sk);
1067
1068 /* Buffer size written by crypto operation. */
1069 resultlen = areq->outlen;
1070
1071 af_alg_free_areq_sgls(areq);
1072 sock_kfree_s(sk, areq, areq->areqlen);
1073 __sock_put(sk);
1074
1075 iocb->ki_complete(iocb, err ? err : resultlen, 0);
1076
1077 release_sock(sk);
1078}
1079EXPORT_SYMBOL_GPL(af_alg_async_cb);
1080
1081/**
1082 * af_alg_poll - poll system call handler
1083 */
1084unsigned int af_alg_poll(struct file *file, struct socket *sock,
1085 poll_table *wait)
1086{
1087 struct sock *sk = sock->sk;
1088 struct alg_sock *ask = alg_sk(sk);
1089 struct af_alg_ctx *ctx = ask->private;
1090 unsigned int mask;
1091
1092 sock_poll_wait(file, sk_sleep(sk), wait);
1093 mask = 0;
1094
1095 if (!ctx->more || ctx->used)
1096 mask |= POLLIN | POLLRDNORM;
1097
1098 if (af_alg_writable(sk))
1099 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
1100
1101 return mask;
1102}
1103EXPORT_SYMBOL_GPL(af_alg_poll);
1104
1105/**
1106 * af_alg_alloc_areq - allocate struct af_alg_async_req
1107 *
1108 * @sk socket of connection to user space
1109 * @areqlen size of struct af_alg_async_req + crypto_*_reqsize
1110 * @return allocated data structure or ERR_PTR upon error
1111 */
1112struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk,
1113 unsigned int areqlen)
1114{
1115 struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
1116
1117 if (unlikely(!areq))
1118 return ERR_PTR(-ENOMEM);
1119
1120 areq->areqlen = areqlen;
1121 areq->sk = sk;
1122 areq->last_rsgl = NULL;
1123 INIT_LIST_HEAD(&areq->rsgl_list);
1124 areq->tsgl = NULL;
1125 areq->tsgl_entries = 0;
1126
1127 return areq;
1128}
1129EXPORT_SYMBOL_GPL(af_alg_alloc_areq);
1130
1131/**
1132 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto
1133 * operation
1134 *
1135 * @sk socket of connection to user space
1136 * @msg user space message
1137 * @flags flags used to invoke recvmsg with
1138 * @areq instance of the cryptographic request that will hold the RX SGL
1139 * @maxsize maximum number of bytes to be pulled from user space
1140 * @outlen number of bytes in the RX SGL
1141 * @return 0 on success, < 0 upon error
1142 */
1143int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags,
1144 struct af_alg_async_req *areq, size_t maxsize,
1145 size_t *outlen)
1146{
1147 struct alg_sock *ask = alg_sk(sk);
1148 struct af_alg_ctx *ctx = ask->private;
1149 size_t len = 0;
1150
1151 while (maxsize > len && msg_data_left(msg)) {
1152 struct af_alg_rsgl *rsgl;
1153 size_t seglen;
1154 int err;
1155
1156 /* limit the amount of readable buffers */
1157 if (!af_alg_readable(sk))
1158 break;
1159
1160 if (!ctx->used) {
1161 err = af_alg_wait_for_data(sk, flags);
1162 if (err)
1163 return err;
1164 }
1165
1166 seglen = min_t(size_t, (maxsize - len),
1167 msg_data_left(msg));
1168
1169 if (list_empty(&areq->rsgl_list)) {
1170 rsgl = &areq->first_rsgl;
1171 } else {
1172 rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
1173 if (unlikely(!rsgl))
1174 return -ENOMEM;
1175 }
1176
1177 rsgl->sgl.npages = 0;
1178 list_add_tail(&rsgl->list, &areq->rsgl_list);
1179
1180 /* make one iovec available as scatterlist */
1181 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
1182 if (err < 0)
1183 return err;
1184
1185 /* chain the new scatterlist with previous one */
1186 if (areq->last_rsgl)
1187 af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl);
1188
1189 areq->last_rsgl = rsgl;
1190 len += err;
1191 ctx->rcvused += err;
1192 rsgl->sg_num_bytes = err;
1193 iov_iter_advance(&msg->msg_iter, err);
1194 }
1195
1196 *outlen = len;
1197 return 0;
1198}
1199EXPORT_SYMBOL_GPL(af_alg_get_rsgl);
1200
510static int __init af_alg_init(void) 1201static int __init af_alg_init(void)
511{ 1202{
512 int err = proto_register(&alg_proto, 0); 1203 int err = proto_register(&alg_proto, 0);
diff --git a/crypto/ahash.c b/crypto/ahash.c
index 826cd7ab4d4a..5e8666e6ccae 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -588,6 +588,35 @@ int crypto_unregister_ahash(struct ahash_alg *alg)
588} 588}
589EXPORT_SYMBOL_GPL(crypto_unregister_ahash); 589EXPORT_SYMBOL_GPL(crypto_unregister_ahash);
590 590
591int crypto_register_ahashes(struct ahash_alg *algs, int count)
592{
593 int i, ret;
594
595 for (i = 0; i < count; i++) {
596 ret = crypto_register_ahash(&algs[i]);
597 if (ret)
598 goto err;
599 }
600
601 return 0;
602
603err:
604 for (--i; i >= 0; --i)
605 crypto_unregister_ahash(&algs[i]);
606
607 return ret;
608}
609EXPORT_SYMBOL_GPL(crypto_register_ahashes);
610
611void crypto_unregister_ahashes(struct ahash_alg *algs, int count)
612{
613 int i;
614
615 for (i = count - 1; i >= 0; --i)
616 crypto_unregister_ahash(&algs[i]);
617}
618EXPORT_SYMBOL_GPL(crypto_unregister_ahashes);
619
591int ahash_register_instance(struct crypto_template *tmpl, 620int ahash_register_instance(struct crypto_template *tmpl,
592 struct ahash_instance *inst) 621 struct ahash_instance *inst)
593{ 622{
diff --git a/crypto/algapi.c b/crypto/algapi.c
index e4cc7615a139..aa699ff6c876 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -975,13 +975,15 @@ void crypto_inc(u8 *a, unsigned int size)
975} 975}
976EXPORT_SYMBOL_GPL(crypto_inc); 976EXPORT_SYMBOL_GPL(crypto_inc);
977 977
978void __crypto_xor(u8 *dst, const u8 *src, unsigned int len) 978void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len)
979{ 979{
980 int relalign = 0; 980 int relalign = 0;
981 981
982 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { 982 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
983 int size = sizeof(unsigned long); 983 int size = sizeof(unsigned long);
984 int d = ((unsigned long)dst ^ (unsigned long)src) & (size - 1); 984 int d = (((unsigned long)dst ^ (unsigned long)src1) |
985 ((unsigned long)dst ^ (unsigned long)src2)) &
986 (size - 1);
985 987
986 relalign = d ? 1 << __ffs(d) : size; 988 relalign = d ? 1 << __ffs(d) : size;
987 989
@@ -992,34 +994,37 @@ void __crypto_xor(u8 *dst, const u8 *src, unsigned int len)
992 * process the remainder of the input using optimal strides. 994 * process the remainder of the input using optimal strides.
993 */ 995 */
994 while (((unsigned long)dst & (relalign - 1)) && len > 0) { 996 while (((unsigned long)dst & (relalign - 1)) && len > 0) {
995 *dst++ ^= *src++; 997 *dst++ = *src1++ ^ *src2++;
996 len--; 998 len--;
997 } 999 }
998 } 1000 }
999 1001
1000 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { 1002 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) {
1001 *(u64 *)dst ^= *(u64 *)src; 1003 *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2;
1002 dst += 8; 1004 dst += 8;
1003 src += 8; 1005 src1 += 8;
1006 src2 += 8;
1004 len -= 8; 1007 len -= 8;
1005 } 1008 }
1006 1009
1007 while (len >= 4 && !(relalign & 3)) { 1010 while (len >= 4 && !(relalign & 3)) {
1008 *(u32 *)dst ^= *(u32 *)src; 1011 *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2;
1009 dst += 4; 1012 dst += 4;
1010 src += 4; 1013 src1 += 4;
1014 src2 += 4;
1011 len -= 4; 1015 len -= 4;
1012 } 1016 }
1013 1017
1014 while (len >= 2 && !(relalign & 1)) { 1018 while (len >= 2 && !(relalign & 1)) {
1015 *(u16 *)dst ^= *(u16 *)src; 1019 *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2;
1016 dst += 2; 1020 dst += 2;
1017 src += 2; 1021 src1 += 2;
1022 src2 += 2;
1018 len -= 2; 1023 len -= 2;
1019 } 1024 }
1020 1025
1021 while (len--) 1026 while (len--)
1022 *dst++ ^= *src++; 1027 *dst++ = *src1++ ^ *src2++;
1023} 1028}
1024EXPORT_SYMBOL_GPL(__crypto_xor); 1029EXPORT_SYMBOL_GPL(__crypto_xor);
1025 1030
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index be117495eb43..516b38c3a169 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -5,88 +5,56 @@
5 * 5 *
6 * This file provides the user-space API for AEAD ciphers. 6 * This file provides the user-space API for AEAD ciphers.
7 * 7 *
8 * This file is derived from algif_skcipher.c.
9 *
10 * This program is free software; you can redistribute it and/or modify it 8 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 9 * under the terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 2 of the License, or (at your option) 10 * Software Foundation; either version 2 of the License, or (at your option)
13 * any later version. 11 * any later version.
12 *
13 * The following concept of the memory management is used:
14 *
15 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
16 * filled by user space with the data submitted via sendpage/sendmsg. Filling
17 * up the TX SGL does not cause a crypto operation -- the data will only be
18 * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
19 * provide a buffer which is tracked with the RX SGL.
20 *
21 * During the processing of the recvmsg operation, the cipher request is
22 * allocated and prepared. As part of the recvmsg operation, the processed
23 * TX buffers are extracted from the TX SGL into a separate SGL.
24 *
25 * After the completion of the crypto operation, the RX SGL and the cipher
26 * request is released. The extracted TX SGL parts are released together with
27 * the RX SGL release.
14 */ 28 */
15 29
16#include <crypto/internal/aead.h> 30#include <crypto/internal/aead.h>
17#include <crypto/scatterwalk.h> 31#include <crypto/scatterwalk.h>
18#include <crypto/if_alg.h> 32#include <crypto/if_alg.h>
33#include <crypto/skcipher.h>
34#include <crypto/null.h>
19#include <linux/init.h> 35#include <linux/init.h>
20#include <linux/list.h> 36#include <linux/list.h>
21#include <linux/kernel.h> 37#include <linux/kernel.h>
22#include <linux/sched/signal.h>
23#include <linux/mm.h> 38#include <linux/mm.h>
24#include <linux/module.h> 39#include <linux/module.h>
25#include <linux/net.h> 40#include <linux/net.h>
26#include <net/sock.h> 41#include <net/sock.h>
27 42
28struct aead_sg_list {
29 unsigned int cur;
30 struct scatterlist sg[ALG_MAX_PAGES];
31};
32
33struct aead_async_rsgl {
34 struct af_alg_sgl sgl;
35 struct list_head list;
36};
37
38struct aead_async_req {
39 struct scatterlist *tsgl;
40 struct aead_async_rsgl first_rsgl;
41 struct list_head list;
42 struct kiocb *iocb;
43 struct sock *sk;
44 unsigned int tsgls;
45 char iv[];
46};
47
48struct aead_tfm { 43struct aead_tfm {
49 struct crypto_aead *aead; 44 struct crypto_aead *aead;
50 bool has_key; 45 bool has_key;
46 struct crypto_skcipher *null_tfm;
51}; 47};
52 48
53struct aead_ctx { 49static inline bool aead_sufficient_data(struct sock *sk)
54 struct aead_sg_list tsgl;
55 struct aead_async_rsgl first_rsgl;
56 struct list_head list;
57
58 void *iv;
59
60 struct af_alg_completion completion;
61
62 unsigned long used;
63
64 unsigned int len;
65 bool more;
66 bool merge;
67 bool enc;
68
69 size_t aead_assoclen;
70 struct aead_request aead_req;
71};
72
73static inline int aead_sndbuf(struct sock *sk)
74{ 50{
75 struct alg_sock *ask = alg_sk(sk); 51 struct alg_sock *ask = alg_sk(sk);
76 struct aead_ctx *ctx = ask->private; 52 struct sock *psk = ask->parent;
77 53 struct alg_sock *pask = alg_sk(psk);
78 return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - 54 struct af_alg_ctx *ctx = ask->private;
79 ctx->used, 0); 55 struct aead_tfm *aeadc = pask->private;
80} 56 struct crypto_aead *tfm = aeadc->aead;
81 57 unsigned int as = crypto_aead_authsize(tfm);
82static inline bool aead_writable(struct sock *sk)
83{
84 return PAGE_SIZE <= aead_sndbuf(sk);
85}
86
87static inline bool aead_sufficient_data(struct aead_ctx *ctx)
88{
89 unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
90 58
91 /* 59 /*
92 * The minimum amount of memory needed for an AEAD cipher is 60 * The minimum amount of memory needed for an AEAD cipher is
@@ -95,484 +63,58 @@ static inline bool aead_sufficient_data(struct aead_ctx *ctx)
95 return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as); 63 return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
96} 64}
97 65
98static void aead_reset_ctx(struct aead_ctx *ctx)
99{
100 struct aead_sg_list *sgl = &ctx->tsgl;
101
102 sg_init_table(sgl->sg, ALG_MAX_PAGES);
103 sgl->cur = 0;
104 ctx->used = 0;
105 ctx->more = 0;
106 ctx->merge = 0;
107}
108
109static void aead_put_sgl(struct sock *sk)
110{
111 struct alg_sock *ask = alg_sk(sk);
112 struct aead_ctx *ctx = ask->private;
113 struct aead_sg_list *sgl = &ctx->tsgl;
114 struct scatterlist *sg = sgl->sg;
115 unsigned int i;
116
117 for (i = 0; i < sgl->cur; i++) {
118 if (!sg_page(sg + i))
119 continue;
120
121 put_page(sg_page(sg + i));
122 sg_assign_page(sg + i, NULL);
123 }
124 aead_reset_ctx(ctx);
125}
126
127static void aead_wmem_wakeup(struct sock *sk)
128{
129 struct socket_wq *wq;
130
131 if (!aead_writable(sk))
132 return;
133
134 rcu_read_lock();
135 wq = rcu_dereference(sk->sk_wq);
136 if (skwq_has_sleeper(wq))
137 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
138 POLLRDNORM |
139 POLLRDBAND);
140 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
141 rcu_read_unlock();
142}
143
144static int aead_wait_for_data(struct sock *sk, unsigned flags)
145{
146 DEFINE_WAIT_FUNC(wait, woken_wake_function);
147 struct alg_sock *ask = alg_sk(sk);
148 struct aead_ctx *ctx = ask->private;
149 long timeout;
150 int err = -ERESTARTSYS;
151
152 if (flags & MSG_DONTWAIT)
153 return -EAGAIN;
154
155 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
156 add_wait_queue(sk_sleep(sk), &wait);
157 for (;;) {
158 if (signal_pending(current))
159 break;
160 timeout = MAX_SCHEDULE_TIMEOUT;
161 if (sk_wait_event(sk, &timeout, !ctx->more, &wait)) {
162 err = 0;
163 break;
164 }
165 }
166 remove_wait_queue(sk_sleep(sk), &wait);
167
168 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
169
170 return err;
171}
172
173static void aead_data_wakeup(struct sock *sk)
174{
175 struct alg_sock *ask = alg_sk(sk);
176 struct aead_ctx *ctx = ask->private;
177 struct socket_wq *wq;
178
179 if (ctx->more)
180 return;
181 if (!ctx->used)
182 return;
183
184 rcu_read_lock();
185 wq = rcu_dereference(sk->sk_wq);
186 if (skwq_has_sleeper(wq))
187 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
188 POLLRDNORM |
189 POLLRDBAND);
190 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
191 rcu_read_unlock();
192}
193
194static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) 66static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
195{ 67{
196 struct sock *sk = sock->sk; 68 struct sock *sk = sock->sk;
197 struct alg_sock *ask = alg_sk(sk); 69 struct alg_sock *ask = alg_sk(sk);
198 struct aead_ctx *ctx = ask->private; 70 struct sock *psk = ask->parent;
199 unsigned ivsize = 71 struct alg_sock *pask = alg_sk(psk);
200 crypto_aead_ivsize(crypto_aead_reqtfm(&ctx->aead_req)); 72 struct aead_tfm *aeadc = pask->private;
201 struct aead_sg_list *sgl = &ctx->tsgl; 73 struct crypto_aead *tfm = aeadc->aead;
202 struct af_alg_control con = {}; 74 unsigned int ivsize = crypto_aead_ivsize(tfm);
203 long copied = 0;
204 bool enc = 0;
205 bool init = 0;
206 int err = -EINVAL;
207
208 if (msg->msg_controllen) {
209 err = af_alg_cmsg_send(msg, &con);
210 if (err)
211 return err;
212
213 init = 1;
214 switch (con.op) {
215 case ALG_OP_ENCRYPT:
216 enc = 1;
217 break;
218 case ALG_OP_DECRYPT:
219 enc = 0;
220 break;
221 default:
222 return -EINVAL;
223 }
224
225 if (con.iv && con.iv->ivlen != ivsize)
226 return -EINVAL;
227 }
228
229 lock_sock(sk);
230 if (!ctx->more && ctx->used)
231 goto unlock;
232
233 if (init) {
234 ctx->enc = enc;
235 if (con.iv)
236 memcpy(ctx->iv, con.iv->iv, ivsize);
237
238 ctx->aead_assoclen = con.aead_assoclen;
239 }
240
241 while (size) {
242 size_t len = size;
243 struct scatterlist *sg = NULL;
244
245 /* use the existing memory in an allocated page */
246 if (ctx->merge) {
247 sg = sgl->sg + sgl->cur - 1;
248 len = min_t(unsigned long, len,
249 PAGE_SIZE - sg->offset - sg->length);
250 err = memcpy_from_msg(page_address(sg_page(sg)) +
251 sg->offset + sg->length,
252 msg, len);
253 if (err)
254 goto unlock;
255
256 sg->length += len;
257 ctx->merge = (sg->offset + sg->length) &
258 (PAGE_SIZE - 1);
259
260 ctx->used += len;
261 copied += len;
262 size -= len;
263 continue;
264 }
265
266 if (!aead_writable(sk)) {
267 /* user space sent too much data */
268 aead_put_sgl(sk);
269 err = -EMSGSIZE;
270 goto unlock;
271 }
272
273 /* allocate a new page */
274 len = min_t(unsigned long, size, aead_sndbuf(sk));
275 while (len) {
276 size_t plen = 0;
277
278 if (sgl->cur >= ALG_MAX_PAGES) {
279 aead_put_sgl(sk);
280 err = -E2BIG;
281 goto unlock;
282 }
283
284 sg = sgl->sg + sgl->cur;
285 plen = min_t(size_t, len, PAGE_SIZE);
286
287 sg_assign_page(sg, alloc_page(GFP_KERNEL));
288 err = -ENOMEM;
289 if (!sg_page(sg))
290 goto unlock;
291
292 err = memcpy_from_msg(page_address(sg_page(sg)),
293 msg, plen);
294 if (err) {
295 __free_page(sg_page(sg));
296 sg_assign_page(sg, NULL);
297 goto unlock;
298 }
299
300 sg->offset = 0;
301 sg->length = plen;
302 len -= plen;
303 ctx->used += plen;
304 copied += plen;
305 sgl->cur++;
306 size -= plen;
307 ctx->merge = plen & (PAGE_SIZE - 1);
308 }
309 }
310
311 err = 0;
312
313 ctx->more = msg->msg_flags & MSG_MORE;
314 if (!ctx->more && !aead_sufficient_data(ctx)) {
315 aead_put_sgl(sk);
316 err = -EMSGSIZE;
317 }
318
319unlock:
320 aead_data_wakeup(sk);
321 release_sock(sk);
322
323 return err ?: copied;
324}
325
326static ssize_t aead_sendpage(struct socket *sock, struct page *page,
327 int offset, size_t size, int flags)
328{
329 struct sock *sk = sock->sk;
330 struct alg_sock *ask = alg_sk(sk);
331 struct aead_ctx *ctx = ask->private;
332 struct aead_sg_list *sgl = &ctx->tsgl;
333 int err = -EINVAL;
334
335 if (flags & MSG_SENDPAGE_NOTLAST)
336 flags |= MSG_MORE;
337
338 if (sgl->cur >= ALG_MAX_PAGES)
339 return -E2BIG;
340
341 lock_sock(sk);
342 if (!ctx->more && ctx->used)
343 goto unlock;
344
345 if (!size)
346 goto done;
347
348 if (!aead_writable(sk)) {
349 /* user space sent too much data */
350 aead_put_sgl(sk);
351 err = -EMSGSIZE;
352 goto unlock;
353 }
354
355 ctx->merge = 0;
356
357 get_page(page);
358 sg_set_page(sgl->sg + sgl->cur, page, size, offset);
359 sgl->cur++;
360 ctx->used += size;
361
362 err = 0;
363
364done:
365 ctx->more = flags & MSG_MORE;
366 if (!ctx->more && !aead_sufficient_data(ctx)) {
367 aead_put_sgl(sk);
368 err = -EMSGSIZE;
369 }
370 75
371unlock: 76 return af_alg_sendmsg(sock, msg, size, ivsize);
372 aead_data_wakeup(sk);
373 release_sock(sk);
374
375 return err ?: size;
376} 77}
377 78
378#define GET_ASYM_REQ(req, tfm) (struct aead_async_req *) \ 79static int crypto_aead_copy_sgl(struct crypto_skcipher *null_tfm,
379 ((char *)req + sizeof(struct aead_request) + \ 80 struct scatterlist *src,
380 crypto_aead_reqsize(tfm)) 81 struct scatterlist *dst, unsigned int len)
381
382 #define GET_REQ_SIZE(tfm) sizeof(struct aead_async_req) + \
383 crypto_aead_reqsize(tfm) + crypto_aead_ivsize(tfm) + \
384 sizeof(struct aead_request)
385
386static void aead_async_cb(struct crypto_async_request *_req, int err)
387{ 82{
388 struct aead_request *req = _req->data; 83 SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
389 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
390 struct aead_async_req *areq = GET_ASYM_REQ(req, tfm);
391 struct sock *sk = areq->sk;
392 struct scatterlist *sg = areq->tsgl;
393 struct aead_async_rsgl *rsgl;
394 struct kiocb *iocb = areq->iocb;
395 unsigned int i, reqlen = GET_REQ_SIZE(tfm);
396
397 list_for_each_entry(rsgl, &areq->list, list) {
398 af_alg_free_sg(&rsgl->sgl);
399 if (rsgl != &areq->first_rsgl)
400 sock_kfree_s(sk, rsgl, sizeof(*rsgl));
401 }
402 84
403 for (i = 0; i < areq->tsgls; i++) 85 skcipher_request_set_tfm(skreq, null_tfm);
404 put_page(sg_page(sg + i)); 86 skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_BACKLOG,
87 NULL, NULL);
88 skcipher_request_set_crypt(skreq, src, dst, len, NULL);
405 89
406 sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls); 90 return crypto_skcipher_encrypt(skreq);
407 sock_kfree_s(sk, req, reqlen);
408 __sock_put(sk);
409 iocb->ki_complete(iocb, err, err);
410} 91}
411 92
412static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg, 93static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
413 int flags) 94 size_t ignored, int flags)
414{ 95{
415 struct sock *sk = sock->sk; 96 struct sock *sk = sock->sk;
416 struct alg_sock *ask = alg_sk(sk); 97 struct alg_sock *ask = alg_sk(sk);
417 struct aead_ctx *ctx = ask->private; 98 struct sock *psk = ask->parent;
418 struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req); 99 struct alg_sock *pask = alg_sk(psk);
419 struct aead_async_req *areq; 100 struct af_alg_ctx *ctx = ask->private;
420 struct aead_request *req = NULL; 101 struct aead_tfm *aeadc = pask->private;
421 struct aead_sg_list *sgl = &ctx->tsgl; 102 struct crypto_aead *tfm = aeadc->aead;
422 struct aead_async_rsgl *last_rsgl = NULL, *rsgl; 103 struct crypto_skcipher *null_tfm = aeadc->null_tfm;
423 unsigned int as = crypto_aead_authsize(tfm); 104 unsigned int as = crypto_aead_authsize(tfm);
424 unsigned int i, reqlen = GET_REQ_SIZE(tfm); 105 struct af_alg_async_req *areq;
425 int err = -ENOMEM; 106 struct af_alg_tsgl *tsgl;
426 unsigned long used; 107 struct scatterlist *src;
427 size_t outlen = 0; 108 int err = 0;
428 size_t usedpages = 0; 109 size_t used = 0; /* [in] TX bufs to be en/decrypted */
429 110 size_t outlen = 0; /* [out] RX bufs produced by kernel */
430 lock_sock(sk); 111 size_t usedpages = 0; /* [in] RX bufs to be used from user */
431 if (ctx->more) { 112 size_t processed = 0; /* [in] TX bufs to be consumed */
432 err = aead_wait_for_data(sk, flags);
433 if (err)
434 goto unlock;
435 }
436
437 if (!aead_sufficient_data(ctx))
438 goto unlock;
439
440 used = ctx->used;
441 if (ctx->enc)
442 outlen = used + as;
443 else
444 outlen = used - as;
445
446 req = sock_kmalloc(sk, reqlen, GFP_KERNEL);
447 if (unlikely(!req))
448 goto unlock;
449
450 areq = GET_ASYM_REQ(req, tfm);
451 memset(&areq->first_rsgl, '\0', sizeof(areq->first_rsgl));
452 INIT_LIST_HEAD(&areq->list);
453 areq->iocb = msg->msg_iocb;
454 areq->sk = sk;
455 memcpy(areq->iv, ctx->iv, crypto_aead_ivsize(tfm));
456 aead_request_set_tfm(req, tfm);
457 aead_request_set_ad(req, ctx->aead_assoclen);
458 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
459 aead_async_cb, req);
460 used -= ctx->aead_assoclen;
461
462 /* take over all tx sgls from ctx */
463 areq->tsgl = sock_kmalloc(sk,
464 sizeof(*areq->tsgl) * max_t(u32, sgl->cur, 1),
465 GFP_KERNEL);
466 if (unlikely(!areq->tsgl))
467 goto free;
468
469 sg_init_table(areq->tsgl, max_t(u32, sgl->cur, 1));
470 for (i = 0; i < sgl->cur; i++)
471 sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]),
472 sgl->sg[i].length, sgl->sg[i].offset);
473
474 areq->tsgls = sgl->cur;
475
476 /* create rx sgls */
477 while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
478 size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
479 (outlen - usedpages));
480
481 if (list_empty(&areq->list)) {
482 rsgl = &areq->first_rsgl;
483
484 } else {
485 rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
486 if (unlikely(!rsgl)) {
487 err = -ENOMEM;
488 goto free;
489 }
490 }
491 rsgl->sgl.npages = 0;
492 list_add_tail(&rsgl->list, &areq->list);
493
494 /* make one iovec available as scatterlist */
495 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
496 if (err < 0)
497 goto free;
498
499 usedpages += err;
500
501 /* chain the new scatterlist with previous one */
502 if (last_rsgl)
503 af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
504
505 last_rsgl = rsgl;
506
507 iov_iter_advance(&msg->msg_iter, err);
508 }
509
510 /* ensure output buffer is sufficiently large */
511 if (usedpages < outlen) {
512 err = -EINVAL;
513 goto unlock;
514 }
515
516 aead_request_set_crypt(req, areq->tsgl, areq->first_rsgl.sgl.sg, used,
517 areq->iv);
518 err = ctx->enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
519 if (err) {
520 if (err == -EINPROGRESS) {
521 sock_hold(sk);
522 err = -EIOCBQUEUED;
523 aead_reset_ctx(ctx);
524 goto unlock;
525 } else if (err == -EBADMSG) {
526 aead_put_sgl(sk);
527 }
528 goto free;
529 }
530 aead_put_sgl(sk);
531
532free:
533 list_for_each_entry(rsgl, &areq->list, list) {
534 af_alg_free_sg(&rsgl->sgl);
535 if (rsgl != &areq->first_rsgl)
536 sock_kfree_s(sk, rsgl, sizeof(*rsgl));
537 }
538 if (areq->tsgl)
539 sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
540 if (req)
541 sock_kfree_s(sk, req, reqlen);
542unlock:
543 aead_wmem_wakeup(sk);
544 release_sock(sk);
545 return err ? err : outlen;
546}
547
548static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
549{
550 struct sock *sk = sock->sk;
551 struct alg_sock *ask = alg_sk(sk);
552 struct aead_ctx *ctx = ask->private;
553 unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
554 struct aead_sg_list *sgl = &ctx->tsgl;
555 struct aead_async_rsgl *last_rsgl = NULL;
556 struct aead_async_rsgl *rsgl, *tmp;
557 int err = -EINVAL;
558 unsigned long used = 0;
559 size_t outlen = 0;
560 size_t usedpages = 0;
561
562 lock_sock(sk);
563 113
564 /* 114 /*
565 * Please see documentation of aead_request_set_crypt for the 115 * Data length provided by caller via sendmsg/sendpage that has not
566 * description of the AEAD memory structure expected from the caller. 116 * yet been processed.
567 */ 117 */
568
569 if (ctx->more) {
570 err = aead_wait_for_data(sk, flags);
571 if (err)
572 goto unlock;
573 }
574
575 /* data length provided by caller via sendmsg/sendpage */
576 used = ctx->used; 118 used = ctx->used;
577 119
578 /* 120 /*
@@ -584,8 +126,8 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
584 * the error message in sendmsg/sendpage and still call recvmsg. This 126 * the error message in sendmsg/sendpage and still call recvmsg. This
585 * check here protects the kernel integrity. 127 * check here protects the kernel integrity.
586 */ 128 */
587 if (!aead_sufficient_data(ctx)) 129 if (!aead_sufficient_data(sk))
588 goto unlock; 130 return -EINVAL;
589 131
590 /* 132 /*
591 * Calculate the minimum output buffer size holding the result of the 133 * Calculate the minimum output buffer size holding the result of the
@@ -606,104 +148,191 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
606 */ 148 */
607 used -= ctx->aead_assoclen; 149 used -= ctx->aead_assoclen;
608 150
609 /* convert iovecs of output buffers into scatterlists */ 151 /* Allocate cipher request for current operation. */
610 while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) { 152 areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
611 size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter), 153 crypto_aead_reqsize(tfm));
612 (outlen - usedpages)); 154 if (IS_ERR(areq))
613 155 return PTR_ERR(areq);
614 if (list_empty(&ctx->list)) {
615 rsgl = &ctx->first_rsgl;
616 } else {
617 rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
618 if (unlikely(!rsgl)) {
619 err = -ENOMEM;
620 goto unlock;
621 }
622 }
623 rsgl->sgl.npages = 0;
624 list_add_tail(&rsgl->list, &ctx->list);
625 156
626 /* make one iovec available as scatterlist */ 157 /* convert iovecs of output buffers into RX SGL */
627 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); 158 err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);
628 if (err < 0) 159 if (err)
629 goto unlock; 160 goto free;
630 usedpages += err;
631 /* chain the new scatterlist with previous one */
632 if (last_rsgl)
633 af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
634 161
635 last_rsgl = rsgl; 162 /*
163 * Ensure output buffer is sufficiently large. If the caller provides
164 * less buffer space, only use the relative required input size. This
165 * allows AIO operation where the caller sent all data to be processed
166 * and the AIO operation performs the operation on the different chunks
167 * of the input data.
168 */
169 if (usedpages < outlen) {
170 size_t less = outlen - usedpages;
636 171
637 iov_iter_advance(&msg->msg_iter, err); 172 if (used < less) {
173 err = -EINVAL;
174 goto free;
175 }
176 used -= less;
177 outlen -= less;
638 } 178 }
639 179
640 /* ensure output buffer is sufficiently large */ 180 processed = used + ctx->aead_assoclen;
641 if (usedpages < outlen) { 181 tsgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, list);
642 err = -EINVAL;
643 goto unlock;
644 }
645 182
646 sg_mark_end(sgl->sg + sgl->cur - 1); 183 /*
647 aead_request_set_crypt(&ctx->aead_req, sgl->sg, ctx->first_rsgl.sgl.sg, 184 * Copy of AAD from source to destination
648 used, ctx->iv); 185 *
649 aead_request_set_ad(&ctx->aead_req, ctx->aead_assoclen); 186 * The AAD is copied to the destination buffer without change. Even
187 * when user space uses an in-place cipher operation, the kernel
188 * will copy the data as it does not see whether such in-place operation
189 * is initiated.
190 *
191 * To ensure efficiency, the following implementation ensure that the
192 * ciphers are invoked to perform a crypto operation in-place. This
193 * is achieved by memory management specified as follows.
194 */
650 195
651 err = af_alg_wait_for_completion(ctx->enc ? 196 /* Use the RX SGL as source (and destination) for crypto op. */
652 crypto_aead_encrypt(&ctx->aead_req) : 197 src = areq->first_rsgl.sgl.sg;
653 crypto_aead_decrypt(&ctx->aead_req), 198
654 &ctx->completion); 199 if (ctx->enc) {
200 /*
201 * Encryption operation - The in-place cipher operation is
202 * achieved by the following operation:
203 *
204 * TX SGL: AAD || PT
205 * | |
206 * | copy |
207 * v v
208 * RX SGL: AAD || PT || Tag
209 */
210 err = crypto_aead_copy_sgl(null_tfm, tsgl->sg,
211 areq->first_rsgl.sgl.sg, processed);
212 if (err)
213 goto free;
214 af_alg_pull_tsgl(sk, processed, NULL, 0);
215 } else {
216 /*
217 * Decryption operation - To achieve an in-place cipher
218 * operation, the following SGL structure is used:
219 *
220 * TX SGL: AAD || CT || Tag
221 * | | ^
222 * | copy | | Create SGL link.
223 * v v |
224 * RX SGL: AAD || CT ----+
225 */
226
227 /* Copy AAD || CT to RX SGL buffer for in-place operation. */
228 err = crypto_aead_copy_sgl(null_tfm, tsgl->sg,
229 areq->first_rsgl.sgl.sg, outlen);
230 if (err)
231 goto free;
655 232
656 if (err) { 233 /* Create TX SGL for tag and chain it to RX SGL. */
657 /* EBADMSG implies a valid cipher operation took place */ 234 areq->tsgl_entries = af_alg_count_tsgl(sk, processed,
658 if (err == -EBADMSG) 235 processed - as);
659 aead_put_sgl(sk); 236 if (!areq->tsgl_entries)
237 areq->tsgl_entries = 1;
238 areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) *
239 areq->tsgl_entries,
240 GFP_KERNEL);
241 if (!areq->tsgl) {
242 err = -ENOMEM;
243 goto free;
244 }
245 sg_init_table(areq->tsgl, areq->tsgl_entries);
246
247 /* Release TX SGL, except for tag data and reassign tag data. */
248 af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as);
249
250 /* chain the areq TX SGL holding the tag with RX SGL */
251 if (usedpages) {
252 /* RX SGL present */
253 struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl;
254
255 sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
256 sg_chain(sgl_prev->sg, sgl_prev->npages + 1,
257 areq->tsgl);
258 } else
259 /* no RX SGL present (e.g. authentication only) */
260 src = areq->tsgl;
261 }
660 262
661 goto unlock; 263 /* Initialize the crypto operation */
264 aead_request_set_crypt(&areq->cra_u.aead_req, src,
265 areq->first_rsgl.sgl.sg, used, ctx->iv);
266 aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);
267 aead_request_set_tfm(&areq->cra_u.aead_req, tfm);
268
269 if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
270 /* AIO operation */
271 areq->iocb = msg->msg_iocb;
272 aead_request_set_callback(&areq->cra_u.aead_req,
273 CRYPTO_TFM_REQ_MAY_BACKLOG,
274 af_alg_async_cb, areq);
275 err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
276 crypto_aead_decrypt(&areq->cra_u.aead_req);
277 } else {
278 /* Synchronous operation */
279 aead_request_set_callback(&areq->cra_u.aead_req,
280 CRYPTO_TFM_REQ_MAY_BACKLOG,
281 af_alg_complete, &ctx->completion);
282 err = af_alg_wait_for_completion(ctx->enc ?
283 crypto_aead_encrypt(&areq->cra_u.aead_req) :
284 crypto_aead_decrypt(&areq->cra_u.aead_req),
285 &ctx->completion);
662 } 286 }
663 287
664 aead_put_sgl(sk); 288 /* AIO operation in progress */
665 err = 0; 289 if (err == -EINPROGRESS) {
290 sock_hold(sk);
666 291
667unlock: 292 /* Remember output size that will be generated. */
668 list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) { 293 areq->outlen = outlen;
669 af_alg_free_sg(&rsgl->sgl); 294
670 list_del(&rsgl->list); 295 return -EIOCBQUEUED;
671 if (rsgl != &ctx->first_rsgl)
672 sock_kfree_s(sk, rsgl, sizeof(*rsgl));
673 } 296 }
674 INIT_LIST_HEAD(&ctx->list);
675 aead_wmem_wakeup(sk);
676 release_sock(sk);
677 297
678 return err ? err : outlen; 298free:
679} 299 af_alg_free_areq_sgls(areq);
300 sock_kfree_s(sk, areq, areq->areqlen);
680 301
681static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, 302 return err ? err : outlen;
682 int flags)
683{
684 return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
685 aead_recvmsg_async(sock, msg, flags) :
686 aead_recvmsg_sync(sock, msg, flags);
687} 303}
688 304
689static unsigned int aead_poll(struct file *file, struct socket *sock, 305static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
690 poll_table *wait) 306 size_t ignored, int flags)
691{ 307{
692 struct sock *sk = sock->sk; 308 struct sock *sk = sock->sk;
693 struct alg_sock *ask = alg_sk(sk); 309 int ret = 0;
694 struct aead_ctx *ctx = ask->private;
695 unsigned int mask;
696
697 sock_poll_wait(file, sk_sleep(sk), wait);
698 mask = 0;
699 310
700 if (!ctx->more) 311 lock_sock(sk);
701 mask |= POLLIN | POLLRDNORM; 312 while (msg_data_left(msg)) {
313 int err = _aead_recvmsg(sock, msg, ignored, flags);
314
315 /*
316 * This error covers -EIOCBQUEUED which implies that we can
317 * only handle one AIO request. If the caller wants to have
318 * multiple AIO requests in parallel, he must make multiple
319 * separate AIO calls.
320 *
321 * Also return the error if no data has been processed so far.
322 */
323 if (err <= 0) {
324 if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)
325 ret = err;
326 goto out;
327 }
702 328
703 if (aead_writable(sk)) 329 ret += err;
704 mask |= POLLOUT | POLLWRNORM | POLLWRBAND; 330 }
705 331
706 return mask; 332out:
333 af_alg_wmem_wakeup(sk);
334 release_sock(sk);
335 return ret;
707} 336}
708 337
709static struct proto_ops algif_aead_ops = { 338static struct proto_ops algif_aead_ops = {
@@ -723,9 +352,9 @@ static struct proto_ops algif_aead_ops = {
723 352
724 .release = af_alg_release, 353 .release = af_alg_release,
725 .sendmsg = aead_sendmsg, 354 .sendmsg = aead_sendmsg,
726 .sendpage = aead_sendpage, 355 .sendpage = af_alg_sendpage,
727 .recvmsg = aead_recvmsg, 356 .recvmsg = aead_recvmsg,
728 .poll = aead_poll, 357 .poll = af_alg_poll,
729}; 358};
730 359
731static int aead_check_key(struct socket *sock) 360static int aead_check_key(struct socket *sock)
@@ -787,7 +416,7 @@ static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page,
787 if (err) 416 if (err)
788 return err; 417 return err;
789 418
790 return aead_sendpage(sock, page, offset, size, flags); 419 return af_alg_sendpage(sock, page, offset, size, flags);
791} 420}
792 421
793static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg, 422static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
@@ -821,13 +450,14 @@ static struct proto_ops algif_aead_ops_nokey = {
821 .sendmsg = aead_sendmsg_nokey, 450 .sendmsg = aead_sendmsg_nokey,
822 .sendpage = aead_sendpage_nokey, 451 .sendpage = aead_sendpage_nokey,
823 .recvmsg = aead_recvmsg_nokey, 452 .recvmsg = aead_recvmsg_nokey,
824 .poll = aead_poll, 453 .poll = af_alg_poll,
825}; 454};
826 455
827static void *aead_bind(const char *name, u32 type, u32 mask) 456static void *aead_bind(const char *name, u32 type, u32 mask)
828{ 457{
829 struct aead_tfm *tfm; 458 struct aead_tfm *tfm;
830 struct crypto_aead *aead; 459 struct crypto_aead *aead;
460 struct crypto_skcipher *null_tfm;
831 461
832 tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); 462 tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
833 if (!tfm) 463 if (!tfm)
@@ -839,7 +469,15 @@ static void *aead_bind(const char *name, u32 type, u32 mask)
839 return ERR_CAST(aead); 469 return ERR_CAST(aead);
840 } 470 }
841 471
472 null_tfm = crypto_get_default_null_skcipher2();
473 if (IS_ERR(null_tfm)) {
474 crypto_free_aead(aead);
475 kfree(tfm);
476 return ERR_CAST(null_tfm);
477 }
478
842 tfm->aead = aead; 479 tfm->aead = aead;
480 tfm->null_tfm = null_tfm;
843 481
844 return tfm; 482 return tfm;
845} 483}
@@ -873,12 +511,15 @@ static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
873static void aead_sock_destruct(struct sock *sk) 511static void aead_sock_destruct(struct sock *sk)
874{ 512{
875 struct alg_sock *ask = alg_sk(sk); 513 struct alg_sock *ask = alg_sk(sk);
876 struct aead_ctx *ctx = ask->private; 514 struct af_alg_ctx *ctx = ask->private;
877 unsigned int ivlen = crypto_aead_ivsize( 515 struct sock *psk = ask->parent;
878 crypto_aead_reqtfm(&ctx->aead_req)); 516 struct alg_sock *pask = alg_sk(psk);
879 517 struct aead_tfm *aeadc = pask->private;
880 WARN_ON(refcount_read(&sk->sk_refcnt) != 0); 518 struct crypto_aead *tfm = aeadc->aead;
881 aead_put_sgl(sk); 519 unsigned int ivlen = crypto_aead_ivsize(tfm);
520
521 af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
522 crypto_put_default_null_skcipher2();
882 sock_kzfree_s(sk, ctx->iv, ivlen); 523 sock_kzfree_s(sk, ctx->iv, ivlen);
883 sock_kfree_s(sk, ctx, ctx->len); 524 sock_kfree_s(sk, ctx, ctx->len);
884 af_alg_release_parent(sk); 525 af_alg_release_parent(sk);
@@ -886,11 +527,11 @@ static void aead_sock_destruct(struct sock *sk)
886 527
887static int aead_accept_parent_nokey(void *private, struct sock *sk) 528static int aead_accept_parent_nokey(void *private, struct sock *sk)
888{ 529{
889 struct aead_ctx *ctx; 530 struct af_alg_ctx *ctx;
890 struct alg_sock *ask = alg_sk(sk); 531 struct alg_sock *ask = alg_sk(sk);
891 struct aead_tfm *tfm = private; 532 struct aead_tfm *tfm = private;
892 struct crypto_aead *aead = tfm->aead; 533 struct crypto_aead *aead = tfm->aead;
893 unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(aead); 534 unsigned int len = sizeof(*ctx);
894 unsigned int ivlen = crypto_aead_ivsize(aead); 535 unsigned int ivlen = crypto_aead_ivsize(aead);
895 536
896 ctx = sock_kmalloc(sk, len, GFP_KERNEL); 537 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
@@ -905,23 +546,18 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk)
905 } 546 }
906 memset(ctx->iv, 0, ivlen); 547 memset(ctx->iv, 0, ivlen);
907 548
549 INIT_LIST_HEAD(&ctx->tsgl_list);
908 ctx->len = len; 550 ctx->len = len;
909 ctx->used = 0; 551 ctx->used = 0;
552 ctx->rcvused = 0;
910 ctx->more = 0; 553 ctx->more = 0;
911 ctx->merge = 0; 554 ctx->merge = 0;
912 ctx->enc = 0; 555 ctx->enc = 0;
913 ctx->tsgl.cur = 0;
914 ctx->aead_assoclen = 0; 556 ctx->aead_assoclen = 0;
915 af_alg_init_completion(&ctx->completion); 557 af_alg_init_completion(&ctx->completion);
916 sg_init_table(ctx->tsgl.sg, ALG_MAX_PAGES);
917 INIT_LIST_HEAD(&ctx->list);
918 558
919 ask->private = ctx; 559 ask->private = ctx;
920 560
921 aead_request_set_tfm(&ctx->aead_req, aead);
922 aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
923 af_alg_complete, &ctx->completion);
924
925 sk->sk_destruct = aead_sock_destruct; 561 sk->sk_destruct = aead_sock_destruct;
926 562
927 return 0; 563 return 0;
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index 903605dbc1a5..8ae4170aaeb4 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -10,6 +10,21 @@
10 * Software Foundation; either version 2 of the License, or (at your option) 10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version. 11 * any later version.
12 * 12 *
13 * The following concept of the memory management is used:
14 *
15 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
16 * filled by user space with the data submitted via sendpage/sendmsg. Filling
17 * up the TX SGL does not cause a crypto operation -- the data will only be
18 * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
19 * provide a buffer which is tracked with the RX SGL.
20 *
21 * During the processing of the recvmsg operation, the cipher request is
22 * allocated and prepared. As part of the recvmsg operation, the processed
23 * TX buffers are extracted from the TX SGL into a separate SGL.
24 *
25 * After the completion of the crypto operation, the RX SGL and the cipher
26 * request is released. The extracted TX SGL parts are released together with
27 * the RX SGL release.
13 */ 28 */
14 29
15#include <crypto/scatterwalk.h> 30#include <crypto/scatterwalk.h>
@@ -18,284 +33,16 @@
18#include <linux/init.h> 33#include <linux/init.h>
19#include <linux/list.h> 34#include <linux/list.h>
20#include <linux/kernel.h> 35#include <linux/kernel.h>
21#include <linux/sched/signal.h>
22#include <linux/mm.h> 36#include <linux/mm.h>
23#include <linux/module.h> 37#include <linux/module.h>
24#include <linux/net.h> 38#include <linux/net.h>
25#include <net/sock.h> 39#include <net/sock.h>
26 40
27struct skcipher_sg_list {
28 struct list_head list;
29
30 int cur;
31
32 struct scatterlist sg[0];
33};
34
35struct skcipher_tfm { 41struct skcipher_tfm {
36 struct crypto_skcipher *skcipher; 42 struct crypto_skcipher *skcipher;
37 bool has_key; 43 bool has_key;
38}; 44};
39 45
40struct skcipher_ctx {
41 struct list_head tsgl;
42 struct af_alg_sgl rsgl;
43
44 void *iv;
45
46 struct af_alg_completion completion;
47
48 atomic_t inflight;
49 size_t used;
50
51 unsigned int len;
52 bool more;
53 bool merge;
54 bool enc;
55
56 struct skcipher_request req;
57};
58
59struct skcipher_async_rsgl {
60 struct af_alg_sgl sgl;
61 struct list_head list;
62};
63
64struct skcipher_async_req {
65 struct kiocb *iocb;
66 struct skcipher_async_rsgl first_sgl;
67 struct list_head list;
68 struct scatterlist *tsg;
69 atomic_t *inflight;
70 struct skcipher_request req;
71};
72
73#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
74 sizeof(struct scatterlist) - 1)
75
76static void skcipher_free_async_sgls(struct skcipher_async_req *sreq)
77{
78 struct skcipher_async_rsgl *rsgl, *tmp;
79 struct scatterlist *sgl;
80 struct scatterlist *sg;
81 int i, n;
82
83 list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) {
84 af_alg_free_sg(&rsgl->sgl);
85 if (rsgl != &sreq->first_sgl)
86 kfree(rsgl);
87 }
88 sgl = sreq->tsg;
89 n = sg_nents(sgl);
90 for_each_sg(sgl, sg, n, i) {
91 struct page *page = sg_page(sg);
92
93 /* some SGs may not have a page mapped */
94 if (page && page_ref_count(page))
95 put_page(page);
96 }
97
98 kfree(sreq->tsg);
99}
100
101static void skcipher_async_cb(struct crypto_async_request *req, int err)
102{
103 struct skcipher_async_req *sreq = req->data;
104 struct kiocb *iocb = sreq->iocb;
105
106 atomic_dec(sreq->inflight);
107 skcipher_free_async_sgls(sreq);
108 kzfree(sreq);
109 iocb->ki_complete(iocb, err, err);
110}
111
112static inline int skcipher_sndbuf(struct sock *sk)
113{
114 struct alg_sock *ask = alg_sk(sk);
115 struct skcipher_ctx *ctx = ask->private;
116
117 return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
118 ctx->used, 0);
119}
120
121static inline bool skcipher_writable(struct sock *sk)
122{
123 return PAGE_SIZE <= skcipher_sndbuf(sk);
124}
125
126static int skcipher_alloc_sgl(struct sock *sk)
127{
128 struct alg_sock *ask = alg_sk(sk);
129 struct skcipher_ctx *ctx = ask->private;
130 struct skcipher_sg_list *sgl;
131 struct scatterlist *sg = NULL;
132
133 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
134 if (!list_empty(&ctx->tsgl))
135 sg = sgl->sg;
136
137 if (!sg || sgl->cur >= MAX_SGL_ENTS) {
138 sgl = sock_kmalloc(sk, sizeof(*sgl) +
139 sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
140 GFP_KERNEL);
141 if (!sgl)
142 return -ENOMEM;
143
144 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
145 sgl->cur = 0;
146
147 if (sg)
148 sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
149
150 list_add_tail(&sgl->list, &ctx->tsgl);
151 }
152
153 return 0;
154}
155
156static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
157{
158 struct alg_sock *ask = alg_sk(sk);
159 struct skcipher_ctx *ctx = ask->private;
160 struct skcipher_sg_list *sgl;
161 struct scatterlist *sg;
162 int i;
163
164 while (!list_empty(&ctx->tsgl)) {
165 sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list,
166 list);
167 sg = sgl->sg;
168
169 for (i = 0; i < sgl->cur; i++) {
170 size_t plen = min_t(size_t, used, sg[i].length);
171
172 if (!sg_page(sg + i))
173 continue;
174
175 sg[i].length -= plen;
176 sg[i].offset += plen;
177
178 used -= plen;
179 ctx->used -= plen;
180
181 if (sg[i].length)
182 return;
183 if (put)
184 put_page(sg_page(sg + i));
185 sg_assign_page(sg + i, NULL);
186 }
187
188 list_del(&sgl->list);
189 sock_kfree_s(sk, sgl,
190 sizeof(*sgl) + sizeof(sgl->sg[0]) *
191 (MAX_SGL_ENTS + 1));
192 }
193
194 if (!ctx->used)
195 ctx->merge = 0;
196}
197
198static void skcipher_free_sgl(struct sock *sk)
199{
200 struct alg_sock *ask = alg_sk(sk);
201 struct skcipher_ctx *ctx = ask->private;
202
203 skcipher_pull_sgl(sk, ctx->used, 1);
204}
205
206static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags)
207{
208 DEFINE_WAIT_FUNC(wait, woken_wake_function);
209 int err = -ERESTARTSYS;
210 long timeout;
211
212 if (flags & MSG_DONTWAIT)
213 return -EAGAIN;
214
215 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
216
217 add_wait_queue(sk_sleep(sk), &wait);
218 for (;;) {
219 if (signal_pending(current))
220 break;
221 timeout = MAX_SCHEDULE_TIMEOUT;
222 if (sk_wait_event(sk, &timeout, skcipher_writable(sk), &wait)) {
223 err = 0;
224 break;
225 }
226 }
227 remove_wait_queue(sk_sleep(sk), &wait);
228
229 return err;
230}
231
232static void skcipher_wmem_wakeup(struct sock *sk)
233{
234 struct socket_wq *wq;
235
236 if (!skcipher_writable(sk))
237 return;
238
239 rcu_read_lock();
240 wq = rcu_dereference(sk->sk_wq);
241 if (skwq_has_sleeper(wq))
242 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
243 POLLRDNORM |
244 POLLRDBAND);
245 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
246 rcu_read_unlock();
247}
248
249static int skcipher_wait_for_data(struct sock *sk, unsigned flags)
250{
251 DEFINE_WAIT_FUNC(wait, woken_wake_function);
252 struct alg_sock *ask = alg_sk(sk);
253 struct skcipher_ctx *ctx = ask->private;
254 long timeout;
255 int err = -ERESTARTSYS;
256
257 if (flags & MSG_DONTWAIT) {
258 return -EAGAIN;
259 }
260
261 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
262
263 add_wait_queue(sk_sleep(sk), &wait);
264 for (;;) {
265 if (signal_pending(current))
266 break;
267 timeout = MAX_SCHEDULE_TIMEOUT;
268 if (sk_wait_event(sk, &timeout, ctx->used, &wait)) {
269 err = 0;
270 break;
271 }
272 }
273 remove_wait_queue(sk_sleep(sk), &wait);
274
275 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
276
277 return err;
278}
279
280static void skcipher_data_wakeup(struct sock *sk)
281{
282 struct alg_sock *ask = alg_sk(sk);
283 struct skcipher_ctx *ctx = ask->private;
284 struct socket_wq *wq;
285
286 if (!ctx->used)
287 return;
288
289 rcu_read_lock();
290 wq = rcu_dereference(sk->sk_wq);
291 if (skwq_has_sleeper(wq))
292 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
293 POLLRDNORM |
294 POLLRDBAND);
295 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
296 rcu_read_unlock();
297}
298
299static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, 46static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
300 size_t size) 47 size_t size)
301{ 48{
@@ -303,445 +50,143 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
303 struct alg_sock *ask = alg_sk(sk); 50 struct alg_sock *ask = alg_sk(sk);
304 struct sock *psk = ask->parent; 51 struct sock *psk = ask->parent;
305 struct alg_sock *pask = alg_sk(psk); 52 struct alg_sock *pask = alg_sk(psk);
306 struct skcipher_ctx *ctx = ask->private;
307 struct skcipher_tfm *skc = pask->private; 53 struct skcipher_tfm *skc = pask->private;
308 struct crypto_skcipher *tfm = skc->skcipher; 54 struct crypto_skcipher *tfm = skc->skcipher;
309 unsigned ivsize = crypto_skcipher_ivsize(tfm); 55 unsigned ivsize = crypto_skcipher_ivsize(tfm);
310 struct skcipher_sg_list *sgl;
311 struct af_alg_control con = {};
312 long copied = 0;
313 bool enc = 0;
314 bool init = 0;
315 int err;
316 int i;
317
318 if (msg->msg_controllen) {
319 err = af_alg_cmsg_send(msg, &con);
320 if (err)
321 return err;
322
323 init = 1;
324 switch (con.op) {
325 case ALG_OP_ENCRYPT:
326 enc = 1;
327 break;
328 case ALG_OP_DECRYPT:
329 enc = 0;
330 break;
331 default:
332 return -EINVAL;
333 }
334
335 if (con.iv && con.iv->ivlen != ivsize)
336 return -EINVAL;
337 }
338
339 err = -EINVAL;
340
341 lock_sock(sk);
342 if (!ctx->more && ctx->used)
343 goto unlock;
344
345 if (init) {
346 ctx->enc = enc;
347 if (con.iv)
348 memcpy(ctx->iv, con.iv->iv, ivsize);
349 }
350
351 while (size) {
352 struct scatterlist *sg;
353 unsigned long len = size;
354 size_t plen;
355
356 if (ctx->merge) {
357 sgl = list_entry(ctx->tsgl.prev,
358 struct skcipher_sg_list, list);
359 sg = sgl->sg + sgl->cur - 1;
360 len = min_t(unsigned long, len,
361 PAGE_SIZE - sg->offset - sg->length);
362
363 err = memcpy_from_msg(page_address(sg_page(sg)) +
364 sg->offset + sg->length,
365 msg, len);
366 if (err)
367 goto unlock;
368
369 sg->length += len;
370 ctx->merge = (sg->offset + sg->length) &
371 (PAGE_SIZE - 1);
372
373 ctx->used += len;
374 copied += len;
375 size -= len;
376 continue;
377 }
378 56
379 if (!skcipher_writable(sk)) { 57 return af_alg_sendmsg(sock, msg, size, ivsize);
380 err = skcipher_wait_for_wmem(sk, msg->msg_flags);
381 if (err)
382 goto unlock;
383 }
384
385 len = min_t(unsigned long, len, skcipher_sndbuf(sk));
386
387 err = skcipher_alloc_sgl(sk);
388 if (err)
389 goto unlock;
390
391 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
392 sg = sgl->sg;
393 if (sgl->cur)
394 sg_unmark_end(sg + sgl->cur - 1);
395 do {
396 i = sgl->cur;
397 plen = min_t(size_t, len, PAGE_SIZE);
398
399 sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
400 err = -ENOMEM;
401 if (!sg_page(sg + i))
402 goto unlock;
403
404 err = memcpy_from_msg(page_address(sg_page(sg + i)),
405 msg, plen);
406 if (err) {
407 __free_page(sg_page(sg + i));
408 sg_assign_page(sg + i, NULL);
409 goto unlock;
410 }
411
412 sg[i].length = plen;
413 len -= plen;
414 ctx->used += plen;
415 copied += plen;
416 size -= plen;
417 sgl->cur++;
418 } while (len && sgl->cur < MAX_SGL_ENTS);
419
420 if (!size)
421 sg_mark_end(sg + sgl->cur - 1);
422
423 ctx->merge = plen & (PAGE_SIZE - 1);
424 }
425
426 err = 0;
427
428 ctx->more = msg->msg_flags & MSG_MORE;
429
430unlock:
431 skcipher_data_wakeup(sk);
432 release_sock(sk);
433
434 return copied ?: err;
435} 58}
436 59
437static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, 60static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
438 int offset, size_t size, int flags) 61 size_t ignored, int flags)
439{
440 struct sock *sk = sock->sk;
441 struct alg_sock *ask = alg_sk(sk);
442 struct skcipher_ctx *ctx = ask->private;
443 struct skcipher_sg_list *sgl;
444 int err = -EINVAL;
445
446 if (flags & MSG_SENDPAGE_NOTLAST)
447 flags |= MSG_MORE;
448
449 lock_sock(sk);
450 if (!ctx->more && ctx->used)
451 goto unlock;
452
453 if (!size)
454 goto done;
455
456 if (!skcipher_writable(sk)) {
457 err = skcipher_wait_for_wmem(sk, flags);
458 if (err)
459 goto unlock;
460 }
461
462 err = skcipher_alloc_sgl(sk);
463 if (err)
464 goto unlock;
465
466 ctx->merge = 0;
467 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
468
469 if (sgl->cur)
470 sg_unmark_end(sgl->sg + sgl->cur - 1);
471
472 sg_mark_end(sgl->sg + sgl->cur);
473 get_page(page);
474 sg_set_page(sgl->sg + sgl->cur, page, size, offset);
475 sgl->cur++;
476 ctx->used += size;
477
478done:
479 ctx->more = flags & MSG_MORE;
480
481unlock:
482 skcipher_data_wakeup(sk);
483 release_sock(sk);
484
485 return err ?: size;
486}
487
488static int skcipher_all_sg_nents(struct skcipher_ctx *ctx)
489{
490 struct skcipher_sg_list *sgl;
491 struct scatterlist *sg;
492 int nents = 0;
493
494 list_for_each_entry(sgl, &ctx->tsgl, list) {
495 sg = sgl->sg;
496
497 while (!sg->length)
498 sg++;
499
500 nents += sg_nents(sg);
501 }
502 return nents;
503}
504
505static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
506 int flags)
507{ 62{
508 struct sock *sk = sock->sk; 63 struct sock *sk = sock->sk;
509 struct alg_sock *ask = alg_sk(sk); 64 struct alg_sock *ask = alg_sk(sk);
510 struct sock *psk = ask->parent; 65 struct sock *psk = ask->parent;
511 struct alg_sock *pask = alg_sk(psk); 66 struct alg_sock *pask = alg_sk(psk);
512 struct skcipher_ctx *ctx = ask->private; 67 struct af_alg_ctx *ctx = ask->private;
513 struct skcipher_tfm *skc = pask->private; 68 struct skcipher_tfm *skc = pask->private;
514 struct crypto_skcipher *tfm = skc->skcipher; 69 struct crypto_skcipher *tfm = skc->skcipher;
515 struct skcipher_sg_list *sgl; 70 unsigned int bs = crypto_skcipher_blocksize(tfm);
516 struct scatterlist *sg; 71 struct af_alg_async_req *areq;
517 struct skcipher_async_req *sreq; 72 int err = 0;
518 struct skcipher_request *req; 73 size_t len = 0;
519 struct skcipher_async_rsgl *last_rsgl = NULL;
520 unsigned int txbufs = 0, len = 0, tx_nents;
521 unsigned int reqsize = crypto_skcipher_reqsize(tfm);
522 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
523 int err = -ENOMEM;
524 bool mark = false;
525 char *iv;
526
527 sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL);
528 if (unlikely(!sreq))
529 goto out;
530
531 req = &sreq->req;
532 iv = (char *)(req + 1) + reqsize;
533 sreq->iocb = msg->msg_iocb;
534 INIT_LIST_HEAD(&sreq->list);
535 sreq->inflight = &ctx->inflight;
536 74
537 lock_sock(sk); 75 /* Allocate cipher request for current operation. */
538 tx_nents = skcipher_all_sg_nents(ctx); 76 areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
539 sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL); 77 crypto_skcipher_reqsize(tfm));
540 if (unlikely(!sreq->tsg)) 78 if (IS_ERR(areq))
541 goto unlock; 79 return PTR_ERR(areq);
542 sg_init_table(sreq->tsg, tx_nents);
543 memcpy(iv, ctx->iv, ivsize);
544 skcipher_request_set_tfm(req, tfm);
545 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP,
546 skcipher_async_cb, sreq);
547
548 while (iov_iter_count(&msg->msg_iter)) {
549 struct skcipher_async_rsgl *rsgl;
550 int used;
551
552 if (!ctx->used) {
553 err = skcipher_wait_for_data(sk, flags);
554 if (err)
555 goto free;
556 }
557 sgl = list_first_entry(&ctx->tsgl,
558 struct skcipher_sg_list, list);
559 sg = sgl->sg;
560
561 while (!sg->length)
562 sg++;
563
564 used = min_t(unsigned long, ctx->used,
565 iov_iter_count(&msg->msg_iter));
566 used = min_t(unsigned long, used, sg->length);
567
568 if (txbufs == tx_nents) {
569 struct scatterlist *tmp;
570 int x;
571 /* Ran out of tx slots in async request
572 * need to expand */
573 tmp = kcalloc(tx_nents * 2, sizeof(*tmp),
574 GFP_KERNEL);
575 if (!tmp) {
576 err = -ENOMEM;
577 goto free;
578 }
579
580 sg_init_table(tmp, tx_nents * 2);
581 for (x = 0; x < tx_nents; x++)
582 sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]),
583 sreq->tsg[x].length,
584 sreq->tsg[x].offset);
585 kfree(sreq->tsg);
586 sreq->tsg = tmp;
587 tx_nents *= 2;
588 mark = true;
589 }
590 /* Need to take over the tx sgl from ctx
591 * to the asynch req - these sgls will be freed later */
592 sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length,
593 sg->offset);
594
595 if (list_empty(&sreq->list)) {
596 rsgl = &sreq->first_sgl;
597 list_add_tail(&rsgl->list, &sreq->list);
598 } else {
599 rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL);
600 if (!rsgl) {
601 err = -ENOMEM;
602 goto free;
603 }
604 list_add_tail(&rsgl->list, &sreq->list);
605 }
606 80
607 used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used); 81 /* convert iovecs of output buffers into RX SGL */
608 err = used; 82 err = af_alg_get_rsgl(sk, msg, flags, areq, -1, &len);
609 if (used < 0) 83 if (err)
610 goto free; 84 goto free;
611 if (last_rsgl) 85
612 af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); 86 /* Process only as much RX buffers for which we have TX data */
613 87 if (len > ctx->used)
614 last_rsgl = rsgl; 88 len = ctx->used;
615 len += used; 89
616 skcipher_pull_sgl(sk, used, 0); 90 /*
617 iov_iter_advance(&msg->msg_iter, used); 91 * If more buffers are to be expected to be processed, process only
92 * full block size buffers.
93 */
94 if (ctx->more || len < ctx->used)
95 len -= len % bs;
96
97 /*
98 * Create a per request TX SGL for this request which tracks the
99 * SG entries from the global TX SGL.
100 */
101 areq->tsgl_entries = af_alg_count_tsgl(sk, len, 0);
102 if (!areq->tsgl_entries)
103 areq->tsgl_entries = 1;
104 areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * areq->tsgl_entries,
105 GFP_KERNEL);
106 if (!areq->tsgl) {
107 err = -ENOMEM;
108 goto free;
109 }
110 sg_init_table(areq->tsgl, areq->tsgl_entries);
111 af_alg_pull_tsgl(sk, len, areq->tsgl, 0);
112
113 /* Initialize the crypto operation */
114 skcipher_request_set_tfm(&areq->cra_u.skcipher_req, tfm);
115 skcipher_request_set_crypt(&areq->cra_u.skcipher_req, areq->tsgl,
116 areq->first_rsgl.sgl.sg, len, ctx->iv);
117
118 if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
119 /* AIO operation */
120 areq->iocb = msg->msg_iocb;
121 skcipher_request_set_callback(&areq->cra_u.skcipher_req,
122 CRYPTO_TFM_REQ_MAY_SLEEP,
123 af_alg_async_cb, areq);
124 err = ctx->enc ?
125 crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) :
126 crypto_skcipher_decrypt(&areq->cra_u.skcipher_req);
127 } else {
128 /* Synchronous operation */
129 skcipher_request_set_callback(&areq->cra_u.skcipher_req,
130 CRYPTO_TFM_REQ_MAY_SLEEP |
131 CRYPTO_TFM_REQ_MAY_BACKLOG,
132 af_alg_complete,
133 &ctx->completion);
134 err = af_alg_wait_for_completion(ctx->enc ?
135 crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) :
136 crypto_skcipher_decrypt(&areq->cra_u.skcipher_req),
137 &ctx->completion);
618 } 138 }
619 139
620 if (mark) 140 /* AIO operation in progress */
621 sg_mark_end(sreq->tsg + txbufs - 1);
622
623 skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg,
624 len, iv);
625 err = ctx->enc ? crypto_skcipher_encrypt(req) :
626 crypto_skcipher_decrypt(req);
627 if (err == -EINPROGRESS) { 141 if (err == -EINPROGRESS) {
628 atomic_inc(&ctx->inflight); 142 sock_hold(sk);
629 err = -EIOCBQUEUED; 143
630 sreq = NULL; 144 /* Remember output size that will be generated. */
631 goto unlock; 145 areq->outlen = len;
146
147 return -EIOCBQUEUED;
632 } 148 }
149
633free: 150free:
634 skcipher_free_async_sgls(sreq); 151 af_alg_free_areq_sgls(areq);
635unlock: 152 sock_kfree_s(sk, areq, areq->areqlen);
636 skcipher_wmem_wakeup(sk); 153
637 release_sock(sk); 154 return err ? err : len;
638 kzfree(sreq);
639out:
640 return err;
641} 155}
642 156
643static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg, 157static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
644 int flags) 158 size_t ignored, int flags)
645{ 159{
646 struct sock *sk = sock->sk; 160 struct sock *sk = sock->sk;
647 struct alg_sock *ask = alg_sk(sk); 161 int ret = 0;
648 struct sock *psk = ask->parent;
649 struct alg_sock *pask = alg_sk(psk);
650 struct skcipher_ctx *ctx = ask->private;
651 struct skcipher_tfm *skc = pask->private;
652 struct crypto_skcipher *tfm = skc->skcipher;
653 unsigned bs = crypto_skcipher_blocksize(tfm);
654 struct skcipher_sg_list *sgl;
655 struct scatterlist *sg;
656 int err = -EAGAIN;
657 int used;
658 long copied = 0;
659 162
660 lock_sock(sk); 163 lock_sock(sk);
661 while (msg_data_left(msg)) { 164 while (msg_data_left(msg)) {
662 if (!ctx->used) { 165 int err = _skcipher_recvmsg(sock, msg, ignored, flags);
663 err = skcipher_wait_for_data(sk, flags); 166
664 if (err) 167 /*
665 goto unlock; 168 * This error covers -EIOCBQUEUED which implies that we can
169 * only handle one AIO request. If the caller wants to have
170 * multiple AIO requests in parallel, he must make multiple
171 * separate AIO calls.
172 *
173 * Also return the error if no data has been processed so far.
174 */
175 if (err <= 0) {
176 if (err == -EIOCBQUEUED || !ret)
177 ret = err;
178 goto out;
666 } 179 }
667 180
668 used = min_t(unsigned long, ctx->used, msg_data_left(msg)); 181 ret += err;
669
670 used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used);
671 err = used;
672 if (err < 0)
673 goto unlock;
674
675 if (ctx->more || used < ctx->used)
676 used -= used % bs;
677
678 err = -EINVAL;
679 if (!used)
680 goto free;
681
682 sgl = list_first_entry(&ctx->tsgl,
683 struct skcipher_sg_list, list);
684 sg = sgl->sg;
685
686 while (!sg->length)
687 sg++;
688
689 skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used,
690 ctx->iv);
691
692 err = af_alg_wait_for_completion(
693 ctx->enc ?
694 crypto_skcipher_encrypt(&ctx->req) :
695 crypto_skcipher_decrypt(&ctx->req),
696 &ctx->completion);
697
698free:
699 af_alg_free_sg(&ctx->rsgl);
700
701 if (err)
702 goto unlock;
703
704 copied += used;
705 skcipher_pull_sgl(sk, used, 1);
706 iov_iter_advance(&msg->msg_iter, used);
707 } 182 }
708 183
709 err = 0; 184out:
710 185 af_alg_wmem_wakeup(sk);
711unlock:
712 skcipher_wmem_wakeup(sk);
713 release_sock(sk); 186 release_sock(sk);
714 187 return ret;
715 return copied ?: err;
716}
717
718static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
719 size_t ignored, int flags)
720{
721 return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
722 skcipher_recvmsg_async(sock, msg, flags) :
723 skcipher_recvmsg_sync(sock, msg, flags);
724} 188}
725 189
726static unsigned int skcipher_poll(struct file *file, struct socket *sock,
727 poll_table *wait)
728{
729 struct sock *sk = sock->sk;
730 struct alg_sock *ask = alg_sk(sk);
731 struct skcipher_ctx *ctx = ask->private;
732 unsigned int mask;
733
734 sock_poll_wait(file, sk_sleep(sk), wait);
735 mask = 0;
736
737 if (ctx->used)
738 mask |= POLLIN | POLLRDNORM;
739
740 if (skcipher_writable(sk))
741 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
742
743 return mask;
744}
745 190
746static struct proto_ops algif_skcipher_ops = { 191static struct proto_ops algif_skcipher_ops = {
747 .family = PF_ALG, 192 .family = PF_ALG,
@@ -760,9 +205,9 @@ static struct proto_ops algif_skcipher_ops = {
760 205
761 .release = af_alg_release, 206 .release = af_alg_release,
762 .sendmsg = skcipher_sendmsg, 207 .sendmsg = skcipher_sendmsg,
763 .sendpage = skcipher_sendpage, 208 .sendpage = af_alg_sendpage,
764 .recvmsg = skcipher_recvmsg, 209 .recvmsg = skcipher_recvmsg,
765 .poll = skcipher_poll, 210 .poll = af_alg_poll,
766}; 211};
767 212
768static int skcipher_check_key(struct socket *sock) 213static int skcipher_check_key(struct socket *sock)
@@ -824,7 +269,7 @@ static ssize_t skcipher_sendpage_nokey(struct socket *sock, struct page *page,
824 if (err) 269 if (err)
825 return err; 270 return err;
826 271
827 return skcipher_sendpage(sock, page, offset, size, flags); 272 return af_alg_sendpage(sock, page, offset, size, flags);
828} 273}
829 274
830static int skcipher_recvmsg_nokey(struct socket *sock, struct msghdr *msg, 275static int skcipher_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
@@ -858,7 +303,7 @@ static struct proto_ops algif_skcipher_ops_nokey = {
858 .sendmsg = skcipher_sendmsg_nokey, 303 .sendmsg = skcipher_sendmsg_nokey,
859 .sendpage = skcipher_sendpage_nokey, 304 .sendpage = skcipher_sendpage_nokey,
860 .recvmsg = skcipher_recvmsg_nokey, 305 .recvmsg = skcipher_recvmsg_nokey,
861 .poll = skcipher_poll, 306 .poll = af_alg_poll,
862}; 307};
863 308
864static void *skcipher_bind(const char *name, u32 type, u32 mask) 309static void *skcipher_bind(const char *name, u32 type, u32 mask)
@@ -900,26 +345,16 @@ static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
900 return err; 345 return err;
901} 346}
902 347
903static void skcipher_wait(struct sock *sk)
904{
905 struct alg_sock *ask = alg_sk(sk);
906 struct skcipher_ctx *ctx = ask->private;
907 int ctr = 0;
908
909 while (atomic_read(&ctx->inflight) && ctr++ < 100)
910 msleep(100);
911}
912
913static void skcipher_sock_destruct(struct sock *sk) 348static void skcipher_sock_destruct(struct sock *sk)
914{ 349{
915 struct alg_sock *ask = alg_sk(sk); 350 struct alg_sock *ask = alg_sk(sk);
916 struct skcipher_ctx *ctx = ask->private; 351 struct af_alg_ctx *ctx = ask->private;
917 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req); 352 struct sock *psk = ask->parent;
918 353 struct alg_sock *pask = alg_sk(psk);
919 if (atomic_read(&ctx->inflight)) 354 struct skcipher_tfm *skc = pask->private;
920 skcipher_wait(sk); 355 struct crypto_skcipher *tfm = skc->skcipher;
921 356
922 skcipher_free_sgl(sk); 357 af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
923 sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); 358 sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm));
924 sock_kfree_s(sk, ctx, ctx->len); 359 sock_kfree_s(sk, ctx, ctx->len);
925 af_alg_release_parent(sk); 360 af_alg_release_parent(sk);
@@ -927,11 +362,11 @@ static void skcipher_sock_destruct(struct sock *sk)
927 362
928static int skcipher_accept_parent_nokey(void *private, struct sock *sk) 363static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
929{ 364{
930 struct skcipher_ctx *ctx; 365 struct af_alg_ctx *ctx;
931 struct alg_sock *ask = alg_sk(sk); 366 struct alg_sock *ask = alg_sk(sk);
932 struct skcipher_tfm *tfm = private; 367 struct skcipher_tfm *tfm = private;
933 struct crypto_skcipher *skcipher = tfm->skcipher; 368 struct crypto_skcipher *skcipher = tfm->skcipher;
934 unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher); 369 unsigned int len = sizeof(*ctx);
935 370
936 ctx = sock_kmalloc(sk, len, GFP_KERNEL); 371 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
937 if (!ctx) 372 if (!ctx)
@@ -946,22 +381,17 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
946 381
947 memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher)); 382 memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher));
948 383
949 INIT_LIST_HEAD(&ctx->tsgl); 384 INIT_LIST_HEAD(&ctx->tsgl_list);
950 ctx->len = len; 385 ctx->len = len;
951 ctx->used = 0; 386 ctx->used = 0;
387 ctx->rcvused = 0;
952 ctx->more = 0; 388 ctx->more = 0;
953 ctx->merge = 0; 389 ctx->merge = 0;
954 ctx->enc = 0; 390 ctx->enc = 0;
955 atomic_set(&ctx->inflight, 0);
956 af_alg_init_completion(&ctx->completion); 391 af_alg_init_completion(&ctx->completion);
957 392
958 ask->private = ctx; 393 ask->private = ctx;
959 394
960 skcipher_request_set_tfm(&ctx->req, skcipher);
961 skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP |
962 CRYPTO_TFM_REQ_MAY_BACKLOG,
963 af_alg_complete, &ctx->completion);
964
965 sk->sk_destruct = skcipher_sock_destruct; 395 sk->sk_destruct = skcipher_sock_destruct;
966 396
967 return 0; 397 return 0;
diff --git a/crypto/ctr.c b/crypto/ctr.c
index 477d9226ccaa..854d924f9d8e 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -65,8 +65,7 @@ static void crypto_ctr_crypt_final(struct blkcipher_walk *walk,
65 unsigned int nbytes = walk->nbytes; 65 unsigned int nbytes = walk->nbytes;
66 66
67 crypto_cipher_encrypt_one(tfm, keystream, ctrblk); 67 crypto_cipher_encrypt_one(tfm, keystream, ctrblk);
68 crypto_xor(keystream, src, nbytes); 68 crypto_xor_cpy(dst, keystream, src, nbytes);
69 memcpy(dst, keystream, nbytes);
70 69
71 crypto_inc(ctrblk, bsize); 70 crypto_inc(ctrblk, bsize);
72} 71}
diff --git a/crypto/ecdh.c b/crypto/ecdh.c
index 61c7708905d0..4271fc77d261 100644
--- a/crypto/ecdh.c
+++ b/crypto/ecdh.c
@@ -20,8 +20,6 @@ struct ecdh_ctx {
20 unsigned int curve_id; 20 unsigned int curve_id;
21 unsigned int ndigits; 21 unsigned int ndigits;
22 u64 private_key[ECC_MAX_DIGITS]; 22 u64 private_key[ECC_MAX_DIGITS];
23 u64 public_key[2 * ECC_MAX_DIGITS];
24 u64 shared_secret[ECC_MAX_DIGITS];
25}; 23};
26 24
27static inline struct ecdh_ctx *ecdh_get_ctx(struct crypto_kpp *tfm) 25static inline struct ecdh_ctx *ecdh_get_ctx(struct crypto_kpp *tfm)
@@ -70,41 +68,58 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
70 68
71static int ecdh_compute_value(struct kpp_request *req) 69static int ecdh_compute_value(struct kpp_request *req)
72{ 70{
73 int ret = 0;
74 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 71 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
75 struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); 72 struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
76 size_t copied, nbytes; 73 u64 *public_key;
74 u64 *shared_secret = NULL;
77 void *buf; 75 void *buf;
76 size_t copied, nbytes, public_key_sz;
77 int ret = -ENOMEM;
78 78
79 nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT; 79 nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
80 /* Public part is a point thus it has both coordinates */
81 public_key_sz = 2 * nbytes;
82
83 public_key = kmalloc(public_key_sz, GFP_KERNEL);
84 if (!public_key)
85 return -ENOMEM;
80 86
81 if (req->src) { 87 if (req->src) {
82 copied = sg_copy_to_buffer(req->src, 1, ctx->public_key, 88 shared_secret = kmalloc(nbytes, GFP_KERNEL);
83 2 * nbytes); 89 if (!shared_secret)
84 if (copied != 2 * nbytes) 90 goto free_pubkey;
85 return -EINVAL; 91
92 copied = sg_copy_to_buffer(req->src, 1, public_key,
93 public_key_sz);
94 if (copied != public_key_sz) {
95 ret = -EINVAL;
96 goto free_all;
97 }
86 98
87 ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits, 99 ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits,
88 ctx->private_key, 100 ctx->private_key, public_key,
89 ctx->public_key, 101 shared_secret);
90 ctx->shared_secret);
91 102
92 buf = ctx->shared_secret; 103 buf = shared_secret;
93 } else { 104 } else {
94 ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits, 105 ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits,
95 ctx->private_key, ctx->public_key); 106 ctx->private_key, public_key);
96 buf = ctx->public_key; 107 buf = public_key;
97 /* Public part is a point thus it has both coordinates */ 108 nbytes = public_key_sz;
98 nbytes *= 2;
99 } 109 }
100 110
101 if (ret < 0) 111 if (ret < 0)
102 return ret; 112 goto free_all;
103 113
104 copied = sg_copy_from_buffer(req->dst, 1, buf, nbytes); 114 copied = sg_copy_from_buffer(req->dst, 1, buf, nbytes);
105 if (copied != nbytes) 115 if (copied != nbytes)
106 return -EINVAL; 116 ret = -EINVAL;
107 117
118 /* fall through */
119free_all:
120 kzfree(shared_secret);
121free_pubkey:
122 kfree(public_key);
108 return ret; 123 return ret;
109} 124}
110 125
diff --git a/crypto/pcbc.c b/crypto/pcbc.c
index 29dd2b4a3b85..d9e45a958720 100644
--- a/crypto/pcbc.c
+++ b/crypto/pcbc.c
@@ -55,8 +55,7 @@ static int crypto_pcbc_encrypt_segment(struct skcipher_request *req,
55 do { 55 do {
56 crypto_xor(iv, src, bsize); 56 crypto_xor(iv, src, bsize);
57 crypto_cipher_encrypt_one(tfm, dst, iv); 57 crypto_cipher_encrypt_one(tfm, dst, iv);
58 memcpy(iv, dst, bsize); 58 crypto_xor_cpy(iv, dst, src, bsize);
59 crypto_xor(iv, src, bsize);
60 59
61 src += bsize; 60 src += bsize;
62 dst += bsize; 61 dst += bsize;
@@ -79,8 +78,7 @@ static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req,
79 memcpy(tmpbuf, src, bsize); 78 memcpy(tmpbuf, src, bsize);
80 crypto_xor(iv, src, bsize); 79 crypto_xor(iv, src, bsize);
81 crypto_cipher_encrypt_one(tfm, src, iv); 80 crypto_cipher_encrypt_one(tfm, src, iv);
82 memcpy(iv, tmpbuf, bsize); 81 crypto_xor_cpy(iv, tmpbuf, src, bsize);
83 crypto_xor(iv, src, bsize);
84 82
85 src += bsize; 83 src += bsize;
86 } while ((nbytes -= bsize) >= bsize); 84 } while ((nbytes -= bsize) >= bsize);
@@ -127,8 +125,7 @@ static int crypto_pcbc_decrypt_segment(struct skcipher_request *req,
127 do { 125 do {
128 crypto_cipher_decrypt_one(tfm, dst, src); 126 crypto_cipher_decrypt_one(tfm, dst, src);
129 crypto_xor(dst, iv, bsize); 127 crypto_xor(dst, iv, bsize);
130 memcpy(iv, src, bsize); 128 crypto_xor_cpy(iv, dst, src, bsize);
131 crypto_xor(iv, dst, bsize);
132 129
133 src += bsize; 130 src += bsize;
134 dst += bsize; 131 dst += bsize;
@@ -153,8 +150,7 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req,
153 memcpy(tmpbuf, src, bsize); 150 memcpy(tmpbuf, src, bsize);
154 crypto_cipher_decrypt_one(tfm, src, src); 151 crypto_cipher_decrypt_one(tfm, src, src);
155 crypto_xor(src, iv, bsize); 152 crypto_xor(src, iv, bsize);
156 memcpy(iv, tmpbuf, bsize); 153 crypto_xor_cpy(iv, src, tmpbuf, bsize);
157 crypto_xor(iv, src, bsize);
158 154
159 src += bsize; 155 src += bsize;
160 } while ((nbytes -= bsize) >= bsize); 156 } while ((nbytes -= bsize) >= bsize);
diff --git a/crypto/rng.c b/crypto/rng.c
index 5e8469244960..b4a618668161 100644
--- a/crypto/rng.c
+++ b/crypto/rng.c
@@ -43,12 +43,14 @@ int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen)
43 if (!buf) 43 if (!buf)
44 return -ENOMEM; 44 return -ENOMEM;
45 45
46 get_random_bytes(buf, slen); 46 err = get_random_bytes_wait(buf, slen);
47 if (err)
48 goto out;
47 seed = buf; 49 seed = buf;
48 } 50 }
49 51
50 err = crypto_rng_alg(tfm)->seed(tfm, seed, slen); 52 err = crypto_rng_alg(tfm)->seed(tfm, seed, slen);
51 53out:
52 kzfree(buf); 54 kzfree(buf);
53 return err; 55 return err;
54} 56}
diff --git a/crypto/scompress.c b/crypto/scompress.c
index ae1d3cf209e4..2075e2c4e7df 100644
--- a/crypto/scompress.c
+++ b/crypto/scompress.c
@@ -65,11 +65,6 @@ static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
65 seq_puts(m, "type : scomp\n"); 65 seq_puts(m, "type : scomp\n");
66} 66}
67 67
68static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
69{
70 return 0;
71}
72
73static void crypto_scomp_free_scratches(void * __percpu *scratches) 68static void crypto_scomp_free_scratches(void * __percpu *scratches)
74{ 69{
75 int i; 70 int i;
@@ -125,12 +120,26 @@ static int crypto_scomp_alloc_all_scratches(void)
125 if (!scomp_src_scratches) 120 if (!scomp_src_scratches)
126 return -ENOMEM; 121 return -ENOMEM;
127 scomp_dst_scratches = crypto_scomp_alloc_scratches(); 122 scomp_dst_scratches = crypto_scomp_alloc_scratches();
128 if (!scomp_dst_scratches) 123 if (!scomp_dst_scratches) {
124 crypto_scomp_free_scratches(scomp_src_scratches);
125 scomp_src_scratches = NULL;
129 return -ENOMEM; 126 return -ENOMEM;
127 }
130 } 128 }
131 return 0; 129 return 0;
132} 130}
133 131
132static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
133{
134 int ret;
135
136 mutex_lock(&scomp_lock);
137 ret = crypto_scomp_alloc_all_scratches();
138 mutex_unlock(&scomp_lock);
139
140 return ret;
141}
142
134static void crypto_scomp_sg_free(struct scatterlist *sgl) 143static void crypto_scomp_sg_free(struct scatterlist *sgl)
135{ 144{
136 int i, n; 145 int i, n;
@@ -211,9 +220,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
211 scratch_dst, &req->dlen, *ctx); 220 scratch_dst, &req->dlen, *ctx);
212 if (!ret) { 221 if (!ret) {
213 if (!req->dst) { 222 if (!req->dst) {
214 req->dst = crypto_scomp_sg_alloc(req->dlen, 223 req->dst = crypto_scomp_sg_alloc(req->dlen, GFP_ATOMIC);
215 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
216 GFP_KERNEL : GFP_ATOMIC);
217 if (!req->dst) 224 if (!req->dst)
218 goto out; 225 goto out;
219 } 226 }
@@ -240,6 +247,10 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
240 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); 247 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
241 248
242 crypto_free_scomp(*ctx); 249 crypto_free_scomp(*ctx);
250
251 mutex_lock(&scomp_lock);
252 crypto_scomp_free_all_scratches();
253 mutex_unlock(&scomp_lock);
243} 254}
244 255
245int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) 256int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
@@ -316,40 +327,18 @@ static const struct crypto_type crypto_scomp_type = {
316int crypto_register_scomp(struct scomp_alg *alg) 327int crypto_register_scomp(struct scomp_alg *alg)
317{ 328{
318 struct crypto_alg *base = &alg->base; 329 struct crypto_alg *base = &alg->base;
319 int ret = -ENOMEM;
320
321 mutex_lock(&scomp_lock);
322 if (crypto_scomp_alloc_all_scratches())
323 goto error;
324 330
325 base->cra_type = &crypto_scomp_type; 331 base->cra_type = &crypto_scomp_type;
326 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; 332 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
327 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; 333 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS;
328 334
329 ret = crypto_register_alg(base); 335 return crypto_register_alg(base);
330 if (ret)
331 goto error;
332
333 mutex_unlock(&scomp_lock);
334 return ret;
335
336error:
337 crypto_scomp_free_all_scratches();
338 mutex_unlock(&scomp_lock);
339 return ret;
340} 336}
341EXPORT_SYMBOL_GPL(crypto_register_scomp); 337EXPORT_SYMBOL_GPL(crypto_register_scomp);
342 338
343int crypto_unregister_scomp(struct scomp_alg *alg) 339int crypto_unregister_scomp(struct scomp_alg *alg)
344{ 340{
345 int ret; 341 return crypto_unregister_alg(&alg->base);
346
347 mutex_lock(&scomp_lock);
348 ret = crypto_unregister_alg(&alg->base);
349 crypto_scomp_free_all_scratches();
350 mutex_unlock(&scomp_lock);
351
352 return ret;
353} 342}
354EXPORT_SYMBOL_GPL(crypto_unregister_scomp); 343EXPORT_SYMBOL_GPL(crypto_unregister_scomp);
355 344
diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c
index 94970a794975..7c3382facc82 100644
--- a/crypto/serpent_generic.c
+++ b/crypto/serpent_generic.c
@@ -229,6 +229,46 @@
229 x4 ^= x2; \ 229 x4 ^= x2; \
230 }) 230 })
231 231
232static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k)
233{
234 k += 100;
235 S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24);
236 S4(r1, r2, r4, r3, r0); store_and_load_keys(r2, r4, r3, r0, 24, 20);
237 S5(r2, r4, r3, r0, r1); store_and_load_keys(r1, r2, r4, r0, 20, 16);
238 S6(r1, r2, r4, r0, r3); store_and_load_keys(r4, r3, r2, r0, 16, 12);
239 S7(r4, r3, r2, r0, r1); store_and_load_keys(r1, r2, r0, r4, 12, 8);
240 S0(r1, r2, r0, r4, r3); store_and_load_keys(r0, r2, r4, r1, 8, 4);
241 S1(r0, r2, r4, r1, r3); store_and_load_keys(r3, r4, r1, r0, 4, 0);
242 S2(r3, r4, r1, r0, r2); store_and_load_keys(r2, r4, r3, r0, 0, -4);
243 S3(r2, r4, r3, r0, r1); store_and_load_keys(r0, r1, r4, r2, -4, -8);
244 S4(r0, r1, r4, r2, r3); store_and_load_keys(r1, r4, r2, r3, -8, -12);
245 S5(r1, r4, r2, r3, r0); store_and_load_keys(r0, r1, r4, r3, -12, -16);
246 S6(r0, r1, r4, r3, r2); store_and_load_keys(r4, r2, r1, r3, -16, -20);
247 S7(r4, r2, r1, r3, r0); store_and_load_keys(r0, r1, r3, r4, -20, -24);
248 S0(r0, r1, r3, r4, r2); store_and_load_keys(r3, r1, r4, r0, -24, -28);
249 k -= 50;
250 S1(r3, r1, r4, r0, r2); store_and_load_keys(r2, r4, r0, r3, 22, 18);
251 S2(r2, r4, r0, r3, r1); store_and_load_keys(r1, r4, r2, r3, 18, 14);
252 S3(r1, r4, r2, r3, r0); store_and_load_keys(r3, r0, r4, r1, 14, 10);
253 S4(r3, r0, r4, r1, r2); store_and_load_keys(r0, r4, r1, r2, 10, 6);
254 S5(r0, r4, r1, r2, r3); store_and_load_keys(r3, r0, r4, r2, 6, 2);
255 S6(r3, r0, r4, r2, r1); store_and_load_keys(r4, r1, r0, r2, 2, -2);
256 S7(r4, r1, r0, r2, r3); store_and_load_keys(r3, r0, r2, r4, -2, -6);
257 S0(r3, r0, r2, r4, r1); store_and_load_keys(r2, r0, r4, r3, -6, -10);
258 S1(r2, r0, r4, r3, r1); store_and_load_keys(r1, r4, r3, r2, -10, -14);
259 S2(r1, r4, r3, r2, r0); store_and_load_keys(r0, r4, r1, r2, -14, -18);
260 S3(r0, r4, r1, r2, r3); store_and_load_keys(r2, r3, r4, r0, -18, -22);
261 k -= 50;
262 S4(r2, r3, r4, r0, r1); store_and_load_keys(r3, r4, r0, r1, 28, 24);
263 S5(r3, r4, r0, r1, r2); store_and_load_keys(r2, r3, r4, r1, 24, 20);
264 S6(r2, r3, r4, r1, r0); store_and_load_keys(r4, r0, r3, r1, 20, 16);
265 S7(r4, r0, r3, r1, r2); store_and_load_keys(r2, r3, r1, r4, 16, 12);
266 S0(r2, r3, r1, r4, r0); store_and_load_keys(r1, r3, r4, r2, 12, 8);
267 S1(r1, r3, r4, r2, r0); store_and_load_keys(r0, r4, r2, r1, 8, 4);
268 S2(r0, r4, r2, r1, r3); store_and_load_keys(r3, r4, r0, r1, 4, 0);
269 S3(r3, r4, r0, r1, r2); storekeys(r1, r2, r4, r3, 0);
270}
271
232int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key, 272int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key,
233 unsigned int keylen) 273 unsigned int keylen)
234{ 274{
@@ -395,42 +435,7 @@ int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key,
395 keyiter(k[23], r1, r0, r3, 131, 31); 435 keyiter(k[23], r1, r0, r3, 131, 31);
396 436
397 /* Apply S-boxes */ 437 /* Apply S-boxes */
398 438 __serpent_setkey_sbox(r0, r1, r2, r3, r4, ctx->expkey);
399 S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24);
400 S4(r1, r2, r4, r3, r0); store_and_load_keys(r2, r4, r3, r0, 24, 20);
401 S5(r2, r4, r3, r0, r1); store_and_load_keys(r1, r2, r4, r0, 20, 16);
402 S6(r1, r2, r4, r0, r3); store_and_load_keys(r4, r3, r2, r0, 16, 12);
403 S7(r4, r3, r2, r0, r1); store_and_load_keys(r1, r2, r0, r4, 12, 8);
404 S0(r1, r2, r0, r4, r3); store_and_load_keys(r0, r2, r4, r1, 8, 4);
405 S1(r0, r2, r4, r1, r3); store_and_load_keys(r3, r4, r1, r0, 4, 0);
406 S2(r3, r4, r1, r0, r2); store_and_load_keys(r2, r4, r3, r0, 0, -4);
407 S3(r2, r4, r3, r0, r1); store_and_load_keys(r0, r1, r4, r2, -4, -8);
408 S4(r0, r1, r4, r2, r3); store_and_load_keys(r1, r4, r2, r3, -8, -12);
409 S5(r1, r4, r2, r3, r0); store_and_load_keys(r0, r1, r4, r3, -12, -16);
410 S6(r0, r1, r4, r3, r2); store_and_load_keys(r4, r2, r1, r3, -16, -20);
411 S7(r4, r2, r1, r3, r0); store_and_load_keys(r0, r1, r3, r4, -20, -24);
412 S0(r0, r1, r3, r4, r2); store_and_load_keys(r3, r1, r4, r0, -24, -28);
413 k -= 50;
414 S1(r3, r1, r4, r0, r2); store_and_load_keys(r2, r4, r0, r3, 22, 18);
415 S2(r2, r4, r0, r3, r1); store_and_load_keys(r1, r4, r2, r3, 18, 14);
416 S3(r1, r4, r2, r3, r0); store_and_load_keys(r3, r0, r4, r1, 14, 10);
417 S4(r3, r0, r4, r1, r2); store_and_load_keys(r0, r4, r1, r2, 10, 6);
418 S5(r0, r4, r1, r2, r3); store_and_load_keys(r3, r0, r4, r2, 6, 2);
419 S6(r3, r0, r4, r2, r1); store_and_load_keys(r4, r1, r0, r2, 2, -2);
420 S7(r4, r1, r0, r2, r3); store_and_load_keys(r3, r0, r2, r4, -2, -6);
421 S0(r3, r0, r2, r4, r1); store_and_load_keys(r2, r0, r4, r3, -6, -10);
422 S1(r2, r0, r4, r3, r1); store_and_load_keys(r1, r4, r3, r2, -10, -14);
423 S2(r1, r4, r3, r2, r0); store_and_load_keys(r0, r4, r1, r2, -14, -18);
424 S3(r0, r4, r1, r2, r3); store_and_load_keys(r2, r3, r4, r0, -18, -22);
425 k -= 50;
426 S4(r2, r3, r4, r0, r1); store_and_load_keys(r3, r4, r0, r1, 28, 24);
427 S5(r3, r4, r0, r1, r2); store_and_load_keys(r2, r3, r4, r1, 24, 20);
428 S6(r2, r3, r4, r1, r0); store_and_load_keys(r4, r0, r3, r1, 20, 16);
429 S7(r4, r0, r3, r1, r2); store_and_load_keys(r2, r3, r1, r4, 16, 12);
430 S0(r2, r3, r1, r4, r0); store_and_load_keys(r1, r3, r4, r2, 12, 8);
431 S1(r1, r3, r4, r2, r0); store_and_load_keys(r0, r4, r2, r1, 8, 4);
432 S2(r0, r4, r2, r1, r3); store_and_load_keys(r3, r4, r0, r1, 4, 0);
433 S3(r3, r4, r0, r1, r2); storekeys(r1, r2, r4, r3, 0);
434 439
435 return 0; 440 return 0;
436} 441}
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 0dd6a432d6ca..0022a18d36ee 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -1404,9 +1404,9 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
1404 test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, 1404 test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
1405 speed_template_32_40_48); 1405 speed_template_32_40_48);
1406 test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, 1406 test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
1407 speed_template_32_48_64); 1407 speed_template_32_64);
1408 test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, 1408 test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1409 speed_template_32_48_64); 1409 speed_template_32_64);
1410 test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, 1410 test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
1411 speed_template_16_24_32); 1411 speed_template_16_24_32);
1412 test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, 1412 test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
@@ -1837,9 +1837,9 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
1837 test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, 1837 test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
1838 speed_template_32_40_48); 1838 speed_template_32_40_48);
1839 test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, 1839 test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
1840 speed_template_32_48_64); 1840 speed_template_32_64);
1841 test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, 1841 test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1842 speed_template_32_48_64); 1842 speed_template_32_64);
1843 test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, 1843 test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
1844 speed_template_16_24_32); 1844 speed_template_16_24_32);
1845 test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, 1845 test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 1b223c32a8ae..95a031e9eced 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -13,10 +13,8 @@ menuconfig HW_RANDOM
13 that's usually called /dev/hwrng, and which exposes one 13 that's usually called /dev/hwrng, and which exposes one
14 of possibly several hardware random number generators. 14 of possibly several hardware random number generators.
15 15
16 These hardware random number generators do not feed directly 16 These hardware random number generators do feed into the
17 into the kernel's random number generator. That is usually 17 kernel's random number generator entropy pool.
18 handled by the "rngd" daemon. Documentation/hw_random.txt
19 has more information.
20 18
21 If unsure, say Y. 19 If unsure, say Y.
22 20
@@ -255,6 +253,20 @@ config HW_RANDOM_MXC_RNGA
255 253
256 If unsure, say Y. 254 If unsure, say Y.
257 255
256config HW_RANDOM_IMX_RNGC
257 tristate "Freescale i.MX RNGC Random Number Generator"
258 depends on ARCH_MXC
259 default HW_RANDOM
260 ---help---
261 This driver provides kernel-side support for the Random Number
262 Generator Version C hardware found on some Freescale i.MX
263 processors. Version B is also supported by this driver.
264
265 To compile this driver as a module, choose M here: the
266 module will be called imx-rngc.
267
268 If unsure, say Y.
269
258config HW_RANDOM_NOMADIK 270config HW_RANDOM_NOMADIK
259 tristate "ST-Ericsson Nomadik Random Number Generator support" 271 tristate "ST-Ericsson Nomadik Random Number Generator support"
260 depends on ARCH_NOMADIK 272 depends on ARCH_NOMADIK
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index b085975ec1d2..39a67defac67 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_HW_RANDOM_PASEMI) += pasemi-rng.o
20obj-$(CONFIG_HW_RANDOM_VIRTIO) += virtio-rng.o 20obj-$(CONFIG_HW_RANDOM_VIRTIO) += virtio-rng.o
21obj-$(CONFIG_HW_RANDOM_TX4939) += tx4939-rng.o 21obj-$(CONFIG_HW_RANDOM_TX4939) += tx4939-rng.o
22obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o 22obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o
23obj-$(CONFIG_HW_RANDOM_IMX_RNGC) += imx-rngc.o
23obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o 24obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o
24obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o 25obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o
25obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o 26obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 503a41dfa193..9701ac7d8b47 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -28,7 +28,10 @@
28#define RNG_MODULE_NAME "hw_random" 28#define RNG_MODULE_NAME "hw_random"
29 29
30static struct hwrng *current_rng; 30static struct hwrng *current_rng;
31/* the current rng has been explicitly chosen by user via sysfs */
32static int cur_rng_set_by_user;
31static struct task_struct *hwrng_fill; 33static struct task_struct *hwrng_fill;
34/* list of registered rngs, sorted decending by quality */
32static LIST_HEAD(rng_list); 35static LIST_HEAD(rng_list);
33/* Protects rng_list and current_rng */ 36/* Protects rng_list and current_rng */
34static DEFINE_MUTEX(rng_mutex); 37static DEFINE_MUTEX(rng_mutex);
@@ -303,6 +306,7 @@ static ssize_t hwrng_attr_current_store(struct device *dev,
303 list_for_each_entry(rng, &rng_list, list) { 306 list_for_each_entry(rng, &rng_list, list) {
304 if (sysfs_streq(rng->name, buf)) { 307 if (sysfs_streq(rng->name, buf)) {
305 err = 0; 308 err = 0;
309 cur_rng_set_by_user = 1;
306 if (rng != current_rng) 310 if (rng != current_rng)
307 err = set_current_rng(rng); 311 err = set_current_rng(rng);
308 break; 312 break;
@@ -351,16 +355,27 @@ static ssize_t hwrng_attr_available_show(struct device *dev,
351 return strlen(buf); 355 return strlen(buf);
352} 356}
353 357
358static ssize_t hwrng_attr_selected_show(struct device *dev,
359 struct device_attribute *attr,
360 char *buf)
361{
362 return snprintf(buf, PAGE_SIZE, "%d\n", cur_rng_set_by_user);
363}
364
354static DEVICE_ATTR(rng_current, S_IRUGO | S_IWUSR, 365static DEVICE_ATTR(rng_current, S_IRUGO | S_IWUSR,
355 hwrng_attr_current_show, 366 hwrng_attr_current_show,
356 hwrng_attr_current_store); 367 hwrng_attr_current_store);
357static DEVICE_ATTR(rng_available, S_IRUGO, 368static DEVICE_ATTR(rng_available, S_IRUGO,
358 hwrng_attr_available_show, 369 hwrng_attr_available_show,
359 NULL); 370 NULL);
371static DEVICE_ATTR(rng_selected, S_IRUGO,
372 hwrng_attr_selected_show,
373 NULL);
360 374
361static struct attribute *rng_dev_attrs[] = { 375static struct attribute *rng_dev_attrs[] = {
362 &dev_attr_rng_current.attr, 376 &dev_attr_rng_current.attr,
363 &dev_attr_rng_available.attr, 377 &dev_attr_rng_available.attr,
378 &dev_attr_rng_selected.attr,
364 NULL 379 NULL
365}; 380};
366 381
@@ -417,6 +432,7 @@ int hwrng_register(struct hwrng *rng)
417{ 432{
418 int err = -EINVAL; 433 int err = -EINVAL;
419 struct hwrng *old_rng, *tmp; 434 struct hwrng *old_rng, *tmp;
435 struct list_head *rng_list_ptr;
420 436
421 if (!rng->name || (!rng->data_read && !rng->read)) 437 if (!rng->name || (!rng->data_read && !rng->read))
422 goto out; 438 goto out;
@@ -432,14 +448,27 @@ int hwrng_register(struct hwrng *rng)
432 init_completion(&rng->cleanup_done); 448 init_completion(&rng->cleanup_done);
433 complete(&rng->cleanup_done); 449 complete(&rng->cleanup_done);
434 450
451 /* rng_list is sorted by decreasing quality */
452 list_for_each(rng_list_ptr, &rng_list) {
453 tmp = list_entry(rng_list_ptr, struct hwrng, list);
454 if (tmp->quality < rng->quality)
455 break;
456 }
457 list_add_tail(&rng->list, rng_list_ptr);
458
435 old_rng = current_rng; 459 old_rng = current_rng;
436 err = 0; 460 err = 0;
437 if (!old_rng) { 461 if (!old_rng ||
462 (!cur_rng_set_by_user && rng->quality > old_rng->quality)) {
463 /*
464 * Set new rng as current as the new rng source
465 * provides better entropy quality and was not
466 * chosen by userspace.
467 */
438 err = set_current_rng(rng); 468 err = set_current_rng(rng);
439 if (err) 469 if (err)
440 goto out_unlock; 470 goto out_unlock;
441 } 471 }
442 list_add_tail(&rng->list, &rng_list);
443 472
444 if (old_rng && !rng->init) { 473 if (old_rng && !rng->init) {
445 /* 474 /*
@@ -466,12 +495,13 @@ void hwrng_unregister(struct hwrng *rng)
466 list_del(&rng->list); 495 list_del(&rng->list);
467 if (current_rng == rng) { 496 if (current_rng == rng) {
468 drop_current_rng(); 497 drop_current_rng();
498 cur_rng_set_by_user = 0;
499 /* rng_list is sorted by quality, use the best (=first) one */
469 if (!list_empty(&rng_list)) { 500 if (!list_empty(&rng_list)) {
470 struct hwrng *tail; 501 struct hwrng *new_rng;
471
472 tail = list_entry(rng_list.prev, struct hwrng, list);
473 502
474 set_current_rng(tail); 503 new_rng = list_entry(rng_list.next, struct hwrng, list);
504 set_current_rng(new_rng);
475 } 505 }
476 } 506 }
477 507
diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c
new file mode 100644
index 000000000000..88db42d30760
--- /dev/null
+++ b/drivers/char/hw_random/imx-rngc.c
@@ -0,0 +1,331 @@
1/*
2 * RNG driver for Freescale RNGC
3 *
4 * Copyright (C) 2008-2012 Freescale Semiconductor, Inc.
5 * Copyright (C) 2017 Martin Kaiser <martin@kaiser.cx>
6 *
7 * The code contained herein is licensed under the GNU General Public
8 * License. You may obtain a copy of the GNU General Public License
9 * Version 2 or later at the following locations:
10 *
11 * http://www.opensource.org/licenses/gpl-license.html
12 * http://www.gnu.org/copyleft/gpl.html
13 */
14
15#include <linux/module.h>
16#include <linux/init.h>
17#include <linux/kernel.h>
18#include <linux/clk.h>
19#include <linux/err.h>
20#include <linux/platform_device.h>
21#include <linux/interrupt.h>
22#include <linux/hw_random.h>
23#include <linux/completion.h>
24#include <linux/io.h>
25
26#define RNGC_COMMAND 0x0004
27#define RNGC_CONTROL 0x0008
28#define RNGC_STATUS 0x000C
29#define RNGC_ERROR 0x0010
30#define RNGC_FIFO 0x0014
31
32#define RNGC_CMD_CLR_ERR 0x00000020
33#define RNGC_CMD_CLR_INT 0x00000010
34#define RNGC_CMD_SEED 0x00000002
35#define RNGC_CMD_SELF_TEST 0x00000001
36
37#define RNGC_CTRL_MASK_ERROR 0x00000040
38#define RNGC_CTRL_MASK_DONE 0x00000020
39
40#define RNGC_STATUS_ERROR 0x00010000
41#define RNGC_STATUS_FIFO_LEVEL_MASK 0x00000f00
42#define RNGC_STATUS_FIFO_LEVEL_SHIFT 8
43#define RNGC_STATUS_SEED_DONE 0x00000020
44#define RNGC_STATUS_ST_DONE 0x00000010
45
46#define RNGC_ERROR_STATUS_STAT_ERR 0x00000008
47
48#define RNGC_TIMEOUT 3000 /* 3 sec */
49
50
51static bool self_test = true;
52module_param(self_test, bool, 0);
53
54struct imx_rngc {
55 struct device *dev;
56 struct clk *clk;
57 void __iomem *base;
58 struct hwrng rng;
59 struct completion rng_op_done;
60 /*
61 * err_reg is written only by the irq handler and read only
62 * when interrupts are masked, we need no spinlock
63 */
64 u32 err_reg;
65};
66
67
68static inline void imx_rngc_irq_mask_clear(struct imx_rngc *rngc)
69{
70 u32 ctrl, cmd;
71
72 /* mask interrupts */
73 ctrl = readl(rngc->base + RNGC_CONTROL);
74 ctrl |= RNGC_CTRL_MASK_DONE | RNGC_CTRL_MASK_ERROR;
75 writel(ctrl, rngc->base + RNGC_CONTROL);
76
77 /*
78 * CLR_INT clears the interrupt only if there's no error
79 * CLR_ERR clear the interrupt and the error register if there
80 * is an error
81 */
82 cmd = readl(rngc->base + RNGC_COMMAND);
83 cmd |= RNGC_CMD_CLR_INT | RNGC_CMD_CLR_ERR;
84 writel(cmd, rngc->base + RNGC_COMMAND);
85}
86
87static inline void imx_rngc_irq_unmask(struct imx_rngc *rngc)
88{
89 u32 ctrl;
90
91 ctrl = readl(rngc->base + RNGC_CONTROL);
92 ctrl &= ~(RNGC_CTRL_MASK_DONE | RNGC_CTRL_MASK_ERROR);
93 writel(ctrl, rngc->base + RNGC_CONTROL);
94}
95
96static int imx_rngc_self_test(struct imx_rngc *rngc)
97{
98 u32 cmd;
99 int ret;
100
101 imx_rngc_irq_unmask(rngc);
102
103 /* run self test */
104 cmd = readl(rngc->base + RNGC_COMMAND);
105 writel(cmd | RNGC_CMD_SELF_TEST, rngc->base + RNGC_COMMAND);
106
107 ret = wait_for_completion_timeout(&rngc->rng_op_done, RNGC_TIMEOUT);
108 if (!ret) {
109 imx_rngc_irq_mask_clear(rngc);
110 return -ETIMEDOUT;
111 }
112
113 if (rngc->err_reg != 0)
114 return -EIO;
115
116 return 0;
117}
118
119static int imx_rngc_read(struct hwrng *rng, void *data, size_t max, bool wait)
120{
121 struct imx_rngc *rngc = container_of(rng, struct imx_rngc, rng);
122 unsigned int status;
123 unsigned int level;
124 int retval = 0;
125
126 while (max >= sizeof(u32)) {
127 status = readl(rngc->base + RNGC_STATUS);
128
129 /* is there some error while reading this random number? */
130 if (status & RNGC_STATUS_ERROR)
131 break;
132
133 /* how many random numbers are in FIFO? [0-16] */
134 level = (status & RNGC_STATUS_FIFO_LEVEL_MASK) >>
135 RNGC_STATUS_FIFO_LEVEL_SHIFT;
136
137 if (level) {
138 /* retrieve a random number from FIFO */
139 *(u32 *)data = readl(rngc->base + RNGC_FIFO);
140
141 retval += sizeof(u32);
142 data += sizeof(u32);
143 max -= sizeof(u32);
144 }
145 }
146
147 return retval ? retval : -EIO;
148}
149
150static irqreturn_t imx_rngc_irq(int irq, void *priv)
151{
152 struct imx_rngc *rngc = (struct imx_rngc *)priv;
153 u32 status;
154
155 /*
156 * clearing the interrupt will also clear the error register
157 * read error and status before clearing
158 */
159 status = readl(rngc->base + RNGC_STATUS);
160 rngc->err_reg = readl(rngc->base + RNGC_ERROR);
161
162 imx_rngc_irq_mask_clear(rngc);
163
164 if (status & (RNGC_STATUS_SEED_DONE | RNGC_STATUS_ST_DONE))
165 complete(&rngc->rng_op_done);
166
167 return IRQ_HANDLED;
168}
169
170static int imx_rngc_init(struct hwrng *rng)
171{
172 struct imx_rngc *rngc = container_of(rng, struct imx_rngc, rng);
173 u32 cmd;
174 int ret;
175
176 /* clear error */
177 cmd = readl(rngc->base + RNGC_COMMAND);
178 writel(cmd | RNGC_CMD_CLR_ERR, rngc->base + RNGC_COMMAND);
179
180 /* create seed, repeat while there is some statistical error */
181 do {
182 imx_rngc_irq_unmask(rngc);
183
184 /* seed creation */
185 cmd = readl(rngc->base + RNGC_COMMAND);
186 writel(cmd | RNGC_CMD_SEED, rngc->base + RNGC_COMMAND);
187
188 ret = wait_for_completion_timeout(&rngc->rng_op_done,
189 RNGC_TIMEOUT);
190
191 if (!ret) {
192 imx_rngc_irq_mask_clear(rngc);
193 return -ETIMEDOUT;
194 }
195
196 } while (rngc->err_reg == RNGC_ERROR_STATUS_STAT_ERR);
197
198 return rngc->err_reg ? -EIO : 0;
199}
200
201static int imx_rngc_probe(struct platform_device *pdev)
202{
203 struct imx_rngc *rngc;
204 struct resource *res;
205 int ret;
206 int irq;
207
208 rngc = devm_kzalloc(&pdev->dev, sizeof(*rngc), GFP_KERNEL);
209 if (!rngc)
210 return -ENOMEM;
211
212 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
213 rngc->base = devm_ioremap_resource(&pdev->dev, res);
214 if (IS_ERR(rngc->base))
215 return PTR_ERR(rngc->base);
216
217 rngc->clk = devm_clk_get(&pdev->dev, NULL);
218 if (IS_ERR(rngc->clk)) {
219 dev_err(&pdev->dev, "Can not get rng_clk\n");
220 return PTR_ERR(rngc->clk);
221 }
222
223 irq = platform_get_irq(pdev, 0);
224 if (irq <= 0) {
225 dev_err(&pdev->dev, "Couldn't get irq %d\n", irq);
226 return irq;
227 }
228
229 ret = clk_prepare_enable(rngc->clk);
230 if (ret)
231 return ret;
232
233 ret = devm_request_irq(&pdev->dev,
234 irq, imx_rngc_irq, 0, pdev->name, (void *)rngc);
235 if (ret) {
236 dev_err(rngc->dev, "Can't get interrupt working.\n");
237 goto err;
238 }
239
240 init_completion(&rngc->rng_op_done);
241
242 rngc->rng.name = pdev->name;
243 rngc->rng.init = imx_rngc_init;
244 rngc->rng.read = imx_rngc_read;
245
246 rngc->dev = &pdev->dev;
247 platform_set_drvdata(pdev, rngc);
248
249 imx_rngc_irq_mask_clear(rngc);
250
251 if (self_test) {
252 ret = imx_rngc_self_test(rngc);
253 if (ret) {
254 dev_err(rngc->dev, "FSL RNGC self test failed.\n");
255 goto err;
256 }
257 }
258
259 ret = hwrng_register(&rngc->rng);
260 if (ret) {
261 dev_err(&pdev->dev, "FSL RNGC registering failed (%d)\n", ret);
262 goto err;
263 }
264
265 dev_info(&pdev->dev, "Freescale RNGC registered.\n");
266 return 0;
267
268err:
269 clk_disable_unprepare(rngc->clk);
270
271 return ret;
272}
273
274static int __exit imx_rngc_remove(struct platform_device *pdev)
275{
276 struct imx_rngc *rngc = platform_get_drvdata(pdev);
277
278 hwrng_unregister(&rngc->rng);
279
280 clk_disable_unprepare(rngc->clk);
281
282 return 0;
283}
284
285#ifdef CONFIG_PM
286static int imx_rngc_suspend(struct device *dev)
287{
288 struct imx_rngc *rngc = dev_get_drvdata(dev);
289
290 clk_disable_unprepare(rngc->clk);
291
292 return 0;
293}
294
295static int imx_rngc_resume(struct device *dev)
296{
297 struct imx_rngc *rngc = dev_get_drvdata(dev);
298
299 clk_prepare_enable(rngc->clk);
300
301 return 0;
302}
303
304static const struct dev_pm_ops imx_rngc_pm_ops = {
305 .suspend = imx_rngc_suspend,
306 .resume = imx_rngc_resume,
307};
308#endif
309
310static const struct of_device_id imx_rngc_dt_ids[] = {
311 { .compatible = "fsl,imx25-rngb", .data = NULL, },
312 { /* sentinel */ }
313};
314MODULE_DEVICE_TABLE(of, imx_rngc_dt_ids);
315
316static struct platform_driver imx_rngc_driver = {
317 .driver = {
318 .name = "imx_rngc",
319#ifdef CONFIG_PM
320 .pm = &imx_rngc_pm_ops,
321#endif
322 .of_match_table = imx_rngc_dt_ids,
323 },
324 .remove = __exit_p(imx_rngc_remove),
325};
326
327module_platform_driver_probe(imx_rngc_driver, imx_rngc_probe);
328
329MODULE_AUTHOR("Freescale Semiconductor, Inc.");
330MODULE_DESCRIPTION("H/W RNGC driver for i.MX");
331MODULE_LICENSE("GPL");
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 4b75084fabad..fe33c199fc1a 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -525,12 +525,26 @@ config CRYPTO_DEV_ATMEL_SHA
525 To compile this driver as a module, choose M here: the module 525 To compile this driver as a module, choose M here: the module
526 will be called atmel-sha. 526 will be called atmel-sha.
527 527
528config CRYPTO_DEV_ATMEL_ECC
529 tristate "Support for Microchip / Atmel ECC hw accelerator"
530 depends on ARCH_AT91 || COMPILE_TEST
531 depends on I2C
532 select CRYPTO_ECDH
533 select CRC16
534 help
535 Microhip / Atmel ECC hw accelerator.
536 Select this if you want to use the Microchip / Atmel module for
537 ECDH algorithm.
538
539 To compile this driver as a module, choose M here: the module
540 will be called atmel-ecc.
541
528config CRYPTO_DEV_CCP 542config CRYPTO_DEV_CCP
529 bool "Support for AMD Cryptographic Coprocessor" 543 bool "Support for AMD Secure Processor"
530 depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM 544 depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM
531 help 545 help
532 The AMD Cryptographic Coprocessor provides hardware offload support 546 The AMD Secure Processor provides support for the Cryptographic Coprocessor
533 for encryption, hashing and related operations. 547 (CCP) and the Platform Security Processor (PSP) devices.
534 548
535if CRYPTO_DEV_CCP 549if CRYPTO_DEV_CCP
536 source "drivers/crypto/ccp/Kconfig" 550 source "drivers/crypto/ccp/Kconfig"
@@ -616,6 +630,14 @@ config CRYPTO_DEV_SUN4I_SS
616 To compile this driver as a module, choose M here: the module 630 To compile this driver as a module, choose M here: the module
617 will be called sun4i-ss. 631 will be called sun4i-ss.
618 632
633config CRYPTO_DEV_SUN4I_SS_PRNG
634 bool "Support for Allwinner Security System PRNG"
635 depends on CRYPTO_DEV_SUN4I_SS
636 select CRYPTO_RNG
637 help
638 Select this option if you want to provide kernel-side support for
639 the Pseudo-Random Number Generator found in the Security System.
640
619config CRYPTO_DEV_ROCKCHIP 641config CRYPTO_DEV_ROCKCHIP
620 tristate "Rockchip's Cryptographic Engine driver" 642 tristate "Rockchip's Cryptographic Engine driver"
621 depends on OF && ARCH_ROCKCHIP 643 depends on OF && ARCH_ROCKCHIP
@@ -686,4 +708,25 @@ config CRYPTO_DEV_SAFEXCEL
686 chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash 708 chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash
687 algorithms. 709 algorithms.
688 710
711config CRYPTO_DEV_ARTPEC6
712 tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration."
713 depends on ARM && (ARCH_ARTPEC || COMPILE_TEST)
714 depends on HAS_DMA
715 depends on OF
716 select CRYPTO_AEAD
717 select CRYPTO_AES
718 select CRYPTO_ALGAPI
719 select CRYPTO_BLKCIPHER
720 select CRYPTO_CTR
721 select CRYPTO_HASH
722 select CRYPTO_SHA1
723 select CRYPTO_SHA256
724 select CRYPTO_SHA384
725 select CRYPTO_SHA512
726 help
727 Enables the driver for the on-chip crypto accelerator
728 of Axis ARTPEC SoCs.
729
730 To compile this driver as a module, choose M here.
731
689endif # CRYPTO_HW 732endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 2c555a3393b2..808432b44c6b 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,6 +1,7 @@
1obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o 1obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o
2obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o 2obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
3obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o 3obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
4obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o
4obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o 5obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o
5obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ 6obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
6obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ 7obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
@@ -35,7 +36,7 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
35obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/ 36obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
36obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o 37obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
37obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o 38obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
38obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32/ 39obj-$(CONFIG_ARCH_STM32) += stm32/
39obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ 40obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
40obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o 41obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
41obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ 42obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
@@ -43,3 +44,4 @@ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
43obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ 44obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
44obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ 45obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
45obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/ 46obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
47obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
new file mode 100644
index 000000000000..e66f18a0ddd0
--- /dev/null
+++ b/drivers/crypto/atmel-ecc.c
@@ -0,0 +1,781 @@
1/*
2 * Microchip / Atmel ECC (I2C) driver.
3 *
4 * Copyright (c) 2017, Microchip Technology Inc.
5 * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
6 *
7 * This software is licensed under the terms of the GNU General Public
8 * License version 2, as published by the Free Software Foundation, and
9 * may be copied, distributed, and modified under those terms.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 */
17
18#include <linux/bitrev.h>
19#include <linux/crc16.h>
20#include <linux/delay.h>
21#include <linux/device.h>
22#include <linux/err.h>
23#include <linux/errno.h>
24#include <linux/i2c.h>
25#include <linux/init.h>
26#include <linux/kernel.h>
27#include <linux/module.h>
28#include <linux/of_device.h>
29#include <linux/scatterlist.h>
30#include <linux/slab.h>
31#include <linux/workqueue.h>
32#include <crypto/internal/kpp.h>
33#include <crypto/ecdh.h>
34#include <crypto/kpp.h>
35#include "atmel-ecc.h"
36
37/* Used for binding tfm objects to i2c clients. */
38struct atmel_ecc_driver_data {
39 struct list_head i2c_client_list;
40 spinlock_t i2c_list_lock;
41} ____cacheline_aligned;
42
43static struct atmel_ecc_driver_data driver_data;
44
45/**
46 * atmel_ecc_i2c_client_priv - i2c_client private data
47 * @client : pointer to i2c client device
48 * @i2c_client_list_node: part of i2c_client_list
49 * @lock : lock for sending i2c commands
50 * @wake_token : wake token array of zeros
51 * @wake_token_sz : size in bytes of the wake_token
52 * @tfm_count : number of active crypto transformations on i2c client
53 *
54 * Reads and writes from/to the i2c client are sequential. The first byte
55 * transmitted to the device is treated as the byte size. Any attempt to send
56 * more than this number of bytes will cause the device to not ACK those bytes.
57 * After the host writes a single command byte to the input buffer, reads are
58 * prohibited until after the device completes command execution. Use a mutex
59 * when sending i2c commands.
60 */
61struct atmel_ecc_i2c_client_priv {
62 struct i2c_client *client;
63 struct list_head i2c_client_list_node;
64 struct mutex lock;
65 u8 wake_token[WAKE_TOKEN_MAX_SIZE];
66 size_t wake_token_sz;
67 atomic_t tfm_count ____cacheline_aligned;
68};
69
70/**
71 * atmel_ecdh_ctx - transformation context
72 * @client : pointer to i2c client device
73 * @fallback : used for unsupported curves or when user wants to use its own
74 * private key.
75 * @public_key : generated when calling set_secret(). It's the responsibility
76 * of the user to not call set_secret() while
77 * generate_public_key() or compute_shared_secret() are in flight.
78 * @curve_id : elliptic curve id
79 * @n_sz : size in bytes of the n prime
80 * @do_fallback: true when the device doesn't support the curve or when the user
81 * wants to use its own private key.
82 */
83struct atmel_ecdh_ctx {
84 struct i2c_client *client;
85 struct crypto_kpp *fallback;
86 const u8 *public_key;
87 unsigned int curve_id;
88 size_t n_sz;
89 bool do_fallback;
90};
91
92/**
93 * atmel_ecc_work_data - data structure representing the work
94 * @ctx : transformation context.
95 * @cbk : pointer to a callback function to be invoked upon completion of this
96 * request. This has the form:
97 * callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status)
98 * where:
99 * @work_data: data structure representing the work
100 * @areq : optional pointer to an argument passed with the original
101 * request.
102 * @status : status returned from the i2c client device or i2c error.
103 * @areq: optional pointer to a user argument for use at callback time.
104 * @work: describes the task to be executed.
105 * @cmd : structure used for communicating with the device.
106 */
107struct atmel_ecc_work_data {
108 struct atmel_ecdh_ctx *ctx;
109 void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq,
110 int status);
111 void *areq;
112 struct work_struct work;
113 struct atmel_ecc_cmd cmd;
114};
115
116static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len)
117{
118 return cpu_to_le16(bitrev16(crc16(crc, buffer, len)));
119}
120
121/**
122 * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
123 * CRC16 verification of the count, opcode, param1, param2 and data bytes.
124 * The checksum is saved in little-endian format in the least significant
125 * two bytes of the command. CRC polynomial is 0x8005 and the initial register
126 * value should be zero.
127 *
128 * @cmd : structure used for communicating with the device.
129 */
130static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd)
131{
132 u8 *data = &cmd->count;
133 size_t len = cmd->count - CRC_SIZE;
134 u16 *crc16 = (u16 *)(data + len);
135
136 *crc16 = atmel_ecc_crc16(0, data, len);
137}
138
139static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd)
140{
141 cmd->word_addr = COMMAND;
142 cmd->opcode = OPCODE_READ;
143 /*
144 * Read the word from Configuration zone that contains the lock bytes
145 * (UserExtra, Selector, LockValue, LockConfig).
146 */
147 cmd->param1 = CONFIG_ZONE;
148 cmd->param2 = DEVICE_LOCK_ADDR;
149 cmd->count = READ_COUNT;
150
151 atmel_ecc_checksum(cmd);
152
153 cmd->msecs = MAX_EXEC_TIME_READ;
154 cmd->rxsize = READ_RSP_SIZE;
155}
156
157static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid)
158{
159 cmd->word_addr = COMMAND;
160 cmd->count = GENKEY_COUNT;
161 cmd->opcode = OPCODE_GENKEY;
162 cmd->param1 = GENKEY_MODE_PRIVATE;
163 /* a random private key will be generated and stored in slot keyID */
164 cmd->param2 = cpu_to_le16(keyid);
165
166 atmel_ecc_checksum(cmd);
167
168 cmd->msecs = MAX_EXEC_TIME_GENKEY;
169 cmd->rxsize = GENKEY_RSP_SIZE;
170}
171
172static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd,
173 struct scatterlist *pubkey)
174{
175 size_t copied;
176
177 cmd->word_addr = COMMAND;
178 cmd->count = ECDH_COUNT;
179 cmd->opcode = OPCODE_ECDH;
180 cmd->param1 = ECDH_PREFIX_MODE;
181 /* private key slot */
182 cmd->param2 = cpu_to_le16(DATA_SLOT_2);
183
184 /*
185 * The device only supports NIST P256 ECC keys. The public key size will
186 * always be the same. Use a macro for the key size to avoid unnecessary
187 * computations.
188 */
189 copied = sg_copy_to_buffer(pubkey, 1, cmd->data, ATMEL_ECC_PUBKEY_SIZE);
190 if (copied != ATMEL_ECC_PUBKEY_SIZE)
191 return -EINVAL;
192
193 atmel_ecc_checksum(cmd);
194
195 cmd->msecs = MAX_EXEC_TIME_ECDH;
196 cmd->rxsize = ECDH_RSP_SIZE;
197
198 return 0;
199}
200
201/*
202 * After wake and after execution of a command, there will be error, status, or
203 * result bytes in the device's output register that can be retrieved by the
204 * system. When the length of that group is four bytes, the codes returned are
205 * detailed in error_list.
206 */
207static int atmel_ecc_status(struct device *dev, u8 *status)
208{
209 size_t err_list_len = ARRAY_SIZE(error_list);
210 int i;
211 u8 err_id = status[1];
212
213 if (*status != STATUS_SIZE)
214 return 0;
215
216 if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
217 return 0;
218
219 for (i = 0; i < err_list_len; i++)
220 if (error_list[i].value == err_id)
221 break;
222
223 /* if err_id is not in the error_list then ignore it */
224 if (i != err_list_len) {
225 dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
226 return err_id;
227 }
228
229 return 0;
230}
231
232static int atmel_ecc_wakeup(struct i2c_client *client)
233{
234 struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
235 u8 status[STATUS_RSP_SIZE];
236 int ret;
237
238 /*
239 * The device ignores any levels or transitions on the SCL pin when the
240 * device is idle, asleep or during waking up. Don't check for error
241 * when waking up the device.
242 */
243 i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
244
245 /*
246 * Wait to wake the device. Typical execution times for ecdh and genkey
247 * are around tens of milliseconds. Delta is chosen to 50 microseconds.
248 */
249 usleep_range(TWHI_MIN, TWHI_MAX);
250
251 ret = i2c_master_recv(client, status, STATUS_SIZE);
252 if (ret < 0)
253 return ret;
254
255 return atmel_ecc_status(&client->dev, status);
256}
257
258static int atmel_ecc_sleep(struct i2c_client *client)
259{
260 u8 sleep = SLEEP_TOKEN;
261
262 return i2c_master_send(client, &sleep, 1);
263}
264
265static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq,
266 int status)
267{
268 struct kpp_request *req = areq;
269 struct atmel_ecdh_ctx *ctx = work_data->ctx;
270 struct atmel_ecc_cmd *cmd = &work_data->cmd;
271 size_t copied;
272 size_t n_sz = ctx->n_sz;
273
274 if (status)
275 goto free_work_data;
276
277 /* copy the shared secret */
278 copied = sg_copy_from_buffer(req->dst, 1, &cmd->data[RSP_DATA_IDX],
279 n_sz);
280 if (copied != n_sz)
281 status = -EINVAL;
282
283 /* fall through */
284free_work_data:
285 kzfree(work_data);
286 kpp_request_complete(req, status);
287}
288
289/*
290 * atmel_ecc_send_receive() - send a command to the device and receive its
291 * response.
292 * @client: i2c client device
293 * @cmd : structure used to communicate with the device
294 *
295 * After the device receives a Wake token, a watchdog counter starts within the
296 * device. After the watchdog timer expires, the device enters sleep mode
297 * regardless of whether some I/O transmission or command execution is in
298 * progress. If a command is attempted when insufficient time remains prior to
299 * watchdog timer execution, the device will return the watchdog timeout error
300 * code without attempting to execute the command. There is no way to reset the
301 * counter other than to put the device into sleep or idle mode and then
302 * wake it up again.
303 */
304static int atmel_ecc_send_receive(struct i2c_client *client,
305 struct atmel_ecc_cmd *cmd)
306{
307 struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
308 int ret;
309
310 mutex_lock(&i2c_priv->lock);
311
312 ret = atmel_ecc_wakeup(client);
313 if (ret)
314 goto err;
315
316 /* send the command */
317 ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
318 if (ret < 0)
319 goto err;
320
321 /* delay the appropriate amount of time for command to execute */
322 msleep(cmd->msecs);
323
324 /* receive the response */
325 ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
326 if (ret < 0)
327 goto err;
328
329 /* put the device into low-power mode */
330 ret = atmel_ecc_sleep(client);
331 if (ret < 0)
332 goto err;
333
334 mutex_unlock(&i2c_priv->lock);
335 return atmel_ecc_status(&client->dev, cmd->data);
336err:
337 mutex_unlock(&i2c_priv->lock);
338 return ret;
339}
340
341static void atmel_ecc_work_handler(struct work_struct *work)
342{
343 struct atmel_ecc_work_data *work_data =
344 container_of(work, struct atmel_ecc_work_data, work);
345 struct atmel_ecc_cmd *cmd = &work_data->cmd;
346 struct i2c_client *client = work_data->ctx->client;
347 int status;
348
349 status = atmel_ecc_send_receive(client, cmd);
350 work_data->cbk(work_data, work_data->areq, status);
351}
352
353static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data,
354 void (*cbk)(struct atmel_ecc_work_data *work_data,
355 void *areq, int status),
356 void *areq)
357{
358 work_data->cbk = (void *)cbk;
359 work_data->areq = areq;
360
361 INIT_WORK(&work_data->work, atmel_ecc_work_handler);
362 schedule_work(&work_data->work);
363}
364
365static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id)
366{
367 if (curve_id == ECC_CURVE_NIST_P256)
368 return ATMEL_ECC_NIST_P256_N_SIZE;
369
370 return 0;
371}
372
373/*
374 * A random private key is generated and stored in the device. The device
375 * returns the pair public key.
376 */
377static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
378 unsigned int len)
379{
380 struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
381 struct atmel_ecc_cmd *cmd;
382 void *public_key;
383 struct ecdh params;
384 int ret = -ENOMEM;
385
386 /* free the old public key, if any */
387 kfree(ctx->public_key);
388 /* make sure you don't free the old public key twice */
389 ctx->public_key = NULL;
390
391 if (crypto_ecdh_decode_key(buf, len, &params) < 0) {
392 dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n");
393 return -EINVAL;
394 }
395
396 ctx->n_sz = atmel_ecdh_supported_curve(params.curve_id);
397 if (!ctx->n_sz || params.key_size) {
398 /* fallback to ecdh software implementation */
399 ctx->do_fallback = true;
400 return crypto_kpp_set_secret(ctx->fallback, buf, len);
401 }
402
403 cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
404 if (!cmd)
405 return -ENOMEM;
406
407 /*
408 * The device only supports NIST P256 ECC keys. The public key size will
409 * always be the same. Use a macro for the key size to avoid unnecessary
410 * computations.
411 */
412 public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL);
413 if (!public_key)
414 goto free_cmd;
415
416 ctx->do_fallback = false;
417 ctx->curve_id = params.curve_id;
418
419 atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2);
420
421 ret = atmel_ecc_send_receive(ctx->client, cmd);
422 if (ret)
423 goto free_public_key;
424
425 /* save the public key */
426 memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE);
427 ctx->public_key = public_key;
428
429 kfree(cmd);
430 return 0;
431
432free_public_key:
433 kfree(public_key);
434free_cmd:
435 kfree(cmd);
436 return ret;
437}
438
439static int atmel_ecdh_generate_public_key(struct kpp_request *req)
440{
441 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
442 struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
443 size_t copied;
444 int ret = 0;
445
446 if (ctx->do_fallback) {
447 kpp_request_set_tfm(req, ctx->fallback);
448 return crypto_kpp_generate_public_key(req);
449 }
450
451 /* public key was saved at private key generation */
452 copied = sg_copy_from_buffer(req->dst, 1, ctx->public_key,
453 ATMEL_ECC_PUBKEY_SIZE);
454 if (copied != ATMEL_ECC_PUBKEY_SIZE)
455 ret = -EINVAL;
456
457 return ret;
458}
459
460static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
461{
462 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
463 struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
464 struct atmel_ecc_work_data *work_data;
465 gfp_t gfp;
466 int ret;
467
468 if (ctx->do_fallback) {
469 kpp_request_set_tfm(req, ctx->fallback);
470 return crypto_kpp_compute_shared_secret(req);
471 }
472
473 gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
474 GFP_ATOMIC;
475
476 work_data = kmalloc(sizeof(*work_data), gfp);
477 if (!work_data)
478 return -ENOMEM;
479
480 work_data->ctx = ctx;
481
482 ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src);
483 if (ret)
484 goto free_work_data;
485
486 atmel_ecc_enqueue(work_data, atmel_ecdh_done, req);
487
488 return -EINPROGRESS;
489
490free_work_data:
491 kfree(work_data);
492 return ret;
493}
494
495static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
496{
497 struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
498 struct i2c_client *client = ERR_PTR(-ENODEV);
499 int min_tfm_cnt = INT_MAX;
500 int tfm_cnt;
501
502 spin_lock(&driver_data.i2c_list_lock);
503
504 if (list_empty(&driver_data.i2c_client_list)) {
505 spin_unlock(&driver_data.i2c_list_lock);
506 return ERR_PTR(-ENODEV);
507 }
508
509 list_for_each_entry(i2c_priv, &driver_data.i2c_client_list,
510 i2c_client_list_node) {
511 tfm_cnt = atomic_read(&i2c_priv->tfm_count);
512 if (tfm_cnt < min_tfm_cnt) {
513 min_tfm_cnt = tfm_cnt;
514 min_i2c_priv = i2c_priv;
515 }
516 if (!min_tfm_cnt)
517 break;
518 }
519
520 if (min_i2c_priv) {
521 atomic_inc(&min_i2c_priv->tfm_count);
522 client = min_i2c_priv->client;
523 }
524
525 spin_unlock(&driver_data.i2c_list_lock);
526
527 return client;
528}
529
530static void atmel_ecc_i2c_client_free(struct i2c_client *client)
531{
532 struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
533
534 atomic_dec(&i2c_priv->tfm_count);
535}
536
537static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm)
538{
539 const char *alg = kpp_alg_name(tfm);
540 struct crypto_kpp *fallback;
541 struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
542
543 ctx->client = atmel_ecc_i2c_client_alloc();
544 if (IS_ERR(ctx->client)) {
545 pr_err("tfm - i2c_client binding failed\n");
546 return PTR_ERR(ctx->client);
547 }
548
549 fallback = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
550 if (IS_ERR(fallback)) {
551 dev_err(&ctx->client->dev, "Failed to allocate transformation for '%s': %ld\n",
552 alg, PTR_ERR(fallback));
553 return PTR_ERR(fallback);
554 }
555
556 crypto_kpp_set_flags(fallback, crypto_kpp_get_flags(tfm));
557
558 dev_info(&ctx->client->dev, "Using '%s' as fallback implementation.\n",
559 crypto_tfm_alg_driver_name(crypto_kpp_tfm(fallback)));
560
561 ctx->fallback = fallback;
562
563 return 0;
564}
565
566static void atmel_ecdh_exit_tfm(struct crypto_kpp *tfm)
567{
568 struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
569
570 kfree(ctx->public_key);
571 crypto_free_kpp(ctx->fallback);
572 atmel_ecc_i2c_client_free(ctx->client);
573}
574
575static unsigned int atmel_ecdh_max_size(struct crypto_kpp *tfm)
576{
577 struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
578
579 if (ctx->fallback)
580 return crypto_kpp_maxsize(ctx->fallback);
581
582 /*
583 * The device only supports NIST P256 ECC keys. The public key size will
584 * always be the same. Use a macro for the key size to avoid unnecessary
585 * computations.
586 */
587 return ATMEL_ECC_PUBKEY_SIZE;
588}
589
590static struct kpp_alg atmel_ecdh = {
591 .set_secret = atmel_ecdh_set_secret,
592 .generate_public_key = atmel_ecdh_generate_public_key,
593 .compute_shared_secret = atmel_ecdh_compute_shared_secret,
594 .init = atmel_ecdh_init_tfm,
595 .exit = atmel_ecdh_exit_tfm,
596 .max_size = atmel_ecdh_max_size,
597 .base = {
598 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
599 .cra_name = "ecdh",
600 .cra_driver_name = "atmel-ecdh",
601 .cra_priority = ATMEL_ECC_PRIORITY,
602 .cra_module = THIS_MODULE,
603 .cra_ctxsize = sizeof(struct atmel_ecdh_ctx),
604 },
605};
606
607static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate)
608{
609 u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
610
611 /* return the size of the wake_token in bytes */
612 return DIV_ROUND_UP(no_of_bits, 8);
613}
614
615static int device_sanity_check(struct i2c_client *client)
616{
617 struct atmel_ecc_cmd *cmd;
618 int ret;
619
620 cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
621 if (!cmd)
622 return -ENOMEM;
623
624 atmel_ecc_init_read_cmd(cmd);
625
626 ret = atmel_ecc_send_receive(client, cmd);
627 if (ret)
628 goto free_cmd;
629
630 /*
631 * It is vital that the Configuration, Data and OTP zones be locked
632 * prior to release into the field of the system containing the device.
633 * Failure to lock these zones may permit modification of any secret
634 * keys and may lead to other security problems.
635 */
636 if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
637 dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
638 ret = -ENOTSUPP;
639 }
640
641 /* fall through */
642free_cmd:
643 kfree(cmd);
644 return ret;
645}
646
647static int atmel_ecc_probe(struct i2c_client *client,
648 const struct i2c_device_id *id)
649{
650 struct atmel_ecc_i2c_client_priv *i2c_priv;
651 struct device *dev = &client->dev;
652 int ret;
653 u32 bus_clk_rate;
654
655 if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
656 dev_err(dev, "I2C_FUNC_I2C not supported\n");
657 return -ENODEV;
658 }
659
660 ret = of_property_read_u32(client->adapter->dev.of_node,
661 "clock-frequency", &bus_clk_rate);
662 if (ret) {
663 dev_err(dev, "of: failed to read clock-frequency property\n");
664 return ret;
665 }
666
667 if (bus_clk_rate > 1000000L) {
668 dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
669 bus_clk_rate);
670 return -EINVAL;
671 }
672
673 i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
674 if (!i2c_priv)
675 return -ENOMEM;
676
677 i2c_priv->client = client;
678 mutex_init(&i2c_priv->lock);
679
680 /*
681 * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
682 * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
683 * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
684 */
685 i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate);
686
687 memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
688
689 atomic_set(&i2c_priv->tfm_count, 0);
690
691 i2c_set_clientdata(client, i2c_priv);
692
693 ret = device_sanity_check(client);
694 if (ret)
695 return ret;
696
697 spin_lock(&driver_data.i2c_list_lock);
698 list_add_tail(&i2c_priv->i2c_client_list_node,
699 &driver_data.i2c_client_list);
700 spin_unlock(&driver_data.i2c_list_lock);
701
702 ret = crypto_register_kpp(&atmel_ecdh);
703 if (ret) {
704 spin_lock(&driver_data.i2c_list_lock);
705 list_del(&i2c_priv->i2c_client_list_node);
706 spin_unlock(&driver_data.i2c_list_lock);
707
708 dev_err(dev, "%s alg registration failed\n",
709 atmel_ecdh.base.cra_driver_name);
710 } else {
711 dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n");
712 }
713
714 return ret;
715}
716
717static int atmel_ecc_remove(struct i2c_client *client)
718{
719 struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
720
721 /* Return EBUSY if i2c client already allocated. */
722 if (atomic_read(&i2c_priv->tfm_count)) {
723 dev_err(&client->dev, "Device is busy\n");
724 return -EBUSY;
725 }
726
727 crypto_unregister_kpp(&atmel_ecdh);
728
729 spin_lock(&driver_data.i2c_list_lock);
730 list_del(&i2c_priv->i2c_client_list_node);
731 spin_unlock(&driver_data.i2c_list_lock);
732
733 return 0;
734}
735
736#ifdef CONFIG_OF
737static const struct of_device_id atmel_ecc_dt_ids[] = {
738 {
739 .compatible = "atmel,atecc508a",
740 }, {
741 /* sentinel */
742 }
743};
744MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids);
745#endif
746
747static const struct i2c_device_id atmel_ecc_id[] = {
748 { "atecc508a", 0 },
749 { }
750};
751MODULE_DEVICE_TABLE(i2c, atmel_ecc_id);
752
753static struct i2c_driver atmel_ecc_driver = {
754 .driver = {
755 .name = "atmel-ecc",
756 .of_match_table = of_match_ptr(atmel_ecc_dt_ids),
757 },
758 .probe = atmel_ecc_probe,
759 .remove = atmel_ecc_remove,
760 .id_table = atmel_ecc_id,
761};
762
763static int __init atmel_ecc_init(void)
764{
765 spin_lock_init(&driver_data.i2c_list_lock);
766 INIT_LIST_HEAD(&driver_data.i2c_client_list);
767 return i2c_add_driver(&atmel_ecc_driver);
768}
769
770static void __exit atmel_ecc_exit(void)
771{
772 flush_scheduled_work();
773 i2c_del_driver(&atmel_ecc_driver);
774}
775
776module_init(atmel_ecc_init);
777module_exit(atmel_ecc_exit);
778
779MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>");
780MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver");
781MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h
new file mode 100644
index 000000000000..25232c8abcc2
--- /dev/null
+++ b/drivers/crypto/atmel-ecc.h
@@ -0,0 +1,128 @@
1/*
2 * Copyright (c) 2017, Microchip Technology Inc.
3 * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
16 *
17 */
18
19#ifndef __ATMEL_ECC_H__
20#define __ATMEL_ECC_H__
21
22#define ATMEL_ECC_PRIORITY 300
23
24#define COMMAND 0x03 /* packet function */
25#define SLEEP_TOKEN 0x01
26#define WAKE_TOKEN_MAX_SIZE 8
27
28/* Definitions of Data and Command sizes */
29#define WORD_ADDR_SIZE 1
30#define COUNT_SIZE 1
31#define CRC_SIZE 2
32#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE)
33
34/* size in bytes of the n prime */
35#define ATMEL_ECC_NIST_P256_N_SIZE 32
36#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE)
37
38#define STATUS_RSP_SIZE 4
39#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
40#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \
41 CMD_OVERHEAD_SIZE)
42#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE)
43#define MAX_RSP_SIZE GENKEY_RSP_SIZE
44
45/**
46 * atmel_ecc_cmd - structure used for communicating with the device.
47 * @word_addr: indicates the function of the packet sent to the device. This
48 * byte should have a value of COMMAND for normal operation.
49 * @count : number of bytes to be transferred to (or from) the device.
50 * @opcode : the command code.
51 * @param1 : the first parameter; always present.
52 * @param2 : the second parameter; always present.
53 * @data : optional remaining input data. Includes a 2-byte CRC.
54 * @rxsize : size of the data received from i2c client.
55 * @msecs : command execution time in milliseconds
56 */
57struct atmel_ecc_cmd {
58 u8 word_addr;
59 u8 count;
60 u8 opcode;
61 u8 param1;
62 u16 param2;
63 u8 data[MAX_RSP_SIZE];
64 u8 msecs;
65 u16 rxsize;
66} __packed;
67
68/* Status/Error codes */
69#define STATUS_SIZE 0x04
70#define STATUS_NOERR 0x00
71#define STATUS_WAKE_SUCCESSFUL 0x11
72
73static const struct {
74 u8 value;
75 const char *error_text;
76} error_list[] = {
77 { 0x01, "CheckMac or Verify miscompare" },
78 { 0x03, "Parse Error" },
79 { 0x05, "ECC Fault" },
80 { 0x0F, "Execution Error" },
81 { 0xEE, "Watchdog about to expire" },
82 { 0xFF, "CRC or other communication error" },
83};
84
85/* Definitions for eeprom organization */
86#define CONFIG_ZONE 0
87
88/* Definitions for Indexes common to all commands */
89#define RSP_DATA_IDX 1 /* buffer index of data in response */
90#define DATA_SLOT_2 2 /* used for ECDH private key */
91
92/* Definitions for the device lock state */
93#define DEVICE_LOCK_ADDR 0x15
94#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2)
95#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3)
96
97/*
98 * Wake High delay to data communication (microseconds). SDA should be stable
99 * high for this entire duration.
100 */
101#define TWHI_MIN 1500
102#define TWHI_MAX 1550
103
104/* Wake Low duration */
105#define TWLO_USEC 60
106
107/* Command execution time (milliseconds) */
108#define MAX_EXEC_TIME_ECDH 58
109#define MAX_EXEC_TIME_GENKEY 115
110#define MAX_EXEC_TIME_READ 1
111
112/* Command opcode */
113#define OPCODE_ECDH 0x43
114#define OPCODE_GENKEY 0x40
115#define OPCODE_READ 0x02
116
117/* Definitions for the READ Command */
118#define READ_COUNT 7
119
120/* Definitions for the GenKey Command */
121#define GENKEY_COUNT 7
122#define GENKEY_MODE_PRIVATE 0x04
123
124/* Definitions for the ECDH Command */
125#define ECDH_COUNT 71
126#define ECDH_PREFIX_MODE 0x00
127
128#endif /* __ATMEL_ECC_H__ */
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index dad4e5bad827..3e2f41b3eaf3 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -2883,7 +2883,7 @@ sha_dd_err:
2883 2883
2884static int atmel_sha_remove(struct platform_device *pdev) 2884static int atmel_sha_remove(struct platform_device *pdev)
2885{ 2885{
2886 static struct atmel_sha_dev *sha_dd; 2886 struct atmel_sha_dev *sha_dd;
2887 2887
2888 sha_dd = platform_get_drvdata(pdev); 2888 sha_dd = platform_get_drvdata(pdev);
2889 if (!sha_dd) 2889 if (!sha_dd)
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index b25f1b3c981f..f4b335dda568 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -1487,7 +1487,7 @@ tdes_dd_err:
1487 1487
1488static int atmel_tdes_remove(struct platform_device *pdev) 1488static int atmel_tdes_remove(struct platform_device *pdev)
1489{ 1489{
1490 static struct atmel_tdes_dev *tdes_dd; 1490 struct atmel_tdes_dev *tdes_dd;
1491 1491
1492 tdes_dd = platform_get_drvdata(pdev); 1492 tdes_dd = platform_get_drvdata(pdev);
1493 if (!tdes_dd) 1493 if (!tdes_dd)
diff --git a/drivers/crypto/axis/Makefile b/drivers/crypto/axis/Makefile
new file mode 100644
index 000000000000..be9a84a4b667
--- /dev/null
+++ b/drivers/crypto/axis/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) := artpec6_crypto.o
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c
new file mode 100644
index 000000000000..d9fbbf01062b
--- /dev/null
+++ b/drivers/crypto/axis/artpec6_crypto.c
@@ -0,0 +1,3192 @@
1/*
2 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api.
3 *
4 * Copyright (C) 2014-2017 Axis Communications AB
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/bitfield.h>
9#include <linux/crypto.h>
10#include <linux/debugfs.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/fault-inject.h>
14#include <linux/init.h>
15#include <linux/interrupt.h>
16#include <linux/kernel.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/platform_device.h>
21#include <linux/scatterlist.h>
22#include <linux/slab.h>
23
24#include <crypto/aes.h>
25#include <crypto/internal/aead.h>
26#include <crypto/internal/hash.h>
27#include <crypto/internal/skcipher.h>
28#include <crypto/scatterwalk.h>
29#include <crypto/sha.h>
30#include <crypto/xts.h>
31
32/* Max length of a line in all cache levels for Artpec SoCs. */
33#define ARTPEC_CACHE_LINE_MAX 32
34
35#define PDMA_OUT_CFG 0x0000
36#define PDMA_OUT_BUF_CFG 0x0004
37#define PDMA_OUT_CMD 0x0008
38#define PDMA_OUT_DESCRQ_PUSH 0x0010
39#define PDMA_OUT_DESCRQ_STAT 0x0014
40
41#define A6_PDMA_IN_CFG 0x0028
42#define A6_PDMA_IN_BUF_CFG 0x002c
43#define A6_PDMA_IN_CMD 0x0030
44#define A6_PDMA_IN_STATQ_PUSH 0x0038
45#define A6_PDMA_IN_DESCRQ_PUSH 0x0044
46#define A6_PDMA_IN_DESCRQ_STAT 0x0048
47#define A6_PDMA_INTR_MASK 0x0068
48#define A6_PDMA_ACK_INTR 0x006c
49#define A6_PDMA_MASKED_INTR 0x0074
50
51#define A7_PDMA_IN_CFG 0x002c
52#define A7_PDMA_IN_BUF_CFG 0x0030
53#define A7_PDMA_IN_CMD 0x0034
54#define A7_PDMA_IN_STATQ_PUSH 0x003c
55#define A7_PDMA_IN_DESCRQ_PUSH 0x0048
56#define A7_PDMA_IN_DESCRQ_STAT 0x004C
57#define A7_PDMA_INTR_MASK 0x006c
58#define A7_PDMA_ACK_INTR 0x0070
59#define A7_PDMA_MASKED_INTR 0x0078
60
61#define PDMA_OUT_CFG_EN BIT(0)
62
63#define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
64#define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
65
66#define PDMA_OUT_CMD_START BIT(0)
67#define A6_PDMA_OUT_CMD_STOP BIT(3)
68#define A7_PDMA_OUT_CMD_STOP BIT(2)
69
70#define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0)
71#define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6)
72
73#define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0)
74#define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4)
75
76#define PDMA_IN_CFG_EN BIT(0)
77
78#define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
79#define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
80#define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10)
81
82#define PDMA_IN_CMD_START BIT(0)
83#define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2)
84#define A6_PDMA_IN_CMD_STOP BIT(3)
85#define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1)
86#define A7_PDMA_IN_CMD_STOP BIT(2)
87
88#define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0)
89#define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6)
90
91#define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0)
92#define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6)
93
94#define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0)
95#define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4)
96
97#define A6_PDMA_INTR_MASK_IN_DATA BIT(2)
98#define A6_PDMA_INTR_MASK_IN_EOP BIT(3)
99#define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4)
100
101#define A7_PDMA_INTR_MASK_IN_DATA BIT(3)
102#define A7_PDMA_INTR_MASK_IN_EOP BIT(4)
103#define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5)
104
105#define A6_CRY_MD_OPER GENMASK(19, 16)
106
107#define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20)
108#define A6_CRY_MD_HASH_HMAC_FIN BIT(23)
109
110#define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20)
111#define A6_CRY_MD_CIPHER_DECR BIT(22)
112#define A6_CRY_MD_CIPHER_TWEAK BIT(23)
113#define A6_CRY_MD_CIPHER_DSEQ BIT(24)
114
115#define A7_CRY_MD_OPER GENMASK(11, 8)
116
117#define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12)
118#define A7_CRY_MD_HASH_HMAC_FIN BIT(15)
119
120#define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12)
121#define A7_CRY_MD_CIPHER_DECR BIT(14)
122#define A7_CRY_MD_CIPHER_TWEAK BIT(15)
123#define A7_CRY_MD_CIPHER_DSEQ BIT(16)
124
125/* DMA metadata constants */
126#define regk_crypto_aes_cbc 0x00000002
127#define regk_crypto_aes_ctr 0x00000003
128#define regk_crypto_aes_ecb 0x00000001
129#define regk_crypto_aes_gcm 0x00000004
130#define regk_crypto_aes_xts 0x00000005
131#define regk_crypto_cache 0x00000002
132#define a6_regk_crypto_dlkey 0x0000000a
133#define a7_regk_crypto_dlkey 0x0000000e
134#define regk_crypto_ext 0x00000001
135#define regk_crypto_hmac_sha1 0x00000007
136#define regk_crypto_hmac_sha256 0x00000009
137#define regk_crypto_hmac_sha384 0x0000000b
138#define regk_crypto_hmac_sha512 0x0000000d
139#define regk_crypto_init 0x00000000
140#define regk_crypto_key_128 0x00000000
141#define regk_crypto_key_192 0x00000001
142#define regk_crypto_key_256 0x00000002
143#define regk_crypto_null 0x00000000
144#define regk_crypto_sha1 0x00000006
145#define regk_crypto_sha256 0x00000008
146#define regk_crypto_sha384 0x0000000a
147#define regk_crypto_sha512 0x0000000c
148
149/* DMA descriptor structures */
150struct pdma_descr_ctrl {
151 unsigned char short_descr : 1;
152 unsigned char pad1 : 1;
153 unsigned char eop : 1;
154 unsigned char intr : 1;
155 unsigned char short_len : 3;
156 unsigned char pad2 : 1;
157} __packed;
158
159struct pdma_data_descr {
160 unsigned int len : 24;
161 unsigned int buf : 32;
162} __packed;
163
164struct pdma_short_descr {
165 unsigned char data[7];
166} __packed;
167
168struct pdma_descr {
169 struct pdma_descr_ctrl ctrl;
170 union {
171 struct pdma_data_descr data;
172 struct pdma_short_descr shrt;
173 };
174};
175
176struct pdma_stat_descr {
177 unsigned char pad1 : 1;
178 unsigned char pad2 : 1;
179 unsigned char eop : 1;
180 unsigned char pad3 : 5;
181 unsigned int len : 24;
182};
183
184/* Each descriptor array can hold max 64 entries */
185#define PDMA_DESCR_COUNT 64
186
187#define MODULE_NAME "Artpec-6 CA"
188
189/* Hash modes (including HMAC variants) */
190#define ARTPEC6_CRYPTO_HASH_SHA1 1
191#define ARTPEC6_CRYPTO_HASH_SHA256 2
192#define ARTPEC6_CRYPTO_HASH_SHA384 3
193#define ARTPEC6_CRYPTO_HASH_SHA512 4
194
195/* Crypto modes */
196#define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1
197#define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2
198#define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3
199#define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5
200
201/* The PDMA is a DMA-engine tightly coupled with a ciphering engine.
202 * It operates on a descriptor array with up to 64 descriptor entries.
203 * The arrays must be 64 byte aligned in memory.
204 *
205 * The ciphering unit has no registers and is completely controlled by
206 * a 4-byte metadata that is inserted at the beginning of each dma packet.
207 *
208 * A dma packet is a sequence of descriptors terminated by setting the .eop
209 * field in the final descriptor of the packet.
210 *
211 * Multiple packets are used for providing context data, key data and
212 * the plain/ciphertext.
213 *
214 * PDMA Descriptors (Array)
215 * +------+------+------+~~+-------+------+----
216 * | 0 | 1 | 2 |~~| 11 EOP| 12 | ....
217 * +--+---+--+---+----+-+~~+-------+----+-+----
218 * | | | | |
219 * | | | | |
220 * __|__ +-------++-------++-------+ +----+
221 * | MD | |Payload||Payload||Payload| | MD |
222 * +-----+ +-------++-------++-------+ +----+
223 */
224
225struct artpec6_crypto_bounce_buffer {
226 struct list_head list;
227 size_t length;
228 struct scatterlist *sg;
229 size_t offset;
230 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and
231 * holds up to ARTPEC_CACHE_LINE_MAX bytes data.
232 */
233 void *buf;
234};
235
236struct artpec6_crypto_dma_map {
237 dma_addr_t dma_addr;
238 size_t size;
239 enum dma_data_direction dir;
240};
241
242struct artpec6_crypto_dma_descriptors {
243 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64);
244 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64);
245 u32 stat[PDMA_DESCR_COUNT] __aligned(64);
246 struct list_head bounce_buffers;
247 /* Enough maps for all out/in buffers, and all three descr. arrays */
248 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2];
249 dma_addr_t out_dma_addr;
250 dma_addr_t in_dma_addr;
251 dma_addr_t stat_dma_addr;
252 size_t out_cnt;
253 size_t in_cnt;
254 size_t map_count;
255};
256
257enum artpec6_crypto_variant {
258 ARTPEC6_CRYPTO,
259 ARTPEC7_CRYPTO,
260};
261
262struct artpec6_crypto {
263 void __iomem *base;
264 spinlock_t queue_lock;
265 struct list_head queue; /* waiting for pdma fifo space */
266 struct list_head pending; /* submitted to pdma fifo */
267 struct tasklet_struct task;
268 struct kmem_cache *dma_cache;
269 int pending_count;
270 struct timer_list timer;
271 enum artpec6_crypto_variant variant;
272 void *pad_buffer; /* cache-aligned block padding buffer */
273 void *zero_buffer;
274};
275
276enum artpec6_crypto_hash_flags {
277 HASH_FLAG_INIT_CTX = 2,
278 HASH_FLAG_UPDATE = 4,
279 HASH_FLAG_FINALIZE = 8,
280 HASH_FLAG_HMAC = 16,
281 HASH_FLAG_UPDATE_KEY = 32,
282};
283
284struct artpec6_crypto_req_common {
285 struct list_head list;
286 struct artpec6_crypto_dma_descriptors *dma;
287 struct crypto_async_request *req;
288 void (*complete)(struct crypto_async_request *req);
289 gfp_t gfp_flags;
290};
291
292struct artpec6_hash_request_context {
293 char partial_buffer[SHA512_BLOCK_SIZE];
294 char partial_buffer_out[SHA512_BLOCK_SIZE];
295 char key_buffer[SHA512_BLOCK_SIZE];
296 char pad_buffer[SHA512_BLOCK_SIZE + 32];
297 unsigned char digeststate[SHA512_DIGEST_SIZE];
298 size_t partial_bytes;
299 u64 digcnt;
300 u32 key_md;
301 u32 hash_md;
302 enum artpec6_crypto_hash_flags hash_flags;
303 struct artpec6_crypto_req_common common;
304};
305
306struct artpec6_hash_export_state {
307 char partial_buffer[SHA512_BLOCK_SIZE];
308 unsigned char digeststate[SHA512_DIGEST_SIZE];
309 size_t partial_bytes;
310 u64 digcnt;
311 int oper;
312 unsigned int hash_flags;
313};
314
315struct artpec6_hashalg_context {
316 char hmac_key[SHA512_BLOCK_SIZE];
317 size_t hmac_key_length;
318 struct crypto_shash *child_hash;
319};
320
321struct artpec6_crypto_request_context {
322 u32 cipher_md;
323 bool decrypt;
324 struct artpec6_crypto_req_common common;
325};
326
327struct artpec6_cryptotfm_context {
328 unsigned char aes_key[2*AES_MAX_KEY_SIZE];
329 size_t key_length;
330 u32 key_md;
331 int crypto_type;
332 struct crypto_skcipher *fallback;
333};
334
335struct artpec6_crypto_aead_hw_ctx {
336 __be64 aad_length_bits;
337 __be64 text_length_bits;
338 __u8 J0[AES_BLOCK_SIZE];
339};
340
341struct artpec6_crypto_aead_req_ctx {
342 struct artpec6_crypto_aead_hw_ctx hw_ctx;
343 u32 cipher_md;
344 bool decrypt;
345 struct artpec6_crypto_req_common common;
346 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned;
347};
348
349/* The crypto framework makes it hard to avoid this global. */
350static struct device *artpec6_crypto_dev;
351
352static struct dentry *dbgfs_root;
353
354#ifdef CONFIG_FAULT_INJECTION
355static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read);
356static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full);
357#endif
358
359enum {
360 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START,
361 ARTPEC6_CRYPTO_PREPARE_HASH_START,
362};
363
364static int artpec6_crypto_prepare_aead(struct aead_request *areq);
365static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq);
366static int artpec6_crypto_prepare_hash(struct ahash_request *areq);
367
368static void
369artpec6_crypto_complete_crypto(struct crypto_async_request *req);
370static void
371artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req);
372static void
373artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req);
374static void
375artpec6_crypto_complete_aead(struct crypto_async_request *req);
376static void
377artpec6_crypto_complete_hash(struct crypto_async_request *req);
378
379static int
380artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common);
381
382static void
383artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common);
384
385struct artpec6_crypto_walk {
386 struct scatterlist *sg;
387 size_t offset;
388};
389
390static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk,
391 struct scatterlist *sg)
392{
393 awalk->sg = sg;
394 awalk->offset = 0;
395}
396
397static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk,
398 size_t nbytes)
399{
400 while (nbytes && awalk->sg) {
401 size_t piece;
402
403 WARN_ON(awalk->offset > awalk->sg->length);
404
405 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset);
406 nbytes -= piece;
407 awalk->offset += piece;
408 if (awalk->offset == awalk->sg->length) {
409 awalk->sg = sg_next(awalk->sg);
410 awalk->offset = 0;
411 }
412
413 }
414
415 return nbytes;
416}
417
418static size_t
419artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk)
420{
421 WARN_ON(awalk->sg->length == awalk->offset);
422
423 return awalk->sg->length - awalk->offset;
424}
425
426static dma_addr_t
427artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk)
428{
429 return sg_phys(awalk->sg) + awalk->offset;
430}
431
432static void
433artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common)
434{
435 struct artpec6_crypto_dma_descriptors *dma = common->dma;
436 struct artpec6_crypto_bounce_buffer *b;
437 struct artpec6_crypto_bounce_buffer *next;
438
439 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
440 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n",
441 b, b->length, b->offset, b->buf);
442 sg_pcopy_from_buffer(b->sg,
443 1,
444 b->buf,
445 b->length,
446 b->offset);
447
448 list_del(&b->list);
449 kfree(b);
450 }
451}
452
453static inline bool artpec6_crypto_busy(void)
454{
455 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
456 int fifo_count = ac->pending_count;
457
458 return fifo_count > 6;
459}
460
461static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req)
462{
463 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
464 int ret = -EBUSY;
465
466 spin_lock_bh(&ac->queue_lock);
467
468 if (!artpec6_crypto_busy()) {
469 list_add_tail(&req->list, &ac->pending);
470 artpec6_crypto_start_dma(req);
471 ret = -EINPROGRESS;
472 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
473 list_add_tail(&req->list, &ac->queue);
474 } else {
475 artpec6_crypto_common_destroy(req);
476 }
477
478 spin_unlock_bh(&ac->queue_lock);
479
480 return ret;
481}
482
483static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common)
484{
485 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
486 enum artpec6_crypto_variant variant = ac->variant;
487 void __iomem *base = ac->base;
488 struct artpec6_crypto_dma_descriptors *dma = common->dma;
489 u32 ind, statd, outd;
490
491 /* Make descriptor content visible to the DMA before starting it. */
492 wmb();
493
494 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |
495 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6);
496
497 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |
498 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6);
499
500 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |
501 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6);
502
503 if (variant == ARTPEC6_CRYPTO) {
504 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH);
505 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH);
506 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD);
507 } else {
508 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH);
509 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH);
510 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD);
511 }
512
513 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH);
514 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD);
515
516 ac->pending_count++;
517}
518
519static void
520artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common)
521{
522 struct artpec6_crypto_dma_descriptors *dma = common->dma;
523
524 dma->out_cnt = 0;
525 dma->in_cnt = 0;
526 dma->map_count = 0;
527 INIT_LIST_HEAD(&dma->bounce_buffers);
528}
529
530static bool fault_inject_dma_descr(void)
531{
532#ifdef CONFIG_FAULT_INJECTION
533 return should_fail(&artpec6_crypto_fail_dma_array_full, 1);
534#else
535 return false;
536#endif
537}
538
539/** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a
540 * physical address
541 *
542 * @addr: The physical address of the data buffer
543 * @len: The length of the data buffer
544 * @eop: True if this is the last buffer in the packet
545 *
546 * @return 0 on success or -ENOSPC if there are no more descriptors available
547 */
548static int
549artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common,
550 dma_addr_t addr, size_t len, bool eop)
551{
552 struct artpec6_crypto_dma_descriptors *dma = common->dma;
553 struct pdma_descr *d;
554
555 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
556 fault_inject_dma_descr()) {
557 pr_err("No free OUT DMA descriptors available!\n");
558 return -ENOSPC;
559 }
560
561 d = &dma->out[dma->out_cnt++];
562 memset(d, 0, sizeof(*d));
563
564 d->ctrl.short_descr = 0;
565 d->ctrl.eop = eop;
566 d->data.len = len;
567 d->data.buf = addr;
568 return 0;
569}
570
571/** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor
572 *
573 * @dst: The virtual address of the data
574 * @len: The length of the data, must be between 1 to 7 bytes
575 * @eop: True if this is the last buffer in the packet
576 *
577 * @return 0 on success
578 * -ENOSPC if no more descriptors are available
579 * -EINVAL if the data length exceeds 7 bytes
580 */
581static int
582artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common,
583 void *dst, unsigned int len, bool eop)
584{
585 struct artpec6_crypto_dma_descriptors *dma = common->dma;
586 struct pdma_descr *d;
587
588 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
589 fault_inject_dma_descr()) {
590 pr_err("No free OUT DMA descriptors available!\n");
591 return -ENOSPC;
592 } else if (len > 7 || len < 1) {
593 return -EINVAL;
594 }
595 d = &dma->out[dma->out_cnt++];
596 memset(d, 0, sizeof(*d));
597
598 d->ctrl.short_descr = 1;
599 d->ctrl.short_len = len;
600 d->ctrl.eop = eop;
601 memcpy(d->shrt.data, dst, len);
602 return 0;
603}
604
605static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common,
606 struct page *page, size_t offset,
607 size_t size,
608 enum dma_data_direction dir,
609 dma_addr_t *dma_addr_out)
610{
611 struct artpec6_crypto_dma_descriptors *dma = common->dma;
612 struct device *dev = artpec6_crypto_dev;
613 struct artpec6_crypto_dma_map *map;
614 dma_addr_t dma_addr;
615
616 *dma_addr_out = 0;
617
618 if (dma->map_count >= ARRAY_SIZE(dma->maps))
619 return -ENOMEM;
620
621 dma_addr = dma_map_page(dev, page, offset, size, dir);
622 if (dma_mapping_error(dev, dma_addr))
623 return -ENOMEM;
624
625 map = &dma->maps[dma->map_count++];
626 map->size = size;
627 map->dma_addr = dma_addr;
628 map->dir = dir;
629
630 *dma_addr_out = dma_addr;
631
632 return 0;
633}
634
635static int
636artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common,
637 void *ptr, size_t size,
638 enum dma_data_direction dir,
639 dma_addr_t *dma_addr_out)
640{
641 struct page *page = virt_to_page(ptr);
642 size_t offset = (uintptr_t)ptr & ~PAGE_MASK;
643
644 return artpec6_crypto_dma_map_page(common, page, offset, size, dir,
645 dma_addr_out);
646}
647
648static int
649artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common)
650{
651 struct artpec6_crypto_dma_descriptors *dma = common->dma;
652 int ret;
653
654 ret = artpec6_crypto_dma_map_single(common, dma->in,
655 sizeof(dma->in[0]) * dma->in_cnt,
656 DMA_TO_DEVICE, &dma->in_dma_addr);
657 if (ret)
658 return ret;
659
660 ret = artpec6_crypto_dma_map_single(common, dma->out,
661 sizeof(dma->out[0]) * dma->out_cnt,
662 DMA_TO_DEVICE, &dma->out_dma_addr);
663 if (ret)
664 return ret;
665
666 /* We only read one stat descriptor */
667 dma->stat[dma->in_cnt - 1] = 0;
668
669 /*
670 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor
671 * to be written.
672 */
673 return artpec6_crypto_dma_map_single(common,
674 dma->stat + dma->in_cnt - 1,
675 sizeof(dma->stat[0]),
676 DMA_BIDIRECTIONAL,
677 &dma->stat_dma_addr);
678}
679
680static void
681artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common)
682{
683 struct artpec6_crypto_dma_descriptors *dma = common->dma;
684 struct device *dev = artpec6_crypto_dev;
685 int i;
686
687 for (i = 0; i < dma->map_count; i++) {
688 struct artpec6_crypto_dma_map *map = &dma->maps[i];
689
690 dma_unmap_page(dev, map->dma_addr, map->size, map->dir);
691 }
692
693 dma->map_count = 0;
694}
695
696/** artpec6_crypto_setup_out_descr - Setup an out descriptor
697 *
698 * @dst: The virtual address of the data
699 * @len: The length of the data
700 * @eop: True if this is the last buffer in the packet
701 * @use_short: If this is true and the data length is 7 bytes or less then
702 * a short descriptor will be used
703 *
704 * @return 0 on success
705 * Any errors from artpec6_crypto_setup_out_descr_short() or
706 * setup_out_descr_phys()
707 */
708static int
709artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common,
710 void *dst, unsigned int len, bool eop,
711 bool use_short)
712{
713 if (use_short && len < 7) {
714 return artpec6_crypto_setup_out_descr_short(common, dst, len,
715 eop);
716 } else {
717 int ret;
718 dma_addr_t dma_addr;
719
720 ret = artpec6_crypto_dma_map_single(common, dst, len,
721 DMA_TO_DEVICE,
722 &dma_addr);
723 if (ret)
724 return ret;
725
726 return artpec6_crypto_setup_out_descr_phys(common, dma_addr,
727 len, eop);
728 }
729}
730
731/** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
732 * physical address
733 *
734 * @addr: The physical address of the data buffer
735 * @len: The length of the data buffer
736 * @intr: True if an interrupt should be fired after HW processing of this
737 * descriptor
738 *
739 */
740static int
741artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common,
742 dma_addr_t addr, unsigned int len, bool intr)
743{
744 struct artpec6_crypto_dma_descriptors *dma = common->dma;
745 struct pdma_descr *d;
746
747 if (dma->in_cnt >= PDMA_DESCR_COUNT ||
748 fault_inject_dma_descr()) {
749 pr_err("No free IN DMA descriptors available!\n");
750 return -ENOSPC;
751 }
752 d = &dma->in[dma->in_cnt++];
753 memset(d, 0, sizeof(*d));
754
755 d->ctrl.intr = intr;
756 d->data.len = len;
757 d->data.buf = addr;
758 return 0;
759}
760
761/** artpec6_crypto_setup_in_descr - Setup an in channel descriptor
762 *
763 * @buffer: The virtual address to of the data buffer
764 * @len: The length of the data buffer
765 * @last: If this is the last data buffer in the request (i.e. an interrupt
766 * is needed
767 *
768 * Short descriptors are not used for the in channel
769 */
770static int
771artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common,
772 void *buffer, unsigned int len, bool last)
773{
774 dma_addr_t dma_addr;
775 int ret;
776
777 ret = artpec6_crypto_dma_map_single(common, buffer, len,
778 DMA_FROM_DEVICE, &dma_addr);
779 if (ret)
780 return ret;
781
782 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last);
783}
784
785static struct artpec6_crypto_bounce_buffer *
786artpec6_crypto_alloc_bounce(gfp_t flags)
787{
788 void *base;
789 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) +
790 2 * ARTPEC_CACHE_LINE_MAX;
791 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags);
792
793 if (!bbuf)
794 return NULL;
795
796 base = bbuf + 1;
797 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX);
798 return bbuf;
799}
800
801static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common,
802 struct artpec6_crypto_walk *walk, size_t size)
803{
804 struct artpec6_crypto_bounce_buffer *bbuf;
805 int ret;
806
807 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags);
808 if (!bbuf)
809 return -ENOMEM;
810
811 bbuf->length = size;
812 bbuf->sg = walk->sg;
813 bbuf->offset = walk->offset;
814
815 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false);
816 if (ret) {
817 kfree(bbuf);
818 return ret;
819 }
820
821 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset);
822 list_add_tail(&bbuf->list, &common->dma->bounce_buffers);
823 return 0;
824}
825
826static int
827artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common,
828 struct artpec6_crypto_walk *walk,
829 size_t count)
830{
831 size_t chunk;
832 int ret;
833 dma_addr_t addr;
834
835 while (walk->sg && count) {
836 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
837 addr = artpec6_crypto_walk_chunk_phys(walk);
838
839 /* When destination buffers are not aligned to the cache line
840 * size we need bounce buffers. The DMA-API requires that the
841 * entire line is owned by the DMA buffer and this holds also
842 * for the case when coherent DMA is used.
843 */
844 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) {
845 chunk = min_t(dma_addr_t, chunk,
846 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) -
847 addr);
848
849 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
850 ret = setup_bounce_buffer_in(common, walk, chunk);
851 } else if (chunk < ARTPEC_CACHE_LINE_MAX) {
852 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
853 ret = setup_bounce_buffer_in(common, walk, chunk);
854 } else {
855 dma_addr_t dma_addr;
856
857 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);
858
859 pr_debug("CHUNK %pad:%zu\n", &addr, chunk);
860
861 ret = artpec6_crypto_dma_map_page(common,
862 sg_page(walk->sg),
863 walk->sg->offset +
864 walk->offset,
865 chunk,
866 DMA_FROM_DEVICE,
867 &dma_addr);
868 if (ret)
869 return ret;
870
871 ret = artpec6_crypto_setup_in_descr_phys(common,
872 dma_addr,
873 chunk, false);
874 }
875
876 if (ret)
877 return ret;
878
879 count = count - chunk;
880 artpec6_crypto_walk_advance(walk, chunk);
881 }
882
883 if (count)
884 pr_err("EOL unexpected %zu bytes left\n", count);
885
886 return count ? -EINVAL : 0;
887}
888
889static int
890artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common,
891 struct artpec6_crypto_walk *walk,
892 size_t count)
893{
894 size_t chunk;
895 int ret;
896 dma_addr_t addr;
897
898 while (walk->sg && count) {
899 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
900 addr = artpec6_crypto_walk_chunk_phys(walk);
901
902 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk);
903
904 if (addr & 3) {
905 char buf[3];
906
907 chunk = min_t(size_t, chunk, (4-(addr&3)));
908
909 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,
910 walk->offset);
911
912 ret = artpec6_crypto_setup_out_descr_short(common, buf,
913 chunk,
914 false);
915 } else {
916 dma_addr_t dma_addr;
917
918 ret = artpec6_crypto_dma_map_page(common,
919 sg_page(walk->sg),
920 walk->sg->offset +
921 walk->offset,
922 chunk,
923 DMA_TO_DEVICE,
924 &dma_addr);
925 if (ret)
926 return ret;
927
928 ret = artpec6_crypto_setup_out_descr_phys(common,
929 dma_addr,
930 chunk, false);
931 }
932
933 if (ret)
934 return ret;
935
936 count = count - chunk;
937 artpec6_crypto_walk_advance(walk, chunk);
938 }
939
940 if (count)
941 pr_err("EOL unexpected %zu bytes left\n", count);
942
943 return count ? -EINVAL : 0;
944}
945
946
947/** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor
948 *
949 * If the out descriptor list is non-empty, then the eop flag on the
950 * last used out descriptor will be set.
951 *
952 * @return 0 on success
953 * -EINVAL if the out descriptor is empty or has overflown
954 */
955static int
956artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common)
957{
958 struct artpec6_crypto_dma_descriptors *dma = common->dma;
959 struct pdma_descr *d;
960
961 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) {
962 pr_err("%s: OUT descriptor list is %s\n",
963 MODULE_NAME, dma->out_cnt ? "empty" : "full");
964 return -EINVAL;
965
966 }
967
968 d = &dma->out[dma->out_cnt-1];
969 d->ctrl.eop = 1;
970
971 return 0;
972}
973
974/** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last
975 * in descriptor
976 *
977 * See artpec6_crypto_terminate_out_descrs() for return values
978 */
979static int
980artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common)
981{
982 struct artpec6_crypto_dma_descriptors *dma = common->dma;
983 struct pdma_descr *d;
984
985 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) {
986 pr_err("%s: IN descriptor list is %s\n",
987 MODULE_NAME, dma->in_cnt ? "empty" : "full");
988 return -EINVAL;
989 }
990
991 d = &dma->in[dma->in_cnt-1];
992 d->ctrl.intr = 1;
993 return 0;
994}
995
996/** create_hash_pad - Create a Secure Hash conformant pad
997 *
998 * @dst: The destination buffer to write the pad. Must be at least 64 bytes
999 * @dgstlen: The total length of the hash digest in bytes
1000 * @bitcount: The total length of the digest in bits
1001 *
1002 * @return The total number of padding bytes written to @dst
1003 */
1004static size_t
1005create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount)
1006{
1007 unsigned int mod, target, diff, pad_bytes, size_bytes;
1008 __be64 bits = __cpu_to_be64(bitcount);
1009
1010 switch (oper) {
1011 case regk_crypto_sha1:
1012 case regk_crypto_sha256:
1013 case regk_crypto_hmac_sha1:
1014 case regk_crypto_hmac_sha256:
1015 target = 448 / 8;
1016 mod = 512 / 8;
1017 size_bytes = 8;
1018 break;
1019 default:
1020 target = 896 / 8;
1021 mod = 1024 / 8;
1022 size_bytes = 16;
1023 break;
1024 }
1025
1026 target -= 1;
1027 diff = dgstlen & (mod - 1);
1028 pad_bytes = diff > target ? target + mod - diff : target - diff;
1029
1030 memset(dst + 1, 0, pad_bytes);
1031 dst[0] = 0x80;
1032
1033 if (size_bytes == 16) {
1034 memset(dst + 1 + pad_bytes, 0, 8);
1035 memcpy(dst + 1 + pad_bytes + 8, &bits, 8);
1036 } else {
1037 memcpy(dst + 1 + pad_bytes, &bits, 8);
1038 }
1039
1040 return pad_bytes + size_bytes + 1;
1041}
1042
1043static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common,
1044 struct crypto_async_request *parent,
1045 void (*complete)(struct crypto_async_request *req),
1046 struct scatterlist *dstsg, unsigned int nbytes)
1047{
1048 gfp_t flags;
1049 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1050
1051 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1052 GFP_KERNEL : GFP_ATOMIC;
1053
1054 common->gfp_flags = flags;
1055 common->dma = kmem_cache_alloc(ac->dma_cache, flags);
1056 if (!common->dma)
1057 return -ENOMEM;
1058
1059 common->req = parent;
1060 common->complete = complete;
1061 return 0;
1062}
1063
1064static void
1065artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma)
1066{
1067 struct artpec6_crypto_bounce_buffer *b;
1068 struct artpec6_crypto_bounce_buffer *next;
1069
1070 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
1071 kfree(b);
1072 }
1073}
1074
1075static int
1076artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common)
1077{
1078 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1079
1080 artpec6_crypto_dma_unmap_all(common);
1081 artpec6_crypto_bounce_destroy(common->dma);
1082 kmem_cache_free(ac->dma_cache, common->dma);
1083 common->dma = NULL;
1084 return 0;
1085}
1086
1087/*
1088 * Ciphering functions.
1089 */
1090static int artpec6_crypto_encrypt(struct skcipher_request *req)
1091{
1092 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1093 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1094 struct artpec6_crypto_request_context *req_ctx = NULL;
1095 void (*complete)(struct crypto_async_request *req);
1096 int ret;
1097
1098 req_ctx = skcipher_request_ctx(req);
1099
1100 switch (ctx->crypto_type) {
1101 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1102 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1103 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1104 req_ctx->decrypt = 0;
1105 break;
1106 default:
1107 break;
1108 }
1109
1110 switch (ctx->crypto_type) {
1111 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1112 complete = artpec6_crypto_complete_cbc_encrypt;
1113 break;
1114 default:
1115 complete = artpec6_crypto_complete_crypto;
1116 break;
1117 }
1118
1119 ret = artpec6_crypto_common_init(&req_ctx->common,
1120 &req->base,
1121 complete,
1122 req->dst, req->cryptlen);
1123 if (ret)
1124 return ret;
1125
1126 ret = artpec6_crypto_prepare_crypto(req);
1127 if (ret) {
1128 artpec6_crypto_common_destroy(&req_ctx->common);
1129 return ret;
1130 }
1131
1132 return artpec6_crypto_submit(&req_ctx->common);
1133}
1134
1135static int artpec6_crypto_decrypt(struct skcipher_request *req)
1136{
1137 int ret;
1138 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1139 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1140 struct artpec6_crypto_request_context *req_ctx = NULL;
1141 void (*complete)(struct crypto_async_request *req);
1142
1143 req_ctx = skcipher_request_ctx(req);
1144
1145 switch (ctx->crypto_type) {
1146 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1147 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1148 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1149 req_ctx->decrypt = 1;
1150 break;
1151 default:
1152 break;
1153 }
1154
1155
1156 switch (ctx->crypto_type) {
1157 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1158 complete = artpec6_crypto_complete_cbc_decrypt;
1159 break;
1160 default:
1161 complete = artpec6_crypto_complete_crypto;
1162 break;
1163 }
1164
1165 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1166 complete,
1167 req->dst, req->cryptlen);
1168 if (ret)
1169 return ret;
1170
1171 ret = artpec6_crypto_prepare_crypto(req);
1172 if (ret) {
1173 artpec6_crypto_common_destroy(&req_ctx->common);
1174 return ret;
1175 }
1176
1177 return artpec6_crypto_submit(&req_ctx->common);
1178}
1179
1180static int
1181artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt)
1182{
1183 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1184 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1185 size_t iv_len = crypto_skcipher_ivsize(cipher);
1186 unsigned int counter = be32_to_cpup((__be32 *)
1187 (req->iv + iv_len - 4));
1188 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
1189 AES_BLOCK_SIZE;
1190
1191 /*
1192 * The hardware uses only the last 32-bits as the counter while the
1193 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
1194 * the whole IV is a counter. So fallback if the counter is going to
1195 * overlow.
1196 */
1197 if (counter + nblks < counter) {
1198 int ret;
1199
1200 pr_debug("counter %x will overflow (nblks %u), falling back\n",
1201 counter, counter + nblks);
1202
1203 ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key,
1204 ctx->key_length);
1205 if (ret)
1206 return ret;
1207
1208 {
1209 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
1210
1211 skcipher_request_set_tfm(subreq, ctx->fallback);
1212 skcipher_request_set_callback(subreq, req->base.flags,
1213 NULL, NULL);
1214 skcipher_request_set_crypt(subreq, req->src, req->dst,
1215 req->cryptlen, req->iv);
1216 ret = encrypt ? crypto_skcipher_encrypt(subreq)
1217 : crypto_skcipher_decrypt(subreq);
1218 skcipher_request_zero(subreq);
1219 }
1220 return ret;
1221 }
1222
1223 return encrypt ? artpec6_crypto_encrypt(req)
1224 : artpec6_crypto_decrypt(req);
1225}
1226
1227static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req)
1228{
1229 return artpec6_crypto_ctr_crypt(req, true);
1230}
1231
1232static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req)
1233{
1234 return artpec6_crypto_ctr_crypt(req, false);
1235}
1236
1237/*
1238 * AEAD functions
1239 */
1240static int artpec6_crypto_aead_init(struct crypto_aead *tfm)
1241{
1242 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm);
1243
1244 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
1245
1246 crypto_aead_set_reqsize(tfm,
1247 sizeof(struct artpec6_crypto_aead_req_ctx));
1248
1249 return 0;
1250}
1251
1252static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key,
1253 unsigned int len)
1254{
1255 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base);
1256
1257 if (len != 16 && len != 24 && len != 32) {
1258 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1259 return -1;
1260 }
1261
1262 ctx->key_length = len;
1263
1264 memcpy(ctx->aes_key, key, len);
1265 return 0;
1266}
1267
1268static int artpec6_crypto_aead_encrypt(struct aead_request *req)
1269{
1270 int ret;
1271 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1272
1273 req_ctx->decrypt = false;
1274 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1275 artpec6_crypto_complete_aead,
1276 NULL, 0);
1277 if (ret)
1278 return ret;
1279
1280 ret = artpec6_crypto_prepare_aead(req);
1281 if (ret) {
1282 artpec6_crypto_common_destroy(&req_ctx->common);
1283 return ret;
1284 }
1285
1286 return artpec6_crypto_submit(&req_ctx->common);
1287}
1288
1289static int artpec6_crypto_aead_decrypt(struct aead_request *req)
1290{
1291 int ret;
1292 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1293
1294 req_ctx->decrypt = true;
1295 if (req->cryptlen < AES_BLOCK_SIZE)
1296 return -EINVAL;
1297
1298 ret = artpec6_crypto_common_init(&req_ctx->common,
1299 &req->base,
1300 artpec6_crypto_complete_aead,
1301 NULL, 0);
1302 if (ret)
1303 return ret;
1304
1305 ret = artpec6_crypto_prepare_aead(req);
1306 if (ret) {
1307 artpec6_crypto_common_destroy(&req_ctx->common);
1308 return ret;
1309 }
1310
1311 return artpec6_crypto_submit(&req_ctx->common);
1312}
1313
1314static int artpec6_crypto_prepare_hash(struct ahash_request *areq)
1315{
1316 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1317 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq);
1318 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1319 size_t contextsize = digestsize == SHA384_DIGEST_SIZE ?
1320 SHA512_DIGEST_SIZE : digestsize;
1321 size_t blocksize = crypto_tfm_alg_blocksize(
1322 crypto_ahash_tfm(crypto_ahash_reqtfm(areq)));
1323 struct artpec6_crypto_req_common *common = &req_ctx->common;
1324 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1325 enum artpec6_crypto_variant variant = ac->variant;
1326 u32 sel_ctx;
1327 bool ext_ctx = false;
1328 bool run_hw = false;
1329 int error = 0;
1330
1331 artpec6_crypto_init_dma_operation(common);
1332
1333 /* Upload HMAC key, must be first the first packet */
1334 if (req_ctx->hash_flags & HASH_FLAG_HMAC) {
1335 if (variant == ARTPEC6_CRYPTO) {
1336 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1337 a6_regk_crypto_dlkey);
1338 } else {
1339 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1340 a7_regk_crypto_dlkey);
1341 }
1342
1343 /* Copy and pad up the key */
1344 memcpy(req_ctx->key_buffer, ctx->hmac_key,
1345 ctx->hmac_key_length);
1346 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0,
1347 blocksize - ctx->hmac_key_length);
1348
1349 error = artpec6_crypto_setup_out_descr(common,
1350 (void *)&req_ctx->key_md,
1351 sizeof(req_ctx->key_md), false, false);
1352 if (error)
1353 return error;
1354
1355 error = artpec6_crypto_setup_out_descr(common,
1356 req_ctx->key_buffer, blocksize,
1357 true, false);
1358 if (error)
1359 return error;
1360 }
1361
1362 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) {
1363 /* Restore context */
1364 sel_ctx = regk_crypto_ext;
1365 ext_ctx = true;
1366 } else {
1367 sel_ctx = regk_crypto_init;
1368 }
1369
1370 if (variant == ARTPEC6_CRYPTO) {
1371 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX;
1372 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx);
1373
1374 /* If this is the final round, set the final flag */
1375 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1376 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN;
1377 } else {
1378 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX;
1379 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx);
1380
1381 /* If this is the final round, set the final flag */
1382 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1383 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN;
1384 }
1385
1386 /* Setup up metadata descriptors */
1387 error = artpec6_crypto_setup_out_descr(common,
1388 (void *)&req_ctx->hash_md,
1389 sizeof(req_ctx->hash_md), false, false);
1390 if (error)
1391 return error;
1392
1393 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1394 if (error)
1395 return error;
1396
1397 if (ext_ctx) {
1398 error = artpec6_crypto_setup_out_descr(common,
1399 req_ctx->digeststate,
1400 contextsize, false, false);
1401
1402 if (error)
1403 return error;
1404 }
1405
1406 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) {
1407 size_t done_bytes = 0;
1408 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes;
1409 size_t ready_bytes = round_down(total_bytes, blocksize);
1410 struct artpec6_crypto_walk walk;
1411
1412 run_hw = ready_bytes > 0;
1413 if (req_ctx->partial_bytes && ready_bytes) {
1414 /* We have a partial buffer and will at least some bytes
1415 * to the HW. Empty this partial buffer before tackling
1416 * the SG lists
1417 */
1418 memcpy(req_ctx->partial_buffer_out,
1419 req_ctx->partial_buffer,
1420 req_ctx->partial_bytes);
1421
1422 error = artpec6_crypto_setup_out_descr(common,
1423 req_ctx->partial_buffer_out,
1424 req_ctx->partial_bytes,
1425 false, true);
1426 if (error)
1427 return error;
1428
1429 /* Reset partial buffer */
1430 done_bytes += req_ctx->partial_bytes;
1431 req_ctx->partial_bytes = 0;
1432 }
1433
1434 artpec6_crypto_walk_init(&walk, areq->src);
1435
1436 error = artpec6_crypto_setup_sg_descrs_out(common, &walk,
1437 ready_bytes -
1438 done_bytes);
1439 if (error)
1440 return error;
1441
1442 if (walk.sg) {
1443 size_t sg_skip = ready_bytes - done_bytes;
1444 size_t sg_rem = areq->nbytes - sg_skip;
1445
1446 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
1447 req_ctx->partial_buffer +
1448 req_ctx->partial_bytes,
1449 sg_rem, sg_skip);
1450
1451 req_ctx->partial_bytes += sg_rem;
1452 }
1453
1454 req_ctx->digcnt += ready_bytes;
1455 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE);
1456 }
1457
1458 /* Finalize */
1459 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) {
1460 bool needtrim = contextsize != digestsize;
1461 size_t hash_pad_len;
1462 u64 digest_bits;
1463 u32 oper;
1464
1465 if (variant == ARTPEC6_CRYPTO)
1466 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md);
1467 else
1468 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md);
1469
1470 /* Write out the partial buffer if present */
1471 if (req_ctx->partial_bytes) {
1472 memcpy(req_ctx->partial_buffer_out,
1473 req_ctx->partial_buffer,
1474 req_ctx->partial_bytes);
1475 error = artpec6_crypto_setup_out_descr(common,
1476 req_ctx->partial_buffer_out,
1477 req_ctx->partial_bytes,
1478 false, true);
1479 if (error)
1480 return error;
1481
1482 req_ctx->digcnt += req_ctx->partial_bytes;
1483 req_ctx->partial_bytes = 0;
1484 }
1485
1486 if (req_ctx->hash_flags & HASH_FLAG_HMAC)
1487 digest_bits = 8 * (req_ctx->digcnt + blocksize);
1488 else
1489 digest_bits = 8 * req_ctx->digcnt;
1490
1491 /* Add the hash pad */
1492 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer,
1493 req_ctx->digcnt, digest_bits);
1494 error = artpec6_crypto_setup_out_descr(common,
1495 req_ctx->pad_buffer,
1496 hash_pad_len, false,
1497 true);
1498 req_ctx->digcnt = 0;
1499
1500 if (error)
1501 return error;
1502
1503 /* Descriptor for the final result */
1504 error = artpec6_crypto_setup_in_descr(common, areq->result,
1505 digestsize,
1506 !needtrim);
1507 if (error)
1508 return error;
1509
1510 if (needtrim) {
1511 /* Discard the extra context bytes for SHA-384 */
1512 error = artpec6_crypto_setup_in_descr(common,
1513 req_ctx->partial_buffer,
1514 digestsize - contextsize, true);
1515 if (error)
1516 return error;
1517 }
1518
1519 } else { /* This is not the final operation for this request */
1520 if (!run_hw)
1521 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START;
1522
1523 /* Save the result to the context */
1524 error = artpec6_crypto_setup_in_descr(common,
1525 req_ctx->digeststate,
1526 contextsize, false);
1527 if (error)
1528 return error;
1529 /* fall through */
1530 }
1531
1532 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE |
1533 HASH_FLAG_FINALIZE);
1534
1535 error = artpec6_crypto_terminate_in_descrs(common);
1536 if (error)
1537 return error;
1538
1539 error = artpec6_crypto_terminate_out_descrs(common);
1540 if (error)
1541 return error;
1542
1543 error = artpec6_crypto_dma_map_descs(common);
1544 if (error)
1545 return error;
1546
1547 return ARTPEC6_CRYPTO_PREPARE_HASH_START;
1548}
1549
1550
1551static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm)
1552{
1553 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1554
1555 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1556 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB;
1557
1558 return 0;
1559}
1560
1561static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm)
1562{
1563 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1564
1565 ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base),
1566 0,
1567 CRYPTO_ALG_ASYNC |
1568 CRYPTO_ALG_NEED_FALLBACK);
1569 if (IS_ERR(ctx->fallback))
1570 return PTR_ERR(ctx->fallback);
1571
1572 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1573 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR;
1574
1575 return 0;
1576}
1577
1578static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm)
1579{
1580 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1581
1582 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1583 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC;
1584
1585 return 0;
1586}
1587
1588static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm)
1589{
1590 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1591
1592 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1593 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS;
1594
1595 return 0;
1596}
1597
1598static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm)
1599{
1600 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1601
1602 memset(ctx, 0, sizeof(*ctx));
1603}
1604
1605static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm)
1606{
1607 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1608
1609 crypto_free_skcipher(ctx->fallback);
1610 artpec6_crypto_aes_exit(tfm);
1611}
1612
1613static int
1614artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key,
1615 unsigned int keylen)
1616{
1617 struct artpec6_cryptotfm_context *ctx =
1618 crypto_skcipher_ctx(cipher);
1619
1620 switch (keylen) {
1621 case 16:
1622 case 24:
1623 case 32:
1624 break;
1625 default:
1626 crypto_skcipher_set_flags(cipher,
1627 CRYPTO_TFM_RES_BAD_KEY_LEN);
1628 return -EINVAL;
1629 }
1630
1631 memcpy(ctx->aes_key, key, keylen);
1632 ctx->key_length = keylen;
1633 return 0;
1634}
1635
1636static int
1637artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key,
1638 unsigned int keylen)
1639{
1640 struct artpec6_cryptotfm_context *ctx =
1641 crypto_skcipher_ctx(cipher);
1642 int ret;
1643
1644 ret = xts_check_key(&cipher->base, key, keylen);
1645 if (ret)
1646 return ret;
1647
1648 switch (keylen) {
1649 case 32:
1650 case 48:
1651 case 64:
1652 break;
1653 default:
1654 crypto_skcipher_set_flags(cipher,
1655 CRYPTO_TFM_RES_BAD_KEY_LEN);
1656 return -EINVAL;
1657 }
1658
1659 memcpy(ctx->aes_key, key, keylen);
1660 ctx->key_length = keylen;
1661 return 0;
1662}
1663
1664/** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request
1665 *
1666 * @req: The asynch request to process
1667 *
1668 * @return 0 if the dma job was successfully prepared
1669 * <0 on error
1670 *
1671 * This function sets up the PDMA descriptors for a block cipher request.
1672 *
1673 * The required padding is added for AES-CTR using a statically defined
1674 * buffer.
1675 *
1676 * The PDMA descriptor list will be as follows:
1677 *
1678 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop>
1679 * IN: <CIPHER_MD><data_0>...[data_n]<intr>
1680 *
1681 */
1682static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq)
1683{
1684 int ret;
1685 struct artpec6_crypto_walk walk;
1686 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1687 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1688 struct artpec6_crypto_request_context *req_ctx = NULL;
1689 size_t iv_len = crypto_skcipher_ivsize(cipher);
1690 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1691 enum artpec6_crypto_variant variant = ac->variant;
1692 struct artpec6_crypto_req_common *common;
1693 bool cipher_decr = false;
1694 size_t cipher_klen;
1695 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */
1696 u32 oper;
1697
1698 req_ctx = skcipher_request_ctx(areq);
1699 common = &req_ctx->common;
1700
1701 artpec6_crypto_init_dma_operation(common);
1702
1703 if (variant == ARTPEC6_CRYPTO)
1704 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey);
1705 else
1706 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey);
1707
1708 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1709 sizeof(ctx->key_md), false, false);
1710 if (ret)
1711 return ret;
1712
1713 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1714 ctx->key_length, true, false);
1715 if (ret)
1716 return ret;
1717
1718 req_ctx->cipher_md = 0;
1719
1720 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS)
1721 cipher_klen = ctx->key_length/2;
1722 else
1723 cipher_klen = ctx->key_length;
1724
1725 /* Metadata */
1726 switch (cipher_klen) {
1727 case 16:
1728 cipher_len = regk_crypto_key_128;
1729 break;
1730 case 24:
1731 cipher_len = regk_crypto_key_192;
1732 break;
1733 case 32:
1734 cipher_len = regk_crypto_key_256;
1735 break;
1736 default:
1737 pr_err("%s: Invalid key length %d!\n",
1738 MODULE_NAME, ctx->key_length);
1739 return -EINVAL;
1740 }
1741
1742 switch (ctx->crypto_type) {
1743 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1744 oper = regk_crypto_aes_ecb;
1745 cipher_decr = req_ctx->decrypt;
1746 break;
1747
1748 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1749 oper = regk_crypto_aes_cbc;
1750 cipher_decr = req_ctx->decrypt;
1751 break;
1752
1753 case ARTPEC6_CRYPTO_CIPHER_AES_CTR:
1754 oper = regk_crypto_aes_ctr;
1755 cipher_decr = false;
1756 break;
1757
1758 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1759 oper = regk_crypto_aes_xts;
1760 cipher_decr = req_ctx->decrypt;
1761
1762 if (variant == ARTPEC6_CRYPTO)
1763 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ;
1764 else
1765 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ;
1766 break;
1767
1768 default:
1769 pr_err("%s: Invalid cipher mode %d!\n",
1770 MODULE_NAME, ctx->crypto_type);
1771 return -EINVAL;
1772 }
1773
1774 if (variant == ARTPEC6_CRYPTO) {
1775 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper);
1776 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1777 cipher_len);
1778 if (cipher_decr)
1779 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1780 } else {
1781 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper);
1782 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1783 cipher_len);
1784 if (cipher_decr)
1785 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1786 }
1787
1788 ret = artpec6_crypto_setup_out_descr(common,
1789 &req_ctx->cipher_md,
1790 sizeof(req_ctx->cipher_md),
1791 false, false);
1792 if (ret)
1793 return ret;
1794
1795 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1796 if (ret)
1797 return ret;
1798
1799 if (iv_len) {
1800 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len,
1801 false, false);
1802 if (ret)
1803 return ret;
1804 }
1805 /* Data out */
1806 artpec6_crypto_walk_init(&walk, areq->src);
1807 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen);
1808 if (ret)
1809 return ret;
1810
1811 /* Data in */
1812 artpec6_crypto_walk_init(&walk, areq->dst);
1813 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen);
1814 if (ret)
1815 return ret;
1816
1817 /* CTR-mode padding required by the HW. */
1818 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR ||
1819 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) {
1820 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) -
1821 areq->cryptlen;
1822
1823 if (pad) {
1824 ret = artpec6_crypto_setup_out_descr(common,
1825 ac->pad_buffer,
1826 pad, false, false);
1827 if (ret)
1828 return ret;
1829
1830 ret = artpec6_crypto_setup_in_descr(common,
1831 ac->pad_buffer, pad,
1832 false);
1833 if (ret)
1834 return ret;
1835 }
1836 }
1837
1838 ret = artpec6_crypto_terminate_out_descrs(common);
1839 if (ret)
1840 return ret;
1841
1842 ret = artpec6_crypto_terminate_in_descrs(common);
1843 if (ret)
1844 return ret;
1845
1846 return artpec6_crypto_dma_map_descs(common);
1847}
1848
1849static int artpec6_crypto_prepare_aead(struct aead_request *areq)
1850{
1851 size_t count;
1852 int ret;
1853 size_t input_length;
1854 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1855 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
1856 struct crypto_aead *cipher = crypto_aead_reqtfm(areq);
1857 struct artpec6_crypto_req_common *common = &req_ctx->common;
1858 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1859 enum artpec6_crypto_variant variant = ac->variant;
1860 u32 md_cipher_len;
1861
1862 artpec6_crypto_init_dma_operation(common);
1863
1864 /* Key */
1865 if (variant == ARTPEC6_CRYPTO) {
1866 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1867 a6_regk_crypto_dlkey);
1868 } else {
1869 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1870 a7_regk_crypto_dlkey);
1871 }
1872 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1873 sizeof(ctx->key_md), false, false);
1874 if (ret)
1875 return ret;
1876
1877 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1878 ctx->key_length, true, false);
1879 if (ret)
1880 return ret;
1881
1882 req_ctx->cipher_md = 0;
1883
1884 switch (ctx->key_length) {
1885 case 16:
1886 md_cipher_len = regk_crypto_key_128;
1887 break;
1888 case 24:
1889 md_cipher_len = regk_crypto_key_192;
1890 break;
1891 case 32:
1892 md_cipher_len = regk_crypto_key_256;
1893 break;
1894 default:
1895 return -EINVAL;
1896 }
1897
1898 if (variant == ARTPEC6_CRYPTO) {
1899 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER,
1900 regk_crypto_aes_gcm);
1901 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1902 md_cipher_len);
1903 if (req_ctx->decrypt)
1904 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1905 } else {
1906 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER,
1907 regk_crypto_aes_gcm);
1908 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1909 md_cipher_len);
1910 if (req_ctx->decrypt)
1911 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1912 }
1913
1914 ret = artpec6_crypto_setup_out_descr(common,
1915 (void *) &req_ctx->cipher_md,
1916 sizeof(req_ctx->cipher_md), false,
1917 false);
1918 if (ret)
1919 return ret;
1920
1921 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1922 if (ret)
1923 return ret;
1924
1925 /* For the decryption, cryptlen includes the tag. */
1926 input_length = areq->cryptlen;
1927 if (req_ctx->decrypt)
1928 input_length -= AES_BLOCK_SIZE;
1929
1930 /* Prepare the context buffer */
1931 req_ctx->hw_ctx.aad_length_bits =
1932 __cpu_to_be64(8*areq->assoclen);
1933
1934 req_ctx->hw_ctx.text_length_bits =
1935 __cpu_to_be64(8*input_length);
1936
1937 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher));
1938 // The HW omits the initial increment of the counter field.
1939 crypto_inc(req_ctx->hw_ctx.J0+12, 4);
1940
1941 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx,
1942 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false);
1943 if (ret)
1944 return ret;
1945
1946 {
1947 struct artpec6_crypto_walk walk;
1948
1949 artpec6_crypto_walk_init(&walk, areq->src);
1950
1951 /* Associated data */
1952 count = areq->assoclen;
1953 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1954 if (ret)
1955 return ret;
1956
1957 if (!IS_ALIGNED(areq->assoclen, 16)) {
1958 size_t assoc_pad = 16 - (areq->assoclen % 16);
1959 /* The HW mandates zero padding here */
1960 ret = artpec6_crypto_setup_out_descr(common,
1961 ac->zero_buffer,
1962 assoc_pad, false,
1963 false);
1964 if (ret)
1965 return ret;
1966 }
1967
1968 /* Data to crypto */
1969 count = input_length;
1970 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1971 if (ret)
1972 return ret;
1973
1974 if (!IS_ALIGNED(input_length, 16)) {
1975 size_t crypto_pad = 16 - (input_length % 16);
1976 /* The HW mandates zero padding here */
1977 ret = artpec6_crypto_setup_out_descr(common,
1978 ac->zero_buffer,
1979 crypto_pad,
1980 false,
1981 false);
1982 if (ret)
1983 return ret;
1984 }
1985 }
1986
1987 /* Data from crypto */
1988 {
1989 struct artpec6_crypto_walk walk;
1990 size_t output_len = areq->cryptlen;
1991
1992 if (req_ctx->decrypt)
1993 output_len -= AES_BLOCK_SIZE;
1994
1995 artpec6_crypto_walk_init(&walk, areq->dst);
1996
1997 /* skip associated data in the output */
1998 count = artpec6_crypto_walk_advance(&walk, areq->assoclen);
1999 if (count)
2000 return -EINVAL;
2001
2002 count = output_len;
2003 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count);
2004 if (ret)
2005 return ret;
2006
2007 /* Put padding between the cryptotext and the auth tag */
2008 if (!IS_ALIGNED(output_len, 16)) {
2009 size_t crypto_pad = 16 - (output_len % 16);
2010
2011 ret = artpec6_crypto_setup_in_descr(common,
2012 ac->pad_buffer,
2013 crypto_pad, false);
2014 if (ret)
2015 return ret;
2016 }
2017
2018 /* The authentication tag shall follow immediately after
2019 * the output ciphertext. For decryption it is put in a context
2020 * buffer for later compare against the input tag.
2021 */
2022 count = AES_BLOCK_SIZE;
2023
2024 if (req_ctx->decrypt) {
2025 ret = artpec6_crypto_setup_in_descr(common,
2026 req_ctx->decryption_tag, count, false);
2027 if (ret)
2028 return ret;
2029
2030 } else {
2031 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk,
2032 count);
2033 if (ret)
2034 return ret;
2035 }
2036
2037 }
2038
2039 ret = artpec6_crypto_terminate_in_descrs(common);
2040 if (ret)
2041 return ret;
2042
2043 ret = artpec6_crypto_terminate_out_descrs(common);
2044 if (ret)
2045 return ret;
2046
2047 return artpec6_crypto_dma_map_descs(common);
2048}
2049
2050static void artpec6_crypto_process_queue(struct artpec6_crypto *ac)
2051{
2052 struct artpec6_crypto_req_common *req;
2053
2054 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {
2055 req = list_first_entry(&ac->queue,
2056 struct artpec6_crypto_req_common,
2057 list);
2058 list_move_tail(&req->list, &ac->pending);
2059 artpec6_crypto_start_dma(req);
2060
2061 req->req->complete(req->req, -EINPROGRESS);
2062 }
2063
2064 /*
2065 * In some cases, the hardware can raise an in_eop_flush interrupt
2066 * before actually updating the status, so we have an timer which will
2067 * recheck the status on timeout. Since the cases are expected to be
2068 * very rare, we use a relatively large timeout value. There should be
2069 * no noticeable negative effect if we timeout spuriously.
2070 */
2071 if (ac->pending_count)
2072 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));
2073 else
2074 del_timer(&ac->timer);
2075}
2076
2077static void artpec6_crypto_timeout(unsigned long data)
2078{
2079 struct artpec6_crypto *ac = (struct artpec6_crypto *) data;
2080
2081 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n");
2082
2083 tasklet_schedule(&ac->task);
2084}
2085
2086static void artpec6_crypto_task(unsigned long data)
2087{
2088 struct artpec6_crypto *ac = (struct artpec6_crypto *)data;
2089 struct artpec6_crypto_req_common *req;
2090 struct artpec6_crypto_req_common *n;
2091
2092 if (list_empty(&ac->pending)) {
2093 pr_debug("Spurious IRQ\n");
2094 return;
2095 }
2096
2097 spin_lock_bh(&ac->queue_lock);
2098
2099 list_for_each_entry_safe(req, n, &ac->pending, list) {
2100 struct artpec6_crypto_dma_descriptors *dma = req->dma;
2101 u32 stat;
2102
2103 dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr,
2104 sizeof(dma->stat[0]),
2105 DMA_BIDIRECTIONAL);
2106
2107 stat = req->dma->stat[req->dma->in_cnt-1];
2108
2109 /* A non-zero final status descriptor indicates
2110 * this job has finished.
2111 */
2112 pr_debug("Request %p status is %X\n", req, stat);
2113 if (!stat)
2114 break;
2115
2116 /* Allow testing of timeout handling with fault injection */
2117#ifdef CONFIG_FAULT_INJECTION
2118 if (should_fail(&artpec6_crypto_fail_status_read, 1))
2119 continue;
2120#endif
2121
2122 pr_debug("Completing request %p\n", req);
2123
2124 list_del(&req->list);
2125
2126 artpec6_crypto_dma_unmap_all(req);
2127 artpec6_crypto_copy_bounce_buffers(req);
2128
2129 ac->pending_count--;
2130 artpec6_crypto_common_destroy(req);
2131 req->complete(req->req);
2132 }
2133
2134 artpec6_crypto_process_queue(ac);
2135
2136 spin_unlock_bh(&ac->queue_lock);
2137}
2138
2139static void artpec6_crypto_complete_crypto(struct crypto_async_request *req)
2140{
2141 req->complete(req, 0);
2142}
2143
2144static void
2145artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req)
2146{
2147 struct skcipher_request *cipher_req = container_of(req,
2148 struct skcipher_request, base);
2149
2150 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src,
2151 cipher_req->cryptlen - AES_BLOCK_SIZE,
2152 AES_BLOCK_SIZE, 0);
2153 req->complete(req, 0);
2154}
2155
2156static void
2157artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req)
2158{
2159 struct skcipher_request *cipher_req = container_of(req,
2160 struct skcipher_request, base);
2161
2162 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst,
2163 cipher_req->cryptlen - AES_BLOCK_SIZE,
2164 AES_BLOCK_SIZE, 0);
2165 req->complete(req, 0);
2166}
2167
2168static void artpec6_crypto_complete_aead(struct crypto_async_request *req)
2169{
2170 int result = 0;
2171
2172 /* Verify GCM hashtag. */
2173 struct aead_request *areq = container_of(req,
2174 struct aead_request, base);
2175 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
2176
2177 if (req_ctx->decrypt) {
2178 u8 input_tag[AES_BLOCK_SIZE];
2179
2180 sg_pcopy_to_buffer(areq->src,
2181 sg_nents(areq->src),
2182 input_tag,
2183 AES_BLOCK_SIZE,
2184 areq->assoclen + areq->cryptlen -
2185 AES_BLOCK_SIZE);
2186
2187 if (memcmp(req_ctx->decryption_tag,
2188 input_tag,
2189 AES_BLOCK_SIZE)) {
2190 pr_debug("***EBADMSG:\n");
2191 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,
2192 input_tag, AES_BLOCK_SIZE, true);
2193 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,
2194 req_ctx->decryption_tag,
2195 AES_BLOCK_SIZE, true);
2196
2197 result = -EBADMSG;
2198 }
2199 }
2200
2201 req->complete(req, result);
2202}
2203
2204static void artpec6_crypto_complete_hash(struct crypto_async_request *req)
2205{
2206 req->complete(req, 0);
2207}
2208
2209
2210/*------------------- Hash functions -----------------------------------------*/
2211static int
2212artpec6_crypto_hash_set_key(struct crypto_ahash *tfm,
2213 const u8 *key, unsigned int keylen)
2214{
2215 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base);
2216 size_t blocksize;
2217 int ret;
2218
2219 if (!keylen) {
2220 pr_err("Invalid length (%d) of HMAC key\n",
2221 keylen);
2222 return -EINVAL;
2223 }
2224
2225 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2226
2227 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2228
2229 if (keylen > blocksize) {
2230 SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash);
2231
2232 hdesc->tfm = tfm_ctx->child_hash;
2233 hdesc->flags = crypto_ahash_get_flags(tfm) &
2234 CRYPTO_TFM_REQ_MAY_SLEEP;
2235
2236 tfm_ctx->hmac_key_length = blocksize;
2237 ret = crypto_shash_digest(hdesc, key, keylen,
2238 tfm_ctx->hmac_key);
2239 if (ret)
2240 return ret;
2241
2242 } else {
2243 memcpy(tfm_ctx->hmac_key, key, keylen);
2244 tfm_ctx->hmac_key_length = keylen;
2245 }
2246
2247 return 0;
2248}
2249
2250static int
2251artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac)
2252{
2253 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2254 enum artpec6_crypto_variant variant = ac->variant;
2255 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2256 u32 oper;
2257
2258 memset(req_ctx, 0, sizeof(*req_ctx));
2259
2260 req_ctx->hash_flags = HASH_FLAG_INIT_CTX;
2261 if (hmac)
2262 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY);
2263
2264 switch (type) {
2265 case ARTPEC6_CRYPTO_HASH_SHA1:
2266 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1;
2267 break;
2268 case ARTPEC6_CRYPTO_HASH_SHA256:
2269 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256;
2270 break;
2271 case ARTPEC6_CRYPTO_HASH_SHA384:
2272 oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384;
2273 break;
2274 case ARTPEC6_CRYPTO_HASH_SHA512:
2275 oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512;
2276 break;
2277
2278 default:
2279 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type);
2280 return -EINVAL;
2281 }
2282
2283 if (variant == ARTPEC6_CRYPTO)
2284 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper);
2285 else
2286 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper);
2287
2288 return 0;
2289}
2290
2291static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req)
2292{
2293 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2294 int ret;
2295
2296 if (!req_ctx->common.dma) {
2297 ret = artpec6_crypto_common_init(&req_ctx->common,
2298 &req->base,
2299 artpec6_crypto_complete_hash,
2300 NULL, 0);
2301
2302 if (ret)
2303 return ret;
2304 }
2305
2306 ret = artpec6_crypto_prepare_hash(req);
2307 switch (ret) {
2308 case ARTPEC6_CRYPTO_PREPARE_HASH_START:
2309 ret = artpec6_crypto_submit(&req_ctx->common);
2310 break;
2311
2312 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START:
2313 ret = 0;
2314 /* Fallthrough */
2315
2316 default:
2317 artpec6_crypto_common_destroy(&req_ctx->common);
2318 break;
2319 }
2320
2321 return ret;
2322}
2323
2324static int artpec6_crypto_hash_final(struct ahash_request *req)
2325{
2326 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2327
2328 req_ctx->hash_flags |= HASH_FLAG_FINALIZE;
2329
2330 return artpec6_crypto_prepare_submit_hash(req);
2331}
2332
2333static int artpec6_crypto_hash_update(struct ahash_request *req)
2334{
2335 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2336
2337 req_ctx->hash_flags |= HASH_FLAG_UPDATE;
2338
2339 return artpec6_crypto_prepare_submit_hash(req);
2340}
2341
2342static int artpec6_crypto_sha1_init(struct ahash_request *req)
2343{
2344 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2345}
2346
2347static int artpec6_crypto_sha1_digest(struct ahash_request *req)
2348{
2349 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2350
2351 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2352
2353 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2354
2355 return artpec6_crypto_prepare_submit_hash(req);
2356}
2357
2358static int artpec6_crypto_sha256_init(struct ahash_request *req)
2359{
2360 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2361}
2362
2363static int artpec6_crypto_sha256_digest(struct ahash_request *req)
2364{
2365 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2366
2367 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2368 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2369
2370 return artpec6_crypto_prepare_submit_hash(req);
2371}
2372
2373static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req)
2374{
2375 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0);
2376}
2377
2378static int __maybe_unused
2379artpec6_crypto_sha384_digest(struct ahash_request *req)
2380{
2381 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2382
2383 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0);
2384 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2385
2386 return artpec6_crypto_prepare_submit_hash(req);
2387}
2388
2389static int artpec6_crypto_sha512_init(struct ahash_request *req)
2390{
2391 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0);
2392}
2393
2394static int artpec6_crypto_sha512_digest(struct ahash_request *req)
2395{
2396 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2397
2398 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0);
2399 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2400
2401 return artpec6_crypto_prepare_submit_hash(req);
2402}
2403
2404static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req)
2405{
2406 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2407}
2408
2409static int __maybe_unused
2410artpec6_crypto_hmac_sha384_init(struct ahash_request *req)
2411{
2412 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1);
2413}
2414
2415static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req)
2416{
2417 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1);
2418}
2419
2420static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req)
2421{
2422 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2423
2424 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2425 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2426
2427 return artpec6_crypto_prepare_submit_hash(req);
2428}
2429
2430static int __maybe_unused
2431artpec6_crypto_hmac_sha384_digest(struct ahash_request *req)
2432{
2433 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2434
2435 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1);
2436 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2437
2438 return artpec6_crypto_prepare_submit_hash(req);
2439}
2440
2441static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req)
2442{
2443 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2444
2445 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1);
2446 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2447
2448 return artpec6_crypto_prepare_submit_hash(req);
2449}
2450
2451static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm,
2452 const char *base_hash_name)
2453{
2454 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2455
2456 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2457 sizeof(struct artpec6_hash_request_context));
2458 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
2459
2460 if (base_hash_name) {
2461 struct crypto_shash *child;
2462
2463 child = crypto_alloc_shash(base_hash_name, 0,
2464 CRYPTO_ALG_NEED_FALLBACK);
2465
2466 if (IS_ERR(child))
2467 return PTR_ERR(child);
2468
2469 tfm_ctx->child_hash = child;
2470 }
2471
2472 return 0;
2473}
2474
2475static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm)
2476{
2477 return artpec6_crypto_ahash_init_common(tfm, NULL);
2478}
2479
2480static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm)
2481{
2482 return artpec6_crypto_ahash_init_common(tfm, "sha256");
2483}
2484
2485static int __maybe_unused
2486artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm)
2487{
2488 return artpec6_crypto_ahash_init_common(tfm, "sha384");
2489}
2490
2491static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm)
2492{
2493 return artpec6_crypto_ahash_init_common(tfm, "sha512");
2494}
2495
2496static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm)
2497{
2498 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2499
2500 if (tfm_ctx->child_hash)
2501 crypto_free_shash(tfm_ctx->child_hash);
2502
2503 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2504 tfm_ctx->hmac_key_length = 0;
2505}
2506
2507static int artpec6_crypto_hash_export(struct ahash_request *req, void *out)
2508{
2509 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2510 struct artpec6_hash_export_state *state = out;
2511 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2512 enum artpec6_crypto_variant variant = ac->variant;
2513
2514 BUILD_BUG_ON(sizeof(state->partial_buffer) !=
2515 sizeof(ctx->partial_buffer));
2516 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate));
2517
2518 state->digcnt = ctx->digcnt;
2519 state->partial_bytes = ctx->partial_bytes;
2520 state->hash_flags = ctx->hash_flags;
2521
2522 if (variant == ARTPEC6_CRYPTO)
2523 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md);
2524 else
2525 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md);
2526
2527 memcpy(state->partial_buffer, ctx->partial_buffer,
2528 sizeof(state->partial_buffer));
2529 memcpy(state->digeststate, ctx->digeststate,
2530 sizeof(state->digeststate));
2531
2532 return 0;
2533}
2534
2535static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in)
2536{
2537 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2538 const struct artpec6_hash_export_state *state = in;
2539 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2540 enum artpec6_crypto_variant variant = ac->variant;
2541
2542 memset(ctx, 0, sizeof(*ctx));
2543
2544 ctx->digcnt = state->digcnt;
2545 ctx->partial_bytes = state->partial_bytes;
2546 ctx->hash_flags = state->hash_flags;
2547
2548 if (variant == ARTPEC6_CRYPTO)
2549 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper);
2550 else
2551 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper);
2552
2553 memcpy(ctx->partial_buffer, state->partial_buffer,
2554 sizeof(state->partial_buffer));
2555 memcpy(ctx->digeststate, state->digeststate,
2556 sizeof(state->digeststate));
2557
2558 return 0;
2559}
2560
2561static int init_crypto_hw(struct artpec6_crypto *ac)
2562{
2563 enum artpec6_crypto_variant variant = ac->variant;
2564 void __iomem *base = ac->base;
2565 u32 out_descr_buf_size;
2566 u32 out_data_buf_size;
2567 u32 in_data_buf_size;
2568 u32 in_descr_buf_size;
2569 u32 in_stat_buf_size;
2570 u32 in, out;
2571
2572 /*
2573 * The PDMA unit contains 1984 bytes of internal memory for the OUT
2574 * channels and 1024 bytes for the IN channel. This is an elastic
2575 * memory used to internally store the descriptors and data. The values
2576 * ares specified in 64 byte incremements. Trustzone buffers are not
2577 * used at this stage.
2578 */
2579 out_data_buf_size = 16; /* 1024 bytes for data */
2580 out_descr_buf_size = 15; /* 960 bytes for descriptors */
2581 in_data_buf_size = 8; /* 512 bytes for data */
2582 in_descr_buf_size = 4; /* 256 bytes for descriptors */
2583 in_stat_buf_size = 4; /* 256 bytes for stat descrs */
2584
2585 BUILD_BUG_ON_MSG((out_data_buf_size
2586 + out_descr_buf_size) * 64 > 1984,
2587 "Invalid OUT configuration");
2588
2589 BUILD_BUG_ON_MSG((in_data_buf_size
2590 + in_descr_buf_size
2591 + in_stat_buf_size) * 64 > 1024,
2592 "Invalid IN configuration");
2593
2594 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) |
2595 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) |
2596 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size);
2597
2598 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) |
2599 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size);
2600
2601 writel_relaxed(out, base + PDMA_OUT_BUF_CFG);
2602 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG);
2603
2604 if (variant == ARTPEC6_CRYPTO) {
2605 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG);
2606 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG);
2607 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA |
2608 A6_PDMA_INTR_MASK_IN_EOP_FLUSH,
2609 base + A6_PDMA_INTR_MASK);
2610 } else {
2611 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG);
2612 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG);
2613 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA |
2614 A7_PDMA_INTR_MASK_IN_EOP_FLUSH,
2615 base + A7_PDMA_INTR_MASK);
2616 }
2617
2618 return 0;
2619}
2620
2621static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)
2622{
2623 enum artpec6_crypto_variant variant = ac->variant;
2624 void __iomem *base = ac->base;
2625
2626 if (variant == ARTPEC6_CRYPTO) {
2627 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD);
2628 writel_relaxed(0, base + A6_PDMA_IN_CFG);
2629 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2630 } else {
2631 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD);
2632 writel_relaxed(0, base + A7_PDMA_IN_CFG);
2633 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2634 }
2635
2636 writel_relaxed(0, base + PDMA_OUT_CFG);
2637
2638}
2639
2640static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id)
2641{
2642 struct artpec6_crypto *ac = dev_id;
2643 enum artpec6_crypto_variant variant = ac->variant;
2644 void __iomem *base = ac->base;
2645 u32 mask_in_data, mask_in_eop_flush;
2646 u32 in_cmd_flush_stat, in_cmd_reg;
2647 u32 ack_intr_reg;
2648 u32 ack = 0;
2649 u32 intr;
2650
2651 if (variant == ARTPEC6_CRYPTO) {
2652 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR);
2653 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA;
2654 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH;
2655 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT;
2656 in_cmd_reg = A6_PDMA_IN_CMD;
2657 ack_intr_reg = A6_PDMA_ACK_INTR;
2658 } else {
2659 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR);
2660 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA;
2661 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH;
2662 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT;
2663 in_cmd_reg = A7_PDMA_IN_CMD;
2664 ack_intr_reg = A7_PDMA_ACK_INTR;
2665 }
2666
2667 /* We get two interrupt notifications from each job.
2668 * The in_data means all data was sent to memory and then
2669 * we request a status flush command to write the per-job
2670 * status to its status vector. This ensures that the
2671 * tasklet can detect exactly how many submitted jobs
2672 * that have finished.
2673 */
2674 if (intr & mask_in_data)
2675 ack |= mask_in_data;
2676
2677 if (intr & mask_in_eop_flush)
2678 ack |= mask_in_eop_flush;
2679 else
2680 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg);
2681
2682 writel_relaxed(ack, base + ack_intr_reg);
2683
2684 if (intr & mask_in_eop_flush)
2685 tasklet_schedule(&ac->task);
2686
2687 return IRQ_HANDLED;
2688}
2689
2690/*------------------- Algorithm definitions ----------------------------------*/
2691
2692/* Hashes */
2693static struct ahash_alg hash_algos[] = {
2694 /* SHA-1 */
2695 {
2696 .init = artpec6_crypto_sha1_init,
2697 .update = artpec6_crypto_hash_update,
2698 .final = artpec6_crypto_hash_final,
2699 .digest = artpec6_crypto_sha1_digest,
2700 .import = artpec6_crypto_hash_import,
2701 .export = artpec6_crypto_hash_export,
2702 .halg.digestsize = SHA1_DIGEST_SIZE,
2703 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2704 .halg.base = {
2705 .cra_name = "sha1",
2706 .cra_driver_name = "artpec-sha1",
2707 .cra_priority = 300,
2708 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2709 .cra_blocksize = SHA1_BLOCK_SIZE,
2710 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2711 .cra_alignmask = 3,
2712 .cra_module = THIS_MODULE,
2713 .cra_init = artpec6_crypto_ahash_init,
2714 .cra_exit = artpec6_crypto_ahash_exit,
2715 }
2716 },
2717 /* SHA-256 */
2718 {
2719 .init = artpec6_crypto_sha256_init,
2720 .update = artpec6_crypto_hash_update,
2721 .final = artpec6_crypto_hash_final,
2722 .digest = artpec6_crypto_sha256_digest,
2723 .import = artpec6_crypto_hash_import,
2724 .export = artpec6_crypto_hash_export,
2725 .halg.digestsize = SHA256_DIGEST_SIZE,
2726 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2727 .halg.base = {
2728 .cra_name = "sha256",
2729 .cra_driver_name = "artpec-sha256",
2730 .cra_priority = 300,
2731 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2732 .cra_blocksize = SHA256_BLOCK_SIZE,
2733 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2734 .cra_alignmask = 3,
2735 .cra_module = THIS_MODULE,
2736 .cra_init = artpec6_crypto_ahash_init,
2737 .cra_exit = artpec6_crypto_ahash_exit,
2738 }
2739 },
2740 /* HMAC SHA-256 */
2741 {
2742 .init = artpec6_crypto_hmac_sha256_init,
2743 .update = artpec6_crypto_hash_update,
2744 .final = artpec6_crypto_hash_final,
2745 .digest = artpec6_crypto_hmac_sha256_digest,
2746 .import = artpec6_crypto_hash_import,
2747 .export = artpec6_crypto_hash_export,
2748 .setkey = artpec6_crypto_hash_set_key,
2749 .halg.digestsize = SHA256_DIGEST_SIZE,
2750 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2751 .halg.base = {
2752 .cra_name = "hmac(sha256)",
2753 .cra_driver_name = "artpec-hmac-sha256",
2754 .cra_priority = 300,
2755 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2756 .cra_blocksize = SHA256_BLOCK_SIZE,
2757 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2758 .cra_alignmask = 3,
2759 .cra_module = THIS_MODULE,
2760 .cra_init = artpec6_crypto_ahash_init_hmac_sha256,
2761 .cra_exit = artpec6_crypto_ahash_exit,
2762 }
2763 },
2764};
2765
2766static struct ahash_alg artpec7_hash_algos[] = {
2767 /* SHA-384 */
2768 {
2769 .init = artpec6_crypto_sha384_init,
2770 .update = artpec6_crypto_hash_update,
2771 .final = artpec6_crypto_hash_final,
2772 .digest = artpec6_crypto_sha384_digest,
2773 .import = artpec6_crypto_hash_import,
2774 .export = artpec6_crypto_hash_export,
2775 .halg.digestsize = SHA384_DIGEST_SIZE,
2776 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2777 .halg.base = {
2778 .cra_name = "sha384",
2779 .cra_driver_name = "artpec-sha384",
2780 .cra_priority = 300,
2781 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2782 .cra_blocksize = SHA384_BLOCK_SIZE,
2783 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2784 .cra_alignmask = 3,
2785 .cra_module = THIS_MODULE,
2786 .cra_init = artpec6_crypto_ahash_init,
2787 .cra_exit = artpec6_crypto_ahash_exit,
2788 }
2789 },
2790 /* HMAC SHA-384 */
2791 {
2792 .init = artpec6_crypto_hmac_sha384_init,
2793 .update = artpec6_crypto_hash_update,
2794 .final = artpec6_crypto_hash_final,
2795 .digest = artpec6_crypto_hmac_sha384_digest,
2796 .import = artpec6_crypto_hash_import,
2797 .export = artpec6_crypto_hash_export,
2798 .setkey = artpec6_crypto_hash_set_key,
2799 .halg.digestsize = SHA384_DIGEST_SIZE,
2800 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2801 .halg.base = {
2802 .cra_name = "hmac(sha384)",
2803 .cra_driver_name = "artpec-hmac-sha384",
2804 .cra_priority = 300,
2805 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2806 .cra_blocksize = SHA384_BLOCK_SIZE,
2807 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2808 .cra_alignmask = 3,
2809 .cra_module = THIS_MODULE,
2810 .cra_init = artpec6_crypto_ahash_init_hmac_sha384,
2811 .cra_exit = artpec6_crypto_ahash_exit,
2812 }
2813 },
2814 /* SHA-512 */
2815 {
2816 .init = artpec6_crypto_sha512_init,
2817 .update = artpec6_crypto_hash_update,
2818 .final = artpec6_crypto_hash_final,
2819 .digest = artpec6_crypto_sha512_digest,
2820 .import = artpec6_crypto_hash_import,
2821 .export = artpec6_crypto_hash_export,
2822 .halg.digestsize = SHA512_DIGEST_SIZE,
2823 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2824 .halg.base = {
2825 .cra_name = "sha512",
2826 .cra_driver_name = "artpec-sha512",
2827 .cra_priority = 300,
2828 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2829 .cra_blocksize = SHA512_BLOCK_SIZE,
2830 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2831 .cra_alignmask = 3,
2832 .cra_module = THIS_MODULE,
2833 .cra_init = artpec6_crypto_ahash_init,
2834 .cra_exit = artpec6_crypto_ahash_exit,
2835 }
2836 },
2837 /* HMAC SHA-512 */
2838 {
2839 .init = artpec6_crypto_hmac_sha512_init,
2840 .update = artpec6_crypto_hash_update,
2841 .final = artpec6_crypto_hash_final,
2842 .digest = artpec6_crypto_hmac_sha512_digest,
2843 .import = artpec6_crypto_hash_import,
2844 .export = artpec6_crypto_hash_export,
2845 .setkey = artpec6_crypto_hash_set_key,
2846 .halg.digestsize = SHA512_DIGEST_SIZE,
2847 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2848 .halg.base = {
2849 .cra_name = "hmac(sha512)",
2850 .cra_driver_name = "artpec-hmac-sha512",
2851 .cra_priority = 300,
2852 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2853 .cra_blocksize = SHA512_BLOCK_SIZE,
2854 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2855 .cra_alignmask = 3,
2856 .cra_module = THIS_MODULE,
2857 .cra_init = artpec6_crypto_ahash_init_hmac_sha512,
2858 .cra_exit = artpec6_crypto_ahash_exit,
2859 }
2860 },
2861};
2862
2863/* Crypto */
2864static struct skcipher_alg crypto_algos[] = {
2865 /* AES - ECB */
2866 {
2867 .base = {
2868 .cra_name = "ecb(aes)",
2869 .cra_driver_name = "artpec6-ecb-aes",
2870 .cra_priority = 300,
2871 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2872 CRYPTO_ALG_ASYNC,
2873 .cra_blocksize = AES_BLOCK_SIZE,
2874 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2875 .cra_alignmask = 3,
2876 .cra_module = THIS_MODULE,
2877 },
2878 .min_keysize = AES_MIN_KEY_SIZE,
2879 .max_keysize = AES_MAX_KEY_SIZE,
2880 .setkey = artpec6_crypto_cipher_set_key,
2881 .encrypt = artpec6_crypto_encrypt,
2882 .decrypt = artpec6_crypto_decrypt,
2883 .init = artpec6_crypto_aes_ecb_init,
2884 .exit = artpec6_crypto_aes_exit,
2885 },
2886 /* AES - CTR */
2887 {
2888 .base = {
2889 .cra_name = "ctr(aes)",
2890 .cra_driver_name = "artpec6-ctr-aes",
2891 .cra_priority = 300,
2892 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2893 CRYPTO_ALG_ASYNC |
2894 CRYPTO_ALG_NEED_FALLBACK,
2895 .cra_blocksize = 1,
2896 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2897 .cra_alignmask = 3,
2898 .cra_module = THIS_MODULE,
2899 },
2900 .min_keysize = AES_MIN_KEY_SIZE,
2901 .max_keysize = AES_MAX_KEY_SIZE,
2902 .ivsize = AES_BLOCK_SIZE,
2903 .setkey = artpec6_crypto_cipher_set_key,
2904 .encrypt = artpec6_crypto_ctr_encrypt,
2905 .decrypt = artpec6_crypto_ctr_decrypt,
2906 .init = artpec6_crypto_aes_ctr_init,
2907 .exit = artpec6_crypto_aes_ctr_exit,
2908 },
2909 /* AES - CBC */
2910 {
2911 .base = {
2912 .cra_name = "cbc(aes)",
2913 .cra_driver_name = "artpec6-cbc-aes",
2914 .cra_priority = 300,
2915 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2916 CRYPTO_ALG_ASYNC,
2917 .cra_blocksize = AES_BLOCK_SIZE,
2918 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2919 .cra_alignmask = 3,
2920 .cra_module = THIS_MODULE,
2921 },
2922 .min_keysize = AES_MIN_KEY_SIZE,
2923 .max_keysize = AES_MAX_KEY_SIZE,
2924 .ivsize = AES_BLOCK_SIZE,
2925 .setkey = artpec6_crypto_cipher_set_key,
2926 .encrypt = artpec6_crypto_encrypt,
2927 .decrypt = artpec6_crypto_decrypt,
2928 .init = artpec6_crypto_aes_cbc_init,
2929 .exit = artpec6_crypto_aes_exit
2930 },
2931 /* AES - XTS */
2932 {
2933 .base = {
2934 .cra_name = "xts(aes)",
2935 .cra_driver_name = "artpec6-xts-aes",
2936 .cra_priority = 300,
2937 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2938 CRYPTO_ALG_ASYNC,
2939 .cra_blocksize = 1,
2940 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2941 .cra_alignmask = 3,
2942 .cra_module = THIS_MODULE,
2943 },
2944 .min_keysize = 2*AES_MIN_KEY_SIZE,
2945 .max_keysize = 2*AES_MAX_KEY_SIZE,
2946 .ivsize = 16,
2947 .setkey = artpec6_crypto_xts_set_key,
2948 .encrypt = artpec6_crypto_encrypt,
2949 .decrypt = artpec6_crypto_decrypt,
2950 .init = artpec6_crypto_aes_xts_init,
2951 .exit = artpec6_crypto_aes_exit,
2952 },
2953};
2954
2955static struct aead_alg aead_algos[] = {
2956 {
2957 .init = artpec6_crypto_aead_init,
2958 .setkey = artpec6_crypto_aead_set_key,
2959 .encrypt = artpec6_crypto_aead_encrypt,
2960 .decrypt = artpec6_crypto_aead_decrypt,
2961 .ivsize = AES_BLOCK_SIZE,
2962 .maxauthsize = AES_BLOCK_SIZE,
2963
2964 .base = {
2965 .cra_name = "gcm(aes)",
2966 .cra_driver_name = "artpec-gcm-aes",
2967 .cra_priority = 300,
2968 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
2969 CRYPTO_ALG_KERN_DRIVER_ONLY,
2970 .cra_blocksize = 1,
2971 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2972 .cra_alignmask = 3,
2973 .cra_module = THIS_MODULE,
2974 },
2975 }
2976};
2977
2978#ifdef CONFIG_DEBUG_FS
2979
2980struct dbgfs_u32 {
2981 char *name;
2982 mode_t mode;
2983 u32 *flag;
2984 char *desc;
2985};
2986
2987static void artpec6_crypto_init_debugfs(void)
2988{
2989 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL);
2990
2991 if (!dbgfs_root || IS_ERR(dbgfs_root)) {
2992 dbgfs_root = NULL;
2993 pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME);
2994 return;
2995 }
2996
2997#ifdef CONFIG_FAULT_INJECTION
2998 fault_create_debugfs_attr("fail_status_read", dbgfs_root,
2999 &artpec6_crypto_fail_status_read);
3000
3001 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root,
3002 &artpec6_crypto_fail_dma_array_full);
3003#endif
3004}
3005
3006static void artpec6_crypto_free_debugfs(void)
3007{
3008 if (!dbgfs_root)
3009 return;
3010
3011 debugfs_remove_recursive(dbgfs_root);
3012 dbgfs_root = NULL;
3013}
3014#endif
3015
3016static const struct of_device_id artpec6_crypto_of_match[] = {
3017 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO },
3018 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO },
3019 {}
3020};
3021MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match);
3022
3023static int artpec6_crypto_probe(struct platform_device *pdev)
3024{
3025 const struct of_device_id *match;
3026 enum artpec6_crypto_variant variant;
3027 struct artpec6_crypto *ac;
3028 struct device *dev = &pdev->dev;
3029 void __iomem *base;
3030 struct resource *res;
3031 int irq;
3032 int err;
3033
3034 if (artpec6_crypto_dev)
3035 return -ENODEV;
3036
3037 match = of_match_node(artpec6_crypto_of_match, dev->of_node);
3038 if (!match)
3039 return -EINVAL;
3040
3041 variant = (enum artpec6_crypto_variant)match->data;
3042
3043 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3044 if (!res)
3045 return -ENODEV;
3046
3047 base = devm_ioremap_resource(&pdev->dev, res);
3048 if (IS_ERR(base))
3049 return PTR_ERR(base);
3050
3051 irq = platform_get_irq(pdev, 0);
3052 if (irq < 0)
3053 return -ENODEV;
3054
3055 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),
3056 GFP_KERNEL);
3057 if (!ac)
3058 return -ENOMEM;
3059
3060 platform_set_drvdata(pdev, ac);
3061 ac->variant = variant;
3062
3063 spin_lock_init(&ac->queue_lock);
3064 INIT_LIST_HEAD(&ac->queue);
3065 INIT_LIST_HEAD(&ac->pending);
3066 setup_timer(&ac->timer, artpec6_crypto_timeout, (unsigned long) ac);
3067
3068 ac->base = base;
3069
3070 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",
3071 sizeof(struct artpec6_crypto_dma_descriptors),
3072 64,
3073 0,
3074 NULL);
3075 if (!ac->dma_cache)
3076 return -ENOMEM;
3077
3078#ifdef CONFIG_DEBUG_FS
3079 artpec6_crypto_init_debugfs();
3080#endif
3081
3082 tasklet_init(&ac->task, artpec6_crypto_task,
3083 (unsigned long)ac);
3084
3085 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
3086 GFP_KERNEL);
3087 if (!ac->pad_buffer)
3088 return -ENOMEM;
3089 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);
3090
3091 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
3092 GFP_KERNEL);
3093 if (!ac->zero_buffer)
3094 return -ENOMEM;
3095 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);
3096
3097 err = init_crypto_hw(ac);
3098 if (err)
3099 goto free_cache;
3100
3101 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0,
3102 "artpec6-crypto", ac);
3103 if (err)
3104 goto disable_hw;
3105
3106 artpec6_crypto_dev = &pdev->dev;
3107
3108 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
3109 if (err) {
3110 dev_err(dev, "Failed to register ahashes\n");
3111 goto disable_hw;
3112 }
3113
3114 if (variant != ARTPEC6_CRYPTO) {
3115 err = crypto_register_ahashes(artpec7_hash_algos,
3116 ARRAY_SIZE(artpec7_hash_algos));
3117 if (err) {
3118 dev_err(dev, "Failed to register ahashes\n");
3119 goto unregister_ahashes;
3120 }
3121 }
3122
3123 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
3124 if (err) {
3125 dev_err(dev, "Failed to register ciphers\n");
3126 goto unregister_a7_ahashes;
3127 }
3128
3129 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos));
3130 if (err) {
3131 dev_err(dev, "Failed to register aeads\n");
3132 goto unregister_algs;
3133 }
3134
3135 return 0;
3136
3137unregister_algs:
3138 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
3139unregister_a7_ahashes:
3140 if (variant != ARTPEC6_CRYPTO)
3141 crypto_unregister_ahashes(artpec7_hash_algos,
3142 ARRAY_SIZE(artpec7_hash_algos));
3143unregister_ahashes:
3144 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
3145disable_hw:
3146 artpec6_crypto_disable_hw(ac);
3147free_cache:
3148 kmem_cache_destroy(ac->dma_cache);
3149 return err;
3150}
3151
3152static int artpec6_crypto_remove(struct platform_device *pdev)
3153{
3154 struct artpec6_crypto *ac = platform_get_drvdata(pdev);
3155 int irq = platform_get_irq(pdev, 0);
3156
3157 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
3158 if (ac->variant != ARTPEC6_CRYPTO)
3159 crypto_unregister_ahashes(artpec7_hash_algos,
3160 ARRAY_SIZE(artpec7_hash_algos));
3161 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
3162 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos));
3163
3164 tasklet_disable(&ac->task);
3165 devm_free_irq(&pdev->dev, irq, ac);
3166 tasklet_kill(&ac->task);
3167 del_timer_sync(&ac->timer);
3168
3169 artpec6_crypto_disable_hw(ac);
3170
3171 kmem_cache_destroy(ac->dma_cache);
3172#ifdef CONFIG_DEBUG_FS
3173 artpec6_crypto_free_debugfs();
3174#endif
3175 return 0;
3176}
3177
3178static struct platform_driver artpec6_crypto_driver = {
3179 .probe = artpec6_crypto_probe,
3180 .remove = artpec6_crypto_remove,
3181 .driver = {
3182 .name = "artpec6-crypto",
3183 .owner = THIS_MODULE,
3184 .of_match_table = artpec6_crypto_of_match,
3185 },
3186};
3187
3188module_platform_driver(artpec6_crypto_driver);
3189
3190MODULE_AUTHOR("Axis Communications AB");
3191MODULE_DESCRIPTION("ARTPEC-6 Crypto driver");
3192MODULE_LICENSE("GPL");
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
index 9cfd36c1bcb6..8685c7e4debd 100644
--- a/drivers/crypto/bcm/cipher.c
+++ b/drivers/crypto/bcm/cipher.c
@@ -90,8 +90,6 @@ static int aead_pri = 150;
90module_param(aead_pri, int, 0644); 90module_param(aead_pri, int, 0644);
91MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); 91MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
92 92
93#define MAX_SPUS 16
94
95/* A type 3 BCM header, expected to precede the SPU header for SPU-M. 93/* A type 3 BCM header, expected to precede the SPU header for SPU-M.
96 * Bits 3 and 4 in the first byte encode the channel number (the dma ringset). 94 * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
97 * 0x60 - ring 0 95 * 0x60 - ring 0
@@ -120,7 +118,7 @@ static u8 select_channel(void)
120{ 118{
121 u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan); 119 u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
122 120
123 return chan_idx % iproc_priv.spu.num_spu; 121 return chan_idx % iproc_priv.spu.num_chan;
124} 122}
125 123
126/** 124/**
@@ -4528,8 +4526,13 @@ static void spu_functions_register(struct device *dev,
4528 */ 4526 */
4529static int spu_mb_init(struct device *dev) 4527static int spu_mb_init(struct device *dev)
4530{ 4528{
4531 struct mbox_client *mcl = &iproc_priv.mcl[iproc_priv.spu.num_spu]; 4529 struct mbox_client *mcl = &iproc_priv.mcl;
4532 int err; 4530 int err, i;
4531
4532 iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan,
4533 sizeof(struct mbox_chan *), GFP_KERNEL);
4534 if (!iproc_priv.mbox)
4535 return -ENOMEM;
4533 4536
4534 mcl->dev = dev; 4537 mcl->dev = dev;
4535 mcl->tx_block = false; 4538 mcl->tx_block = false;
@@ -4538,25 +4541,33 @@ static int spu_mb_init(struct device *dev)
4538 mcl->rx_callback = spu_rx_callback; 4541 mcl->rx_callback = spu_rx_callback;
4539 mcl->tx_done = NULL; 4542 mcl->tx_done = NULL;
4540 4543
4541 iproc_priv.mbox[iproc_priv.spu.num_spu] = 4544 for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4542 mbox_request_channel(mcl, 0); 4545 iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
4543 if (IS_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu])) { 4546 if (IS_ERR(iproc_priv.mbox[i])) {
4544 err = (int)PTR_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu]); 4547 err = (int)PTR_ERR(iproc_priv.mbox[i]);
4545 dev_err(dev, 4548 dev_err(dev,
4546 "Mbox channel %d request failed with err %d", 4549 "Mbox channel %d request failed with err %d",
4547 iproc_priv.spu.num_spu, err); 4550 i, err);
4548 iproc_priv.mbox[iproc_priv.spu.num_spu] = NULL; 4551 iproc_priv.mbox[i] = NULL;
4549 return err; 4552 goto free_channels;
4553 }
4550 } 4554 }
4551 4555
4552 return 0; 4556 return 0;
4557free_channels:
4558 for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4559 if (iproc_priv.mbox[i])
4560 mbox_free_channel(iproc_priv.mbox[i]);
4561 }
4562
4563 return err;
4553} 4564}
4554 4565
4555static void spu_mb_release(struct platform_device *pdev) 4566static void spu_mb_release(struct platform_device *pdev)
4556{ 4567{
4557 int i; 4568 int i;
4558 4569
4559 for (i = 0; i < iproc_priv.spu.num_spu; i++) 4570 for (i = 0; i < iproc_priv.spu.num_chan; i++)
4560 mbox_free_channel(iproc_priv.mbox[i]); 4571 mbox_free_channel(iproc_priv.mbox[i]);
4561} 4572}
4562 4573
@@ -4567,7 +4578,7 @@ static void spu_counters_init(void)
4567 4578
4568 atomic_set(&iproc_priv.session_count, 0); 4579 atomic_set(&iproc_priv.session_count, 0);
4569 atomic_set(&iproc_priv.stream_count, 0); 4580 atomic_set(&iproc_priv.stream_count, 0);
4570 atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_spu); 4581 atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan);
4571 atomic64_set(&iproc_priv.bytes_in, 0); 4582 atomic64_set(&iproc_priv.bytes_in, 0);
4572 atomic64_set(&iproc_priv.bytes_out, 0); 4583 atomic64_set(&iproc_priv.bytes_out, 0);
4573 for (i = 0; i < SPU_OP_NUM; i++) { 4584 for (i = 0; i < SPU_OP_NUM; i++) {
@@ -4809,47 +4820,38 @@ static int spu_dt_read(struct platform_device *pdev)
4809 struct resource *spu_ctrl_regs; 4820 struct resource *spu_ctrl_regs;
4810 const struct of_device_id *match; 4821 const struct of_device_id *match;
4811 const struct spu_type_subtype *matched_spu_type; 4822 const struct spu_type_subtype *matched_spu_type;
4812 void __iomem *spu_reg_vbase[MAX_SPUS]; 4823 struct device_node *dn = pdev->dev.of_node;
4813 int err; 4824 int err, i;
4814 4825
4815 match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev); 4826 /* Count number of mailbox channels */
4816 matched_spu_type = match->data; 4827 spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells");
4817 4828
4818 if (iproc_priv.spu.num_spu > 1) { 4829 match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev);
4819 /* If this is 2nd or later SPU, make sure it's same type */ 4830 if (!match) {
4820 if ((spu->spu_type != matched_spu_type->type) || 4831 dev_err(&pdev->dev, "Failed to match device\n");
4821 (spu->spu_subtype != matched_spu_type->subtype)) { 4832 return -ENODEV;
4822 err = -EINVAL;
4823 dev_err(&pdev->dev, "Multiple SPU types not allowed");
4824 return err;
4825 }
4826 } else {
4827 /* Record type of first SPU */
4828 spu->spu_type = matched_spu_type->type;
4829 spu->spu_subtype = matched_spu_type->subtype;
4830 } 4833 }
4831 4834
4832 /* Get and map SPU registers */ 4835 matched_spu_type = match->data;
4833 spu_ctrl_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4834 if (!spu_ctrl_regs) {
4835 err = -EINVAL;
4836 dev_err(&pdev->dev, "Invalid/missing registers for SPU\n");
4837 return err;
4838 }
4839 4836
4840 spu_reg_vbase[iproc_priv.spu.num_spu] = 4837 spu->spu_type = matched_spu_type->type;
4841 devm_ioremap_resource(dev, spu_ctrl_regs); 4838 spu->spu_subtype = matched_spu_type->subtype;
4842 if (IS_ERR(spu_reg_vbase[iproc_priv.spu.num_spu])) {
4843 err = PTR_ERR(spu_reg_vbase[iproc_priv.spu.num_spu]);
4844 dev_err(&pdev->dev, "Failed to map registers: %d\n",
4845 err);
4846 spu_reg_vbase[iproc_priv.spu.num_spu] = NULL;
4847 return err;
4848 }
4849 4839
4850 dev_dbg(dev, "SPU %d detected.", iproc_priv.spu.num_spu); 4840 i = 0;
4841 for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
4842 platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
4851 4843
4852 spu->reg_vbase[iproc_priv.spu.num_spu] = spu_reg_vbase; 4844 spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
4845 if (IS_ERR(spu->reg_vbase[i])) {
4846 err = PTR_ERR(spu->reg_vbase[i]);
4847 dev_err(&pdev->dev, "Failed to map registers: %d\n",
4848 err);
4849 spu->reg_vbase[i] = NULL;
4850 return err;
4851 }
4852 }
4853 spu->num_spu = i;
4854 dev_dbg(dev, "Device has %d SPUs", spu->num_spu);
4853 4855
4854 return 0; 4856 return 0;
4855} 4857}
@@ -4860,8 +4862,8 @@ int bcm_spu_probe(struct platform_device *pdev)
4860 struct spu_hw *spu = &iproc_priv.spu; 4862 struct spu_hw *spu = &iproc_priv.spu;
4861 int err = 0; 4863 int err = 0;
4862 4864
4863 iproc_priv.pdev[iproc_priv.spu.num_spu] = pdev; 4865 iproc_priv.pdev = pdev;
4864 platform_set_drvdata(iproc_priv.pdev[iproc_priv.spu.num_spu], 4866 platform_set_drvdata(iproc_priv.pdev,
4865 &iproc_priv); 4867 &iproc_priv);
4866 4868
4867 err = spu_dt_read(pdev); 4869 err = spu_dt_read(pdev);
@@ -4872,12 +4874,6 @@ int bcm_spu_probe(struct platform_device *pdev)
4872 if (err < 0) 4874 if (err < 0)
4873 goto failure; 4875 goto failure;
4874 4876
4875 iproc_priv.spu.num_spu++;
4876
4877 /* If already initialized, we've just added another SPU and are done */
4878 if (iproc_priv.inited)
4879 return 0;
4880
4881 if (spu->spu_type == SPU_TYPE_SPUM) 4877 if (spu->spu_type == SPU_TYPE_SPUM)
4882 iproc_priv.bcm_hdr_len = 8; 4878 iproc_priv.bcm_hdr_len = 8;
4883 else if (spu->spu_type == SPU_TYPE_SPU2) 4879 else if (spu->spu_type == SPU_TYPE_SPU2)
@@ -4893,8 +4889,6 @@ int bcm_spu_probe(struct platform_device *pdev)
4893 if (err < 0) 4889 if (err < 0)
4894 goto fail_reg; 4890 goto fail_reg;
4895 4891
4896 iproc_priv.inited = true;
4897
4898 return 0; 4892 return 0;
4899 4893
4900fail_reg: 4894fail_reg:
diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h
index 51dca529ce8f..57a55eb2a255 100644
--- a/drivers/crypto/bcm/cipher.h
+++ b/drivers/crypto/bcm/cipher.h
@@ -427,10 +427,13 @@ struct spu_hw {
427 427
428 /* The number of SPUs on this platform */ 428 /* The number of SPUs on this platform */
429 u32 num_spu; 429 u32 num_spu;
430
431 /* The number of SPU channels on this platform */
432 u32 num_chan;
430}; 433};
431 434
432struct device_private { 435struct device_private {
433 struct platform_device *pdev[MAX_SPUS]; 436 struct platform_device *pdev;
434 437
435 struct spu_hw spu; 438 struct spu_hw spu;
436 439
@@ -470,12 +473,10 @@ struct device_private {
470 /* Number of ICV check failures for AEAD messages */ 473 /* Number of ICV check failures for AEAD messages */
471 atomic_t bad_icv; 474 atomic_t bad_icv;
472 475
473 struct mbox_client mcl[MAX_SPUS]; 476 struct mbox_client mcl;
474 /* Array of mailbox channel pointers, one for each channel */
475 struct mbox_chan *mbox[MAX_SPUS];
476 477
477 /* Driver initialized */ 478 /* Array of mailbox channel pointers, one for each channel */
478 bool inited; 479 struct mbox_chan **mbox;
479}; 480};
480 481
481extern struct device_private iproc_priv; 482extern struct device_private iproc_priv;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 0488b7f81dcf..54f3b375a453 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -81,40 +81,6 @@
81#define debug(format, arg...) 81#define debug(format, arg...)
82#endif 82#endif
83 83
84#ifdef DEBUG
85#include <linux/highmem.h>
86
87static void dbg_dump_sg(const char *level, const char *prefix_str,
88 int prefix_type, int rowsize, int groupsize,
89 struct scatterlist *sg, size_t tlen, bool ascii)
90{
91 struct scatterlist *it;
92 void *it_page;
93 size_t len;
94 void *buf;
95
96 for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) {
97 /*
98 * make sure the scatterlist's page
99 * has a valid virtual memory mapping
100 */
101 it_page = kmap_atomic(sg_page(it));
102 if (unlikely(!it_page)) {
103 printk(KERN_ERR "dbg_dump_sg: kmap failed\n");
104 return;
105 }
106
107 buf = it_page + it->offset;
108 len = min_t(size_t, tlen, it->length);
109 print_hex_dump(level, prefix_str, prefix_type, rowsize,
110 groupsize, buf, len, ascii);
111 tlen -= len;
112
113 kunmap_atomic(it_page);
114 }
115}
116#endif
117
118static struct list_head alg_list; 84static struct list_head alg_list;
119 85
120struct caam_alg_entry { 86struct caam_alg_entry {
@@ -898,10 +864,10 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
898 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 864 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
899 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 865 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
900 edesc->src_nents > 1 ? 100 : ivsize, 1); 866 edesc->src_nents > 1 ? 100 : ivsize, 1);
901 dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ",
902 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
903 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
904#endif 867#endif
868 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
869 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
870 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
905 871
906 ablkcipher_unmap(jrdev, edesc, req); 872 ablkcipher_unmap(jrdev, edesc, req);
907 873
@@ -937,10 +903,10 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
937 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 903 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
938 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 904 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
939 ivsize, 1); 905 ivsize, 1);
940 dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ",
941 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
942 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
943#endif 906#endif
907 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
908 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
909 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
944 910
945 ablkcipher_unmap(jrdev, edesc, req); 911 ablkcipher_unmap(jrdev, edesc, req);
946 912
@@ -1107,10 +1073,10 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1107 ivsize, 1); 1073 ivsize, 1);
1108 pr_err("asked=%d, nbytes%d\n", 1074 pr_err("asked=%d, nbytes%d\n",
1109 (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes); 1075 (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes);
1110 dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ",
1111 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1112 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1113#endif 1076#endif
1077 caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ",
1078 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1079 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1114 1080
1115 len = desc_len(sh_desc); 1081 len = desc_len(sh_desc);
1116 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1082 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
@@ -1164,10 +1130,10 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
1164 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 1130 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
1165 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1131 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1166 ivsize, 1); 1132 ivsize, 1);
1167 dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ",
1168 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1169 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1170#endif 1133#endif
1134 caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ",
1135 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1136 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1171 1137
1172 len = desc_len(sh_desc); 1138 len = desc_len(sh_desc);
1173 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1139 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
@@ -1449,11 +1415,9 @@ static int aead_decrypt(struct aead_request *req)
1449 u32 *desc; 1415 u32 *desc;
1450 int ret = 0; 1416 int ret = 0;
1451 1417
1452#ifdef DEBUG 1418 caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
1453 dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 1419 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1454 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1420 req->assoclen + req->cryptlen, 1);
1455 req->assoclen + req->cryptlen, 1);
1456#endif
1457 1421
1458 /* allocate extended descriptor */ 1422 /* allocate extended descriptor */
1459 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1423 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 6f9c7ec0e339..530c14ee32de 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -599,7 +599,7 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
599 599
600 /* skip key loading if they are loaded due to sharing */ 600 /* skip key loading if they are loaded due to sharing */
601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
602 JUMP_COND_SHRD | JUMP_COND_SELF); 602 JUMP_COND_SHRD);
603 if (cdata->key_inline) 603 if (cdata->key_inline)
604 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 604 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
605 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 605 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
@@ -688,8 +688,7 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
688 688
689 /* skip key loading if they are loaded due to sharing */ 689 /* skip key loading if they are loaded due to sharing */
690 key_jump_cmd = append_jump(desc, JUMP_JSL | 690 key_jump_cmd = append_jump(desc, JUMP_JSL |
691 JUMP_TEST_ALL | JUMP_COND_SHRD | 691 JUMP_TEST_ALL | JUMP_COND_SHRD);
692 JUMP_COND_SELF);
693 if (cdata->key_inline) 692 if (cdata->key_inline)
694 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 693 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
695 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 694 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index 78c4c0485c58..2eefc4a26bc2 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -12,7 +12,6 @@
12#include "intern.h" 12#include "intern.h"
13#include "desc_constr.h" 13#include "desc_constr.h"
14#include "error.h" 14#include "error.h"
15#include "sg_sw_sec4.h"
16#include "sg_sw_qm.h" 15#include "sg_sw_qm.h"
17#include "key_gen.h" 16#include "key_gen.h"
18#include "qi.h" 17#include "qi.h"
@@ -399,6 +398,7 @@ badkey:
399 * @iv_dma: dma address of iv for checking continuity and link table 398 * @iv_dma: dma address of iv for checking continuity and link table
400 * @qm_sg_bytes: length of dma mapped h/w link table 399 * @qm_sg_bytes: length of dma mapped h/w link table
401 * @qm_sg_dma: bus physical mapped address of h/w link table 400 * @qm_sg_dma: bus physical mapped address of h/w link table
401 * @assoclen: associated data length, in CAAM endianness
402 * @assoclen_dma: bus physical mapped address of req->assoclen 402 * @assoclen_dma: bus physical mapped address of req->assoclen
403 * @drv_req: driver-specific request structure 403 * @drv_req: driver-specific request structure
404 * @sgt: the h/w link table 404 * @sgt: the h/w link table
@@ -409,8 +409,12 @@ struct aead_edesc {
409 dma_addr_t iv_dma; 409 dma_addr_t iv_dma;
410 int qm_sg_bytes; 410 int qm_sg_bytes;
411 dma_addr_t qm_sg_dma; 411 dma_addr_t qm_sg_dma;
412 unsigned int assoclen;
412 dma_addr_t assoclen_dma; 413 dma_addr_t assoclen_dma;
413 struct caam_drv_req drv_req; 414 struct caam_drv_req drv_req;
415#define CAAM_QI_MAX_AEAD_SG \
416 ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \
417 sizeof(struct qm_sg_entry))
414 struct qm_sg_entry sgt[0]; 418 struct qm_sg_entry sgt[0];
415}; 419};
416 420
@@ -431,6 +435,9 @@ struct ablkcipher_edesc {
431 int qm_sg_bytes; 435 int qm_sg_bytes;
432 dma_addr_t qm_sg_dma; 436 dma_addr_t qm_sg_dma;
433 struct caam_drv_req drv_req; 437 struct caam_drv_req drv_req;
438#define CAAM_QI_MAX_ABLKCIPHER_SG \
439 ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \
440 sizeof(struct qm_sg_entry))
434 struct qm_sg_entry sgt[0]; 441 struct qm_sg_entry sgt[0];
435}; 442};
436 443
@@ -660,6 +667,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
660 */ 667 */
661 qm_sg_ents = 1 + !!ivsize + mapped_src_nents + 668 qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
662 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 669 (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
670 if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) {
671 dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
672 qm_sg_ents, CAAM_QI_MAX_AEAD_SG);
673 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
674 iv_dma, ivsize, op_type, 0, 0);
675 qi_cache_free(edesc);
676 return ERR_PTR(-ENOMEM);
677 }
663 sg_table = &edesc->sgt[0]; 678 sg_table = &edesc->sgt[0];
664 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 679 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
665 680
@@ -670,7 +685,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
670 edesc->drv_req.cbk = aead_done; 685 edesc->drv_req.cbk = aead_done;
671 edesc->drv_req.drv_ctx = drv_ctx; 686 edesc->drv_req.drv_ctx = drv_ctx;
672 687
673 edesc->assoclen_dma = dma_map_single(qidev, &req->assoclen, 4, 688 edesc->assoclen = cpu_to_caam32(req->assoclen);
689 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
674 DMA_TO_DEVICE); 690 DMA_TO_DEVICE);
675 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 691 if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
676 dev_err(qidev, "unable to map assoclen\n"); 692 dev_err(qidev, "unable to map assoclen\n");
@@ -776,9 +792,9 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
776 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 792 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
777 struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); 793 struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher);
778 struct device *qidev = caam_ctx->qidev; 794 struct device *qidev = caam_ctx->qidev;
779#ifdef DEBUG
780 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 795 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
781 796
797#ifdef DEBUG
782 dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 798 dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
783#endif 799#endif
784 800
@@ -791,14 +807,21 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
791 print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", 807 print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ",
792 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 808 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
793 edesc->src_nents > 1 ? 100 : ivsize, 1); 809 edesc->src_nents > 1 ? 100 : ivsize, 1);
794 dbg_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 810 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
795 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 811 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
796 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 812 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
797#endif 813#endif
798 814
799 ablkcipher_unmap(qidev, edesc, req); 815 ablkcipher_unmap(qidev, edesc, req);
800 qi_cache_free(edesc); 816 qi_cache_free(edesc);
801 817
818 /*
819 * The crypto API expects us to set the IV (req->info) to the last
820 * ciphertext block. This is used e.g. by the CTS mode.
821 */
822 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize,
823 ivsize, 0);
824
802 ablkcipher_request_complete(req, status); 825 ablkcipher_request_complete(req, status);
803} 826}
804 827
@@ -880,6 +903,15 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
880 } 903 }
881 dst_sg_idx = qm_sg_ents; 904 dst_sg_idx = qm_sg_ents;
882 905
906 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
907 if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
908 dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
909 qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
910 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
911 iv_dma, ivsize, op_type, 0, 0);
912 return ERR_PTR(-ENOMEM);
913 }
914
883 /* allocate space for base edesc and link tables */ 915 /* allocate space for base edesc and link tables */
884 edesc = qi_cache_alloc(GFP_DMA | flags); 916 edesc = qi_cache_alloc(GFP_DMA | flags);
885 if (unlikely(!edesc)) { 917 if (unlikely(!edesc)) {
@@ -892,7 +924,6 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
892 edesc->src_nents = src_nents; 924 edesc->src_nents = src_nents;
893 edesc->dst_nents = dst_nents; 925 edesc->dst_nents = dst_nents;
894 edesc->iv_dma = iv_dma; 926 edesc->iv_dma = iv_dma;
895 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
896 sg_table = &edesc->sgt[0]; 927 sg_table = &edesc->sgt[0];
897 edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 928 edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
898 edesc->drv_req.app_ctx = req; 929 edesc->drv_req.app_ctx = req;
@@ -1026,6 +1057,14 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1026 qm_sg_ents += 1 + mapped_dst_nents; 1057 qm_sg_ents += 1 + mapped_dst_nents;
1027 } 1058 }
1028 1059
1060 if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
1061 dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
1062 qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
1063 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1064 iv_dma, ivsize, GIVENCRYPT, 0, 0);
1065 return ERR_PTR(-ENOMEM);
1066 }
1067
1029 /* allocate space for base edesc and link tables */ 1068 /* allocate space for base edesc and link tables */
1030 edesc = qi_cache_alloc(GFP_DMA | flags); 1069 edesc = qi_cache_alloc(GFP_DMA | flags);
1031 if (!edesc) { 1070 if (!edesc) {
@@ -1968,7 +2007,7 @@ static struct caam_aead_alg driver_aeads[] = {
1968 .cra_name = "echainiv(authenc(hmac(sha256)," 2007 .cra_name = "echainiv(authenc(hmac(sha256),"
1969 "cbc(des)))", 2008 "cbc(des)))",
1970 .cra_driver_name = "echainiv-authenc-" 2009 .cra_driver_name = "echainiv-authenc-"
1971 "hmac-sha256-cbc-desi-" 2010 "hmac-sha256-cbc-des-"
1972 "caam-qi", 2011 "caam-qi",
1973 .cra_blocksize = DES_BLOCK_SIZE, 2012 .cra_blocksize = DES_BLOCK_SIZE,
1974 }, 2013 },
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 910ec61cae09..698580b60b2f 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -791,8 +791,8 @@ static int ahash_update_ctx(struct ahash_request *req)
791 to_hash - *buflen, 791 to_hash - *buflen,
792 *next_buflen, 0); 792 *next_buflen, 0);
793 } else { 793 } else {
794 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 794 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
795 cpu_to_caam32(SEC4_SG_LEN_FIN); 795 1);
796 } 796 }
797 797
798 desc = edesc->hw_desc; 798 desc = edesc->hw_desc;
@@ -882,8 +882,7 @@ static int ahash_final_ctx(struct ahash_request *req)
882 if (ret) 882 if (ret)
883 goto unmap_ctx; 883 goto unmap_ctx;
884 884
885 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 885 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
886 cpu_to_caam32(SEC4_SG_LEN_FIN);
887 886
888 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 887 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
889 sec4_sg_bytes, DMA_TO_DEVICE); 888 sec4_sg_bytes, DMA_TO_DEVICE);
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 41398da3edf4..fde07d4ff019 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -285,11 +285,7 @@ static int caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev)
285 if (err) 285 if (err)
286 return err; 286 return err;
287 287
288 err = caam_init_buf(ctx, 1); 288 return caam_init_buf(ctx, 1);
289 if (err)
290 return err;
291
292 return 0;
293} 289}
294 290
295static struct hwrng caam_rng = { 291static struct hwrng caam_rng = {
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index dd353e342c12..dacb53fb690e 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -17,6 +17,8 @@
17 17
18bool caam_little_end; 18bool caam_little_end;
19EXPORT_SYMBOL(caam_little_end); 19EXPORT_SYMBOL(caam_little_end);
20bool caam_dpaa2;
21EXPORT_SYMBOL(caam_dpaa2);
20 22
21#ifdef CONFIG_CAAM_QI 23#ifdef CONFIG_CAAM_QI
22#include "qi.h" 24#include "qi.h"
@@ -319,8 +321,11 @@ static int caam_remove(struct platform_device *pdev)
319 caam_qi_shutdown(ctrlpriv->qidev); 321 caam_qi_shutdown(ctrlpriv->qidev);
320#endif 322#endif
321 323
322 /* De-initialize RNG state handles initialized by this driver. */ 324 /*
323 if (ctrlpriv->rng4_sh_init) 325 * De-initialize RNG state handles initialized by this driver.
326 * In case of DPAA 2.x, RNG is managed by MC firmware.
327 */
328 if (!caam_dpaa2 && ctrlpriv->rng4_sh_init)
324 deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); 329 deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init);
325 330
326 /* Shut down debug views */ 331 /* Shut down debug views */
@@ -444,7 +449,6 @@ static int caam_probe(struct platform_device *pdev)
444 449
445 dev = &pdev->dev; 450 dev = &pdev->dev;
446 dev_set_drvdata(dev, ctrlpriv); 451 dev_set_drvdata(dev, ctrlpriv);
447 ctrlpriv->pdev = pdev;
448 nprop = pdev->dev.of_node; 452 nprop = pdev->dev.of_node;
449 453
450 /* Enable clocking */ 454 /* Enable clocking */
@@ -553,12 +557,17 @@ static int caam_probe(struct platform_device *pdev)
553 557
554 /* 558 /*
555 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, 559 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
556 * long pointers in master configuration register 560 * long pointers in master configuration register.
561 * In case of DPAA 2.x, Management Complex firmware performs
562 * the configuration.
557 */ 563 */
558 clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, 564 caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
559 MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | 565 if (!caam_dpaa2)
560 MCFGR_WDENABLE | MCFGR_LARGE_BURST | 566 clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR,
561 (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); 567 MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF |
568 MCFGR_WDENABLE | MCFGR_LARGE_BURST |
569 (sizeof(dma_addr_t) == sizeof(u64) ?
570 MCFGR_LONG_PTR : 0));
562 571
563 /* 572 /*
564 * Read the Compile Time paramters and SCFGR to determine 573 * Read the Compile Time paramters and SCFGR to determine
@@ -587,7 +596,9 @@ static int caam_probe(struct platform_device *pdev)
587 JRSTART_JR3_START); 596 JRSTART_JR3_START);
588 597
589 if (sizeof(dma_addr_t) == sizeof(u64)) { 598 if (sizeof(dma_addr_t) == sizeof(u64)) {
590 if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) 599 if (caam_dpaa2)
600 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
601 else if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
591 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); 602 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
592 else 603 else
593 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); 604 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36));
@@ -630,11 +641,9 @@ static int caam_probe(struct platform_device *pdev)
630 ring++; 641 ring++;
631 } 642 }
632 643
633 /* Check to see if QI present. If so, enable */ 644 /* Check to see if (DPAA 1.x) QI present. If so, enable */
634 ctrlpriv->qi_present = 645 ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
635 !!(rd_reg32(&ctrl->perfmon.comp_parms_ms) & 646 if (ctrlpriv->qi_present && !caam_dpaa2) {
636 CTPR_MS_QI_MASK);
637 if (ctrlpriv->qi_present) {
638 ctrlpriv->qi = (struct caam_queue_if __iomem __force *) 647 ctrlpriv->qi = (struct caam_queue_if __iomem __force *)
639 ((__force uint8_t *)ctrl + 648 ((__force uint8_t *)ctrl +
640 BLOCK_OFFSET * QI_BLOCK_NUMBER 649 BLOCK_OFFSET * QI_BLOCK_NUMBER
@@ -662,8 +671,10 @@ static int caam_probe(struct platform_device *pdev)
662 /* 671 /*
663 * If SEC has RNG version >= 4 and RNG state handle has not been 672 * If SEC has RNG version >= 4 and RNG state handle has not been
664 * already instantiated, do RNG instantiation 673 * already instantiated, do RNG instantiation
674 * In case of DPAA 2.x, RNG is managed by MC firmware.
665 */ 675 */
666 if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { 676 if (!caam_dpaa2 &&
677 (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
667 ctrlpriv->rng4_sh_init = 678 ctrlpriv->rng4_sh_init =
668 rd_reg32(&ctrl->r4tst[0].rdsta); 679 rd_reg32(&ctrl->r4tst[0].rdsta);
669 /* 680 /*
@@ -731,63 +742,43 @@ static int caam_probe(struct platform_device *pdev)
731 /* Report "alive" for developer to see */ 742 /* Report "alive" for developer to see */
732 dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, 743 dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
733 caam_get_era()); 744 caam_get_era());
734 dev_info(dev, "job rings = %d, qi = %d\n", 745 dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n",
735 ctrlpriv->total_jobrs, ctrlpriv->qi_present); 746 ctrlpriv->total_jobrs, ctrlpriv->qi_present,
747 caam_dpaa2 ? "yes" : "no");
736 748
737#ifdef CONFIG_DEBUG_FS 749#ifdef CONFIG_DEBUG_FS
738 750 debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH,
739 ctrlpriv->ctl_rq_dequeued = 751 ctrlpriv->ctl, &perfmon->req_dequeued,
740 debugfs_create_file("rq_dequeued", 752 &caam_fops_u64_ro);
741 S_IRUSR | S_IRGRP | S_IROTH, 753 debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH,
742 ctrlpriv->ctl, &perfmon->req_dequeued, 754 ctrlpriv->ctl, &perfmon->ob_enc_req,
743 &caam_fops_u64_ro); 755 &caam_fops_u64_ro);
744 ctrlpriv->ctl_ob_enc_req = 756 debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH,
745 debugfs_create_file("ob_rq_encrypted", 757 ctrlpriv->ctl, &perfmon->ib_dec_req,
746 S_IRUSR | S_IRGRP | S_IROTH, 758 &caam_fops_u64_ro);
747 ctrlpriv->ctl, &perfmon->ob_enc_req, 759 debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH,
748 &caam_fops_u64_ro); 760 ctrlpriv->ctl, &perfmon->ob_enc_bytes,
749 ctrlpriv->ctl_ib_dec_req = 761 &caam_fops_u64_ro);
750 debugfs_create_file("ib_rq_decrypted", 762 debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH,
751 S_IRUSR | S_IRGRP | S_IROTH, 763 ctrlpriv->ctl, &perfmon->ob_prot_bytes,
752 ctrlpriv->ctl, &perfmon->ib_dec_req, 764 &caam_fops_u64_ro);
753 &caam_fops_u64_ro); 765 debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH,
754 ctrlpriv->ctl_ob_enc_bytes = 766 ctrlpriv->ctl, &perfmon->ib_dec_bytes,
755 debugfs_create_file("ob_bytes_encrypted", 767 &caam_fops_u64_ro);
756 S_IRUSR | S_IRGRP | S_IROTH, 768 debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH,
757 ctrlpriv->ctl, &perfmon->ob_enc_bytes, 769 ctrlpriv->ctl, &perfmon->ib_valid_bytes,
758 &caam_fops_u64_ro); 770 &caam_fops_u64_ro);
759 ctrlpriv->ctl_ob_prot_bytes =
760 debugfs_create_file("ob_bytes_protected",
761 S_IRUSR | S_IRGRP | S_IROTH,
762 ctrlpriv->ctl, &perfmon->ob_prot_bytes,
763 &caam_fops_u64_ro);
764 ctrlpriv->ctl_ib_dec_bytes =
765 debugfs_create_file("ib_bytes_decrypted",
766 S_IRUSR | S_IRGRP | S_IROTH,
767 ctrlpriv->ctl, &perfmon->ib_dec_bytes,
768 &caam_fops_u64_ro);
769 ctrlpriv->ctl_ib_valid_bytes =
770 debugfs_create_file("ib_bytes_validated",
771 S_IRUSR | S_IRGRP | S_IROTH,
772 ctrlpriv->ctl, &perfmon->ib_valid_bytes,
773 &caam_fops_u64_ro);
774 771
775 /* Controller level - global status values */ 772 /* Controller level - global status values */
776 ctrlpriv->ctl_faultaddr = 773 debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH,
777 debugfs_create_file("fault_addr", 774 ctrlpriv->ctl, &perfmon->faultaddr,
778 S_IRUSR | S_IRGRP | S_IROTH, 775 &caam_fops_u32_ro);
779 ctrlpriv->ctl, &perfmon->faultaddr, 776 debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH,
780 &caam_fops_u32_ro); 777 ctrlpriv->ctl, &perfmon->faultdetail,
781 ctrlpriv->ctl_faultdetail = 778 &caam_fops_u32_ro);
782 debugfs_create_file("fault_detail", 779 debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH,
783 S_IRUSR | S_IRGRP | S_IROTH, 780 ctrlpriv->ctl, &perfmon->status,
784 ctrlpriv->ctl, &perfmon->faultdetail, 781 &caam_fops_u32_ro);
785 &caam_fops_u32_ro);
786 ctrlpriv->ctl_faultstatus =
787 debugfs_create_file("fault_status",
788 S_IRUSR | S_IRGRP | S_IROTH,
789 ctrlpriv->ctl, &perfmon->status,
790 &caam_fops_u32_ro);
791 782
792 /* Internal covering keys (useful in non-secure mode only) */ 783 /* Internal covering keys (useful in non-secure mode only) */
793 ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; 784 ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
diff --git a/drivers/crypto/caam/ctrl.h b/drivers/crypto/caam/ctrl.h
index cac5402a46eb..7e7bf68c9ef5 100644
--- a/drivers/crypto/caam/ctrl.h
+++ b/drivers/crypto/caam/ctrl.h
@@ -10,4 +10,6 @@
10/* Prototypes for backend-level services exposed to APIs */ 10/* Prototypes for backend-level services exposed to APIs */
11int caam_get_era(void); 11int caam_get_era(void);
12 12
13extern bool caam_dpaa2;
14
13#endif /* CTRL_H */ 15#endif /* CTRL_H */
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 6f44ccb55c63..3d639f3b45aa 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -9,6 +9,46 @@
9#include "desc.h" 9#include "desc.h"
10#include "error.h" 10#include "error.h"
11 11
12#ifdef DEBUG
13#include <linux/highmem.h>
14
15void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
16 int rowsize, int groupsize, struct scatterlist *sg,
17 size_t tlen, bool ascii)
18{
19 struct scatterlist *it;
20 void *it_page;
21 size_t len;
22 void *buf;
23
24 for (it = sg; it && tlen > 0 ; it = sg_next(sg)) {
25 /*
26 * make sure the scatterlist's page
27 * has a valid virtual memory mapping
28 */
29 it_page = kmap_atomic(sg_page(it));
30 if (unlikely(!it_page)) {
31 pr_err("caam_dump_sg: kmap failed\n");
32 return;
33 }
34
35 buf = it_page + it->offset;
36 len = min_t(size_t, tlen, it->length);
37 print_hex_dump(level, prefix_str, prefix_type, rowsize,
38 groupsize, buf, len, ascii);
39 tlen -= len;
40
41 kunmap_atomic(it_page);
42 }
43}
44#else
45void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
46 int rowsize, int groupsize, struct scatterlist *sg,
47 size_t tlen, bool ascii)
48{}
49#endif /* DEBUG */
50EXPORT_SYMBOL(caam_dump_sg);
51
12static const struct { 52static const struct {
13 u8 value; 53 u8 value;
14 const char *error_text; 54 const char *error_text;
diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h
index b6350b0d9153..250e1a21c473 100644
--- a/drivers/crypto/caam/error.h
+++ b/drivers/crypto/caam/error.h
@@ -8,4 +8,8 @@
8#define CAAM_ERROR_H 8#define CAAM_ERROR_H
9#define CAAM_ERROR_STR_MAX 302 9#define CAAM_ERROR_STR_MAX 302
10void caam_jr_strstatus(struct device *jrdev, u32 status); 10void caam_jr_strstatus(struct device *jrdev, u32 status);
11
12void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
13 int rowsize, int groupsize, struct scatterlist *sg,
14 size_t tlen, bool ascii);
11#endif /* CAAM_ERROR_H */ 15#endif /* CAAM_ERROR_H */
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 85b6c5835b8f..a52361258d3a 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -64,12 +64,9 @@ struct caam_drv_private_jr {
64 * Driver-private storage for a single CAAM block instance 64 * Driver-private storage for a single CAAM block instance
65 */ 65 */
66struct caam_drv_private { 66struct caam_drv_private {
67
68 struct device *dev;
69#ifdef CONFIG_CAAM_QI 67#ifdef CONFIG_CAAM_QI
70 struct device *qidev; 68 struct device *qidev;
71#endif 69#endif
72 struct platform_device *pdev;
73 70
74 /* Physical-presence section */ 71 /* Physical-presence section */
75 struct caam_ctrl __iomem *ctrl; /* controller region */ 72 struct caam_ctrl __iomem *ctrl; /* controller region */
@@ -105,16 +102,8 @@ struct caam_drv_private {
105#ifdef CONFIG_DEBUG_FS 102#ifdef CONFIG_DEBUG_FS
106 struct dentry *dfs_root; 103 struct dentry *dfs_root;
107 struct dentry *ctl; /* controller dir */ 104 struct dentry *ctl; /* controller dir */
108 struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req;
109 struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes;
110 struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes;
111 struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus;
112
113 struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap; 105 struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap;
114 struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk; 106 struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk;
115#ifdef CONFIG_CAAM_QI
116 struct dentry *qi_congested;
117#endif
118#endif 107#endif
119}; 108};
120 109
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 1ccfb317d468..d258953ff488 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -9,6 +9,7 @@
9#include <linux/of_address.h> 9#include <linux/of_address.h>
10 10
11#include "compat.h" 11#include "compat.h"
12#include "ctrl.h"
12#include "regs.h" 13#include "regs.h"
13#include "jr.h" 14#include "jr.h"
14#include "desc.h" 15#include "desc.h"
@@ -499,7 +500,11 @@ static int caam_jr_probe(struct platform_device *pdev)
499 jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl; 500 jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
500 501
501 if (sizeof(dma_addr_t) == sizeof(u64)) { 502 if (sizeof(dma_addr_t) == sizeof(u64)) {
502 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) 503 if (caam_dpaa2)
504 error = dma_set_mask_and_coherent(jrdev,
505 DMA_BIT_MASK(49));
506 else if (of_device_is_compatible(nprop,
507 "fsl,sec-v5.0-job-ring"))
503 error = dma_set_mask_and_coherent(jrdev, 508 error = dma_set_mask_and_coherent(jrdev,
504 DMA_BIT_MASK(40)); 509 DMA_BIT_MASK(40));
505 else 510 else
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 1990ed460c46..e4cf00014233 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -24,9 +24,6 @@
24 */ 24 */
25#define MAX_RSP_FQ_BACKLOG_PER_CPU 256 25#define MAX_RSP_FQ_BACKLOG_PER_CPU 256
26 26
27/* Length of a single buffer in the QI driver memory cache */
28#define CAAM_QI_MEMCACHE_SIZE 512
29
30#define CAAM_QI_ENQUEUE_RETRIES 10000 27#define CAAM_QI_ENQUEUE_RETRIES 10000
31 28
32#define CAAM_NAPI_WEIGHT 63 29#define CAAM_NAPI_WEIGHT 63
@@ -55,6 +52,7 @@ struct caam_qi_pcpu_priv {
55} ____cacheline_aligned; 52} ____cacheline_aligned;
56 53
57static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); 54static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
55static DEFINE_PER_CPU(int, last_cpu);
58 56
59/* 57/*
60 * caam_qi_priv - CAAM QI backend private params 58 * caam_qi_priv - CAAM QI backend private params
@@ -203,8 +201,8 @@ static struct qman_fq *create_caam_req_fq(struct device *qidev,
203 goto init_req_fq_fail; 201 goto init_req_fq_fail;
204 } 202 }
205 203
206 dev_info(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, 204 dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
207 smp_processor_id()); 205 smp_processor_id());
208 return req_fq; 206 return req_fq;
209 207
210init_req_fq_fail: 208init_req_fq_fail:
@@ -277,6 +275,7 @@ empty_fq:
277 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid); 275 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
278 276
279 qman_destroy_fq(fq); 277 qman_destroy_fq(fq);
278 kfree(fq);
280 279
281 return ret; 280 return ret;
282} 281}
@@ -342,8 +341,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
342 drv_ctx->req_fq = old_fq; 341 drv_ctx->req_fq = old_fq;
343 342
344 if (kill_fq(qidev, new_fq)) 343 if (kill_fq(qidev, new_fq))
345 dev_warn(qidev, "New CAAM FQ: %u kill failed\n", 344 dev_warn(qidev, "New CAAM FQ kill failed\n");
346 new_fq->fqid);
347 345
348 return ret; 346 return ret;
349 } 347 }
@@ -373,10 +371,9 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
373 drv_ctx->req_fq = old_fq; 371 drv_ctx->req_fq = old_fq;
374 372
375 if (kill_fq(qidev, new_fq)) 373 if (kill_fq(qidev, new_fq))
376 dev_warn(qidev, "New CAAM FQ: %u kill failed\n", 374 dev_warn(qidev, "New CAAM FQ kill failed\n");
377 new_fq->fqid);
378 } else if (kill_fq(qidev, old_fq)) { 375 } else if (kill_fq(qidev, old_fq)) {
379 dev_warn(qidev, "Old CAAM FQ: %u kill failed\n", old_fq->fqid); 376 dev_warn(qidev, "Old CAAM FQ kill failed\n");
380 } 377 }
381 378
382 return 0; 379 return 0;
@@ -392,7 +389,6 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
392 dma_addr_t hwdesc; 389 dma_addr_t hwdesc;
393 struct caam_drv_ctx *drv_ctx; 390 struct caam_drv_ctx *drv_ctx;
394 const cpumask_t *cpus = qman_affine_cpus(); 391 const cpumask_t *cpus = qman_affine_cpus();
395 static DEFINE_PER_CPU(int, last_cpu);
396 392
397 num_words = desc_len(sh_desc); 393 num_words = desc_len(sh_desc);
398 if (num_words > MAX_SDLEN) { 394 if (num_words > MAX_SDLEN) {
@@ -511,7 +507,6 @@ int caam_qi_shutdown(struct device *qidev)
511 507
512 if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) 508 if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
513 dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i); 509 dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
514 kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
515 } 510 }
516 511
517 /* 512 /*
@@ -646,7 +641,7 @@ static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
646 641
647 per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; 642 per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
648 643
649 dev_info(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); 644 dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
650 return 0; 645 return 0;
651} 646}
652 647
@@ -679,7 +674,7 @@ static int init_cgr(struct device *qidev)
679 return ret; 674 return ret;
680 } 675 }
681 676
682 dev_info(qidev, "Congestion threshold set to %llu\n", val); 677 dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
683 return 0; 678 return 0;
684} 679}
685 680
@@ -737,6 +732,7 @@ int caam_qi_init(struct platform_device *caam_pdev)
737 qi_pdev = platform_device_register_full(&qi_pdev_info); 732 qi_pdev = platform_device_register_full(&qi_pdev_info);
738 if (IS_ERR(qi_pdev)) 733 if (IS_ERR(qi_pdev))
739 return PTR_ERR(qi_pdev); 734 return PTR_ERR(qi_pdev);
735 set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
740 736
741 ctrlpriv = dev_get_drvdata(ctrldev); 737 ctrlpriv = dev_get_drvdata(ctrldev);
742 qidev = &qi_pdev->dev; 738 qidev = &qi_pdev->dev;
@@ -795,10 +791,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
795 /* Done with the CGRs; restore the cpus allowed mask */ 791 /* Done with the CGRs; restore the cpus allowed mask */
796 set_cpus_allowed_ptr(current, &old_cpumask); 792 set_cpus_allowed_ptr(current, &old_cpumask);
797#ifdef CONFIG_DEBUG_FS 793#ifdef CONFIG_DEBUG_FS
798 ctrlpriv->qi_congested = debugfs_create_file("qi_congested", 0444, 794 debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
799 ctrlpriv->ctl, 795 &times_congested, &caam_fops_u64_ro);
800 &times_congested,
801 &caam_fops_u64_ro);
802#endif 796#endif
803 dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); 797 dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
804 return 0; 798 return 0;
diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h
index 33b0433f5f22..ecb21f207637 100644
--- a/drivers/crypto/caam/qi.h
+++ b/drivers/crypto/caam/qi.h
@@ -39,6 +39,9 @@
39 */ 39 */
40#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) 40#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
41 41
42/* Length of a single buffer in the QI driver memory cache */
43#define CAAM_QI_MEMCACHE_SIZE 768
44
42extern bool caam_congested __read_mostly; 45extern bool caam_congested __read_mostly;
43 46
44/* 47/*
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index 84d2f838a063..2b5efff9ec3c 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -293,6 +293,7 @@ struct caam_perfmon {
293 u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ 293 u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/
294#define CTPR_MS_QI_SHIFT 25 294#define CTPR_MS_QI_SHIFT 25
295#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) 295#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT)
296#define CTPR_MS_DPAA2 BIT(13)
296#define CTPR_MS_VIRT_EN_INCL 0x00000001 297#define CTPR_MS_VIRT_EN_INCL 0x00000001
297#define CTPR_MS_VIRT_EN_POR 0x00000002 298#define CTPR_MS_VIRT_EN_POR 0x00000002
298#define CTPR_MS_PG_SZ_MASK 0x10 299#define CTPR_MS_PG_SZ_MASK 0x10
diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h
new file mode 100644
index 000000000000..31b440757146
--- /dev/null
+++ b/drivers/crypto/caam/sg_sw_qm2.h
@@ -0,0 +1,81 @@
1/*
2 * Copyright 2015-2016 Freescale Semiconductor, Inc.
3 * Copyright 2017 NXP
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the names of the above-listed copyright holders nor the
13 * names of any contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 *
17 * ALTERNATIVELY, this software may be distributed under the terms of the
18 * GNU General Public License ("GPL") as published by the Free Software
19 * Foundation, either version 2 of that License or (at your option) any
20 * later version.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#ifndef _SG_SW_QM2_H_
36#define _SG_SW_QM2_H_
37
38#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
39
40static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr,
41 dma_addr_t dma, u32 len, u16 offset)
42{
43 dpaa2_sg_set_addr(qm_sg_ptr, dma);
44 dpaa2_sg_set_format(qm_sg_ptr, dpaa2_sg_single);
45 dpaa2_sg_set_final(qm_sg_ptr, false);
46 dpaa2_sg_set_len(qm_sg_ptr, len);
47 dpaa2_sg_set_bpid(qm_sg_ptr, 0);
48 dpaa2_sg_set_offset(qm_sg_ptr, offset);
49}
50
51/*
52 * convert scatterlist to h/w link table format
53 * but does not have final bit; instead, returns last entry
54 */
55static inline struct dpaa2_sg_entry *
56sg_to_qm_sg(struct scatterlist *sg, int sg_count,
57 struct dpaa2_sg_entry *qm_sg_ptr, u16 offset)
58{
59 while (sg_count && sg) {
60 dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
61 sg_dma_len(sg), offset);
62 qm_sg_ptr++;
63 sg = sg_next(sg);
64 sg_count--;
65 }
66 return qm_sg_ptr - 1;
67}
68
69/*
70 * convert scatterlist to h/w link table format
71 * scatterlist must have been previously dma mapped
72 */
73static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
74 struct dpaa2_sg_entry *qm_sg_ptr,
75 u16 offset)
76{
77 qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
78 dpaa2_sg_set_final(qm_sg_ptr, true);
79}
80
81#endif /* _SG_SW_QM2_H_ */
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index c6adad09c972..936b1b630058 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -5,7 +5,13 @@
5 * 5 *
6 */ 6 */
7 7
8#ifndef _SG_SW_SEC4_H_
9#define _SG_SW_SEC4_H_
10
11#include "ctrl.h"
8#include "regs.h" 12#include "regs.h"
13#include "sg_sw_qm2.h"
14#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
9 15
10struct sec4_sg_entry { 16struct sec4_sg_entry {
11 u64 ptr; 17 u64 ptr;
@@ -19,9 +25,15 @@ struct sec4_sg_entry {
19static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, 25static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
20 dma_addr_t dma, u32 len, u16 offset) 26 dma_addr_t dma, u32 len, u16 offset)
21{ 27{
22 sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); 28 if (caam_dpaa2) {
23 sec4_sg_ptr->len = cpu_to_caam32(len); 29 dma_to_qm_sg_one((struct dpaa2_sg_entry *)sec4_sg_ptr, dma, len,
24 sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); 30 offset);
31 } else {
32 sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma);
33 sec4_sg_ptr->len = cpu_to_caam32(len);
34 sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset &
35 SEC4_SG_OFFSET_MASK);
36 }
25#ifdef DEBUG 37#ifdef DEBUG
26 print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", 38 print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ",
27 DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, 39 DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr,
@@ -47,6 +59,14 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
47 return sec4_sg_ptr - 1; 59 return sec4_sg_ptr - 1;
48} 60}
49 61
62static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr)
63{
64 if (caam_dpaa2)
65 dpaa2_sg_set_final((struct dpaa2_sg_entry *)sec4_sg_ptr, true);
66 else
67 sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
68}
69
50/* 70/*
51 * convert scatterlist to h/w link table format 71 * convert scatterlist to h/w link table format
52 * scatterlist must have been previously dma mapped 72 * scatterlist must have been previously dma mapped
@@ -56,20 +76,7 @@ static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count,
56 u16 offset) 76 u16 offset)
57{ 77{
58 sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); 78 sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset);
59 sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 79 sg_to_sec4_set_last(sec4_sg_ptr);
60} 80}
61 81
62static inline struct sec4_sg_entry *sg_to_sec4_sg_len( 82#endif /* _SG_SW_SEC4_H_ */
63 struct scatterlist *sg, unsigned int total,
64 struct sec4_sg_entry *sec4_sg_ptr)
65{
66 do {
67 unsigned int len = min(sg_dma_len(sg), total);
68
69 dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), len, 0);
70 sec4_sg_ptr++;
71 sg = sg_next(sg);
72 total -= len;
73 } while (total);
74 return sec4_sg_ptr - 1;
75}
diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c
index 4119c40e7c4b..34a6d8bf229e 100644
--- a/drivers/crypto/cavium/cpt/cptpf_main.c
+++ b/drivers/crypto/cavium/cpt/cptpf_main.c
@@ -268,8 +268,10 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
268 mcode = &cpt->mcode[cpt->next_mc_idx]; 268 mcode = &cpt->mcode[cpt->next_mc_idx];
269 memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ); 269 memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
270 mcode->code_size = ntohl(ucode->code_length) * 2; 270 mcode->code_size = ntohl(ucode->code_length) * 2;
271 if (!mcode->code_size) 271 if (!mcode->code_size) {
272 return -EINVAL; 272 ret = -EINVAL;
273 goto fw_release;
274 }
273 275
274 mcode->is_ae = is_ae; 276 mcode->is_ae = is_ae;
275 mcode->core_mask = 0ULL; 277 mcode->core_mask = 0ULL;
@@ -280,7 +282,8 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
280 &mcode->phys_base, GFP_KERNEL); 282 &mcode->phys_base, GFP_KERNEL);
281 if (!mcode->code) { 283 if (!mcode->code) {
282 dev_err(dev, "Unable to allocate space for microcode"); 284 dev_err(dev, "Unable to allocate space for microcode");
283 return -ENOMEM; 285 ret = -ENOMEM;
286 goto fw_release;
284 } 287 }
285 288
286 memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)), 289 memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
@@ -302,12 +305,14 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
302 ret = do_cpt_init(cpt, mcode); 305 ret = do_cpt_init(cpt, mcode);
303 if (ret) { 306 if (ret) {
304 dev_err(dev, "do_cpt_init failed with ret: %d\n", ret); 307 dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
305 return ret; 308 goto fw_release;
306 } 309 }
307 310
308 dev_info(dev, "Microcode Loaded %s\n", mcode->version); 311 dev_info(dev, "Microcode Loaded %s\n", mcode->version);
309 mcode->is_mc_valid = 1; 312 mcode->is_mc_valid = 1;
310 cpt->next_mc_idx++; 313 cpt->next_mc_idx++;
314
315fw_release:
311 release_firmware(fw_entry); 316 release_firmware(fw_entry);
312 317
313 return ret; 318 return ret;
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index 9ccefb9b7232..fee7cb2ce747 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -513,8 +513,10 @@ static int nitrox_probe(struct pci_dev *pdev,
513 pci_set_master(pdev); 513 pci_set_master(pdev);
514 514
515 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); 515 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
516 if (!ndev) 516 if (!ndev) {
517 err = -ENOMEM;
517 goto ndev_fail; 518 goto ndev_fail;
519 }
518 520
519 pci_set_drvdata(pdev, ndev); 521 pci_set_drvdata(pdev, ndev);
520 ndev->pdev = pdev; 522 ndev->pdev = pdev;
diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig
index 2238f77aa248..6d626606b9c5 100644
--- a/drivers/crypto/ccp/Kconfig
+++ b/drivers/crypto/ccp/Kconfig
@@ -1,25 +1,33 @@
1config CRYPTO_DEV_CCP_DD 1config CRYPTO_DEV_CCP_DD
2 tristate "Cryptographic Coprocessor device driver" 2 tristate "Secure Processor device driver"
3 depends on CRYPTO_DEV_CCP
4 default m 3 default m
4 help
5 Provides AMD Secure Processor device driver.
6 If you choose 'M' here, this module will be called ccp.
7
8config CRYPTO_DEV_SP_CCP
9 bool "Cryptographic Coprocessor device"
10 default y
11 depends on CRYPTO_DEV_CCP_DD
5 select HW_RANDOM 12 select HW_RANDOM
6 select DMA_ENGINE 13 select DMA_ENGINE
7 select DMADEVICES 14 select DMADEVICES
8 select CRYPTO_SHA1 15 select CRYPTO_SHA1
9 select CRYPTO_SHA256 16 select CRYPTO_SHA256
10 help 17 help
11 Provides the interface to use the AMD Cryptographic Coprocessor 18 Provides the support for AMD Cryptographic Coprocessor (CCP) device
12 which can be used to offload encryption operations such as SHA, 19 which can be used to offload encryption operations such as SHA, AES
13 AES and more. If you choose 'M' here, this module will be called 20 and more.
14 ccp.
15 21
16config CRYPTO_DEV_CCP_CRYPTO 22config CRYPTO_DEV_CCP_CRYPTO
17 tristate "Encryption and hashing offload support" 23 tristate "Encryption and hashing offload support"
18 depends on CRYPTO_DEV_CCP_DD
19 default m 24 default m
25 depends on CRYPTO_DEV_CCP_DD
26 depends on CRYPTO_DEV_SP_CCP
20 select CRYPTO_HASH 27 select CRYPTO_HASH
21 select CRYPTO_BLKCIPHER 28 select CRYPTO_BLKCIPHER
22 select CRYPTO_AUTHENC 29 select CRYPTO_AUTHENC
30 select CRYPTO_RSA
23 help 31 help
24 Support for using the cryptographic API with the AMD Cryptographic 32 Support for using the cryptographic API with the AMD Cryptographic
25 Coprocessor. This module supports offload of SHA and AES algorithms. 33 Coprocessor. This module supports offload of SHA and AES algorithms.
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index 59493fd3a751..57f8debfcfb3 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -1,12 +1,12 @@
1obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o 1obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o
2ccp-objs := ccp-dev.o \ 2ccp-objs := sp-dev.o sp-platform.o
3ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \
3 ccp-ops.o \ 4 ccp-ops.o \
4 ccp-dev-v3.o \ 5 ccp-dev-v3.o \
5 ccp-dev-v5.o \ 6 ccp-dev-v5.o \
6 ccp-platform.o \
7 ccp-dmaengine.o \ 7 ccp-dmaengine.o \
8 ccp-debugfs.o 8 ccp-debugfs.o
9ccp-$(CONFIG_PCI) += ccp-pci.o 9ccp-$(CONFIG_PCI) += sp-pci.o
10 10
11obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o 11obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
12ccp-crypto-objs := ccp-crypto-main.o \ 12ccp-crypto-objs := ccp-crypto-main.o \
@@ -15,4 +15,5 @@ ccp-crypto-objs := ccp-crypto-main.o \
15 ccp-crypto-aes-xts.o \ 15 ccp-crypto-aes-xts.o \
16 ccp-crypto-aes-galois.o \ 16 ccp-crypto-aes-galois.o \
17 ccp-crypto-des3.o \ 17 ccp-crypto-des3.o \
18 ccp-crypto-rsa.o \
18 ccp-crypto-sha.o 19 ccp-crypto-sha.o
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
index 38ee6f348ea9..52313524a4dd 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support 2 * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support
3 * 3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Gary R Hook <gary.hook@amd.com> 6 * Author: Gary R Hook <gary.hook@amd.com>
7 * 7 *
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
index 58a4244b4752..94b5bcf5b628 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
@@ -1,8 +1,9 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support 2 * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support
3 * 3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * 8 *
8 * This program is free software; you can redistribute it and/or modify 9 * This program is free software; you can redistribute it and/or modify
@@ -15,6 +16,7 @@
15#include <linux/delay.h> 16#include <linux/delay.h>
16#include <linux/scatterlist.h> 17#include <linux/scatterlist.h>
17#include <crypto/aes.h> 18#include <crypto/aes.h>
19#include <crypto/xts.h>
18#include <crypto/internal/skcipher.h> 20#include <crypto/internal/skcipher.h>
19#include <crypto/scatterwalk.h> 21#include <crypto/scatterwalk.h>
20 22
@@ -37,46 +39,26 @@ struct ccp_unit_size_map {
37 u32 value; 39 u32 value;
38}; 40};
39 41
40static struct ccp_unit_size_map unit_size_map[] = { 42static struct ccp_unit_size_map xts_unit_sizes[] = {
41 { 43 {
42 .size = 4096, 44 .size = 16,
43 .value = CCP_XTS_AES_UNIT_SIZE_4096, 45 .value = CCP_XTS_AES_UNIT_SIZE_16,
44 },
45 {
46 .size = 2048,
47 .value = CCP_XTS_AES_UNIT_SIZE_2048,
48 },
49 {
50 .size = 1024,
51 .value = CCP_XTS_AES_UNIT_SIZE_1024,
52 }, 46 },
53 { 47 {
54 .size = 512, 48 .size = 512,
55 .value = CCP_XTS_AES_UNIT_SIZE_512, 49 .value = CCP_XTS_AES_UNIT_SIZE_512,
56 }, 50 },
57 { 51 {
58 .size = 256, 52 .size = 1024,
59 .value = CCP_XTS_AES_UNIT_SIZE__LAST, 53 .value = CCP_XTS_AES_UNIT_SIZE_1024,
60 },
61 {
62 .size = 128,
63 .value = CCP_XTS_AES_UNIT_SIZE__LAST,
64 },
65 {
66 .size = 64,
67 .value = CCP_XTS_AES_UNIT_SIZE__LAST,
68 },
69 {
70 .size = 32,
71 .value = CCP_XTS_AES_UNIT_SIZE__LAST,
72 }, 54 },
73 { 55 {
74 .size = 16, 56 .size = 2048,
75 .value = CCP_XTS_AES_UNIT_SIZE_16, 57 .value = CCP_XTS_AES_UNIT_SIZE_2048,
76 }, 58 },
77 { 59 {
78 .size = 1, 60 .size = 4096,
79 .value = CCP_XTS_AES_UNIT_SIZE__LAST, 61 .value = CCP_XTS_AES_UNIT_SIZE_4096,
80 }, 62 },
81}; 63};
82 64
@@ -96,15 +78,26 @@ static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)
96static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 78static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
97 unsigned int key_len) 79 unsigned int key_len)
98{ 80{
99 struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); 81 struct crypto_tfm *xfm = crypto_ablkcipher_tfm(tfm);
82 struct ccp_ctx *ctx = crypto_tfm_ctx(xfm);
83 unsigned int ccpversion = ccp_version();
84 int ret;
100 85
101 /* Only support 128-bit AES key with a 128-bit Tweak key, 86 ret = xts_check_key(xfm, key, key_len);
102 * otherwise use the fallback 87 if (ret)
88 return ret;
89
90 /* Version 3 devices support 128-bit keys; version 5 devices can
91 * accommodate 128- and 256-bit keys.
103 */ 92 */
104 switch (key_len) { 93 switch (key_len) {
105 case AES_KEYSIZE_128 * 2: 94 case AES_KEYSIZE_128 * 2:
106 memcpy(ctx->u.aes.key, key, key_len); 95 memcpy(ctx->u.aes.key, key, key_len);
107 break; 96 break;
97 case AES_KEYSIZE_256 * 2:
98 if (ccpversion > CCP_VERSION(3, 0))
99 memcpy(ctx->u.aes.key, key, key_len);
100 break;
108 } 101 }
109 ctx->u.aes.key_len = key_len / 2; 102 ctx->u.aes.key_len = key_len / 2;
110 sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); 103 sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
@@ -117,6 +110,8 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
117{ 110{
118 struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 111 struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
119 struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); 112 struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
113 unsigned int ccpversion = ccp_version();
114 unsigned int fallback = 0;
120 unsigned int unit; 115 unsigned int unit;
121 u32 unit_size; 116 u32 unit_size;
122 int ret; 117 int ret;
@@ -130,18 +125,32 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
130 if (!req->info) 125 if (!req->info)
131 return -EINVAL; 126 return -EINVAL;
132 127
128 /* Check conditions under which the CCP can fulfill a request. The
129 * device can handle input plaintext of a length that is a multiple
130 * of the unit_size, bug the crypto implementation only supports
131 * the unit_size being equal to the input length. This limits the
132 * number of scenarios we can handle.
133 */
133 unit_size = CCP_XTS_AES_UNIT_SIZE__LAST; 134 unit_size = CCP_XTS_AES_UNIT_SIZE__LAST;
134 if (req->nbytes <= unit_size_map[0].size) { 135 for (unit = 0; unit < ARRAY_SIZE(xts_unit_sizes); unit++) {
135 for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) { 136 if (req->nbytes == xts_unit_sizes[unit].size) {
136 if (!(req->nbytes & (unit_size_map[unit].size - 1))) { 137 unit_size = unit;
137 unit_size = unit_size_map[unit].value; 138 break;
138 break;
139 }
140 } 139 }
141 } 140 }
142 141 /* The CCP has restrictions on block sizes. Also, a version 3 device
143 if ((unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) || 142 * only supports AES-128 operations; version 5 CCPs support both
144 (ctx->u.aes.key_len != AES_KEYSIZE_128)) { 143 * AES-128 and -256 operations.
144 */
145 if (unit_size == CCP_XTS_AES_UNIT_SIZE__LAST)
146 fallback = 1;
147 if ((ccpversion < CCP_VERSION(5, 0)) &&
148 (ctx->u.aes.key_len != AES_KEYSIZE_128))
149 fallback = 1;
150 if ((ctx->u.aes.key_len != AES_KEYSIZE_128) &&
151 (ctx->u.aes.key_len != AES_KEYSIZE_256))
152 fallback = 1;
153 if (fallback) {
145 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher); 154 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher);
146 155
147 /* Use the fallback to process the request for any 156 /* Use the fallback to process the request for any
@@ -164,6 +173,7 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
164 memset(&rctx->cmd, 0, sizeof(rctx->cmd)); 173 memset(&rctx->cmd, 0, sizeof(rctx->cmd));
165 INIT_LIST_HEAD(&rctx->cmd.entry); 174 INIT_LIST_HEAD(&rctx->cmd.entry);
166 rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; 175 rctx->cmd.engine = CCP_ENGINE_XTS_AES_128;
176 rctx->cmd.u.xts.type = CCP_AES_TYPE_128;
167 rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT 177 rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT
168 : CCP_AES_ACTION_DECRYPT; 178 : CCP_AES_ACTION_DECRYPT;
169 rctx->cmd.u.xts.unit_size = unit_size; 179 rctx->cmd.u.xts.unit_size = unit_size;
diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c
index 5af7347ae03c..ae87b741f9d5 100644
--- a/drivers/crypto/ccp/ccp-crypto-des3.c
+++ b/drivers/crypto/ccp/ccp-crypto-des3.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support 2 * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support
3 * 3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Gary R Hook <ghook@amd.com> 6 * Author: Gary R Hook <ghook@amd.com>
7 * 7 *
diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index 8dccbddabef1..35a9de7fd475 100644
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) crypto API support 2 * AMD Cryptographic Coprocessor (CCP) crypto API support
3 * 3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * 7 *
@@ -17,6 +17,7 @@
17#include <linux/ccp.h> 17#include <linux/ccp.h>
18#include <linux/scatterlist.h> 18#include <linux/scatterlist.h>
19#include <crypto/internal/hash.h> 19#include <crypto/internal/hash.h>
20#include <crypto/internal/akcipher.h>
20 21
21#include "ccp-crypto.h" 22#include "ccp-crypto.h"
22 23
@@ -37,10 +38,15 @@ static unsigned int des3_disable;
37module_param(des3_disable, uint, 0444); 38module_param(des3_disable, uint, 0444);
38MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value"); 39MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value");
39 40
41static unsigned int rsa_disable;
42module_param(rsa_disable, uint, 0444);
43MODULE_PARM_DESC(rsa_disable, "Disable use of RSA - any non-zero value");
44
40/* List heads for the supported algorithms */ 45/* List heads for the supported algorithms */
41static LIST_HEAD(hash_algs); 46static LIST_HEAD(hash_algs);
42static LIST_HEAD(cipher_algs); 47static LIST_HEAD(cipher_algs);
43static LIST_HEAD(aead_algs); 48static LIST_HEAD(aead_algs);
49static LIST_HEAD(akcipher_algs);
44 50
45/* For any tfm, requests for that tfm must be returned on the order 51/* For any tfm, requests for that tfm must be returned on the order
46 * received. With multiple queues available, the CCP can process more 52 * received. With multiple queues available, the CCP can process more
@@ -358,6 +364,12 @@ static int ccp_register_algs(void)
358 return ret; 364 return ret;
359 } 365 }
360 366
367 if (!rsa_disable) {
368 ret = ccp_register_rsa_algs(&akcipher_algs);
369 if (ret)
370 return ret;
371 }
372
361 return 0; 373 return 0;
362} 374}
363 375
@@ -366,6 +378,7 @@ static void ccp_unregister_algs(void)
366 struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp; 378 struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp;
367 struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp; 379 struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp;
368 struct ccp_crypto_aead *aead_alg, *aead_tmp; 380 struct ccp_crypto_aead *aead_alg, *aead_tmp;
381 struct ccp_crypto_akcipher_alg *akc_alg, *akc_tmp;
369 382
370 list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) { 383 list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) {
371 crypto_unregister_ahash(&ahash_alg->alg); 384 crypto_unregister_ahash(&ahash_alg->alg);
@@ -384,6 +397,12 @@ static void ccp_unregister_algs(void)
384 list_del(&aead_alg->entry); 397 list_del(&aead_alg->entry);
385 kfree(aead_alg); 398 kfree(aead_alg);
386 } 399 }
400
401 list_for_each_entry_safe(akc_alg, akc_tmp, &akcipher_algs, entry) {
402 crypto_unregister_akcipher(&akc_alg->alg);
403 list_del(&akc_alg->entry);
404 kfree(akc_alg);
405 }
387} 406}
388 407
389static int ccp_crypto_init(void) 408static int ccp_crypto_init(void)
diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c
new file mode 100644
index 000000000000..e6db8672d89c
--- /dev/null
+++ b/drivers/crypto/ccp/ccp-crypto-rsa.c
@@ -0,0 +1,299 @@
1/*
2 * AMD Cryptographic Coprocessor (CCP) RSA crypto API support
3 *
4 * Copyright (C) 2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/sched.h>
15#include <linux/scatterlist.h>
16#include <linux/crypto.h>
17#include <crypto/algapi.h>
18#include <crypto/internal/rsa.h>
19#include <crypto/internal/akcipher.h>
20#include <crypto/akcipher.h>
21#include <crypto/scatterwalk.h>
22
23#include "ccp-crypto.h"
24
25static inline struct akcipher_request *akcipher_request_cast(
26 struct crypto_async_request *req)
27{
28 return container_of(req, struct akcipher_request, base);
29}
30
31static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen,
32 const u8 *buf, size_t sz)
33{
34 int nskip;
35
36 for (nskip = 0; nskip < sz; nskip++)
37 if (buf[nskip])
38 break;
39 *kplen = sz - nskip;
40 *kpbuf = kzalloc(*kplen, GFP_KERNEL);
41 if (!*kpbuf)
42 return -ENOMEM;
43 memcpy(*kpbuf, buf + nskip, *kplen);
44
45 return 0;
46}
47
48static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret)
49{
50 struct akcipher_request *req = akcipher_request_cast(async_req);
51 struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req);
52
53 if (ret)
54 return ret;
55
56 req->dst_len = rctx->cmd.u.rsa.key_size >> 3;
57
58 return 0;
59}
60
61static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm)
62{
63 if (ccp_version() > CCP_VERSION(3, 0))
64 return CCP5_RSA_MAXMOD;
65 else
66 return CCP_RSA_MAXMOD;
67}
68
69static int ccp_rsa_crypt(struct akcipher_request *req, bool encrypt)
70{
71 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
72 struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
73 struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req);
74 int ret = 0;
75
76 memset(&rctx->cmd, 0, sizeof(rctx->cmd));
77 INIT_LIST_HEAD(&rctx->cmd.entry);
78 rctx->cmd.engine = CCP_ENGINE_RSA;
79
80 rctx->cmd.u.rsa.key_size = ctx->u.rsa.key_len; /* in bits */
81 if (encrypt) {
82 rctx->cmd.u.rsa.exp = &ctx->u.rsa.e_sg;
83 rctx->cmd.u.rsa.exp_len = ctx->u.rsa.e_len;
84 } else {
85 rctx->cmd.u.rsa.exp = &ctx->u.rsa.d_sg;
86 rctx->cmd.u.rsa.exp_len = ctx->u.rsa.d_len;
87 }
88 rctx->cmd.u.rsa.mod = &ctx->u.rsa.n_sg;
89 rctx->cmd.u.rsa.mod_len = ctx->u.rsa.n_len;
90 rctx->cmd.u.rsa.src = req->src;
91 rctx->cmd.u.rsa.src_len = req->src_len;
92 rctx->cmd.u.rsa.dst = req->dst;
93
94 ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
95
96 return ret;
97}
98
99static int ccp_rsa_encrypt(struct akcipher_request *req)
100{
101 return ccp_rsa_crypt(req, true);
102}
103
104static int ccp_rsa_decrypt(struct akcipher_request *req)
105{
106 return ccp_rsa_crypt(req, false);
107}
108
109static int ccp_check_key_length(unsigned int len)
110{
111 /* In bits */
112 if (len < 8 || len > 4096)
113 return -EINVAL;
114 return 0;
115}
116
117static void ccp_rsa_free_key_bufs(struct ccp_ctx *ctx)
118{
119 /* Clean up old key data */
120 kzfree(ctx->u.rsa.e_buf);
121 ctx->u.rsa.e_buf = NULL;
122 ctx->u.rsa.e_len = 0;
123 kzfree(ctx->u.rsa.n_buf);
124 ctx->u.rsa.n_buf = NULL;
125 ctx->u.rsa.n_len = 0;
126 kzfree(ctx->u.rsa.d_buf);
127 ctx->u.rsa.d_buf = NULL;
128 ctx->u.rsa.d_len = 0;
129}
130
131static int ccp_rsa_setkey(struct crypto_akcipher *tfm, const void *key,
132 unsigned int keylen, bool private)
133{
134 struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
135 struct rsa_key raw_key;
136 int ret;
137
138 ccp_rsa_free_key_bufs(ctx);
139 memset(&raw_key, 0, sizeof(raw_key));
140
141 /* Code borrowed from crypto/rsa.c */
142 if (private)
143 ret = rsa_parse_priv_key(&raw_key, key, keylen);
144 else
145 ret = rsa_parse_pub_key(&raw_key, key, keylen);
146 if (ret)
147 goto n_key;
148
149 ret = ccp_copy_and_save_keypart(&ctx->u.rsa.n_buf, &ctx->u.rsa.n_len,
150 raw_key.n, raw_key.n_sz);
151 if (ret)
152 goto key_err;
153 sg_init_one(&ctx->u.rsa.n_sg, ctx->u.rsa.n_buf, ctx->u.rsa.n_len);
154
155 ctx->u.rsa.key_len = ctx->u.rsa.n_len << 3; /* convert to bits */
156 if (ccp_check_key_length(ctx->u.rsa.key_len)) {
157 ret = -EINVAL;
158 goto key_err;
159 }
160
161 ret = ccp_copy_and_save_keypart(&ctx->u.rsa.e_buf, &ctx->u.rsa.e_len,
162 raw_key.e, raw_key.e_sz);
163 if (ret)
164 goto key_err;
165 sg_init_one(&ctx->u.rsa.e_sg, ctx->u.rsa.e_buf, ctx->u.rsa.e_len);
166
167 if (private) {
168 ret = ccp_copy_and_save_keypart(&ctx->u.rsa.d_buf,
169 &ctx->u.rsa.d_len,
170 raw_key.d, raw_key.d_sz);
171 if (ret)
172 goto key_err;
173 sg_init_one(&ctx->u.rsa.d_sg,
174 ctx->u.rsa.d_buf, ctx->u.rsa.d_len);
175 }
176
177 return 0;
178
179key_err:
180 ccp_rsa_free_key_bufs(ctx);
181
182n_key:
183 return ret;
184}
185
186static int ccp_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
187 unsigned int keylen)
188{
189 return ccp_rsa_setkey(tfm, key, keylen, true);
190}
191
192static int ccp_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
193 unsigned int keylen)
194{
195 return ccp_rsa_setkey(tfm, key, keylen, false);
196}
197
198static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm)
199{
200 struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
201
202 akcipher_set_reqsize(tfm, sizeof(struct ccp_rsa_req_ctx));
203 ctx->complete = ccp_rsa_complete;
204
205 return 0;
206}
207
208static void ccp_rsa_exit_tfm(struct crypto_akcipher *tfm)
209{
210 struct ccp_ctx *ctx = crypto_tfm_ctx(&tfm->base);
211
212 ccp_rsa_free_key_bufs(ctx);
213}
214
215static struct akcipher_alg ccp_rsa_defaults = {
216 .encrypt = ccp_rsa_encrypt,
217 .decrypt = ccp_rsa_decrypt,
218 .sign = ccp_rsa_decrypt,
219 .verify = ccp_rsa_encrypt,
220 .set_pub_key = ccp_rsa_setpubkey,
221 .set_priv_key = ccp_rsa_setprivkey,
222 .max_size = ccp_rsa_maxsize,
223 .init = ccp_rsa_init_tfm,
224 .exit = ccp_rsa_exit_tfm,
225 .base = {
226 .cra_name = "rsa",
227 .cra_driver_name = "rsa-ccp",
228 .cra_priority = CCP_CRA_PRIORITY,
229 .cra_module = THIS_MODULE,
230 .cra_ctxsize = 2 * sizeof(struct ccp_ctx),
231 },
232};
233
234struct ccp_rsa_def {
235 unsigned int version;
236 const char *name;
237 const char *driver_name;
238 unsigned int reqsize;
239 struct akcipher_alg *alg_defaults;
240};
241
242static struct ccp_rsa_def rsa_algs[] = {
243 {
244 .version = CCP_VERSION(3, 0),
245 .name = "rsa",
246 .driver_name = "rsa-ccp",
247 .reqsize = sizeof(struct ccp_rsa_req_ctx),
248 .alg_defaults = &ccp_rsa_defaults,
249 }
250};
251
252int ccp_register_rsa_alg(struct list_head *head, const struct ccp_rsa_def *def)
253{
254 struct ccp_crypto_akcipher_alg *ccp_alg;
255 struct akcipher_alg *alg;
256 int ret;
257
258 ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
259 if (!ccp_alg)
260 return -ENOMEM;
261
262 INIT_LIST_HEAD(&ccp_alg->entry);
263
264 alg = &ccp_alg->alg;
265 *alg = *def->alg_defaults;
266 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
267 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
268 def->driver_name);
269 ret = crypto_register_akcipher(alg);
270 if (ret) {
271 pr_err("%s akcipher algorithm registration error (%d)\n",
272 alg->base.cra_name, ret);
273 kfree(ccp_alg);
274 return ret;
275 }
276
277 list_add(&ccp_alg->entry, head);
278
279 return 0;
280}
281
282int ccp_register_rsa_algs(struct list_head *head)
283{
284 int i, ret;
285 unsigned int ccpversion = ccp_version();
286
287 /* Register the RSA algorithm in standard mode
288 * This works for CCP v3 and later
289 */
290 for (i = 0; i < ARRAY_SIZE(rsa_algs); i++) {
291 if (rsa_algs[i].version > ccpversion)
292 continue;
293 ret = ccp_register_rsa_alg(head, &rsa_algs[i]);
294 if (ret)
295 return ret;
296 }
297
298 return 0;
299}
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index ce97b3868f4a..8b9b16d433f7 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) SHA crypto API support 2 * AMD Cryptographic Coprocessor (CCP) SHA crypto API support
3 * 3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com>
diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h
index dd5bf15f06e5..b9fd090c46c2 100644
--- a/drivers/crypto/ccp/ccp-crypto.h
+++ b/drivers/crypto/ccp/ccp-crypto.h
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) crypto API support 2 * AMD Cryptographic Coprocessor (CCP) crypto API support
3 * 3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * 7 *
@@ -24,6 +24,8 @@
24#include <crypto/ctr.h> 24#include <crypto/ctr.h>
25#include <crypto/hash.h> 25#include <crypto/hash.h>
26#include <crypto/sha.h> 26#include <crypto/sha.h>
27#include <crypto/akcipher.h>
28#include <crypto/internal/rsa.h>
27 29
28#define CCP_LOG_LEVEL KERN_INFO 30#define CCP_LOG_LEVEL KERN_INFO
29 31
@@ -58,6 +60,12 @@ struct ccp_crypto_ahash_alg {
58 struct ahash_alg alg; 60 struct ahash_alg alg;
59}; 61};
60 62
63struct ccp_crypto_akcipher_alg {
64 struct list_head entry;
65
66 struct akcipher_alg alg;
67};
68
61static inline struct ccp_crypto_ablkcipher_alg * 69static inline struct ccp_crypto_ablkcipher_alg *
62 ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm) 70 ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm)
63{ 71{
@@ -91,7 +99,7 @@ struct ccp_aes_ctx {
91 99
92 struct scatterlist key_sg; 100 struct scatterlist key_sg;
93 unsigned int key_len; 101 unsigned int key_len;
94 u8 key[AES_MAX_KEY_SIZE]; 102 u8 key[AES_MAX_KEY_SIZE * 2];
95 103
96 u8 nonce[CTR_RFC3686_NONCE_SIZE]; 104 u8 nonce[CTR_RFC3686_NONCE_SIZE];
97 105
@@ -227,12 +235,35 @@ struct ccp_sha_exp_ctx {
227 u8 buf[MAX_SHA_BLOCK_SIZE]; 235 u8 buf[MAX_SHA_BLOCK_SIZE];
228}; 236};
229 237
238/***** RSA related defines *****/
239
240struct ccp_rsa_ctx {
241 unsigned int key_len; /* in bits */
242 struct scatterlist e_sg;
243 u8 *e_buf;
244 unsigned int e_len;
245 struct scatterlist n_sg;
246 u8 *n_buf;
247 unsigned int n_len;
248 struct scatterlist d_sg;
249 u8 *d_buf;
250 unsigned int d_len;
251};
252
253struct ccp_rsa_req_ctx {
254 struct ccp_cmd cmd;
255};
256
257#define CCP_RSA_MAXMOD (4 * 1024 / 8)
258#define CCP5_RSA_MAXMOD (16 * 1024 / 8)
259
230/***** Common Context Structure *****/ 260/***** Common Context Structure *****/
231struct ccp_ctx { 261struct ccp_ctx {
232 int (*complete)(struct crypto_async_request *req, int ret); 262 int (*complete)(struct crypto_async_request *req, int ret);
233 263
234 union { 264 union {
235 struct ccp_aes_ctx aes; 265 struct ccp_aes_ctx aes;
266 struct ccp_rsa_ctx rsa;
236 struct ccp_sha_ctx sha; 267 struct ccp_sha_ctx sha;
237 struct ccp_des3_ctx des3; 268 struct ccp_des3_ctx des3;
238 } u; 269 } u;
@@ -249,5 +280,6 @@ int ccp_register_aes_xts_algs(struct list_head *head);
249int ccp_register_aes_aeads(struct list_head *head); 280int ccp_register_aes_aeads(struct list_head *head);
250int ccp_register_sha_algs(struct list_head *head); 281int ccp_register_sha_algs(struct list_head *head);
251int ccp_register_des3_algs(struct list_head *head); 282int ccp_register_des3_algs(struct list_head *head);
283int ccp_register_rsa_algs(struct list_head *head);
252 284
253#endif 285#endif
diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c
index 3cd6c83754e0..59d4ca4e72d8 100644
--- a/drivers/crypto/ccp/ccp-debugfs.c
+++ b/drivers/crypto/ccp/ccp-debugfs.c
@@ -305,19 +305,19 @@ void ccp5_debugfs_setup(struct ccp_device *ccp)
305 305
306 ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir); 306 ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir);
307 if (!ccp->debugfs_instance) 307 if (!ccp->debugfs_instance)
308 return; 308 goto err;
309 309
310 debugfs_info = debugfs_create_file("info", 0400, 310 debugfs_info = debugfs_create_file("info", 0400,
311 ccp->debugfs_instance, ccp, 311 ccp->debugfs_instance, ccp,
312 &ccp_debugfs_info_ops); 312 &ccp_debugfs_info_ops);
313 if (!debugfs_info) 313 if (!debugfs_info)
314 return; 314 goto err;
315 315
316 debugfs_stats = debugfs_create_file("stats", 0600, 316 debugfs_stats = debugfs_create_file("stats", 0600,
317 ccp->debugfs_instance, ccp, 317 ccp->debugfs_instance, ccp,
318 &ccp_debugfs_stats_ops); 318 &ccp_debugfs_stats_ops);
319 if (!debugfs_stats) 319 if (!debugfs_stats)
320 return; 320 goto err;
321 321
322 for (i = 0; i < ccp->cmd_q_count; i++) { 322 for (i = 0; i < ccp->cmd_q_count; i++) {
323 cmd_q = &ccp->cmd_q[i]; 323 cmd_q = &ccp->cmd_q[i];
@@ -327,15 +327,20 @@ void ccp5_debugfs_setup(struct ccp_device *ccp)
327 debugfs_q_instance = 327 debugfs_q_instance =
328 debugfs_create_dir(name, ccp->debugfs_instance); 328 debugfs_create_dir(name, ccp->debugfs_instance);
329 if (!debugfs_q_instance) 329 if (!debugfs_q_instance)
330 return; 330 goto err;
331 331
332 debugfs_q_stats = 332 debugfs_q_stats =
333 debugfs_create_file("stats", 0600, 333 debugfs_create_file("stats", 0600,
334 debugfs_q_instance, cmd_q, 334 debugfs_q_instance, cmd_q,
335 &ccp_debugfs_queue_ops); 335 &ccp_debugfs_queue_ops);
336 if (!debugfs_q_stats) 336 if (!debugfs_q_stats)
337 return; 337 goto err;
338 } 338 }
339
340 return;
341
342err:
343 debugfs_remove_recursive(ccp->debugfs_instance);
339} 344}
340 345
341void ccp5_debugfs_destroy(void) 346void ccp5_debugfs_destroy(void)
diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c
index 367c2e30656f..240bebbcb8ac 100644
--- a/drivers/crypto/ccp/ccp-dev-v3.c
+++ b/drivers/crypto/ccp/ccp-dev-v3.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com>
@@ -359,8 +359,7 @@ static void ccp_irq_bh(unsigned long data)
359 359
360static irqreturn_t ccp_irq_handler(int irq, void *data) 360static irqreturn_t ccp_irq_handler(int irq, void *data)
361{ 361{
362 struct device *dev = data; 362 struct ccp_device *ccp = (struct ccp_device *)data;
363 struct ccp_device *ccp = dev_get_drvdata(dev);
364 363
365 ccp_disable_queue_interrupts(ccp); 364 ccp_disable_queue_interrupts(ccp);
366 if (ccp->use_tasklet) 365 if (ccp->use_tasklet)
@@ -454,7 +453,7 @@ static int ccp_init(struct ccp_device *ccp)
454 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); 453 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
455 454
456 /* Request an irq */ 455 /* Request an irq */
457 ret = ccp->get_irq(ccp); 456 ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
458 if (ret) { 457 if (ret) {
459 dev_err(dev, "unable to allocate an IRQ\n"); 458 dev_err(dev, "unable to allocate an IRQ\n");
460 goto e_pool; 459 goto e_pool;
@@ -511,7 +510,7 @@ e_kthread:
511 if (ccp->cmd_q[i].kthread) 510 if (ccp->cmd_q[i].kthread)
512 kthread_stop(ccp->cmd_q[i].kthread); 511 kthread_stop(ccp->cmd_q[i].kthread);
513 512
514 ccp->free_irq(ccp); 513 sp_free_ccp_irq(ccp->sp, ccp);
515 514
516e_pool: 515e_pool:
517 for (i = 0; i < ccp->cmd_q_count; i++) 516 for (i = 0; i < ccp->cmd_q_count; i++)
@@ -550,7 +549,7 @@ static void ccp_destroy(struct ccp_device *ccp)
550 if (ccp->cmd_q[i].kthread) 549 if (ccp->cmd_q[i].kthread)
551 kthread_stop(ccp->cmd_q[i].kthread); 550 kthread_stop(ccp->cmd_q[i].kthread);
552 551
553 ccp->free_irq(ccp); 552 sp_free_ccp_irq(ccp->sp, ccp);
554 553
555 for (i = 0; i < ccp->cmd_q_count; i++) 554 for (i = 0; i < ccp->cmd_q_count; i++)
556 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 555 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
@@ -586,10 +585,17 @@ static const struct ccp_actions ccp3_actions = {
586 .irqhandler = ccp_irq_handler, 585 .irqhandler = ccp_irq_handler,
587}; 586};
588 587
588const struct ccp_vdata ccpv3_platform = {
589 .version = CCP_VERSION(3, 0),
590 .setup = NULL,
591 .perform = &ccp3_actions,
592 .offset = 0,
593};
594
589const struct ccp_vdata ccpv3 = { 595const struct ccp_vdata ccpv3 = {
590 .version = CCP_VERSION(3, 0), 596 .version = CCP_VERSION(3, 0),
591 .setup = NULL, 597 .setup = NULL,
592 .perform = &ccp3_actions, 598 .perform = &ccp3_actions,
593 .bar = 2,
594 .offset = 0x20000, 599 .offset = 0x20000,
600 .rsamax = CCP_RSA_MAX_WIDTH,
595}; 601};
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index b10d2d2075cb..65604fc65e8f 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Gary R Hook <gary.hook@amd.com> 6 * Author: Gary R Hook <gary.hook@amd.com>
7 * 7 *
@@ -145,6 +145,7 @@ union ccp_function {
145#define CCP_AES_MODE(p) ((p)->aes.mode) 145#define CCP_AES_MODE(p) ((p)->aes.mode)
146#define CCP_AES_TYPE(p) ((p)->aes.type) 146#define CCP_AES_TYPE(p) ((p)->aes.type)
147#define CCP_XTS_SIZE(p) ((p)->aes_xts.size) 147#define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
148#define CCP_XTS_TYPE(p) ((p)->aes_xts.type)
148#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) 149#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
149#define CCP_DES3_SIZE(p) ((p)->des3.size) 150#define CCP_DES3_SIZE(p) ((p)->des3.size)
150#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) 151#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt)
@@ -344,6 +345,7 @@ static int ccp5_perform_xts_aes(struct ccp_op *op)
344 CCP5_CMD_PROT(&desc) = 0; 345 CCP5_CMD_PROT(&desc) = 0;
345 346
346 function.raw = 0; 347 function.raw = 0;
348 CCP_XTS_TYPE(&function) = op->u.xts.type;
347 CCP_XTS_ENCRYPT(&function) = op->u.xts.action; 349 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
348 CCP_XTS_SIZE(&function) = op->u.xts.unit_size; 350 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
349 CCP5_CMD_FUNCTION(&desc) = function.raw; 351 CCP5_CMD_FUNCTION(&desc) = function.raw;
@@ -469,7 +471,7 @@ static int ccp5_perform_rsa(struct ccp_op *op)
469 CCP5_CMD_PROT(&desc) = 0; 471 CCP5_CMD_PROT(&desc) = 0;
470 472
471 function.raw = 0; 473 function.raw = 0;
472 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3; 474 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
473 CCP5_CMD_FUNCTION(&desc) = function.raw; 475 CCP5_CMD_FUNCTION(&desc) = function.raw;
474 476
475 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; 477 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
@@ -484,10 +486,10 @@ static int ccp5_perform_rsa(struct ccp_op *op)
484 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 486 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
485 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 487 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
486 488
487 /* Exponent is in LSB memory */ 489 /* Key (Exponent) is in external memory */
488 CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE; 490 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
489 CCP5_CMD_KEY_HI(&desc) = 0; 491 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
490 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 492 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
491 493
492 return ccp5_do_cmd(&desc, op->cmd_q); 494 return ccp5_do_cmd(&desc, op->cmd_q);
493} 495}
@@ -769,8 +771,7 @@ static void ccp5_irq_bh(unsigned long data)
769 771
770static irqreturn_t ccp5_irq_handler(int irq, void *data) 772static irqreturn_t ccp5_irq_handler(int irq, void *data)
771{ 773{
772 struct device *dev = data; 774 struct ccp_device *ccp = (struct ccp_device *)data;
773 struct ccp_device *ccp = dev_get_drvdata(dev);
774 775
775 ccp5_disable_queue_interrupts(ccp); 776 ccp5_disable_queue_interrupts(ccp);
776 ccp->total_interrupts++; 777 ccp->total_interrupts++;
@@ -881,7 +882,7 @@ static int ccp5_init(struct ccp_device *ccp)
881 882
882 dev_dbg(dev, "Requesting an IRQ...\n"); 883 dev_dbg(dev, "Requesting an IRQ...\n");
883 /* Request an irq */ 884 /* Request an irq */
884 ret = ccp->get_irq(ccp); 885 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
885 if (ret) { 886 if (ret) {
886 dev_err(dev, "unable to allocate an IRQ\n"); 887 dev_err(dev, "unable to allocate an IRQ\n");
887 goto e_pool; 888 goto e_pool;
@@ -987,7 +988,7 @@ e_kthread:
987 kthread_stop(ccp->cmd_q[i].kthread); 988 kthread_stop(ccp->cmd_q[i].kthread);
988 989
989e_irq: 990e_irq:
990 ccp->free_irq(ccp); 991 sp_free_ccp_irq(ccp->sp, ccp);
991 992
992e_pool: 993e_pool:
993 for (i = 0; i < ccp->cmd_q_count; i++) 994 for (i = 0; i < ccp->cmd_q_count; i++)
@@ -1037,7 +1038,7 @@ static void ccp5_destroy(struct ccp_device *ccp)
1037 if (ccp->cmd_q[i].kthread) 1038 if (ccp->cmd_q[i].kthread)
1038 kthread_stop(ccp->cmd_q[i].kthread); 1039 kthread_stop(ccp->cmd_q[i].kthread);
1039 1040
1040 ccp->free_irq(ccp); 1041 sp_free_ccp_irq(ccp->sp, ccp);
1041 1042
1042 for (i = 0; i < ccp->cmd_q_count; i++) { 1043 for (i = 0; i < ccp->cmd_q_count; i++) {
1043 cmd_q = &ccp->cmd_q[i]; 1044 cmd_q = &ccp->cmd_q[i];
@@ -1106,15 +1107,14 @@ static const struct ccp_actions ccp5_actions = {
1106 .init = ccp5_init, 1107 .init = ccp5_init,
1107 .destroy = ccp5_destroy, 1108 .destroy = ccp5_destroy,
1108 .get_free_slots = ccp5_get_free_slots, 1109 .get_free_slots = ccp5_get_free_slots,
1109 .irqhandler = ccp5_irq_handler,
1110}; 1110};
1111 1111
1112const struct ccp_vdata ccpv5a = { 1112const struct ccp_vdata ccpv5a = {
1113 .version = CCP_VERSION(5, 0), 1113 .version = CCP_VERSION(5, 0),
1114 .setup = ccp5_config, 1114 .setup = ccp5_config,
1115 .perform = &ccp5_actions, 1115 .perform = &ccp5_actions,
1116 .bar = 2,
1117 .offset = 0x0, 1116 .offset = 0x0,
1117 .rsamax = CCP5_RSA_MAX_WIDTH,
1118}; 1118};
1119 1119
1120const struct ccp_vdata ccpv5b = { 1120const struct ccp_vdata ccpv5b = {
@@ -1122,6 +1122,6 @@ const struct ccp_vdata ccpv5b = {
1122 .dma_chan_attr = DMA_PRIVATE, 1122 .dma_chan_attr = DMA_PRIVATE,
1123 .setup = ccp5other_config, 1123 .setup = ccp5other_config,
1124 .perform = &ccp5_actions, 1124 .perform = &ccp5_actions,
1125 .bar = 2,
1126 .offset = 0x0, 1125 .offset = 0x0,
1126 .rsamax = CCP5_RSA_MAX_WIDTH,
1127}; 1127};
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index 2506b5025700..4e029b176641 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com>
@@ -11,7 +11,6 @@
11 * published by the Free Software Foundation. 11 * published by the Free Software Foundation.
12 */ 12 */
13 13
14#include <linux/module.h>
15#include <linux/kernel.h> 14#include <linux/kernel.h>
16#include <linux/kthread.h> 15#include <linux/kthread.h>
17#include <linux/sched.h> 16#include <linux/sched.h>
@@ -30,12 +29,6 @@
30 29
31#include "ccp-dev.h" 30#include "ccp-dev.h"
32 31
33MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
34MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>");
35MODULE_LICENSE("GPL");
36MODULE_VERSION("1.1.0");
37MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver");
38
39struct ccp_tasklet_data { 32struct ccp_tasklet_data {
40 struct completion completion; 33 struct completion completion;
41 struct ccp_cmd *cmd; 34 struct ccp_cmd *cmd;
@@ -111,13 +104,6 @@ static LIST_HEAD(ccp_units);
111static DEFINE_SPINLOCK(ccp_rr_lock); 104static DEFINE_SPINLOCK(ccp_rr_lock);
112static struct ccp_device *ccp_rr; 105static struct ccp_device *ccp_rr;
113 106
114/* Ever-increasing value to produce unique unit numbers */
115static atomic_t ccp_unit_ordinal;
116static unsigned int ccp_increment_unit_ordinal(void)
117{
118 return atomic_inc_return(&ccp_unit_ordinal);
119}
120
121/** 107/**
122 * ccp_add_device - add a CCP device to the list 108 * ccp_add_device - add a CCP device to the list
123 * 109 *
@@ -415,6 +401,7 @@ static void ccp_do_cmd_complete(unsigned long data)
415 struct ccp_cmd *cmd = tdata->cmd; 401 struct ccp_cmd *cmd = tdata->cmd;
416 402
417 cmd->callback(cmd->data, cmd->ret); 403 cmd->callback(cmd->data, cmd->ret);
404
418 complete(&tdata->completion); 405 complete(&tdata->completion);
419} 406}
420 407
@@ -464,14 +451,17 @@ int ccp_cmd_queue_thread(void *data)
464 * 451 *
465 * @dev: device struct of the CCP 452 * @dev: device struct of the CCP
466 */ 453 */
467struct ccp_device *ccp_alloc_struct(struct device *dev) 454struct ccp_device *ccp_alloc_struct(struct sp_device *sp)
468{ 455{
456 struct device *dev = sp->dev;
469 struct ccp_device *ccp; 457 struct ccp_device *ccp;
470 458
471 ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL); 459 ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);
472 if (!ccp) 460 if (!ccp)
473 return NULL; 461 return NULL;
474 ccp->dev = dev; 462 ccp->dev = dev;
463 ccp->sp = sp;
464 ccp->axcache = sp->axcache;
475 465
476 INIT_LIST_HEAD(&ccp->cmd); 466 INIT_LIST_HEAD(&ccp->cmd);
477 INIT_LIST_HEAD(&ccp->backlog); 467 INIT_LIST_HEAD(&ccp->backlog);
@@ -486,9 +476,8 @@ struct ccp_device *ccp_alloc_struct(struct device *dev)
486 init_waitqueue_head(&ccp->sb_queue); 476 init_waitqueue_head(&ccp->sb_queue);
487 init_waitqueue_head(&ccp->suspend_queue); 477 init_waitqueue_head(&ccp->suspend_queue);
488 478
489 ccp->ord = ccp_increment_unit_ordinal(); 479 snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord);
490 snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord); 480 snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord);
491 snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord);
492 481
493 return ccp; 482 return ccp;
494} 483}
@@ -538,55 +527,100 @@ bool ccp_queues_suspended(struct ccp_device *ccp)
538 527
539 return ccp->cmd_q_count == suspended; 528 return ccp->cmd_q_count == suspended;
540} 529}
541#endif
542 530
543static int __init ccp_mod_init(void) 531int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
544{ 532{
545#ifdef CONFIG_X86 533 struct ccp_device *ccp = sp->ccp_data;
546 int ret; 534 unsigned long flags;
535 unsigned int i;
547 536
548 ret = ccp_pci_init(); 537 spin_lock_irqsave(&ccp->cmd_lock, flags);
549 if (ret)
550 return ret;
551 538
552 /* Don't leave the driver loaded if init failed */ 539 ccp->suspending = 1;
553 if (ccp_present() != 0) { 540
554 ccp_pci_exit(); 541 /* Wake all the queue kthreads to prepare for suspend */
555 return -ENODEV; 542 for (i = 0; i < ccp->cmd_q_count; i++)
543 wake_up_process(ccp->cmd_q[i].kthread);
544
545 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
546
547 /* Wait for all queue kthreads to say they're done */
548 while (!ccp_queues_suspended(ccp))
549 wait_event_interruptible(ccp->suspend_queue,
550 ccp_queues_suspended(ccp));
551
552 return 0;
553}
554
555int ccp_dev_resume(struct sp_device *sp)
556{
557 struct ccp_device *ccp = sp->ccp_data;
558 unsigned long flags;
559 unsigned int i;
560
561 spin_lock_irqsave(&ccp->cmd_lock, flags);
562
563 ccp->suspending = 0;
564
565 /* Wake up all the kthreads */
566 for (i = 0; i < ccp->cmd_q_count; i++) {
567 ccp->cmd_q[i].suspended = 0;
568 wake_up_process(ccp->cmd_q[i].kthread);
556 } 569 }
557 570
571 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
572
558 return 0; 573 return 0;
574}
559#endif 575#endif
560 576
561#ifdef CONFIG_ARM64 577int ccp_dev_init(struct sp_device *sp)
578{
579 struct device *dev = sp->dev;
580 struct ccp_device *ccp;
562 int ret; 581 int ret;
563 582
564 ret = ccp_platform_init(); 583 ret = -ENOMEM;
584 ccp = ccp_alloc_struct(sp);
585 if (!ccp)
586 goto e_err;
587 sp->ccp_data = ccp;
588
589 ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;
590 if (!ccp->vdata || !ccp->vdata->version) {
591 ret = -ENODEV;
592 dev_err(dev, "missing driver data\n");
593 goto e_err;
594 }
595
596 ccp->use_tasklet = sp->use_tasklet;
597
598 ccp->io_regs = sp->io_map + ccp->vdata->offset;
599 if (ccp->vdata->setup)
600 ccp->vdata->setup(ccp);
601
602 ret = ccp->vdata->perform->init(ccp);
565 if (ret) 603 if (ret)
566 return ret; 604 goto e_err;
567 605
568 /* Don't leave the driver loaded if init failed */ 606 dev_notice(dev, "ccp enabled\n");
569 if (ccp_present() != 0) {
570 ccp_platform_exit();
571 return -ENODEV;
572 }
573 607
574 return 0; 608 return 0;
575#endif
576 609
577 return -ENODEV; 610e_err:
611 sp->ccp_data = NULL;
612
613 dev_notice(dev, "ccp initialization failed\n");
614
615 return ret;
578} 616}
579 617
580static void __exit ccp_mod_exit(void) 618void ccp_dev_destroy(struct sp_device *sp)
581{ 619{
582#ifdef CONFIG_X86 620 struct ccp_device *ccp = sp->ccp_data;
583 ccp_pci_exit();
584#endif
585 621
586#ifdef CONFIG_ARM64 622 if (!ccp)
587 ccp_platform_exit(); 623 return;
588#endif
589}
590 624
591module_init(ccp_mod_init); 625 ccp->vdata->perform->destroy(ccp);
592module_exit(ccp_mod_exit); 626}
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index a70154ac7405..6810b65c1939 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com>
@@ -27,6 +27,8 @@
27#include <linux/irqreturn.h> 27#include <linux/irqreturn.h>
28#include <linux/dmaengine.h> 28#include <linux/dmaengine.h>
29 29
30#include "sp-dev.h"
31
30#define MAX_CCP_NAME_LEN 16 32#define MAX_CCP_NAME_LEN 16
31#define MAX_DMAPOOL_NAME_LEN 32 33#define MAX_DMAPOOL_NAME_LEN 32
32 34
@@ -192,6 +194,7 @@
192#define CCP_AES_CTX_SB_COUNT 1 194#define CCP_AES_CTX_SB_COUNT 1
193 195
194#define CCP_XTS_AES_KEY_SB_COUNT 1 196#define CCP_XTS_AES_KEY_SB_COUNT 1
197#define CCP5_XTS_AES_KEY_SB_COUNT 2
195#define CCP_XTS_AES_CTX_SB_COUNT 1 198#define CCP_XTS_AES_CTX_SB_COUNT 1
196 199
197#define CCP_DES3_KEY_SB_COUNT 1 200#define CCP_DES3_KEY_SB_COUNT 1
@@ -200,6 +203,7 @@
200#define CCP_SHA_SB_COUNT 1 203#define CCP_SHA_SB_COUNT 1
201 204
202#define CCP_RSA_MAX_WIDTH 4096 205#define CCP_RSA_MAX_WIDTH 4096
206#define CCP5_RSA_MAX_WIDTH 16384
203 207
204#define CCP_PASSTHRU_BLOCKSIZE 256 208#define CCP_PASSTHRU_BLOCKSIZE 256
205#define CCP_PASSTHRU_MASKSIZE 32 209#define CCP_PASSTHRU_MASKSIZE 32
@@ -344,12 +348,11 @@ struct ccp_device {
344 char rngname[MAX_CCP_NAME_LEN]; 348 char rngname[MAX_CCP_NAME_LEN];
345 349
346 struct device *dev; 350 struct device *dev;
351 struct sp_device *sp;
347 352
348 /* Bus specific device information 353 /* Bus specific device information
349 */ 354 */
350 void *dev_specific; 355 void *dev_specific;
351 int (*get_irq)(struct ccp_device *ccp);
352 void (*free_irq)(struct ccp_device *ccp);
353 unsigned int qim; 356 unsigned int qim;
354 unsigned int irq; 357 unsigned int irq;
355 bool use_tasklet; 358 bool use_tasklet;
@@ -362,7 +365,6 @@ struct ccp_device {
362 * them. 365 * them.
363 */ 366 */
364 struct mutex req_mutex ____cacheline_aligned; 367 struct mutex req_mutex ____cacheline_aligned;
365 void __iomem *io_map;
366 void __iomem *io_regs; 368 void __iomem *io_regs;
367 369
368 /* Master lists that all cmds are queued on. Because there can be 370 /* Master lists that all cmds are queued on. Because there can be
@@ -497,6 +499,7 @@ struct ccp_aes_op {
497}; 499};
498 500
499struct ccp_xts_aes_op { 501struct ccp_xts_aes_op {
502 enum ccp_aes_type type;
500 enum ccp_aes_action action; 503 enum ccp_aes_action action;
501 enum ccp_xts_aes_unit_size unit_size; 504 enum ccp_xts_aes_unit_size unit_size;
502}; 505};
@@ -626,18 +629,12 @@ struct ccp5_desc {
626 struct dword7 dw7; 629 struct dword7 dw7;
627}; 630};
628 631
629int ccp_pci_init(void);
630void ccp_pci_exit(void);
631
632int ccp_platform_init(void);
633void ccp_platform_exit(void);
634
635void ccp_add_device(struct ccp_device *ccp); 632void ccp_add_device(struct ccp_device *ccp);
636void ccp_del_device(struct ccp_device *ccp); 633void ccp_del_device(struct ccp_device *ccp);
637 634
638extern void ccp_log_error(struct ccp_device *, int); 635extern void ccp_log_error(struct ccp_device *, int);
639 636
640struct ccp_device *ccp_alloc_struct(struct device *dev); 637struct ccp_device *ccp_alloc_struct(struct sp_device *sp);
641bool ccp_queues_suspended(struct ccp_device *ccp); 638bool ccp_queues_suspended(struct ccp_device *ccp);
642int ccp_cmd_queue_thread(void *data); 639int ccp_cmd_queue_thread(void *data);
643int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait); 640int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait);
@@ -669,16 +666,7 @@ struct ccp_actions {
669 irqreturn_t (*irqhandler)(int, void *); 666 irqreturn_t (*irqhandler)(int, void *);
670}; 667};
671 668
672/* Structure to hold CCP version-specific values */ 669extern const struct ccp_vdata ccpv3_platform;
673struct ccp_vdata {
674 const unsigned int version;
675 const unsigned int dma_chan_attr;
676 void (*setup)(struct ccp_device *);
677 const struct ccp_actions *perform;
678 const unsigned int bar;
679 const unsigned int offset;
680};
681
682extern const struct ccp_vdata ccpv3; 670extern const struct ccp_vdata ccpv3;
683extern const struct ccp_vdata ccpv5a; 671extern const struct ccp_vdata ccpv5a;
684extern const struct ccp_vdata ccpv5b; 672extern const struct ccp_vdata ccpv5b;
diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c
index e00be01fbf5a..901343dd513e 100644
--- a/drivers/crypto/ccp/ccp-dmaengine.c
+++ b/drivers/crypto/ccp/ccp-dmaengine.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Gary R Hook <gary.hook@amd.com> 6 * Author: Gary R Hook <gary.hook@amd.com>
7 * 7 *
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index c0dfdacbdff5..406b95329b3d 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com>
@@ -168,7 +168,7 @@ static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
168 168
169 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 169 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
170 dir); 170 dir);
171 if (!wa->dma.address) 171 if (dma_mapping_error(wa->dev, wa->dma.address))
172 return -ENOMEM; 172 return -ENOMEM;
173 173
174 wa->dma.length = len; 174 wa->dma.length = len;
@@ -1038,6 +1038,8 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1038 struct ccp_op op; 1038 struct ccp_op op;
1039 unsigned int unit_size, dm_offset; 1039 unsigned int unit_size, dm_offset;
1040 bool in_place = false; 1040 bool in_place = false;
1041 unsigned int sb_count;
1042 enum ccp_aes_type aestype;
1041 int ret; 1043 int ret;
1042 1044
1043 switch (xts->unit_size) { 1045 switch (xts->unit_size) {
@@ -1061,7 +1063,11 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1061 return -EINVAL; 1063 return -EINVAL;
1062 } 1064 }
1063 1065
1064 if (xts->key_len != AES_KEYSIZE_128) 1066 if (xts->key_len == AES_KEYSIZE_128)
1067 aestype = CCP_AES_TYPE_128;
1068 else if (xts->key_len == AES_KEYSIZE_256)
1069 aestype = CCP_AES_TYPE_256;
1070 else
1065 return -EINVAL; 1071 return -EINVAL;
1066 1072
1067 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1073 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
@@ -1083,23 +1089,44 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1083 op.sb_key = cmd_q->sb_key; 1089 op.sb_key = cmd_q->sb_key;
1084 op.sb_ctx = cmd_q->sb_ctx; 1090 op.sb_ctx = cmd_q->sb_ctx;
1085 op.init = 1; 1091 op.init = 1;
1092 op.u.xts.type = aestype;
1086 op.u.xts.action = xts->action; 1093 op.u.xts.action = xts->action;
1087 op.u.xts.unit_size = xts->unit_size; 1094 op.u.xts.unit_size = xts->unit_size;
1088 1095
1089 /* All supported key sizes fit in a single (32-byte) SB entry 1096 /* A version 3 device only supports 128-bit keys, which fits into a
1090 * and must be in little endian format. Use the 256-bit byte 1097 * single SB entry. A version 5 device uses a 512-bit vector, so two
1091 * swap passthru option to convert from big endian to little 1098 * SB entries.
1092 * endian.
1093 */ 1099 */
1100 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1101 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1102 else
1103 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1094 ret = ccp_init_dm_workarea(&key, cmd_q, 1104 ret = ccp_init_dm_workarea(&key, cmd_q,
1095 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, 1105 sb_count * CCP_SB_BYTES,
1096 DMA_TO_DEVICE); 1106 DMA_TO_DEVICE);
1097 if (ret) 1107 if (ret)
1098 return ret; 1108 return ret;
1099 1109
1100 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1110 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1101 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1111 /* All supported key sizes must be in little endian format.
1102 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 1112 * Use the 256-bit byte swap passthru option to convert from
1113 * big endian to little endian.
1114 */
1115 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1116 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1117 ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1118 } else {
1119 /* Version 5 CCPs use a 512-bit space for the key: each portion
1120 * occupies 256 bits, or one entire slot, and is zero-padded.
1121 */
1122 unsigned int pad;
1123
1124 dm_offset = CCP_SB_BYTES;
1125 pad = dm_offset - xts->key_len;
1126 ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1127 ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len,
1128 xts->key_len);
1129 }
1103 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1130 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1104 CCP_PASSTHRU_BYTESWAP_256BIT); 1131 CCP_PASSTHRU_BYTESWAP_256BIT);
1105 if (ret) { 1132 if (ret) {
@@ -1731,42 +1758,53 @@ e_ctx:
1731static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1758static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1732{ 1759{
1733 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1760 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1734 struct ccp_dm_workarea exp, src; 1761 struct ccp_dm_workarea exp, src, dst;
1735 struct ccp_data dst;
1736 struct ccp_op op; 1762 struct ccp_op op;
1737 unsigned int sb_count, i_len, o_len; 1763 unsigned int sb_count, i_len, o_len;
1738 int ret; 1764 int ret;
1739 1765
1740 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1766 /* Check against the maximum allowable size, in bits */
1767 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1741 return -EINVAL; 1768 return -EINVAL;
1742 1769
1743 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1770 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1744 return -EINVAL; 1771 return -EINVAL;
1745 1772
1773 memset(&op, 0, sizeof(op));
1774 op.cmd_q = cmd_q;
1775 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1776
1746 /* The RSA modulus must precede the message being acted upon, so 1777 /* The RSA modulus must precede the message being acted upon, so
1747 * it must be copied to a DMA area where the message and the 1778 * it must be copied to a DMA area where the message and the
1748 * modulus can be concatenated. Therefore the input buffer 1779 * modulus can be concatenated. Therefore the input buffer
1749 * length required is twice the output buffer length (which 1780 * length required is twice the output buffer length (which
1750 * must be a multiple of 256-bits). 1781 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1782 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1783 * required.
1751 */ 1784 */
1752 o_len = ((rsa->key_size + 255) / 256) * 32; 1785 o_len = 32 * ((rsa->key_size + 255) / 256);
1753 i_len = o_len * 2; 1786 i_len = o_len * 2;
1754 1787
1755 sb_count = o_len / CCP_SB_BYTES; 1788 sb_count = 0;
1756 1789 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1757 memset(&op, 0, sizeof(op)); 1790 /* sb_count is the number of storage block slots required
1758 op.cmd_q = cmd_q; 1791 * for the modulus.
1759 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1792 */
1760 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count); 1793 sb_count = o_len / CCP_SB_BYTES;
1761 1794 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1762 if (!op.sb_key) 1795 sb_count);
1763 return -EIO; 1796 if (!op.sb_key)
1797 return -EIO;
1798 } else {
1799 /* A version 5 device allows a modulus size that will not fit
1800 * in the LSB, so the command will transfer it from memory.
1801 * Set the sb key to the default, even though it's not used.
1802 */
1803 op.sb_key = cmd_q->sb_key;
1804 }
1764 1805
1765 /* The RSA exponent may span multiple (32-byte) SB entries and must 1806 /* The RSA exponent must be in little endian format. Reverse its
1766 * be in little endian format. Reverse copy each 32-byte chunk 1807 * byte order.
1767 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1768 * and each byte within that chunk and do not perform any byte swap
1769 * operations on the passthru operation.
1770 */ 1808 */
1771 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1809 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1772 if (ret) 1810 if (ret)
@@ -1775,11 +1813,22 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1775 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1813 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1776 if (ret) 1814 if (ret)
1777 goto e_exp; 1815 goto e_exp;
1778 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1816
1779 CCP_PASSTHRU_BYTESWAP_NOOP); 1817 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1780 if (ret) { 1818 /* Copy the exponent to the local storage block, using
1781 cmd->engine_error = cmd_q->cmd_error; 1819 * as many 32-byte blocks as were allocated above. It's
1782 goto e_exp; 1820 * already little endian, so no further change is required.
1821 */
1822 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1823 CCP_PASSTHRU_BYTESWAP_NOOP);
1824 if (ret) {
1825 cmd->engine_error = cmd_q->cmd_error;
1826 goto e_exp;
1827 }
1828 } else {
1829 /* The exponent can be retrieved from memory via DMA. */
1830 op.exp.u.dma.address = exp.dma.address;
1831 op.exp.u.dma.offset = 0;
1783 } 1832 }
1784 1833
1785 /* Concatenate the modulus and the message. Both the modulus and 1834 /* Concatenate the modulus and the message. Both the modulus and
@@ -1798,8 +1847,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1798 goto e_src; 1847 goto e_src;
1799 1848
1800 /* Prepare the output area for the operation */ 1849 /* Prepare the output area for the operation */
1801 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1850 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1802 o_len, DMA_FROM_DEVICE);
1803 if (ret) 1851 if (ret)
1804 goto e_src; 1852 goto e_src;
1805 1853
@@ -1807,7 +1855,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1807 op.src.u.dma.address = src.dma.address; 1855 op.src.u.dma.address = src.dma.address;
1808 op.src.u.dma.offset = 0; 1856 op.src.u.dma.offset = 0;
1809 op.src.u.dma.length = i_len; 1857 op.src.u.dma.length = i_len;
1810 op.dst.u.dma.address = dst.dm_wa.dma.address; 1858 op.dst.u.dma.address = dst.dma.address;
1811 op.dst.u.dma.offset = 0; 1859 op.dst.u.dma.offset = 0;
1812 op.dst.u.dma.length = o_len; 1860 op.dst.u.dma.length = o_len;
1813 1861
@@ -1820,10 +1868,10 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1820 goto e_dst; 1868 goto e_dst;
1821 } 1869 }
1822 1870
1823 ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len); 1871 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1824 1872
1825e_dst: 1873e_dst:
1826 ccp_free_data(&dst, cmd_q); 1874 ccp_dm_free(&dst);
1827 1875
1828e_src: 1876e_src:
1829 ccp_dm_free(&src); 1877 ccp_dm_free(&src);
@@ -1832,7 +1880,8 @@ e_exp:
1832 ccp_dm_free(&exp); 1880 ccp_dm_free(&exp);
1833 1881
1834e_sb: 1882e_sb:
1835 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1883 if (sb_count)
1884 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1836 1885
1837 return ret; 1886 return ret;
1838} 1887}
@@ -1992,7 +2041,7 @@ static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
1992 2041
1993 memset(&op, 0, sizeof(op)); 2042 memset(&op, 0, sizeof(op));
1994 op.cmd_q = cmd_q; 2043 op.cmd_q = cmd_q;
1995 op.jobid = ccp_gen_jobid(cmd_q->ccp); 2044 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1996 2045
1997 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2046 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1998 /* Load the mask */ 2047 /* Load the mask */
diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c
deleted file mode 100644
index e880d4cf4ada..000000000000
--- a/drivers/crypto/ccp/ccp-pci.c
+++ /dev/null
@@ -1,356 +0,0 @@
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/module.h>
15#include <linux/kernel.h>
16#include <linux/device.h>
17#include <linux/pci.h>
18#include <linux/pci_ids.h>
19#include <linux/dma-mapping.h>
20#include <linux/kthread.h>
21#include <linux/sched.h>
22#include <linux/interrupt.h>
23#include <linux/spinlock.h>
24#include <linux/delay.h>
25#include <linux/ccp.h>
26
27#include "ccp-dev.h"
28
29#define MSIX_VECTORS 2
30
31struct ccp_msix {
32 u32 vector;
33 char name[16];
34};
35
36struct ccp_pci {
37 int msix_count;
38 struct ccp_msix msix[MSIX_VECTORS];
39};
40
41static int ccp_get_msix_irqs(struct ccp_device *ccp)
42{
43 struct ccp_pci *ccp_pci = ccp->dev_specific;
44 struct device *dev = ccp->dev;
45 struct pci_dev *pdev = to_pci_dev(dev);
46 struct msix_entry msix_entry[MSIX_VECTORS];
47 unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1;
48 int v, ret;
49
50 for (v = 0; v < ARRAY_SIZE(msix_entry); v++)
51 msix_entry[v].entry = v;
52
53 ret = pci_enable_msix_range(pdev, msix_entry, 1, v);
54 if (ret < 0)
55 return ret;
56
57 ccp_pci->msix_count = ret;
58 for (v = 0; v < ccp_pci->msix_count; v++) {
59 /* Set the interrupt names and request the irqs */
60 snprintf(ccp_pci->msix[v].name, name_len, "%s-%u",
61 ccp->name, v);
62 ccp_pci->msix[v].vector = msix_entry[v].vector;
63 ret = request_irq(ccp_pci->msix[v].vector,
64 ccp->vdata->perform->irqhandler,
65 0, ccp_pci->msix[v].name, dev);
66 if (ret) {
67 dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n",
68 ret);
69 goto e_irq;
70 }
71 }
72 ccp->use_tasklet = true;
73
74 return 0;
75
76e_irq:
77 while (v--)
78 free_irq(ccp_pci->msix[v].vector, dev);
79
80 pci_disable_msix(pdev);
81
82 ccp_pci->msix_count = 0;
83
84 return ret;
85}
86
87static int ccp_get_msi_irq(struct ccp_device *ccp)
88{
89 struct device *dev = ccp->dev;
90 struct pci_dev *pdev = to_pci_dev(dev);
91 int ret;
92
93 ret = pci_enable_msi(pdev);
94 if (ret)
95 return ret;
96
97 ccp->irq = pdev->irq;
98 ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0,
99 ccp->name, dev);
100 if (ret) {
101 dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret);
102 goto e_msi;
103 }
104 ccp->use_tasklet = true;
105
106 return 0;
107
108e_msi:
109 pci_disable_msi(pdev);
110
111 return ret;
112}
113
114static int ccp_get_irqs(struct ccp_device *ccp)
115{
116 struct device *dev = ccp->dev;
117 int ret;
118
119 ret = ccp_get_msix_irqs(ccp);
120 if (!ret)
121 return 0;
122
123 /* Couldn't get MSI-X vectors, try MSI */
124 dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
125 ret = ccp_get_msi_irq(ccp);
126 if (!ret)
127 return 0;
128
129 /* Couldn't get MSI interrupt */
130 dev_notice(dev, "could not enable MSI (%d)\n", ret);
131
132 return ret;
133}
134
135static void ccp_free_irqs(struct ccp_device *ccp)
136{
137 struct ccp_pci *ccp_pci = ccp->dev_specific;
138 struct device *dev = ccp->dev;
139 struct pci_dev *pdev = to_pci_dev(dev);
140
141 if (ccp_pci->msix_count) {
142 while (ccp_pci->msix_count--)
143 free_irq(ccp_pci->msix[ccp_pci->msix_count].vector,
144 dev);
145 pci_disable_msix(pdev);
146 } else if (ccp->irq) {
147 free_irq(ccp->irq, dev);
148 pci_disable_msi(pdev);
149 }
150 ccp->irq = 0;
151}
152
153static int ccp_find_mmio_area(struct ccp_device *ccp)
154{
155 struct device *dev = ccp->dev;
156 struct pci_dev *pdev = to_pci_dev(dev);
157 resource_size_t io_len;
158 unsigned long io_flags;
159
160 io_flags = pci_resource_flags(pdev, ccp->vdata->bar);
161 io_len = pci_resource_len(pdev, ccp->vdata->bar);
162 if ((io_flags & IORESOURCE_MEM) &&
163 (io_len >= (ccp->vdata->offset + 0x800)))
164 return ccp->vdata->bar;
165
166 return -EIO;
167}
168
169static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
170{
171 struct ccp_device *ccp;
172 struct ccp_pci *ccp_pci;
173 struct device *dev = &pdev->dev;
174 unsigned int bar;
175 int ret;
176
177 ret = -ENOMEM;
178 ccp = ccp_alloc_struct(dev);
179 if (!ccp)
180 goto e_err;
181
182 ccp_pci = devm_kzalloc(dev, sizeof(*ccp_pci), GFP_KERNEL);
183 if (!ccp_pci)
184 goto e_err;
185
186 ccp->dev_specific = ccp_pci;
187 ccp->vdata = (struct ccp_vdata *)id->driver_data;
188 if (!ccp->vdata || !ccp->vdata->version) {
189 ret = -ENODEV;
190 dev_err(dev, "missing driver data\n");
191 goto e_err;
192 }
193 ccp->get_irq = ccp_get_irqs;
194 ccp->free_irq = ccp_free_irqs;
195
196 ret = pci_request_regions(pdev, "ccp");
197 if (ret) {
198 dev_err(dev, "pci_request_regions failed (%d)\n", ret);
199 goto e_err;
200 }
201
202 ret = pci_enable_device(pdev);
203 if (ret) {
204 dev_err(dev, "pci_enable_device failed (%d)\n", ret);
205 goto e_regions;
206 }
207
208 pci_set_master(pdev);
209
210 ret = ccp_find_mmio_area(ccp);
211 if (ret < 0)
212 goto e_device;
213 bar = ret;
214
215 ret = -EIO;
216 ccp->io_map = pci_iomap(pdev, bar, 0);
217 if (!ccp->io_map) {
218 dev_err(dev, "pci_iomap failed\n");
219 goto e_device;
220 }
221 ccp->io_regs = ccp->io_map + ccp->vdata->offset;
222
223 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
224 if (ret) {
225 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
226 if (ret) {
227 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
228 ret);
229 goto e_iomap;
230 }
231 }
232
233 dev_set_drvdata(dev, ccp);
234
235 if (ccp->vdata->setup)
236 ccp->vdata->setup(ccp);
237
238 ret = ccp->vdata->perform->init(ccp);
239 if (ret)
240 goto e_iomap;
241
242 dev_notice(dev, "enabled\n");
243
244 return 0;
245
246e_iomap:
247 pci_iounmap(pdev, ccp->io_map);
248
249e_device:
250 pci_disable_device(pdev);
251
252e_regions:
253 pci_release_regions(pdev);
254
255e_err:
256 dev_notice(dev, "initialization failed\n");
257 return ret;
258}
259
260static void ccp_pci_remove(struct pci_dev *pdev)
261{
262 struct device *dev = &pdev->dev;
263 struct ccp_device *ccp = dev_get_drvdata(dev);
264
265 if (!ccp)
266 return;
267
268 ccp->vdata->perform->destroy(ccp);
269
270 pci_iounmap(pdev, ccp->io_map);
271
272 pci_disable_device(pdev);
273
274 pci_release_regions(pdev);
275
276 dev_notice(dev, "disabled\n");
277}
278
279#ifdef CONFIG_PM
280static int ccp_pci_suspend(struct pci_dev *pdev, pm_message_t state)
281{
282 struct device *dev = &pdev->dev;
283 struct ccp_device *ccp = dev_get_drvdata(dev);
284 unsigned long flags;
285 unsigned int i;
286
287 spin_lock_irqsave(&ccp->cmd_lock, flags);
288
289 ccp->suspending = 1;
290
291 /* Wake all the queue kthreads to prepare for suspend */
292 for (i = 0; i < ccp->cmd_q_count; i++)
293 wake_up_process(ccp->cmd_q[i].kthread);
294
295 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
296
297 /* Wait for all queue kthreads to say they're done */
298 while (!ccp_queues_suspended(ccp))
299 wait_event_interruptible(ccp->suspend_queue,
300 ccp_queues_suspended(ccp));
301
302 return 0;
303}
304
305static int ccp_pci_resume(struct pci_dev *pdev)
306{
307 struct device *dev = &pdev->dev;
308 struct ccp_device *ccp = dev_get_drvdata(dev);
309 unsigned long flags;
310 unsigned int i;
311
312 spin_lock_irqsave(&ccp->cmd_lock, flags);
313
314 ccp->suspending = 0;
315
316 /* Wake up all the kthreads */
317 for (i = 0; i < ccp->cmd_q_count; i++) {
318 ccp->cmd_q[i].suspended = 0;
319 wake_up_process(ccp->cmd_q[i].kthread);
320 }
321
322 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
323
324 return 0;
325}
326#endif
327
328static const struct pci_device_id ccp_pci_table[] = {
329 { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&ccpv3 },
330 { PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&ccpv5a },
331 { PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&ccpv5b },
332 /* Last entry must be zero */
333 { 0, }
334};
335MODULE_DEVICE_TABLE(pci, ccp_pci_table);
336
337static struct pci_driver ccp_pci_driver = {
338 .name = "ccp",
339 .id_table = ccp_pci_table,
340 .probe = ccp_pci_probe,
341 .remove = ccp_pci_remove,
342#ifdef CONFIG_PM
343 .suspend = ccp_pci_suspend,
344 .resume = ccp_pci_resume,
345#endif
346};
347
348int ccp_pci_init(void)
349{
350 return pci_register_driver(&ccp_pci_driver);
351}
352
353void ccp_pci_exit(void)
354{
355 pci_unregister_driver(&ccp_pci_driver);
356}
diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c
deleted file mode 100644
index e26969e601ad..000000000000
--- a/drivers/crypto/ccp/ccp-platform.c
+++ /dev/null
@@ -1,293 +0,0 @@
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2014,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/device.h>
16#include <linux/platform_device.h>
17#include <linux/ioport.h>
18#include <linux/dma-mapping.h>
19#include <linux/kthread.h>
20#include <linux/sched.h>
21#include <linux/interrupt.h>
22#include <linux/spinlock.h>
23#include <linux/delay.h>
24#include <linux/ccp.h>
25#include <linux/of.h>
26#include <linux/of_address.h>
27#include <linux/acpi.h>
28
29#include "ccp-dev.h"
30
31struct ccp_platform {
32 int coherent;
33};
34
35static const struct acpi_device_id ccp_acpi_match[];
36static const struct of_device_id ccp_of_match[];
37
38static struct ccp_vdata *ccp_get_of_version(struct platform_device *pdev)
39{
40#ifdef CONFIG_OF
41 const struct of_device_id *match;
42
43 match = of_match_node(ccp_of_match, pdev->dev.of_node);
44 if (match && match->data)
45 return (struct ccp_vdata *)match->data;
46#endif
47 return NULL;
48}
49
50static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev)
51{
52#ifdef CONFIG_ACPI
53 const struct acpi_device_id *match;
54
55 match = acpi_match_device(ccp_acpi_match, &pdev->dev);
56 if (match && match->driver_data)
57 return (struct ccp_vdata *)match->driver_data;
58#endif
59 return NULL;
60}
61
62static int ccp_get_irq(struct ccp_device *ccp)
63{
64 struct device *dev = ccp->dev;
65 struct platform_device *pdev = to_platform_device(dev);
66 int ret;
67
68 ret = platform_get_irq(pdev, 0);
69 if (ret < 0)
70 return ret;
71
72 ccp->irq = ret;
73 ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0,
74 ccp->name, dev);
75 if (ret) {
76 dev_notice(dev, "unable to allocate IRQ (%d)\n", ret);
77 return ret;
78 }
79
80 return 0;
81}
82
83static int ccp_get_irqs(struct ccp_device *ccp)
84{
85 struct device *dev = ccp->dev;
86 int ret;
87
88 ret = ccp_get_irq(ccp);
89 if (!ret)
90 return 0;
91
92 /* Couldn't get an interrupt */
93 dev_notice(dev, "could not enable interrupts (%d)\n", ret);
94
95 return ret;
96}
97
98static void ccp_free_irqs(struct ccp_device *ccp)
99{
100 struct device *dev = ccp->dev;
101
102 free_irq(ccp->irq, dev);
103}
104
105static struct resource *ccp_find_mmio_area(struct ccp_device *ccp)
106{
107 struct device *dev = ccp->dev;
108 struct platform_device *pdev = to_platform_device(dev);
109 struct resource *ior;
110
111 ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
112 if (ior && (resource_size(ior) >= 0x800))
113 return ior;
114
115 return NULL;
116}
117
118static int ccp_platform_probe(struct platform_device *pdev)
119{
120 struct ccp_device *ccp;
121 struct ccp_platform *ccp_platform;
122 struct device *dev = &pdev->dev;
123 enum dev_dma_attr attr;
124 struct resource *ior;
125 int ret;
126
127 ret = -ENOMEM;
128 ccp = ccp_alloc_struct(dev);
129 if (!ccp)
130 goto e_err;
131
132 ccp_platform = devm_kzalloc(dev, sizeof(*ccp_platform), GFP_KERNEL);
133 if (!ccp_platform)
134 goto e_err;
135
136 ccp->dev_specific = ccp_platform;
137 ccp->vdata = pdev->dev.of_node ? ccp_get_of_version(pdev)
138 : ccp_get_acpi_version(pdev);
139 if (!ccp->vdata || !ccp->vdata->version) {
140 ret = -ENODEV;
141 dev_err(dev, "missing driver data\n");
142 goto e_err;
143 }
144 ccp->get_irq = ccp_get_irqs;
145 ccp->free_irq = ccp_free_irqs;
146
147 ior = ccp_find_mmio_area(ccp);
148 ccp->io_map = devm_ioremap_resource(dev, ior);
149 if (IS_ERR(ccp->io_map)) {
150 ret = PTR_ERR(ccp->io_map);
151 goto e_err;
152 }
153 ccp->io_regs = ccp->io_map;
154
155 attr = device_get_dma_attr(dev);
156 if (attr == DEV_DMA_NOT_SUPPORTED) {
157 dev_err(dev, "DMA is not supported");
158 goto e_err;
159 }
160
161 ccp_platform->coherent = (attr == DEV_DMA_COHERENT);
162 if (ccp_platform->coherent)
163 ccp->axcache = CACHE_WB_NO_ALLOC;
164 else
165 ccp->axcache = CACHE_NONE;
166
167 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
168 if (ret) {
169 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
170 goto e_err;
171 }
172
173 dev_set_drvdata(dev, ccp);
174
175 ret = ccp->vdata->perform->init(ccp);
176 if (ret)
177 goto e_err;
178
179 dev_notice(dev, "enabled\n");
180
181 return 0;
182
183e_err:
184 dev_notice(dev, "initialization failed\n");
185 return ret;
186}
187
188static int ccp_platform_remove(struct platform_device *pdev)
189{
190 struct device *dev = &pdev->dev;
191 struct ccp_device *ccp = dev_get_drvdata(dev);
192
193 ccp->vdata->perform->destroy(ccp);
194
195 dev_notice(dev, "disabled\n");
196
197 return 0;
198}
199
200#ifdef CONFIG_PM
201static int ccp_platform_suspend(struct platform_device *pdev,
202 pm_message_t state)
203{
204 struct device *dev = &pdev->dev;
205 struct ccp_device *ccp = dev_get_drvdata(dev);
206 unsigned long flags;
207 unsigned int i;
208
209 spin_lock_irqsave(&ccp->cmd_lock, flags);
210
211 ccp->suspending = 1;
212
213 /* Wake all the queue kthreads to prepare for suspend */
214 for (i = 0; i < ccp->cmd_q_count; i++)
215 wake_up_process(ccp->cmd_q[i].kthread);
216
217 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
218
219 /* Wait for all queue kthreads to say they're done */
220 while (!ccp_queues_suspended(ccp))
221 wait_event_interruptible(ccp->suspend_queue,
222 ccp_queues_suspended(ccp));
223
224 return 0;
225}
226
227static int ccp_platform_resume(struct platform_device *pdev)
228{
229 struct device *dev = &pdev->dev;
230 struct ccp_device *ccp = dev_get_drvdata(dev);
231 unsigned long flags;
232 unsigned int i;
233
234 spin_lock_irqsave(&ccp->cmd_lock, flags);
235
236 ccp->suspending = 0;
237
238 /* Wake up all the kthreads */
239 for (i = 0; i < ccp->cmd_q_count; i++) {
240 ccp->cmd_q[i].suspended = 0;
241 wake_up_process(ccp->cmd_q[i].kthread);
242 }
243
244 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
245
246 return 0;
247}
248#endif
249
250#ifdef CONFIG_ACPI
251static const struct acpi_device_id ccp_acpi_match[] = {
252 { "AMDI0C00", (kernel_ulong_t)&ccpv3 },
253 { },
254};
255MODULE_DEVICE_TABLE(acpi, ccp_acpi_match);
256#endif
257
258#ifdef CONFIG_OF
259static const struct of_device_id ccp_of_match[] = {
260 { .compatible = "amd,ccp-seattle-v1a",
261 .data = (const void *)&ccpv3 },
262 { },
263};
264MODULE_DEVICE_TABLE(of, ccp_of_match);
265#endif
266
267static struct platform_driver ccp_platform_driver = {
268 .driver = {
269 .name = "ccp",
270#ifdef CONFIG_ACPI
271 .acpi_match_table = ccp_acpi_match,
272#endif
273#ifdef CONFIG_OF
274 .of_match_table = ccp_of_match,
275#endif
276 },
277 .probe = ccp_platform_probe,
278 .remove = ccp_platform_remove,
279#ifdef CONFIG_PM
280 .suspend = ccp_platform_suspend,
281 .resume = ccp_platform_resume,
282#endif
283};
284
285int ccp_platform_init(void)
286{
287 return platform_driver_register(&ccp_platform_driver);
288}
289
290void ccp_platform_exit(void)
291{
292 platform_driver_unregister(&ccp_platform_driver);
293}
diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c
new file mode 100644
index 000000000000..bef387c8abfd
--- /dev/null
+++ b/drivers/crypto/ccp/sp-dev.c
@@ -0,0 +1,277 @@
1/*
2 * AMD Secure Processor driver
3 *
4 * Copyright (C) 2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 * Author: Brijesh Singh <brijesh.singh@amd.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 */
14
15#include <linux/module.h>
16#include <linux/kernel.h>
17#include <linux/kthread.h>
18#include <linux/sched.h>
19#include <linux/interrupt.h>
20#include <linux/spinlock.h>
21#include <linux/spinlock_types.h>
22#include <linux/types.h>
23#include <linux/ccp.h>
24
25#include "ccp-dev.h"
26#include "sp-dev.h"
27
28MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
29MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>");
30MODULE_LICENSE("GPL");
31MODULE_VERSION("1.1.0");
32MODULE_DESCRIPTION("AMD Secure Processor driver");
33
34/* List of SPs, SP count, read-write access lock, and access functions
35 *
36 * Lock structure: get sp_unit_lock for reading whenever we need to
37 * examine the SP list.
38 */
39static DEFINE_RWLOCK(sp_unit_lock);
40static LIST_HEAD(sp_units);
41
42/* Ever-increasing value to produce unique unit numbers */
43static atomic_t sp_ordinal;
44
45static void sp_add_device(struct sp_device *sp)
46{
47 unsigned long flags;
48
49 write_lock_irqsave(&sp_unit_lock, flags);
50
51 list_add_tail(&sp->entry, &sp_units);
52
53 write_unlock_irqrestore(&sp_unit_lock, flags);
54}
55
56static void sp_del_device(struct sp_device *sp)
57{
58 unsigned long flags;
59
60 write_lock_irqsave(&sp_unit_lock, flags);
61
62 list_del(&sp->entry);
63
64 write_unlock_irqrestore(&sp_unit_lock, flags);
65}
66
67static irqreturn_t sp_irq_handler(int irq, void *data)
68{
69 struct sp_device *sp = data;
70
71 if (sp->ccp_irq_handler)
72 sp->ccp_irq_handler(irq, sp->ccp_irq_data);
73
74 if (sp->psp_irq_handler)
75 sp->psp_irq_handler(irq, sp->psp_irq_data);
76
77 return IRQ_HANDLED;
78}
79
80int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler,
81 const char *name, void *data)
82{
83 int ret;
84
85 if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) {
86 /* Need a common routine to manage all interrupts */
87 sp->ccp_irq_data = data;
88 sp->ccp_irq_handler = handler;
89
90 if (!sp->irq_registered) {
91 ret = request_irq(sp->ccp_irq, sp_irq_handler, 0,
92 sp->name, sp);
93 if (ret)
94 return ret;
95
96 sp->irq_registered = true;
97 }
98 } else {
99 /* Each sub-device can manage it's own interrupt */
100 ret = request_irq(sp->ccp_irq, handler, 0, name, data);
101 if (ret)
102 return ret;
103 }
104
105 return 0;
106}
107
108int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler,
109 const char *name, void *data)
110{
111 int ret;
112
113 if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) {
114 /* Need a common routine to manage all interrupts */
115 sp->psp_irq_data = data;
116 sp->psp_irq_handler = handler;
117
118 if (!sp->irq_registered) {
119 ret = request_irq(sp->psp_irq, sp_irq_handler, 0,
120 sp->name, sp);
121 if (ret)
122 return ret;
123
124 sp->irq_registered = true;
125 }
126 } else {
127 /* Each sub-device can manage it's own interrupt */
128 ret = request_irq(sp->psp_irq, handler, 0, name, data);
129 if (ret)
130 return ret;
131 }
132
133 return 0;
134}
135
136void sp_free_ccp_irq(struct sp_device *sp, void *data)
137{
138 if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) {
139 /* Using common routine to manage all interrupts */
140 if (!sp->psp_irq_handler) {
141 /* Nothing else using it, so free it */
142 free_irq(sp->ccp_irq, sp);
143
144 sp->irq_registered = false;
145 }
146
147 sp->ccp_irq_handler = NULL;
148 sp->ccp_irq_data = NULL;
149 } else {
150 /* Each sub-device can manage it's own interrupt */
151 free_irq(sp->ccp_irq, data);
152 }
153}
154
155void sp_free_psp_irq(struct sp_device *sp, void *data)
156{
157 if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) {
158 /* Using common routine to manage all interrupts */
159 if (!sp->ccp_irq_handler) {
160 /* Nothing else using it, so free it */
161 free_irq(sp->psp_irq, sp);
162
163 sp->irq_registered = false;
164 }
165
166 sp->psp_irq_handler = NULL;
167 sp->psp_irq_data = NULL;
168 } else {
169 /* Each sub-device can manage it's own interrupt */
170 free_irq(sp->psp_irq, data);
171 }
172}
173
174/**
175 * sp_alloc_struct - allocate and initialize the sp_device struct
176 *
177 * @dev: device struct of the SP
178 */
179struct sp_device *sp_alloc_struct(struct device *dev)
180{
181 struct sp_device *sp;
182
183 sp = devm_kzalloc(dev, sizeof(*sp), GFP_KERNEL);
184 if (!sp)
185 return NULL;
186
187 sp->dev = dev;
188 sp->ord = atomic_inc_return(&sp_ordinal);
189 snprintf(sp->name, SP_MAX_NAME_LEN, "sp-%u", sp->ord);
190
191 return sp;
192}
193
194int sp_init(struct sp_device *sp)
195{
196 sp_add_device(sp);
197
198 if (sp->dev_vdata->ccp_vdata)
199 ccp_dev_init(sp);
200
201 return 0;
202}
203
204void sp_destroy(struct sp_device *sp)
205{
206 if (sp->dev_vdata->ccp_vdata)
207 ccp_dev_destroy(sp);
208
209 sp_del_device(sp);
210}
211
212#ifdef CONFIG_PM
213int sp_suspend(struct sp_device *sp, pm_message_t state)
214{
215 int ret;
216
217 if (sp->dev_vdata->ccp_vdata) {
218 ret = ccp_dev_suspend(sp, state);
219 if (ret)
220 return ret;
221 }
222
223 return 0;
224}
225
226int sp_resume(struct sp_device *sp)
227{
228 int ret;
229
230 if (sp->dev_vdata->ccp_vdata) {
231 ret = ccp_dev_resume(sp);
232 if (ret)
233 return ret;
234 }
235
236 return 0;
237}
238#endif
239
240static int __init sp_mod_init(void)
241{
242#ifdef CONFIG_X86
243 int ret;
244
245 ret = sp_pci_init();
246 if (ret)
247 return ret;
248
249 return 0;
250#endif
251
252#ifdef CONFIG_ARM64
253 int ret;
254
255 ret = sp_platform_init();
256 if (ret)
257 return ret;
258
259 return 0;
260#endif
261
262 return -ENODEV;
263}
264
265static void __exit sp_mod_exit(void)
266{
267#ifdef CONFIG_X86
268 sp_pci_exit();
269#endif
270
271#ifdef CONFIG_ARM64
272 sp_platform_exit();
273#endif
274}
275
276module_init(sp_mod_init);
277module_exit(sp_mod_exit);
diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h
new file mode 100644
index 000000000000..5ab486ade1ad
--- /dev/null
+++ b/drivers/crypto/ccp/sp-dev.h
@@ -0,0 +1,133 @@
1/*
2 * AMD Secure Processor driver
3 *
4 * Copyright (C) 2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 * Author: Brijesh Singh <brijesh.singh@amd.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 */
14
15#ifndef __SP_DEV_H__
16#define __SP_DEV_H__
17
18#include <linux/device.h>
19#include <linux/pci.h>
20#include <linux/spinlock.h>
21#include <linux/mutex.h>
22#include <linux/list.h>
23#include <linux/wait.h>
24#include <linux/dmapool.h>
25#include <linux/hw_random.h>
26#include <linux/bitops.h>
27#include <linux/interrupt.h>
28#include <linux/irqreturn.h>
29
30#define SP_MAX_NAME_LEN 32
31
32#define CACHE_NONE 0x00
33#define CACHE_WB_NO_ALLOC 0xb7
34
35/* Structure to hold CCP device data */
36struct ccp_device;
37struct ccp_vdata {
38 const unsigned int version;
39 const unsigned int dma_chan_attr;
40 void (*setup)(struct ccp_device *);
41 const struct ccp_actions *perform;
42 const unsigned int offset;
43 const unsigned int rsamax;
44};
45/* Structure to hold SP device data */
46struct sp_dev_vdata {
47 const unsigned int bar;
48
49 const struct ccp_vdata *ccp_vdata;
50 void *psp_vdata;
51};
52
53struct sp_device {
54 struct list_head entry;
55
56 struct device *dev;
57
58 struct sp_dev_vdata *dev_vdata;
59 unsigned int ord;
60 char name[SP_MAX_NAME_LEN];
61
62 /* Bus specific device information */
63 void *dev_specific;
64
65 /* I/O area used for device communication. */
66 void __iomem *io_map;
67
68 /* DMA caching attribute support */
69 unsigned int axcache;
70
71 bool irq_registered;
72 bool use_tasklet;
73
74 unsigned int ccp_irq;
75 irq_handler_t ccp_irq_handler;
76 void *ccp_irq_data;
77
78 unsigned int psp_irq;
79 irq_handler_t psp_irq_handler;
80 void *psp_irq_data;
81
82 void *ccp_data;
83 void *psp_data;
84};
85
86int sp_pci_init(void);
87void sp_pci_exit(void);
88
89int sp_platform_init(void);
90void sp_platform_exit(void);
91
92struct sp_device *sp_alloc_struct(struct device *dev);
93
94int sp_init(struct sp_device *sp);
95void sp_destroy(struct sp_device *sp);
96struct sp_device *sp_get_master(void);
97
98int sp_suspend(struct sp_device *sp, pm_message_t state);
99int sp_resume(struct sp_device *sp);
100int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler,
101 const char *name, void *data);
102void sp_free_ccp_irq(struct sp_device *sp, void *data);
103int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler,
104 const char *name, void *data);
105void sp_free_psp_irq(struct sp_device *sp, void *data);
106
107#ifdef CONFIG_CRYPTO_DEV_SP_CCP
108
109int ccp_dev_init(struct sp_device *sp);
110void ccp_dev_destroy(struct sp_device *sp);
111
112int ccp_dev_suspend(struct sp_device *sp, pm_message_t state);
113int ccp_dev_resume(struct sp_device *sp);
114
115#else /* !CONFIG_CRYPTO_DEV_SP_CCP */
116
117static inline int ccp_dev_init(struct sp_device *sp)
118{
119 return 0;
120}
121static inline void ccp_dev_destroy(struct sp_device *sp) { }
122
123static inline int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
124{
125 return 0;
126}
127static inline int ccp_dev_resume(struct sp_device *sp)
128{
129 return 0;
130}
131#endif /* CONFIG_CRYPTO_DEV_SP_CCP */
132
133#endif
diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c
new file mode 100644
index 000000000000..9859aa683a28
--- /dev/null
+++ b/drivers/crypto/ccp/sp-pci.c
@@ -0,0 +1,276 @@
1/*
2 * AMD Secure Processor device driver
3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/module.h>
15#include <linux/kernel.h>
16#include <linux/device.h>
17#include <linux/pci.h>
18#include <linux/pci_ids.h>
19#include <linux/dma-mapping.h>
20#include <linux/kthread.h>
21#include <linux/sched.h>
22#include <linux/interrupt.h>
23#include <linux/spinlock.h>
24#include <linux/delay.h>
25#include <linux/ccp.h>
26
27#include "ccp-dev.h"
28
29#define MSIX_VECTORS 2
30
31struct sp_pci {
32 int msix_count;
33 struct msix_entry msix_entry[MSIX_VECTORS];
34};
35
36static int sp_get_msix_irqs(struct sp_device *sp)
37{
38 struct sp_pci *sp_pci = sp->dev_specific;
39 struct device *dev = sp->dev;
40 struct pci_dev *pdev = to_pci_dev(dev);
41 int v, ret;
42
43 for (v = 0; v < ARRAY_SIZE(sp_pci->msix_entry); v++)
44 sp_pci->msix_entry[v].entry = v;
45
46 ret = pci_enable_msix_range(pdev, sp_pci->msix_entry, 1, v);
47 if (ret < 0)
48 return ret;
49
50 sp_pci->msix_count = ret;
51 sp->use_tasklet = true;
52
53 sp->psp_irq = sp_pci->msix_entry[0].vector;
54 sp->ccp_irq = (sp_pci->msix_count > 1) ? sp_pci->msix_entry[1].vector
55 : sp_pci->msix_entry[0].vector;
56 return 0;
57}
58
59static int sp_get_msi_irq(struct sp_device *sp)
60{
61 struct device *dev = sp->dev;
62 struct pci_dev *pdev = to_pci_dev(dev);
63 int ret;
64
65 ret = pci_enable_msi(pdev);
66 if (ret)
67 return ret;
68
69 sp->ccp_irq = pdev->irq;
70 sp->psp_irq = pdev->irq;
71
72 return 0;
73}
74
75static int sp_get_irqs(struct sp_device *sp)
76{
77 struct device *dev = sp->dev;
78 int ret;
79
80 ret = sp_get_msix_irqs(sp);
81 if (!ret)
82 return 0;
83
84 /* Couldn't get MSI-X vectors, try MSI */
85 dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
86 ret = sp_get_msi_irq(sp);
87 if (!ret)
88 return 0;
89
90 /* Couldn't get MSI interrupt */
91 dev_notice(dev, "could not enable MSI (%d)\n", ret);
92
93 return ret;
94}
95
96static void sp_free_irqs(struct sp_device *sp)
97{
98 struct sp_pci *sp_pci = sp->dev_specific;
99 struct device *dev = sp->dev;
100 struct pci_dev *pdev = to_pci_dev(dev);
101
102 if (sp_pci->msix_count)
103 pci_disable_msix(pdev);
104 else if (sp->psp_irq)
105 pci_disable_msi(pdev);
106
107 sp->ccp_irq = 0;
108 sp->psp_irq = 0;
109}
110
111static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
112{
113 struct sp_device *sp;
114 struct sp_pci *sp_pci;
115 struct device *dev = &pdev->dev;
116 void __iomem * const *iomap_table;
117 int bar_mask;
118 int ret;
119
120 ret = -ENOMEM;
121 sp = sp_alloc_struct(dev);
122 if (!sp)
123 goto e_err;
124
125 sp_pci = devm_kzalloc(dev, sizeof(*sp_pci), GFP_KERNEL);
126 if (!sp_pci)
127 goto e_err;
128
129 sp->dev_specific = sp_pci;
130 sp->dev_vdata = (struct sp_dev_vdata *)id->driver_data;
131 if (!sp->dev_vdata) {
132 ret = -ENODEV;
133 dev_err(dev, "missing driver data\n");
134 goto e_err;
135 }
136
137 ret = pcim_enable_device(pdev);
138 if (ret) {
139 dev_err(dev, "pcim_enable_device failed (%d)\n", ret);
140 goto e_err;
141 }
142
143 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
144 ret = pcim_iomap_regions(pdev, bar_mask, "ccp");
145 if (ret) {
146 dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret);
147 goto e_err;
148 }
149
150 iomap_table = pcim_iomap_table(pdev);
151 if (!iomap_table) {
152 dev_err(dev, "pcim_iomap_table failed\n");
153 ret = -ENOMEM;
154 goto e_err;
155 }
156
157 sp->io_map = iomap_table[sp->dev_vdata->bar];
158 if (!sp->io_map) {
159 dev_err(dev, "ioremap failed\n");
160 ret = -ENOMEM;
161 goto e_err;
162 }
163
164 ret = sp_get_irqs(sp);
165 if (ret)
166 goto e_err;
167
168 pci_set_master(pdev);
169
170 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
171 if (ret) {
172 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
173 if (ret) {
174 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
175 ret);
176 goto e_err;
177 }
178 }
179
180 dev_set_drvdata(dev, sp);
181
182 ret = sp_init(sp);
183 if (ret)
184 goto e_err;
185
186 dev_notice(dev, "enabled\n");
187
188 return 0;
189
190e_err:
191 dev_notice(dev, "initialization failed\n");
192 return ret;
193}
194
195static void sp_pci_remove(struct pci_dev *pdev)
196{
197 struct device *dev = &pdev->dev;
198 struct sp_device *sp = dev_get_drvdata(dev);
199
200 if (!sp)
201 return;
202
203 sp_destroy(sp);
204
205 sp_free_irqs(sp);
206
207 dev_notice(dev, "disabled\n");
208}
209
210#ifdef CONFIG_PM
211static int sp_pci_suspend(struct pci_dev *pdev, pm_message_t state)
212{
213 struct device *dev = &pdev->dev;
214 struct sp_device *sp = dev_get_drvdata(dev);
215
216 return sp_suspend(sp, state);
217}
218
219static int sp_pci_resume(struct pci_dev *pdev)
220{
221 struct device *dev = &pdev->dev;
222 struct sp_device *sp = dev_get_drvdata(dev);
223
224 return sp_resume(sp);
225}
226#endif
227
228static const struct sp_dev_vdata dev_vdata[] = {
229 {
230 .bar = 2,
231#ifdef CONFIG_CRYPTO_DEV_SP_CCP
232 .ccp_vdata = &ccpv3,
233#endif
234 },
235 {
236 .bar = 2,
237#ifdef CONFIG_CRYPTO_DEV_SP_CCP
238 .ccp_vdata = &ccpv5a,
239#endif
240 },
241 {
242 .bar = 2,
243#ifdef CONFIG_CRYPTO_DEV_SP_CCP
244 .ccp_vdata = &ccpv5b,
245#endif
246 },
247};
248static const struct pci_device_id sp_pci_table[] = {
249 { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] },
250 { PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] },
251 { PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] },
252 /* Last entry must be zero */
253 { 0, }
254};
255MODULE_DEVICE_TABLE(pci, sp_pci_table);
256
257static struct pci_driver sp_pci_driver = {
258 .name = "ccp",
259 .id_table = sp_pci_table,
260 .probe = sp_pci_probe,
261 .remove = sp_pci_remove,
262#ifdef CONFIG_PM
263 .suspend = sp_pci_suspend,
264 .resume = sp_pci_resume,
265#endif
266};
267
268int sp_pci_init(void)
269{
270 return pci_register_driver(&sp_pci_driver);
271}
272
273void sp_pci_exit(void)
274{
275 pci_unregister_driver(&sp_pci_driver);
276}
diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c
new file mode 100644
index 000000000000..71734f254fd1
--- /dev/null
+++ b/drivers/crypto/ccp/sp-platform.c
@@ -0,0 +1,256 @@
1/*
2 * AMD Secure Processor device driver
3 *
4 * Copyright (C) 2014,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/device.h>
16#include <linux/platform_device.h>
17#include <linux/ioport.h>
18#include <linux/dma-mapping.h>
19#include <linux/kthread.h>
20#include <linux/sched.h>
21#include <linux/interrupt.h>
22#include <linux/spinlock.h>
23#include <linux/delay.h>
24#include <linux/ccp.h>
25#include <linux/of.h>
26#include <linux/of_address.h>
27#include <linux/acpi.h>
28
29#include "ccp-dev.h"
30
31struct sp_platform {
32 int coherent;
33 unsigned int irq_count;
34};
35
36static const struct acpi_device_id sp_acpi_match[];
37static const struct of_device_id sp_of_match[];
38
39static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev)
40{
41#ifdef CONFIG_OF
42 const struct of_device_id *match;
43
44 match = of_match_node(sp_of_match, pdev->dev.of_node);
45 if (match && match->data)
46 return (struct sp_dev_vdata *)match->data;
47#endif
48 return NULL;
49}
50
51static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev)
52{
53#ifdef CONFIG_ACPI
54 const struct acpi_device_id *match;
55
56 match = acpi_match_device(sp_acpi_match, &pdev->dev);
57 if (match && match->driver_data)
58 return (struct sp_dev_vdata *)match->driver_data;
59#endif
60 return NULL;
61}
62
63static int sp_get_irqs(struct sp_device *sp)
64{
65 struct sp_platform *sp_platform = sp->dev_specific;
66 struct device *dev = sp->dev;
67 struct platform_device *pdev = to_platform_device(dev);
68 unsigned int i, count;
69 int ret;
70
71 for (i = 0, count = 0; i < pdev->num_resources; i++) {
72 struct resource *res = &pdev->resource[i];
73
74 if (resource_type(res) == IORESOURCE_IRQ)
75 count++;
76 }
77
78 sp_platform->irq_count = count;
79
80 ret = platform_get_irq(pdev, 0);
81 if (ret < 0) {
82 dev_notice(dev, "unable to get IRQ (%d)\n", ret);
83 return ret;
84 }
85
86 sp->psp_irq = ret;
87 if (count == 1) {
88 sp->ccp_irq = ret;
89 } else {
90 ret = platform_get_irq(pdev, 1);
91 if (ret < 0) {
92 dev_notice(dev, "unable to get IRQ (%d)\n", ret);
93 return ret;
94 }
95
96 sp->ccp_irq = ret;
97 }
98
99 return 0;
100}
101
102static int sp_platform_probe(struct platform_device *pdev)
103{
104 struct sp_device *sp;
105 struct sp_platform *sp_platform;
106 struct device *dev = &pdev->dev;
107 enum dev_dma_attr attr;
108 struct resource *ior;
109 int ret;
110
111 ret = -ENOMEM;
112 sp = sp_alloc_struct(dev);
113 if (!sp)
114 goto e_err;
115
116 sp_platform = devm_kzalloc(dev, sizeof(*sp_platform), GFP_KERNEL);
117 if (!sp_platform)
118 goto e_err;
119
120 sp->dev_specific = sp_platform;
121 sp->dev_vdata = pdev->dev.of_node ? sp_get_of_version(pdev)
122 : sp_get_acpi_version(pdev);
123 if (!sp->dev_vdata) {
124 ret = -ENODEV;
125 dev_err(dev, "missing driver data\n");
126 goto e_err;
127 }
128
129 ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
130 sp->io_map = devm_ioremap_resource(dev, ior);
131 if (IS_ERR(sp->io_map)) {
132 ret = PTR_ERR(sp->io_map);
133 goto e_err;
134 }
135
136 attr = device_get_dma_attr(dev);
137 if (attr == DEV_DMA_NOT_SUPPORTED) {
138 dev_err(dev, "DMA is not supported");
139 goto e_err;
140 }
141
142 sp_platform->coherent = (attr == DEV_DMA_COHERENT);
143 if (sp_platform->coherent)
144 sp->axcache = CACHE_WB_NO_ALLOC;
145 else
146 sp->axcache = CACHE_NONE;
147
148 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
149 if (ret) {
150 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
151 goto e_err;
152 }
153
154 ret = sp_get_irqs(sp);
155 if (ret)
156 goto e_err;
157
158 dev_set_drvdata(dev, sp);
159
160 ret = sp_init(sp);
161 if (ret)
162 goto e_err;
163
164 dev_notice(dev, "enabled\n");
165
166 return 0;
167
168e_err:
169 dev_notice(dev, "initialization failed\n");
170 return ret;
171}
172
173static int sp_platform_remove(struct platform_device *pdev)
174{
175 struct device *dev = &pdev->dev;
176 struct sp_device *sp = dev_get_drvdata(dev);
177
178 sp_destroy(sp);
179
180 dev_notice(dev, "disabled\n");
181
182 return 0;
183}
184
185#ifdef CONFIG_PM
186static int sp_platform_suspend(struct platform_device *pdev,
187 pm_message_t state)
188{
189 struct device *dev = &pdev->dev;
190 struct sp_device *sp = dev_get_drvdata(dev);
191
192 return sp_suspend(sp, state);
193}
194
195static int sp_platform_resume(struct platform_device *pdev)
196{
197 struct device *dev = &pdev->dev;
198 struct sp_device *sp = dev_get_drvdata(dev);
199
200 return sp_resume(sp);
201}
202#endif
203
204static const struct sp_dev_vdata dev_vdata[] = {
205 {
206 .bar = 0,
207#ifdef CONFIG_CRYPTO_DEV_SP_CCP
208 .ccp_vdata = &ccpv3_platform,
209#endif
210 },
211};
212
213#ifdef CONFIG_ACPI
214static const struct acpi_device_id sp_acpi_match[] = {
215 { "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] },
216 { },
217};
218MODULE_DEVICE_TABLE(acpi, sp_acpi_match);
219#endif
220
221#ifdef CONFIG_OF
222static const struct of_device_id sp_of_match[] = {
223 { .compatible = "amd,ccp-seattle-v1a",
224 .data = (const void *)&dev_vdata[0] },
225 { },
226};
227MODULE_DEVICE_TABLE(of, sp_of_match);
228#endif
229
230static struct platform_driver sp_platform_driver = {
231 .driver = {
232 .name = "ccp",
233#ifdef CONFIG_ACPI
234 .acpi_match_table = sp_acpi_match,
235#endif
236#ifdef CONFIG_OF
237 .of_match_table = sp_of_match,
238#endif
239 },
240 .probe = sp_platform_probe,
241 .remove = sp_platform_remove,
242#ifdef CONFIG_PM
243 .suspend = sp_platform_suspend,
244 .resume = sp_platform_resume,
245#endif
246};
247
248int sp_platform_init(void)
249{
250 return platform_driver_register(&sp_platform_driver);
251}
252
253void sp_platform_exit(void)
254{
255 platform_driver_unregister(&sp_platform_driver);
256}
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
index fe538e5287a5..eb2a0a73cbed 100644
--- a/drivers/crypto/geode-aes.c
+++ b/drivers/crypto/geode-aes.c
@@ -1,10 +1,10 @@
1 /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc. 1 /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
2 * 2 *
3 * This program is free software; you can redistribute it and/or modify 3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by 4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or 5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version. 6 * (at your option) any later version.
7 */ 7 */
8 8
9#include <linux/module.h> 9#include <linux/module.h>
10#include <linux/kernel.h> 10#include <linux/kernel.h>
@@ -30,6 +30,7 @@ static inline void
30_writefield(u32 offset, void *value) 30_writefield(u32 offset, void *value)
31{ 31{
32 int i; 32 int i;
33
33 for (i = 0; i < 4; i++) 34 for (i = 0; i < 4; i++)
34 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4)); 35 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
35} 36}
@@ -39,6 +40,7 @@ static inline void
39_readfield(u32 offset, void *value) 40_readfield(u32 offset, void *value)
40{ 41{
41 int i; 42 int i;
43
42 for (i = 0; i < 4; i++) 44 for (i = 0; i < 4; i++)
43 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4)); 45 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
44} 46}
@@ -515,6 +517,7 @@ static void geode_aes_remove(struct pci_dev *dev)
515static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) 517static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
516{ 518{
517 int ret; 519 int ret;
520
518 ret = pci_enable_device(dev); 521 ret = pci_enable_device(dev);
519 if (ret) 522 if (ret)
520 return ret; 523 return ret;
@@ -570,7 +573,7 @@ static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
570} 573}
571 574
572static struct pci_device_id geode_aes_tbl[] = { 575static struct pci_device_id geode_aes_tbl[] = {
573 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), } , 576 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), },
574 { 0, } 577 { 0, }
575}; 578};
576 579
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index 0c6a917a9ab8..b87000a0a01c 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -1054,7 +1054,7 @@ res_err:
1054 1054
1055static int img_hash_remove(struct platform_device *pdev) 1055static int img_hash_remove(struct platform_device *pdev)
1056{ 1056{
1057 static struct img_hash_dev *hdev; 1057 struct img_hash_dev *hdev;
1058 1058
1059 hdev = platform_get_drvdata(pdev); 1059 hdev = platform_get_drvdata(pdev);
1060 spin_lock(&img_hash.lock); 1060 spin_lock(&img_hash.lock);
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 1fabd4aee81b..89ba9e85c0f3 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -839,9 +839,10 @@ static int safexcel_probe(struct platform_device *pdev)
839 snprintf(irq_name, 6, "ring%d", i); 839 snprintf(irq_name, 6, "ring%d", i);
840 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, 840 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
841 ring_irq); 841 ring_irq);
842 842 if (irq < 0) {
843 if (irq < 0) 843 ret = irq;
844 goto err_clk; 844 goto err_clk;
845 }
845 846
846 priv->ring[i].work_data.priv = priv; 847 priv->ring[i].work_data.priv = priv;
847 priv->ring[i].work_data.ring = i; 848 priv->ring[i].work_data.ring = i;
diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c
index 000b6500a22d..b182e941b0cd 100644
--- a/drivers/crypto/mediatek/mtk-platform.c
+++ b/drivers/crypto/mediatek/mtk-platform.c
@@ -500,7 +500,7 @@ static int mtk_crypto_probe(struct platform_device *pdev)
500 cryp->irq[i] = platform_get_irq(pdev, i); 500 cryp->irq[i] = platform_get_irq(pdev, i);
501 if (cryp->irq[i] < 0) { 501 if (cryp->irq[i] < 0) {
502 dev_err(cryp->dev, "no IRQ:%d resource info\n", i); 502 dev_err(cryp->dev, "no IRQ:%d resource info\n", i);
503 return -ENXIO; 503 return cryp->irq[i];
504 } 504 }
505 } 505 }
506 506
diff --git a/drivers/crypto/mxc-scc.c b/drivers/crypto/mxc-scc.c
index ee4be1b0d30b..e01c46387df8 100644
--- a/drivers/crypto/mxc-scc.c
+++ b/drivers/crypto/mxc-scc.c
@@ -708,8 +708,8 @@ static int mxc_scc_probe(struct platform_device *pdev)
708 for (i = 0; i < 2; i++) { 708 for (i = 0; i < 2; i++) {
709 irq = platform_get_irq(pdev, i); 709 irq = platform_get_irq(pdev, i);
710 if (irq < 0) { 710 if (irq < 0) {
711 dev_err(dev, "failed to get irq resource\n"); 711 dev_err(dev, "failed to get irq resource: %d\n", irq);
712 ret = -EINVAL; 712 ret = irq;
713 goto err_out; 713 goto err_out;
714 } 714 }
715 715
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index 625ee50fd78b..764be3e6933c 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -908,12 +908,16 @@ static int mxs_dcp_probe(struct platform_device *pdev)
908 908
909 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); 909 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
910 dcp_vmi_irq = platform_get_irq(pdev, 0); 910 dcp_vmi_irq = platform_get_irq(pdev, 0);
911 if (dcp_vmi_irq < 0) 911 if (dcp_vmi_irq < 0) {
912 dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq);
912 return dcp_vmi_irq; 913 return dcp_vmi_irq;
914 }
913 915
914 dcp_irq = platform_get_irq(pdev, 1); 916 dcp_irq = platform_get_irq(pdev, 1);
915 if (dcp_irq < 0) 917 if (dcp_irq < 0) {
918 dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_irq);
916 return dcp_irq; 919 return dcp_irq;
920 }
917 921
918 sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL); 922 sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL);
919 if (!sdcp) 923 if (!sdcp)
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index 269451375b63..a9fd8b9e86cd 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -1730,8 +1730,8 @@ static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1730 continue; 1730 continue;
1731 id = mdesc_get_property(mdesc, tgt, "id", NULL); 1731 id = mdesc_get_property(mdesc, tgt, "id", NULL);
1732 if (table[*id] != NULL) { 1732 if (table[*id] != NULL) {
1733 dev_err(&dev->dev, "%s: SPU cpu slot already set.\n", 1733 dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n",
1734 dev->dev.of_node->full_name); 1734 dev->dev.of_node);
1735 return -EINVAL; 1735 return -EINVAL;
1736 } 1736 }
1737 cpumask_set_cpu(*id, &p->sharing); 1737 cpumask_set_cpu(*id, &p->sharing);
@@ -1751,8 +1751,8 @@ static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1751 1751
1752 p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); 1752 p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1753 if (!p) { 1753 if (!p) {
1754 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n", 1754 dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n",
1755 dev->dev.of_node->full_name); 1755 dev->dev.of_node);
1756 return -ENOMEM; 1756 return -ENOMEM;
1757 } 1757 }
1758 1758
@@ -1981,41 +1981,39 @@ static void n2_spu_driver_version(void)
1981static int n2_crypto_probe(struct platform_device *dev) 1981static int n2_crypto_probe(struct platform_device *dev)
1982{ 1982{
1983 struct mdesc_handle *mdesc; 1983 struct mdesc_handle *mdesc;
1984 const char *full_name;
1985 struct n2_crypto *np; 1984 struct n2_crypto *np;
1986 int err; 1985 int err;
1987 1986
1988 n2_spu_driver_version(); 1987 n2_spu_driver_version();
1989 1988
1990 full_name = dev->dev.of_node->full_name; 1989 pr_info("Found N2CP at %pOF\n", dev->dev.of_node);
1991 pr_info("Found N2CP at %s\n", full_name);
1992 1990
1993 np = alloc_n2cp(); 1991 np = alloc_n2cp();
1994 if (!np) { 1992 if (!np) {
1995 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n", 1993 dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n",
1996 full_name); 1994 dev->dev.of_node);
1997 return -ENOMEM; 1995 return -ENOMEM;
1998 } 1996 }
1999 1997
2000 err = grab_global_resources(); 1998 err = grab_global_resources();
2001 if (err) { 1999 if (err) {
2002 dev_err(&dev->dev, "%s: Unable to grab " 2000 dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
2003 "global resources.\n", full_name); 2001 dev->dev.of_node);
2004 goto out_free_n2cp; 2002 goto out_free_n2cp;
2005 } 2003 }
2006 2004
2007 mdesc = mdesc_grab(); 2005 mdesc = mdesc_grab();
2008 2006
2009 if (!mdesc) { 2007 if (!mdesc) {
2010 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 2008 dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
2011 full_name); 2009 dev->dev.of_node);
2012 err = -ENODEV; 2010 err = -ENODEV;
2013 goto out_free_global; 2011 goto out_free_global;
2014 } 2012 }
2015 err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); 2013 err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
2016 if (err) { 2014 if (err) {
2017 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 2015 dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
2018 full_name); 2016 dev->dev.of_node);
2019 mdesc_release(mdesc); 2017 mdesc_release(mdesc);
2020 goto out_free_global; 2018 goto out_free_global;
2021 } 2019 }
@@ -2026,15 +2024,15 @@ static int n2_crypto_probe(struct platform_device *dev)
2026 mdesc_release(mdesc); 2024 mdesc_release(mdesc);
2027 2025
2028 if (err) { 2026 if (err) {
2029 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n", 2027 dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n",
2030 full_name); 2028 dev->dev.of_node);
2031 goto out_free_global; 2029 goto out_free_global;
2032 } 2030 }
2033 2031
2034 err = n2_register_algs(); 2032 err = n2_register_algs();
2035 if (err) { 2033 if (err) {
2036 dev_err(&dev->dev, "%s: Unable to register algorithms.\n", 2034 dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n",
2037 full_name); 2035 dev->dev.of_node);
2038 goto out_free_spu_list; 2036 goto out_free_spu_list;
2039 } 2037 }
2040 2038
@@ -2092,42 +2090,40 @@ static void free_ncp(struct n2_mau *mp)
2092static int n2_mau_probe(struct platform_device *dev) 2090static int n2_mau_probe(struct platform_device *dev)
2093{ 2091{
2094 struct mdesc_handle *mdesc; 2092 struct mdesc_handle *mdesc;
2095 const char *full_name;
2096 struct n2_mau *mp; 2093 struct n2_mau *mp;
2097 int err; 2094 int err;
2098 2095
2099 n2_spu_driver_version(); 2096 n2_spu_driver_version();
2100 2097
2101 full_name = dev->dev.of_node->full_name; 2098 pr_info("Found NCP at %pOF\n", dev->dev.of_node);
2102 pr_info("Found NCP at %s\n", full_name);
2103 2099
2104 mp = alloc_ncp(); 2100 mp = alloc_ncp();
2105 if (!mp) { 2101 if (!mp) {
2106 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n", 2102 dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n",
2107 full_name); 2103 dev->dev.of_node);
2108 return -ENOMEM; 2104 return -ENOMEM;
2109 } 2105 }
2110 2106
2111 err = grab_global_resources(); 2107 err = grab_global_resources();
2112 if (err) { 2108 if (err) {
2113 dev_err(&dev->dev, "%s: Unable to grab " 2109 dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
2114 "global resources.\n", full_name); 2110 dev->dev.of_node);
2115 goto out_free_ncp; 2111 goto out_free_ncp;
2116 } 2112 }
2117 2113
2118 mdesc = mdesc_grab(); 2114 mdesc = mdesc_grab();
2119 2115
2120 if (!mdesc) { 2116 if (!mdesc) {
2121 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 2117 dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
2122 full_name); 2118 dev->dev.of_node);
2123 err = -ENODEV; 2119 err = -ENODEV;
2124 goto out_free_global; 2120 goto out_free_global;
2125 } 2121 }
2126 2122
2127 err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); 2123 err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
2128 if (err) { 2124 if (err) {
2129 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 2125 dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
2130 full_name); 2126 dev->dev.of_node);
2131 mdesc_release(mdesc); 2127 mdesc_release(mdesc);
2132 goto out_free_global; 2128 goto out_free_global;
2133 } 2129 }
@@ -2138,8 +2134,8 @@ static int n2_mau_probe(struct platform_device *dev)
2138 mdesc_release(mdesc); 2134 mdesc_release(mdesc);
2139 2135
2140 if (err) { 2136 if (err) {
2141 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n", 2137 dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n",
2142 full_name); 2138 dev->dev.of_node);
2143 goto out_free_global; 2139 goto out_free_global;
2144 } 2140 }
2145 2141
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 5120a17731d0..c376a3ee7c2c 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -1095,6 +1095,7 @@ static int omap_aes_probe(struct platform_device *pdev)
1095 irq = platform_get_irq(pdev, 0); 1095 irq = platform_get_irq(pdev, 0);
1096 if (irq < 0) { 1096 if (irq < 0) {
1097 dev_err(dev, "can't get IRQ resource\n"); 1097 dev_err(dev, "can't get IRQ resource\n");
1098 err = irq;
1098 goto err_irq; 1099 goto err_irq;
1099 } 1100 }
1100 1101
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index 0bcab00e0ff5..d37c9506c36c 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -1023,7 +1023,8 @@ static int omap_des_probe(struct platform_device *pdev)
1023 1023
1024 irq = platform_get_irq(pdev, 0); 1024 irq = platform_get_irq(pdev, 0);
1025 if (irq < 0) { 1025 if (irq < 0) {
1026 dev_err(dev, "can't get IRQ resource\n"); 1026 dev_err(dev, "can't get IRQ resource: %d\n", irq);
1027 err = irq;
1027 goto err_irq; 1028 goto err_irq;
1028 } 1029 }
1029 1030
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 9ad9d399daf1..c40ac30ec002 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -2133,7 +2133,7 @@ data_err:
2133 2133
2134static int omap_sham_remove(struct platform_device *pdev) 2134static int omap_sham_remove(struct platform_device *pdev)
2135{ 2135{
2136 static struct omap_sham_dev *dd; 2136 struct omap_sham_dev *dd;
2137 int i, j; 2137 int i, j;
2138 2138
2139 dd = platform_get_drvdata(pdev); 2139 dd = platform_get_drvdata(pdev);
diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c
index d3e25c37dc33..da8a2d3b5e9a 100644
--- a/drivers/crypto/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/qat/qat_common/adf_aer.c
@@ -208,7 +208,7 @@ static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev)
208static void adf_resume(struct pci_dev *pdev) 208static void adf_resume(struct pci_dev *pdev)
209{ 209{
210 dev_info(&pdev->dev, "Acceleration driver reset completed\n"); 210 dev_info(&pdev->dev, "Acceleration driver reset completed\n");
211 dev_info(&pdev->dev, "Device is up and runnig\n"); 211 dev_info(&pdev->dev, "Device is up and running\n");
212} 212}
213 213
214static const struct pci_error_handlers adf_err_handler = { 214static const struct pci_error_handlers adf_err_handler = {
diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c
index d0f80c6241f9..c9d622abd90c 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.c
+++ b/drivers/crypto/rockchip/rk3288_crypto.c
@@ -169,50 +169,82 @@ static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id)
169{ 169{
170 struct rk_crypto_info *dev = platform_get_drvdata(dev_id); 170 struct rk_crypto_info *dev = platform_get_drvdata(dev_id);
171 u32 interrupt_status; 171 u32 interrupt_status;
172 int err = 0;
173 172
174 spin_lock(&dev->lock); 173 spin_lock(&dev->lock);
175 interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS); 174 interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS);
176 CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status); 175 CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status);
176
177 if (interrupt_status & 0x0a) { 177 if (interrupt_status & 0x0a) {
178 dev_warn(dev->dev, "DMA Error\n"); 178 dev_warn(dev->dev, "DMA Error\n");
179 err = -EFAULT; 179 dev->err = -EFAULT;
180 } else if (interrupt_status & 0x05) {
181 err = dev->update(dev);
182 } 180 }
183 if (err) 181 tasklet_schedule(&dev->done_task);
184 dev->complete(dev, err); 182
185 spin_unlock(&dev->lock); 183 spin_unlock(&dev->lock);
186 return IRQ_HANDLED; 184 return IRQ_HANDLED;
187} 185}
188 186
189static void rk_crypto_tasklet_cb(unsigned long data) 187static int rk_crypto_enqueue(struct rk_crypto_info *dev,
188 struct crypto_async_request *async_req)
189{
190 unsigned long flags;
191 int ret;
192
193 spin_lock_irqsave(&dev->lock, flags);
194 ret = crypto_enqueue_request(&dev->queue, async_req);
195 if (dev->busy) {
196 spin_unlock_irqrestore(&dev->lock, flags);
197 return ret;
198 }
199 dev->busy = true;
200 spin_unlock_irqrestore(&dev->lock, flags);
201 tasklet_schedule(&dev->queue_task);
202
203 return ret;
204}
205
206static void rk_crypto_queue_task_cb(unsigned long data)
190{ 207{
191 struct rk_crypto_info *dev = (struct rk_crypto_info *)data; 208 struct rk_crypto_info *dev = (struct rk_crypto_info *)data;
192 struct crypto_async_request *async_req, *backlog; 209 struct crypto_async_request *async_req, *backlog;
193 unsigned long flags; 210 unsigned long flags;
194 int err = 0; 211 int err = 0;
195 212
213 dev->err = 0;
196 spin_lock_irqsave(&dev->lock, flags); 214 spin_lock_irqsave(&dev->lock, flags);
197 backlog = crypto_get_backlog(&dev->queue); 215 backlog = crypto_get_backlog(&dev->queue);
198 async_req = crypto_dequeue_request(&dev->queue); 216 async_req = crypto_dequeue_request(&dev->queue);
199 spin_unlock_irqrestore(&dev->lock, flags); 217
200 if (!async_req) { 218 if (!async_req) {
201 dev_err(dev->dev, "async_req is NULL !!\n"); 219 dev->busy = false;
220 spin_unlock_irqrestore(&dev->lock, flags);
202 return; 221 return;
203 } 222 }
223 spin_unlock_irqrestore(&dev->lock, flags);
224
204 if (backlog) { 225 if (backlog) {
205 backlog->complete(backlog, -EINPROGRESS); 226 backlog->complete(backlog, -EINPROGRESS);
206 backlog = NULL; 227 backlog = NULL;
207 } 228 }
208 229
209 if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_ABLKCIPHER) 230 dev->async_req = async_req;
210 dev->ablk_req = ablkcipher_request_cast(async_req);
211 else
212 dev->ahash_req = ahash_request_cast(async_req);
213 err = dev->start(dev); 231 err = dev->start(dev);
214 if (err) 232 if (err)
215 dev->complete(dev, err); 233 dev->complete(dev->async_req, err);
234}
235
236static void rk_crypto_done_task_cb(unsigned long data)
237{
238 struct rk_crypto_info *dev = (struct rk_crypto_info *)data;
239
240 if (dev->err) {
241 dev->complete(dev->async_req, dev->err);
242 return;
243 }
244
245 dev->err = dev->update(dev);
246 if (dev->err)
247 dev->complete(dev->async_req, dev->err);
216} 248}
217 249
218static struct rk_crypto_tmp *rk_cipher_algs[] = { 250static struct rk_crypto_tmp *rk_cipher_algs[] = {
@@ -361,14 +393,18 @@ static int rk_crypto_probe(struct platform_device *pdev)
361 crypto_info->dev = &pdev->dev; 393 crypto_info->dev = &pdev->dev;
362 platform_set_drvdata(pdev, crypto_info); 394 platform_set_drvdata(pdev, crypto_info);
363 395
364 tasklet_init(&crypto_info->crypto_tasklet, 396 tasklet_init(&crypto_info->queue_task,
365 rk_crypto_tasklet_cb, (unsigned long)crypto_info); 397 rk_crypto_queue_task_cb, (unsigned long)crypto_info);
398 tasklet_init(&crypto_info->done_task,
399 rk_crypto_done_task_cb, (unsigned long)crypto_info);
366 crypto_init_queue(&crypto_info->queue, 50); 400 crypto_init_queue(&crypto_info->queue, 50);
367 401
368 crypto_info->enable_clk = rk_crypto_enable_clk; 402 crypto_info->enable_clk = rk_crypto_enable_clk;
369 crypto_info->disable_clk = rk_crypto_disable_clk; 403 crypto_info->disable_clk = rk_crypto_disable_clk;
370 crypto_info->load_data = rk_load_data; 404 crypto_info->load_data = rk_load_data;
371 crypto_info->unload_data = rk_unload_data; 405 crypto_info->unload_data = rk_unload_data;
406 crypto_info->enqueue = rk_crypto_enqueue;
407 crypto_info->busy = false;
372 408
373 err = rk_crypto_register(crypto_info); 409 err = rk_crypto_register(crypto_info);
374 if (err) { 410 if (err) {
@@ -380,7 +416,8 @@ static int rk_crypto_probe(struct platform_device *pdev)
380 return 0; 416 return 0;
381 417
382err_register_alg: 418err_register_alg:
383 tasklet_kill(&crypto_info->crypto_tasklet); 419 tasklet_kill(&crypto_info->queue_task);
420 tasklet_kill(&crypto_info->done_task);
384err_crypto: 421err_crypto:
385 return err; 422 return err;
386} 423}
@@ -390,7 +427,8 @@ static int rk_crypto_remove(struct platform_device *pdev)
390 struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev); 427 struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev);
391 428
392 rk_crypto_unregister(); 429 rk_crypto_unregister();
393 tasklet_kill(&crypto_tmp->crypto_tasklet); 430 tasklet_kill(&crypto_tmp->done_task);
431 tasklet_kill(&crypto_tmp->queue_task);
394 return 0; 432 return 0;
395} 433}
396 434
diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h
index d7b71fea320b..ab6a1b4c40f0 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.h
+++ b/drivers/crypto/rockchip/rk3288_crypto.h
@@ -190,9 +190,10 @@ struct rk_crypto_info {
190 void __iomem *reg; 190 void __iomem *reg;
191 int irq; 191 int irq;
192 struct crypto_queue queue; 192 struct crypto_queue queue;
193 struct tasklet_struct crypto_tasklet; 193 struct tasklet_struct queue_task;
194 struct ablkcipher_request *ablk_req; 194 struct tasklet_struct done_task;
195 struct ahash_request *ahash_req; 195 struct crypto_async_request *async_req;
196 int err;
196 /* device lock */ 197 /* device lock */
197 spinlock_t lock; 198 spinlock_t lock;
198 199
@@ -208,18 +209,20 @@ struct rk_crypto_info {
208 size_t nents; 209 size_t nents;
209 unsigned int total; 210 unsigned int total;
210 unsigned int count; 211 unsigned int count;
211 u32 mode;
212 dma_addr_t addr_in; 212 dma_addr_t addr_in;
213 dma_addr_t addr_out; 213 dma_addr_t addr_out;
214 bool busy;
214 int (*start)(struct rk_crypto_info *dev); 215 int (*start)(struct rk_crypto_info *dev);
215 int (*update)(struct rk_crypto_info *dev); 216 int (*update)(struct rk_crypto_info *dev);
216 void (*complete)(struct rk_crypto_info *dev, int err); 217 void (*complete)(struct crypto_async_request *base, int err);
217 int (*enable_clk)(struct rk_crypto_info *dev); 218 int (*enable_clk)(struct rk_crypto_info *dev);
218 void (*disable_clk)(struct rk_crypto_info *dev); 219 void (*disable_clk)(struct rk_crypto_info *dev);
219 int (*load_data)(struct rk_crypto_info *dev, 220 int (*load_data)(struct rk_crypto_info *dev,
220 struct scatterlist *sg_src, 221 struct scatterlist *sg_src,
221 struct scatterlist *sg_dst); 222 struct scatterlist *sg_dst);
222 void (*unload_data)(struct rk_crypto_info *dev); 223 void (*unload_data)(struct rk_crypto_info *dev);
224 int (*enqueue)(struct rk_crypto_info *dev,
225 struct crypto_async_request *async_req);
223}; 226};
224 227
225/* the private variable of hash */ 228/* the private variable of hash */
@@ -232,12 +235,14 @@ struct rk_ahash_ctx {
232/* the privete variable of hash for fallback */ 235/* the privete variable of hash for fallback */
233struct rk_ahash_rctx { 236struct rk_ahash_rctx {
234 struct ahash_request fallback_req; 237 struct ahash_request fallback_req;
238 u32 mode;
235}; 239};
236 240
237/* the private variable of cipher */ 241/* the private variable of cipher */
238struct rk_cipher_ctx { 242struct rk_cipher_ctx {
239 struct rk_crypto_info *dev; 243 struct rk_crypto_info *dev;
240 unsigned int keylen; 244 unsigned int keylen;
245 u32 mode;
241}; 246};
242 247
243enum alg_type { 248enum alg_type {
diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
index b5a3afe222e4..639c15c5364b 100644
--- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
+++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
@@ -15,35 +15,19 @@
15 15
16#define RK_CRYPTO_DEC BIT(0) 16#define RK_CRYPTO_DEC BIT(0)
17 17
18static void rk_crypto_complete(struct rk_crypto_info *dev, int err) 18static void rk_crypto_complete(struct crypto_async_request *base, int err)
19{ 19{
20 if (dev->ablk_req->base.complete) 20 if (base->complete)
21 dev->ablk_req->base.complete(&dev->ablk_req->base, err); 21 base->complete(base, err);
22} 22}
23 23
24static int rk_handle_req(struct rk_crypto_info *dev, 24static int rk_handle_req(struct rk_crypto_info *dev,
25 struct ablkcipher_request *req) 25 struct ablkcipher_request *req)
26{ 26{
27 unsigned long flags;
28 int err;
29
30 if (!IS_ALIGNED(req->nbytes, dev->align_size)) 27 if (!IS_ALIGNED(req->nbytes, dev->align_size))
31 return -EINVAL; 28 return -EINVAL;
32 29 else
33 dev->left_bytes = req->nbytes; 30 return dev->enqueue(dev, &req->base);
34 dev->total = req->nbytes;
35 dev->sg_src = req->src;
36 dev->first = req->src;
37 dev->nents = sg_nents(req->src);
38 dev->sg_dst = req->dst;
39 dev->aligned = 1;
40 dev->ablk_req = req;
41
42 spin_lock_irqsave(&dev->lock, flags);
43 err = ablkcipher_enqueue_request(&dev->queue, req);
44 spin_unlock_irqrestore(&dev->lock, flags);
45 tasklet_schedule(&dev->crypto_tasklet);
46 return err;
47} 31}
48 32
49static int rk_aes_setkey(struct crypto_ablkcipher *cipher, 33static int rk_aes_setkey(struct crypto_ablkcipher *cipher,
@@ -93,7 +77,7 @@ static int rk_aes_ecb_encrypt(struct ablkcipher_request *req)
93 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 77 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
94 struct rk_crypto_info *dev = ctx->dev; 78 struct rk_crypto_info *dev = ctx->dev;
95 79
96 dev->mode = RK_CRYPTO_AES_ECB_MODE; 80 ctx->mode = RK_CRYPTO_AES_ECB_MODE;
97 return rk_handle_req(dev, req); 81 return rk_handle_req(dev, req);
98} 82}
99 83
@@ -103,7 +87,7 @@ static int rk_aes_ecb_decrypt(struct ablkcipher_request *req)
103 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 87 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
104 struct rk_crypto_info *dev = ctx->dev; 88 struct rk_crypto_info *dev = ctx->dev;
105 89
106 dev->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; 90 ctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
107 return rk_handle_req(dev, req); 91 return rk_handle_req(dev, req);
108} 92}
109 93
@@ -113,7 +97,7 @@ static int rk_aes_cbc_encrypt(struct ablkcipher_request *req)
113 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 97 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
114 struct rk_crypto_info *dev = ctx->dev; 98 struct rk_crypto_info *dev = ctx->dev;
115 99
116 dev->mode = RK_CRYPTO_AES_CBC_MODE; 100 ctx->mode = RK_CRYPTO_AES_CBC_MODE;
117 return rk_handle_req(dev, req); 101 return rk_handle_req(dev, req);
118} 102}
119 103
@@ -123,7 +107,7 @@ static int rk_aes_cbc_decrypt(struct ablkcipher_request *req)
123 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 107 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
124 struct rk_crypto_info *dev = ctx->dev; 108 struct rk_crypto_info *dev = ctx->dev;
125 109
126 dev->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; 110 ctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
127 return rk_handle_req(dev, req); 111 return rk_handle_req(dev, req);
128} 112}
129 113
@@ -133,7 +117,7 @@ static int rk_des_ecb_encrypt(struct ablkcipher_request *req)
133 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 117 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
134 struct rk_crypto_info *dev = ctx->dev; 118 struct rk_crypto_info *dev = ctx->dev;
135 119
136 dev->mode = 0; 120 ctx->mode = 0;
137 return rk_handle_req(dev, req); 121 return rk_handle_req(dev, req);
138} 122}
139 123
@@ -143,7 +127,7 @@ static int rk_des_ecb_decrypt(struct ablkcipher_request *req)
143 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 127 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
144 struct rk_crypto_info *dev = ctx->dev; 128 struct rk_crypto_info *dev = ctx->dev;
145 129
146 dev->mode = RK_CRYPTO_DEC; 130 ctx->mode = RK_CRYPTO_DEC;
147 return rk_handle_req(dev, req); 131 return rk_handle_req(dev, req);
148} 132}
149 133
@@ -153,7 +137,7 @@ static int rk_des_cbc_encrypt(struct ablkcipher_request *req)
153 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 137 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
154 struct rk_crypto_info *dev = ctx->dev; 138 struct rk_crypto_info *dev = ctx->dev;
155 139
156 dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; 140 ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
157 return rk_handle_req(dev, req); 141 return rk_handle_req(dev, req);
158} 142}
159 143
@@ -163,7 +147,7 @@ static int rk_des_cbc_decrypt(struct ablkcipher_request *req)
163 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 147 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
164 struct rk_crypto_info *dev = ctx->dev; 148 struct rk_crypto_info *dev = ctx->dev;
165 149
166 dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; 150 ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
167 return rk_handle_req(dev, req); 151 return rk_handle_req(dev, req);
168} 152}
169 153
@@ -173,7 +157,7 @@ static int rk_des3_ede_ecb_encrypt(struct ablkcipher_request *req)
173 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 157 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
174 struct rk_crypto_info *dev = ctx->dev; 158 struct rk_crypto_info *dev = ctx->dev;
175 159
176 dev->mode = RK_CRYPTO_TDES_SELECT; 160 ctx->mode = RK_CRYPTO_TDES_SELECT;
177 return rk_handle_req(dev, req); 161 return rk_handle_req(dev, req);
178} 162}
179 163
@@ -183,7 +167,7 @@ static int rk_des3_ede_ecb_decrypt(struct ablkcipher_request *req)
183 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 167 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
184 struct rk_crypto_info *dev = ctx->dev; 168 struct rk_crypto_info *dev = ctx->dev;
185 169
186 dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; 170 ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
187 return rk_handle_req(dev, req); 171 return rk_handle_req(dev, req);
188} 172}
189 173
@@ -193,7 +177,7 @@ static int rk_des3_ede_cbc_encrypt(struct ablkcipher_request *req)
193 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 177 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
194 struct rk_crypto_info *dev = ctx->dev; 178 struct rk_crypto_info *dev = ctx->dev;
195 179
196 dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; 180 ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
197 return rk_handle_req(dev, req); 181 return rk_handle_req(dev, req);
198} 182}
199 183
@@ -203,15 +187,16 @@ static int rk_des3_ede_cbc_decrypt(struct ablkcipher_request *req)
203 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 187 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
204 struct rk_crypto_info *dev = ctx->dev; 188 struct rk_crypto_info *dev = ctx->dev;
205 189
206 dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | 190 ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
207 RK_CRYPTO_DEC; 191 RK_CRYPTO_DEC;
208 return rk_handle_req(dev, req); 192 return rk_handle_req(dev, req);
209} 193}
210 194
211static void rk_ablk_hw_init(struct rk_crypto_info *dev) 195static void rk_ablk_hw_init(struct rk_crypto_info *dev)
212{ 196{
213 struct crypto_ablkcipher *cipher = 197 struct ablkcipher_request *req =
214 crypto_ablkcipher_reqtfm(dev->ablk_req); 198 ablkcipher_request_cast(dev->async_req);
199 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
215 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 200 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
216 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher); 201 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher);
217 u32 ivsize, block, conf_reg = 0; 202 u32 ivsize, block, conf_reg = 0;
@@ -220,25 +205,23 @@ static void rk_ablk_hw_init(struct rk_crypto_info *dev)
220 ivsize = crypto_ablkcipher_ivsize(cipher); 205 ivsize = crypto_ablkcipher_ivsize(cipher);
221 206
222 if (block == DES_BLOCK_SIZE) { 207 if (block == DES_BLOCK_SIZE) {
223 dev->mode |= RK_CRYPTO_TDES_FIFO_MODE | 208 ctx->mode |= RK_CRYPTO_TDES_FIFO_MODE |
224 RK_CRYPTO_TDES_BYTESWAP_KEY | 209 RK_CRYPTO_TDES_BYTESWAP_KEY |
225 RK_CRYPTO_TDES_BYTESWAP_IV; 210 RK_CRYPTO_TDES_BYTESWAP_IV;
226 CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, dev->mode); 211 CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, ctx->mode);
227 memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, 212 memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, req->info, ivsize);
228 dev->ablk_req->info, ivsize);
229 conf_reg = RK_CRYPTO_DESSEL; 213 conf_reg = RK_CRYPTO_DESSEL;
230 } else { 214 } else {
231 dev->mode |= RK_CRYPTO_AES_FIFO_MODE | 215 ctx->mode |= RK_CRYPTO_AES_FIFO_MODE |
232 RK_CRYPTO_AES_KEY_CHANGE | 216 RK_CRYPTO_AES_KEY_CHANGE |
233 RK_CRYPTO_AES_BYTESWAP_KEY | 217 RK_CRYPTO_AES_BYTESWAP_KEY |
234 RK_CRYPTO_AES_BYTESWAP_IV; 218 RK_CRYPTO_AES_BYTESWAP_IV;
235 if (ctx->keylen == AES_KEYSIZE_192) 219 if (ctx->keylen == AES_KEYSIZE_192)
236 dev->mode |= RK_CRYPTO_AES_192BIT_key; 220 ctx->mode |= RK_CRYPTO_AES_192BIT_key;
237 else if (ctx->keylen == AES_KEYSIZE_256) 221 else if (ctx->keylen == AES_KEYSIZE_256)
238 dev->mode |= RK_CRYPTO_AES_256BIT_key; 222 ctx->mode |= RK_CRYPTO_AES_256BIT_key;
239 CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, dev->mode); 223 CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, ctx->mode);
240 memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, 224 memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, req->info, ivsize);
241 dev->ablk_req->info, ivsize);
242 } 225 }
243 conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO | 226 conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |
244 RK_CRYPTO_BYTESWAP_BRFIFO; 227 RK_CRYPTO_BYTESWAP_BRFIFO;
@@ -268,8 +251,18 @@ static int rk_set_data_start(struct rk_crypto_info *dev)
268 251
269static int rk_ablk_start(struct rk_crypto_info *dev) 252static int rk_ablk_start(struct rk_crypto_info *dev)
270{ 253{
254 struct ablkcipher_request *req =
255 ablkcipher_request_cast(dev->async_req);
271 unsigned long flags; 256 unsigned long flags;
272 int err; 257 int err = 0;
258
259 dev->left_bytes = req->nbytes;
260 dev->total = req->nbytes;
261 dev->sg_src = req->src;
262 dev->first = req->src;
263 dev->nents = sg_nents(req->src);
264 dev->sg_dst = req->dst;
265 dev->aligned = 1;
273 266
274 spin_lock_irqsave(&dev->lock, flags); 267 spin_lock_irqsave(&dev->lock, flags);
275 rk_ablk_hw_init(dev); 268 rk_ablk_hw_init(dev);
@@ -280,15 +273,16 @@ static int rk_ablk_start(struct rk_crypto_info *dev)
280 273
281static void rk_iv_copyback(struct rk_crypto_info *dev) 274static void rk_iv_copyback(struct rk_crypto_info *dev)
282{ 275{
283 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(dev->ablk_req); 276 struct ablkcipher_request *req =
277 ablkcipher_request_cast(dev->async_req);
278 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
284 u32 ivsize = crypto_ablkcipher_ivsize(tfm); 279 u32 ivsize = crypto_ablkcipher_ivsize(tfm);
285 280
286 if (ivsize == DES_BLOCK_SIZE) 281 if (ivsize == DES_BLOCK_SIZE)
287 memcpy_fromio(dev->ablk_req->info, 282 memcpy_fromio(req->info, dev->reg + RK_CRYPTO_TDES_IV_0,
288 dev->reg + RK_CRYPTO_TDES_IV_0, ivsize); 283 ivsize);
289 else if (ivsize == AES_BLOCK_SIZE) 284 else if (ivsize == AES_BLOCK_SIZE)
290 memcpy_fromio(dev->ablk_req->info, 285 memcpy_fromio(req->info, dev->reg + RK_CRYPTO_AES_IV_0, ivsize);
291 dev->reg + RK_CRYPTO_AES_IV_0, ivsize);
292} 286}
293 287
294/* return: 288/* return:
@@ -298,10 +292,12 @@ static void rk_iv_copyback(struct rk_crypto_info *dev)
298static int rk_ablk_rx(struct rk_crypto_info *dev) 292static int rk_ablk_rx(struct rk_crypto_info *dev)
299{ 293{
300 int err = 0; 294 int err = 0;
295 struct ablkcipher_request *req =
296 ablkcipher_request_cast(dev->async_req);
301 297
302 dev->unload_data(dev); 298 dev->unload_data(dev);
303 if (!dev->aligned) { 299 if (!dev->aligned) {
304 if (!sg_pcopy_from_buffer(dev->ablk_req->dst, dev->nents, 300 if (!sg_pcopy_from_buffer(req->dst, dev->nents,
305 dev->addr_vir, dev->count, 301 dev->addr_vir, dev->count,
306 dev->total - dev->left_bytes - 302 dev->total - dev->left_bytes -
307 dev->count)) { 303 dev->count)) {
@@ -324,7 +320,8 @@ static int rk_ablk_rx(struct rk_crypto_info *dev)
324 } else { 320 } else {
325 rk_iv_copyback(dev); 321 rk_iv_copyback(dev);
326 /* here show the calculation is over without any err */ 322 /* here show the calculation is over without any err */
327 dev->complete(dev, 0); 323 dev->complete(dev->async_req, 0);
324 tasklet_schedule(&dev->queue_task);
328 } 325 }
329out_rx: 326out_rx:
330 return err; 327 return err;
diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c
index 718588219f75..821a506b9e17 100644
--- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c
+++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c
@@ -40,14 +40,16 @@ static int zero_message_process(struct ahash_request *req)
40 return 0; 40 return 0;
41} 41}
42 42
43static void rk_ahash_crypto_complete(struct rk_crypto_info *dev, int err) 43static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
44{ 44{
45 if (dev->ahash_req->base.complete) 45 if (base->complete)
46 dev->ahash_req->base.complete(&dev->ahash_req->base, err); 46 base->complete(base, err);
47} 47}
48 48
49static void rk_ahash_reg_init(struct rk_crypto_info *dev) 49static void rk_ahash_reg_init(struct rk_crypto_info *dev)
50{ 50{
51 struct ahash_request *req = ahash_request_cast(dev->async_req);
52 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
51 int reg_status = 0; 53 int reg_status = 0;
52 54
53 reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) | 55 reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) |
@@ -67,7 +69,7 @@ static void rk_ahash_reg_init(struct rk_crypto_info *dev)
67 CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT | 69 CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT |
68 RK_CRYPTO_HRDMA_DONE_INT); 70 RK_CRYPTO_HRDMA_DONE_INT);
69 71
70 CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, dev->mode | 72 CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode |
71 RK_CRYPTO_HASH_SWAP_DO); 73 RK_CRYPTO_HASH_SWAP_DO);
72 74
73 CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO | 75 CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO |
@@ -164,64 +166,13 @@ static int rk_ahash_export(struct ahash_request *req, void *out)
164 166
165static int rk_ahash_digest(struct ahash_request *req) 167static int rk_ahash_digest(struct ahash_request *req)
166{ 168{
167 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
168 struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 169 struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
169 struct rk_crypto_info *dev = NULL; 170 struct rk_crypto_info *dev = tctx->dev;
170 unsigned long flags;
171 int ret;
172 171
173 if (!req->nbytes) 172 if (!req->nbytes)
174 return zero_message_process(req); 173 return zero_message_process(req);
175 174 else
176 dev = tctx->dev; 175 return dev->enqueue(dev, &req->base);
177 dev->total = req->nbytes;
178 dev->left_bytes = req->nbytes;
179 dev->aligned = 0;
180 dev->mode = 0;
181 dev->align_size = 4;
182 dev->sg_dst = NULL;
183 dev->sg_src = req->src;
184 dev->first = req->src;
185 dev->nents = sg_nents(req->src);
186
187 switch (crypto_ahash_digestsize(tfm)) {
188 case SHA1_DIGEST_SIZE:
189 dev->mode = RK_CRYPTO_HASH_SHA1;
190 break;
191 case SHA256_DIGEST_SIZE:
192 dev->mode = RK_CRYPTO_HASH_SHA256;
193 break;
194 case MD5_DIGEST_SIZE:
195 dev->mode = RK_CRYPTO_HASH_MD5;
196 break;
197 default:
198 return -EINVAL;
199 }
200
201 rk_ahash_reg_init(dev);
202
203 spin_lock_irqsave(&dev->lock, flags);
204 ret = crypto_enqueue_request(&dev->queue, &req->base);
205 spin_unlock_irqrestore(&dev->lock, flags);
206
207 tasklet_schedule(&dev->crypto_tasklet);
208
209 /*
210 * it will take some time to process date after last dma transmission.
211 *
212 * waiting time is relative with the last date len,
213 * so cannot set a fixed time here.
214 * 10-50 makes system not call here frequently wasting
215 * efficiency, and make it response quickly when dma
216 * complete.
217 */
218 while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS))
219 usleep_range(10, 50);
220
221 memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0,
222 crypto_ahash_digestsize(tfm));
223
224 return 0;
225} 176}
226 177
227static void crypto_ahash_dma_start(struct rk_crypto_info *dev) 178static void crypto_ahash_dma_start(struct rk_crypto_info *dev)
@@ -244,12 +195,45 @@ static int rk_ahash_set_data_start(struct rk_crypto_info *dev)
244 195
245static int rk_ahash_start(struct rk_crypto_info *dev) 196static int rk_ahash_start(struct rk_crypto_info *dev)
246{ 197{
198 struct ahash_request *req = ahash_request_cast(dev->async_req);
199 struct crypto_ahash *tfm;
200 struct rk_ahash_rctx *rctx;
201
202 dev->total = req->nbytes;
203 dev->left_bytes = req->nbytes;
204 dev->aligned = 0;
205 dev->align_size = 4;
206 dev->sg_dst = NULL;
207 dev->sg_src = req->src;
208 dev->first = req->src;
209 dev->nents = sg_nents(req->src);
210 rctx = ahash_request_ctx(req);
211 rctx->mode = 0;
212
213 tfm = crypto_ahash_reqtfm(req);
214 switch (crypto_ahash_digestsize(tfm)) {
215 case SHA1_DIGEST_SIZE:
216 rctx->mode = RK_CRYPTO_HASH_SHA1;
217 break;
218 case SHA256_DIGEST_SIZE:
219 rctx->mode = RK_CRYPTO_HASH_SHA256;
220 break;
221 case MD5_DIGEST_SIZE:
222 rctx->mode = RK_CRYPTO_HASH_MD5;
223 break;
224 default:
225 return -EINVAL;
226 }
227
228 rk_ahash_reg_init(dev);
247 return rk_ahash_set_data_start(dev); 229 return rk_ahash_set_data_start(dev);
248} 230}
249 231
250static int rk_ahash_crypto_rx(struct rk_crypto_info *dev) 232static int rk_ahash_crypto_rx(struct rk_crypto_info *dev)
251{ 233{
252 int err = 0; 234 int err = 0;
235 struct ahash_request *req = ahash_request_cast(dev->async_req);
236 struct crypto_ahash *tfm;
253 237
254 dev->unload_data(dev); 238 dev->unload_data(dev);
255 if (dev->left_bytes) { 239 if (dev->left_bytes) {
@@ -264,7 +248,24 @@ static int rk_ahash_crypto_rx(struct rk_crypto_info *dev)
264 } 248 }
265 err = rk_ahash_set_data_start(dev); 249 err = rk_ahash_set_data_start(dev);
266 } else { 250 } else {
267 dev->complete(dev, 0); 251 /*
252 * it will take some time to process date after last dma
253 * transmission.
254 *
255 * waiting time is relative with the last date len,
256 * so cannot set a fixed time here.
257 * 10us makes system not call here frequently wasting
258 * efficiency, and make it response quickly when dma
259 * complete.
260 */
261 while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS))
262 udelay(10);
263
264 tfm = crypto_ahash_reqtfm(req);
265 memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0,
266 crypto_ahash_digestsize(tfm));
267 dev->complete(dev->async_req, 0);
268 tasklet_schedule(&dev->queue_task);
268 } 269 }
269 270
270out_rx: 271out_rx:
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 1d9ecd368b5b..08e7bdcaa6e3 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -202,7 +202,6 @@ struct sahara_dev {
202 struct completion dma_completion; 202 struct completion dma_completion;
203 203
204 struct sahara_ctx *ctx; 204 struct sahara_ctx *ctx;
205 spinlock_t lock;
206 struct crypto_queue queue; 205 struct crypto_queue queue;
207 unsigned long flags; 206 unsigned long flags;
208 207
@@ -543,10 +542,10 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
543 542
544unmap_out: 543unmap_out:
545 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, 544 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
546 DMA_TO_DEVICE); 545 DMA_FROM_DEVICE);
547unmap_in: 546unmap_in:
548 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, 547 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
549 DMA_FROM_DEVICE); 548 DMA_TO_DEVICE);
550 549
551 return -EINVAL; 550 return -EINVAL;
552} 551}
@@ -594,9 +593,9 @@ static int sahara_aes_process(struct ablkcipher_request *req)
594 } 593 }
595 594
596 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, 595 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
597 DMA_TO_DEVICE);
598 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
599 DMA_FROM_DEVICE); 596 DMA_FROM_DEVICE);
597 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
598 DMA_TO_DEVICE);
600 599
601 return 0; 600 return 0;
602} 601}
@@ -1376,13 +1375,13 @@ static void sahara_unregister_algs(struct sahara_dev *dev)
1376 crypto_unregister_ahash(&sha_v4_algs[i]); 1375 crypto_unregister_ahash(&sha_v4_algs[i]);
1377} 1376}
1378 1377
1379static struct platform_device_id sahara_platform_ids[] = { 1378static const struct platform_device_id sahara_platform_ids[] = {
1380 { .name = "sahara-imx27" }, 1379 { .name = "sahara-imx27" },
1381 { /* sentinel */ } 1380 { /* sentinel */ }
1382}; 1381};
1383MODULE_DEVICE_TABLE(platform, sahara_platform_ids); 1382MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1384 1383
1385static struct of_device_id sahara_dt_ids[] = { 1384static const struct of_device_id sahara_dt_ids[] = {
1386 { .compatible = "fsl,imx53-sahara" }, 1385 { .compatible = "fsl,imx53-sahara" },
1387 { .compatible = "fsl,imx27-sahara" }, 1386 { .compatible = "fsl,imx27-sahara" },
1388 { /* sentinel */ } 1387 { /* sentinel */ }
@@ -1487,7 +1486,6 @@ static int sahara_probe(struct platform_device *pdev)
1487 1486
1488 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); 1487 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1489 1488
1490 spin_lock_init(&dev->lock);
1491 mutex_init(&dev->queue_mutex); 1489 mutex_init(&dev->queue_mutex);
1492 1490
1493 dev_ptr = dev; 1491 dev_ptr = dev;
diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig
index 09b4ec87c212..602332e02729 100644
--- a/drivers/crypto/stm32/Kconfig
+++ b/drivers/crypto/stm32/Kconfig
@@ -1,7 +1,20 @@
1config CRYPTO_DEV_STM32 1config CRC_DEV_STM32
2 tristate "Support for STM32 crypto accelerators" 2 tristate "Support for STM32 crc accelerators"
3 depends on ARCH_STM32 3 depends on ARCH_STM32
4 select CRYPTO_HASH 4 select CRYPTO_HASH
5 help 5 help
6 This enables support for the CRC32 hw accelerator which can be found 6 This enables support for the CRC32 hw accelerator which can be found
7 on STMicroelectronis STM32 SOC. 7 on STMicroelectronics STM32 SOC.
8
9config HASH_DEV_STM32
10 tristate "Support for STM32 hash accelerators"
11 depends on ARCH_STM32
12 depends on HAS_DMA
13 select CRYPTO_HASH
14 select CRYPTO_MD5
15 select CRYPTO_SHA1
16 select CRYPTO_SHA256
17 select CRYPTO_ENGINE
18 help
19 This enables support for the HASH hw accelerator which can be found
20 on STMicroelectronics STM32 SOC.
diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile
index 73b4c6e47f5f..73cd56cad0cc 100644
--- a/drivers/crypto/stm32/Makefile
+++ b/drivers/crypto/stm32/Makefile
@@ -1,2 +1,2 @@
1obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32_cryp.o 1obj-$(CONFIG_CRC_DEV_STM32) += stm32_crc32.o
2stm32_cryp-objs := stm32_crc32.o 2obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o \ No newline at end of file
diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
new file mode 100644
index 000000000000..b585ce54a802
--- /dev/null
+++ b/drivers/crypto/stm32/stm32-hash.c
@@ -0,0 +1,1575 @@
1/*
2 * This file is part of STM32 Crypto driver for Linux.
3 *
4 * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
5 * Author(s): Lionel DEBIEVE <lionel.debieve@st.com> for STMicroelectronics.
6 *
7 * License terms: GPL V2.0.
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License version 2 as published by
11 * the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
16 * details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program. If not, see <http://www.gnu.org/licenses/>.
20 *
21 */
22
23#include <linux/clk.h>
24#include <linux/crypto.h>
25#include <linux/delay.h>
26#include <linux/dmaengine.h>
27#include <linux/interrupt.h>
28#include <linux/io.h>
29#include <linux/iopoll.h>
30#include <linux/kernel.h>
31#include <linux/module.h>
32#include <linux/of_device.h>
33#include <linux/platform_device.h>
34#include <linux/reset.h>
35
36#include <crypto/engine.h>
37#include <crypto/hash.h>
38#include <crypto/md5.h>
39#include <crypto/scatterwalk.h>
40#include <crypto/sha.h>
41#include <crypto/internal/hash.h>
42
43#define HASH_CR 0x00
44#define HASH_DIN 0x04
45#define HASH_STR 0x08
46#define HASH_IMR 0x20
47#define HASH_SR 0x24
48#define HASH_CSR(x) (0x0F8 + ((x) * 0x04))
49#define HASH_HREG(x) (0x310 + ((x) * 0x04))
50#define HASH_HWCFGR 0x3F0
51#define HASH_VER 0x3F4
52#define HASH_ID 0x3F8
53
54/* Control Register */
55#define HASH_CR_INIT BIT(2)
56#define HASH_CR_DMAE BIT(3)
57#define HASH_CR_DATATYPE_POS 4
58#define HASH_CR_MODE BIT(6)
59#define HASH_CR_MDMAT BIT(13)
60#define HASH_CR_DMAA BIT(14)
61#define HASH_CR_LKEY BIT(16)
62
63#define HASH_CR_ALGO_SHA1 0x0
64#define HASH_CR_ALGO_MD5 0x80
65#define HASH_CR_ALGO_SHA224 0x40000
66#define HASH_CR_ALGO_SHA256 0x40080
67
68/* Interrupt */
69#define HASH_DINIE BIT(0)
70#define HASH_DCIE BIT(1)
71
72/* Interrupt Mask */
73#define HASH_MASK_CALC_COMPLETION BIT(0)
74#define HASH_MASK_DATA_INPUT BIT(1)
75
76/* Context swap register */
77#define HASH_CSR_REGISTER_NUMBER 53
78
79/* Status Flags */
80#define HASH_SR_DATA_INPUT_READY BIT(0)
81#define HASH_SR_OUTPUT_READY BIT(1)
82#define HASH_SR_DMA_ACTIVE BIT(2)
83#define HASH_SR_BUSY BIT(3)
84
85/* STR Register */
86#define HASH_STR_NBLW_MASK GENMASK(4, 0)
87#define HASH_STR_DCAL BIT(8)
88
89#define HASH_FLAGS_INIT BIT(0)
90#define HASH_FLAGS_OUTPUT_READY BIT(1)
91#define HASH_FLAGS_CPU BIT(2)
92#define HASH_FLAGS_DMA_READY BIT(3)
93#define HASH_FLAGS_DMA_ACTIVE BIT(4)
94#define HASH_FLAGS_HMAC_INIT BIT(5)
95#define HASH_FLAGS_HMAC_FINAL BIT(6)
96#define HASH_FLAGS_HMAC_KEY BIT(7)
97
98#define HASH_FLAGS_FINAL BIT(15)
99#define HASH_FLAGS_FINUP BIT(16)
100#define HASH_FLAGS_ALGO_MASK GENMASK(21, 18)
101#define HASH_FLAGS_MD5 BIT(18)
102#define HASH_FLAGS_SHA1 BIT(19)
103#define HASH_FLAGS_SHA224 BIT(20)
104#define HASH_FLAGS_SHA256 BIT(21)
105#define HASH_FLAGS_ERRORS BIT(22)
106#define HASH_FLAGS_HMAC BIT(23)
107
108#define HASH_OP_UPDATE 1
109#define HASH_OP_FINAL 2
110
111enum stm32_hash_data_format {
112 HASH_DATA_32_BITS = 0x0,
113 HASH_DATA_16_BITS = 0x1,
114 HASH_DATA_8_BITS = 0x2,
115 HASH_DATA_1_BIT = 0x3
116};
117
118#define HASH_BUFLEN 256
119#define HASH_LONG_KEY 64
120#define HASH_MAX_KEY_SIZE (SHA256_BLOCK_SIZE * 8)
121#define HASH_QUEUE_LENGTH 16
122#define HASH_DMA_THRESHOLD 50
123
124struct stm32_hash_ctx {
125 struct stm32_hash_dev *hdev;
126 unsigned long flags;
127
128 u8 key[HASH_MAX_KEY_SIZE];
129 int keylen;
130};
131
132struct stm32_hash_request_ctx {
133 struct stm32_hash_dev *hdev;
134 unsigned long flags;
135 unsigned long op;
136
137 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
138 size_t digcnt;
139 size_t bufcnt;
140 size_t buflen;
141
142 /* DMA */
143 struct scatterlist *sg;
144 unsigned int offset;
145 unsigned int total;
146 struct scatterlist sg_key;
147
148 dma_addr_t dma_addr;
149 size_t dma_ct;
150 int nents;
151
152 u8 data_type;
153
154 u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32));
155
156 /* Export Context */
157 u32 *hw_context;
158};
159
160struct stm32_hash_algs_info {
161 struct ahash_alg *algs_list;
162 size_t size;
163};
164
165struct stm32_hash_pdata {
166 struct stm32_hash_algs_info *algs_info;
167 size_t algs_info_size;
168};
169
170struct stm32_hash_dev {
171 struct list_head list;
172 struct device *dev;
173 struct clk *clk;
174 struct reset_control *rst;
175 void __iomem *io_base;
176 phys_addr_t phys_base;
177 u32 dma_mode;
178 u32 dma_maxburst;
179
180 spinlock_t lock; /* lock to protect queue */
181
182 struct ahash_request *req;
183 struct crypto_engine *engine;
184
185 int err;
186 unsigned long flags;
187
188 struct dma_chan *dma_lch;
189 struct completion dma_completion;
190
191 const struct stm32_hash_pdata *pdata;
192};
193
194struct stm32_hash_drv {
195 struct list_head dev_list;
196 spinlock_t lock; /* List protection access */
197};
198
199static struct stm32_hash_drv stm32_hash = {
200 .dev_list = LIST_HEAD_INIT(stm32_hash.dev_list),
201 .lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock),
202};
203
204static void stm32_hash_dma_callback(void *param);
205
206static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset)
207{
208 return readl_relaxed(hdev->io_base + offset);
209}
210
211static inline void stm32_hash_write(struct stm32_hash_dev *hdev,
212 u32 offset, u32 value)
213{
214 writel_relaxed(value, hdev->io_base + offset);
215}
216
217static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev)
218{
219 u32 status;
220
221 return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status,
222 !(status & HASH_SR_BUSY), 10, 10000);
223}
224
225static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length)
226{
227 u32 reg;
228
229 reg = stm32_hash_read(hdev, HASH_STR);
230 reg &= ~(HASH_STR_NBLW_MASK);
231 reg |= (8U * ((length) % 4U));
232 stm32_hash_write(hdev, HASH_STR, reg);
233}
234
235static int stm32_hash_write_key(struct stm32_hash_dev *hdev)
236{
237 struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
238 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
239 u32 reg;
240 int keylen = ctx->keylen;
241 void *key = ctx->key;
242
243 if (keylen) {
244 stm32_hash_set_nblw(hdev, keylen);
245
246 while (keylen > 0) {
247 stm32_hash_write(hdev, HASH_DIN, *(u32 *)key);
248 keylen -= 4;
249 key += 4;
250 }
251
252 reg = stm32_hash_read(hdev, HASH_STR);
253 reg |= HASH_STR_DCAL;
254 stm32_hash_write(hdev, HASH_STR, reg);
255
256 return -EINPROGRESS;
257 }
258
259 return 0;
260}
261
262static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev)
263{
264 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
265 struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
266 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
267
268 u32 reg = HASH_CR_INIT;
269
270 if (!(hdev->flags & HASH_FLAGS_INIT)) {
271 switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
272 case HASH_FLAGS_MD5:
273 reg |= HASH_CR_ALGO_MD5;
274 break;
275 case HASH_FLAGS_SHA1:
276 reg |= HASH_CR_ALGO_SHA1;
277 break;
278 case HASH_FLAGS_SHA224:
279 reg |= HASH_CR_ALGO_SHA224;
280 break;
281 case HASH_FLAGS_SHA256:
282 reg |= HASH_CR_ALGO_SHA256;
283 break;
284 default:
285 reg |= HASH_CR_ALGO_MD5;
286 }
287
288 reg |= (rctx->data_type << HASH_CR_DATATYPE_POS);
289
290 if (rctx->flags & HASH_FLAGS_HMAC) {
291 hdev->flags |= HASH_FLAGS_HMAC;
292 reg |= HASH_CR_MODE;
293 if (ctx->keylen > HASH_LONG_KEY)
294 reg |= HASH_CR_LKEY;
295 }
296
297 stm32_hash_write(hdev, HASH_IMR, HASH_DCIE);
298
299 stm32_hash_write(hdev, HASH_CR, reg);
300
301 hdev->flags |= HASH_FLAGS_INIT;
302
303 dev_dbg(hdev->dev, "Write Control %x\n", reg);
304 }
305}
306
307static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
308{
309 size_t count;
310
311 while ((rctx->bufcnt < rctx->buflen) && rctx->total) {
312 count = min(rctx->sg->length - rctx->offset, rctx->total);
313 count = min(count, rctx->buflen - rctx->bufcnt);
314
315 if (count <= 0) {
316 if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) {
317 rctx->sg = sg_next(rctx->sg);
318 continue;
319 } else {
320 break;
321 }
322 }
323
324 scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, rctx->sg,
325 rctx->offset, count, 0);
326
327 rctx->bufcnt += count;
328 rctx->offset += count;
329 rctx->total -= count;
330
331 if (rctx->offset == rctx->sg->length) {
332 rctx->sg = sg_next(rctx->sg);
333 if (rctx->sg)
334 rctx->offset = 0;
335 else
336 rctx->total = 0;
337 }
338 }
339}
340
341static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
342 const u8 *buf, size_t length, int final)
343{
344 unsigned int count, len32;
345 const u32 *buffer = (const u32 *)buf;
346 u32 reg;
347
348 if (final)
349 hdev->flags |= HASH_FLAGS_FINAL;
350
351 len32 = DIV_ROUND_UP(length, sizeof(u32));
352
353 dev_dbg(hdev->dev, "%s: length: %d, final: %x len32 %i\n",
354 __func__, length, final, len32);
355
356 hdev->flags |= HASH_FLAGS_CPU;
357
358 stm32_hash_write_ctrl(hdev);
359
360 if (stm32_hash_wait_busy(hdev))
361 return -ETIMEDOUT;
362
363 if ((hdev->flags & HASH_FLAGS_HMAC) &&
364 (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) {
365 hdev->flags |= HASH_FLAGS_HMAC_KEY;
366 stm32_hash_write_key(hdev);
367 if (stm32_hash_wait_busy(hdev))
368 return -ETIMEDOUT;
369 }
370
371 for (count = 0; count < len32; count++)
372 stm32_hash_write(hdev, HASH_DIN, buffer[count]);
373
374 if (final) {
375 stm32_hash_set_nblw(hdev, length);
376 reg = stm32_hash_read(hdev, HASH_STR);
377 reg |= HASH_STR_DCAL;
378 stm32_hash_write(hdev, HASH_STR, reg);
379 if (hdev->flags & HASH_FLAGS_HMAC) {
380 if (stm32_hash_wait_busy(hdev))
381 return -ETIMEDOUT;
382 stm32_hash_write_key(hdev);
383 }
384 return -EINPROGRESS;
385 }
386
387 return 0;
388}
389
390static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev)
391{
392 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
393 int bufcnt, err = 0, final;
394
395 dev_dbg(hdev->dev, "%s flags %lx\n", __func__, rctx->flags);
396
397 final = (rctx->flags & HASH_FLAGS_FINUP);
398
399 while ((rctx->total >= rctx->buflen) ||
400 (rctx->bufcnt + rctx->total >= rctx->buflen)) {
401 stm32_hash_append_sg(rctx);
402 bufcnt = rctx->bufcnt;
403 rctx->bufcnt = 0;
404 err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt, 0);
405 }
406
407 stm32_hash_append_sg(rctx);
408
409 if (final) {
410 bufcnt = rctx->bufcnt;
411 rctx->bufcnt = 0;
412 err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt,
413 (rctx->flags & HASH_FLAGS_FINUP));
414 }
415
416 return err;
417}
418
419static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
420 struct scatterlist *sg, int length, int mdma)
421{
422 struct dma_async_tx_descriptor *in_desc;
423 dma_cookie_t cookie;
424 u32 reg;
425 int err;
426
427 in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1,
428 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT |
429 DMA_CTRL_ACK);
430 if (!in_desc) {
431 dev_err(hdev->dev, "dmaengine_prep_slave error\n");
432 return -ENOMEM;
433 }
434
435 reinit_completion(&hdev->dma_completion);
436 in_desc->callback = stm32_hash_dma_callback;
437 in_desc->callback_param = hdev;
438
439 hdev->flags |= HASH_FLAGS_FINAL;
440 hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
441
442 reg = stm32_hash_read(hdev, HASH_CR);
443
444 if (mdma)
445 reg |= HASH_CR_MDMAT;
446 else
447 reg &= ~HASH_CR_MDMAT;
448
449 reg |= HASH_CR_DMAE;
450
451 stm32_hash_write(hdev, HASH_CR, reg);
452
453 stm32_hash_set_nblw(hdev, length);
454
455 cookie = dmaengine_submit(in_desc);
456 err = dma_submit_error(cookie);
457 if (err)
458 return -ENOMEM;
459
460 dma_async_issue_pending(hdev->dma_lch);
461
462 if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion,
463 msecs_to_jiffies(100)))
464 err = -ETIMEDOUT;
465
466 if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
467 NULL, NULL) != DMA_COMPLETE)
468 err = -ETIMEDOUT;
469
470 if (err) {
471 dev_err(hdev->dev, "DMA Error %i\n", err);
472 dmaengine_terminate_all(hdev->dma_lch);
473 return err;
474 }
475
476 return -EINPROGRESS;
477}
478
479static void stm32_hash_dma_callback(void *param)
480{
481 struct stm32_hash_dev *hdev = param;
482
483 complete(&hdev->dma_completion);
484
485 hdev->flags |= HASH_FLAGS_DMA_READY;
486}
487
488static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev)
489{
490 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
491 struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
492 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
493 int err;
494
495 if (ctx->keylen < HASH_DMA_THRESHOLD || (hdev->dma_mode == 1)) {
496 err = stm32_hash_write_key(hdev);
497 if (stm32_hash_wait_busy(hdev))
498 return -ETIMEDOUT;
499 } else {
500 if (!(hdev->flags & HASH_FLAGS_HMAC_KEY))
501 sg_init_one(&rctx->sg_key, ctx->key,
502 ALIGN(ctx->keylen, sizeof(u32)));
503
504 rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1,
505 DMA_TO_DEVICE);
506 if (rctx->dma_ct == 0) {
507 dev_err(hdev->dev, "dma_map_sg error\n");
508 return -ENOMEM;
509 }
510
511 err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0);
512
513 dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE);
514 }
515
516 return err;
517}
518
519static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
520{
521 struct dma_slave_config dma_conf;
522 int err;
523
524 memset(&dma_conf, 0, sizeof(dma_conf));
525
526 dma_conf.direction = DMA_MEM_TO_DEV;
527 dma_conf.dst_addr = hdev->phys_base + HASH_DIN;
528 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
529 dma_conf.src_maxburst = hdev->dma_maxburst;
530 dma_conf.dst_maxburst = hdev->dma_maxburst;
531 dma_conf.device_fc = false;
532
533 hdev->dma_lch = dma_request_slave_channel(hdev->dev, "in");
534 if (!hdev->dma_lch) {
535 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
536 return -EBUSY;
537 }
538
539 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
540 if (err) {
541 dma_release_channel(hdev->dma_lch);
542 hdev->dma_lch = NULL;
543 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
544 return err;
545 }
546
547 init_completion(&hdev->dma_completion);
548
549 return 0;
550}
551
552static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
553{
554 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
555 struct scatterlist sg[1], *tsg;
556 int err = 0, len = 0, reg, ncp;
557 unsigned int i;
558 const u32 *buffer = (const u32 *)rctx->buffer;
559
560 rctx->sg = hdev->req->src;
561 rctx->total = hdev->req->nbytes;
562
563 rctx->nents = sg_nents(rctx->sg);
564
565 if (rctx->nents < 0)
566 return -EINVAL;
567
568 stm32_hash_write_ctrl(hdev);
569
570 if (hdev->flags & HASH_FLAGS_HMAC) {
571 err = stm32_hash_hmac_dma_send(hdev);
572 if (err != -EINPROGRESS)
573 return err;
574 }
575
576 for_each_sg(rctx->sg, tsg, rctx->nents, i) {
577 len = sg->length;
578
579 sg[0] = *tsg;
580 if (sg_is_last(sg)) {
581 if (hdev->dma_mode == 1) {
582 len = (ALIGN(sg->length, 16) - 16);
583
584 ncp = sg_pcopy_to_buffer(
585 rctx->sg, rctx->nents,
586 rctx->buffer, sg->length - len,
587 rctx->total - sg->length + len);
588
589 sg->length = len;
590 } else {
591 if (!(IS_ALIGNED(sg->length, sizeof(u32)))) {
592 len = sg->length;
593 sg->length = ALIGN(sg->length,
594 sizeof(u32));
595 }
596 }
597 }
598
599 rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1,
600 DMA_TO_DEVICE);
601 if (rctx->dma_ct == 0) {
602 dev_err(hdev->dev, "dma_map_sg error\n");
603 return -ENOMEM;
604 }
605
606 err = stm32_hash_xmit_dma(hdev, sg, len,
607 !sg_is_last(sg));
608
609 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
610
611 if (err == -ENOMEM)
612 return err;
613 }
614
615 if (hdev->dma_mode == 1) {
616 if (stm32_hash_wait_busy(hdev))
617 return -ETIMEDOUT;
618 reg = stm32_hash_read(hdev, HASH_CR);
619 reg &= ~HASH_CR_DMAE;
620 reg |= HASH_CR_DMAA;
621 stm32_hash_write(hdev, HASH_CR, reg);
622
623 for (i = 0; i < DIV_ROUND_UP(ncp, sizeof(u32)); i++)
624 stm32_hash_write(hdev, HASH_DIN, buffer[i]);
625
626 stm32_hash_set_nblw(hdev, ncp);
627 reg = stm32_hash_read(hdev, HASH_STR);
628 reg |= HASH_STR_DCAL;
629 stm32_hash_write(hdev, HASH_STR, reg);
630 err = -EINPROGRESS;
631 }
632
633 if (hdev->flags & HASH_FLAGS_HMAC) {
634 if (stm32_hash_wait_busy(hdev))
635 return -ETIMEDOUT;
636 err = stm32_hash_hmac_dma_send(hdev);
637 }
638
639 return err;
640}
641
642static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx)
643{
644 struct stm32_hash_dev *hdev = NULL, *tmp;
645
646 spin_lock_bh(&stm32_hash.lock);
647 if (!ctx->hdev) {
648 list_for_each_entry(tmp, &stm32_hash.dev_list, list) {
649 hdev = tmp;
650 break;
651 }
652 ctx->hdev = hdev;
653 } else {
654 hdev = ctx->hdev;
655 }
656
657 spin_unlock_bh(&stm32_hash.lock);
658
659 return hdev;
660}
661
662static bool stm32_hash_dma_aligned_data(struct ahash_request *req)
663{
664 struct scatterlist *sg;
665 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
666 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
667 int i;
668
669 if (req->nbytes <= HASH_DMA_THRESHOLD)
670 return false;
671
672 if (sg_nents(req->src) > 1) {
673 if (hdev->dma_mode == 1)
674 return false;
675 for_each_sg(req->src, sg, sg_nents(req->src), i) {
676 if ((!IS_ALIGNED(sg->length, sizeof(u32))) &&
677 (!sg_is_last(sg)))
678 return false;
679 }
680 }
681
682 if (req->src->offset % 4)
683 return false;
684
685 return true;
686}
687
688static int stm32_hash_init(struct ahash_request *req)
689{
690 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
691 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
692 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
693 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
694
695 rctx->hdev = hdev;
696
697 rctx->flags = HASH_FLAGS_CPU;
698
699 rctx->digcnt = crypto_ahash_digestsize(tfm);
700 switch (rctx->digcnt) {
701 case MD5_DIGEST_SIZE:
702 rctx->flags |= HASH_FLAGS_MD5;
703 break;
704 case SHA1_DIGEST_SIZE:
705 rctx->flags |= HASH_FLAGS_SHA1;
706 break;
707 case SHA224_DIGEST_SIZE:
708 rctx->flags |= HASH_FLAGS_SHA224;
709 break;
710 case SHA256_DIGEST_SIZE:
711 rctx->flags |= HASH_FLAGS_SHA256;
712 break;
713 default:
714 return -EINVAL;
715 }
716
717 rctx->bufcnt = 0;
718 rctx->buflen = HASH_BUFLEN;
719 rctx->total = 0;
720 rctx->offset = 0;
721 rctx->data_type = HASH_DATA_8_BITS;
722
723 memset(rctx->buffer, 0, HASH_BUFLEN);
724
725 if (ctx->flags & HASH_FLAGS_HMAC)
726 rctx->flags |= HASH_FLAGS_HMAC;
727
728 dev_dbg(hdev->dev, "%s Flags %lx\n", __func__, rctx->flags);
729
730 return 0;
731}
732
733static int stm32_hash_update_req(struct stm32_hash_dev *hdev)
734{
735 return stm32_hash_update_cpu(hdev);
736}
737
738static int stm32_hash_final_req(struct stm32_hash_dev *hdev)
739{
740 struct ahash_request *req = hdev->req;
741 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
742 int err;
743
744 if (!(rctx->flags & HASH_FLAGS_CPU))
745 err = stm32_hash_dma_send(hdev);
746 else
747 err = stm32_hash_xmit_cpu(hdev, rctx->buffer, rctx->bufcnt, 1);
748
749 rctx->bufcnt = 0;
750
751 return err;
752}
753
754static void stm32_hash_copy_hash(struct ahash_request *req)
755{
756 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
757 u32 *hash = (u32 *)rctx->digest;
758 unsigned int i, hashsize;
759
760 switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
761 case HASH_FLAGS_MD5:
762 hashsize = MD5_DIGEST_SIZE;
763 break;
764 case HASH_FLAGS_SHA1:
765 hashsize = SHA1_DIGEST_SIZE;
766 break;
767 case HASH_FLAGS_SHA224:
768 hashsize = SHA224_DIGEST_SIZE;
769 break;
770 case HASH_FLAGS_SHA256:
771 hashsize = SHA256_DIGEST_SIZE;
772 break;
773 default:
774 return;
775 }
776
777 for (i = 0; i < hashsize / sizeof(u32); i++)
778 hash[i] = be32_to_cpu(stm32_hash_read(rctx->hdev,
779 HASH_HREG(i)));
780}
781
782static int stm32_hash_finish(struct ahash_request *req)
783{
784 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
785
786 if (!req->result)
787 return -EINVAL;
788
789 memcpy(req->result, rctx->digest, rctx->digcnt);
790
791 return 0;
792}
793
794static void stm32_hash_finish_req(struct ahash_request *req, int err)
795{
796 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
797 struct stm32_hash_dev *hdev = rctx->hdev;
798
799 if (!err && (HASH_FLAGS_FINAL & hdev->flags)) {
800 stm32_hash_copy_hash(req);
801 err = stm32_hash_finish(req);
802 hdev->flags &= ~(HASH_FLAGS_FINAL | HASH_FLAGS_CPU |
803 HASH_FLAGS_INIT | HASH_FLAGS_DMA_READY |
804 HASH_FLAGS_OUTPUT_READY | HASH_FLAGS_HMAC |
805 HASH_FLAGS_HMAC_INIT | HASH_FLAGS_HMAC_FINAL |
806 HASH_FLAGS_HMAC_KEY);
807 } else {
808 rctx->flags |= HASH_FLAGS_ERRORS;
809 }
810
811 crypto_finalize_hash_request(hdev->engine, req, err);
812}
813
814static int stm32_hash_hw_init(struct stm32_hash_dev *hdev,
815 struct stm32_hash_request_ctx *rctx)
816{
817 if (!(HASH_FLAGS_INIT & hdev->flags)) {
818 stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT);
819 stm32_hash_write(hdev, HASH_STR, 0);
820 stm32_hash_write(hdev, HASH_DIN, 0);
821 stm32_hash_write(hdev, HASH_IMR, 0);
822 hdev->err = 0;
823 }
824
825 return 0;
826}
827
828static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev,
829 struct ahash_request *req)
830{
831 return crypto_transfer_hash_request_to_engine(hdev->engine, req);
832}
833
834static int stm32_hash_prepare_req(struct crypto_engine *engine,
835 struct ahash_request *req)
836{
837 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
838 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
839 struct stm32_hash_request_ctx *rctx;
840
841 if (!hdev)
842 return -ENODEV;
843
844 hdev->req = req;
845
846 rctx = ahash_request_ctx(req);
847
848 dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n",
849 rctx->op, req->nbytes);
850
851 return stm32_hash_hw_init(hdev, rctx);
852}
853
854static int stm32_hash_one_request(struct crypto_engine *engine,
855 struct ahash_request *req)
856{
857 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
858 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
859 struct stm32_hash_request_ctx *rctx;
860 int err = 0;
861
862 if (!hdev)
863 return -ENODEV;
864
865 hdev->req = req;
866
867 rctx = ahash_request_ctx(req);
868
869 if (rctx->op == HASH_OP_UPDATE)
870 err = stm32_hash_update_req(hdev);
871 else if (rctx->op == HASH_OP_FINAL)
872 err = stm32_hash_final_req(hdev);
873
874 if (err != -EINPROGRESS)
875 /* done task will not finish it, so do it here */
876 stm32_hash_finish_req(req, err);
877
878 return 0;
879}
880
881static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op)
882{
883 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
884 struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
885 struct stm32_hash_dev *hdev = ctx->hdev;
886
887 rctx->op = op;
888
889 return stm32_hash_handle_queue(hdev, req);
890}
891
892static int stm32_hash_update(struct ahash_request *req)
893{
894 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
895 int ret;
896
897 if (!req->nbytes || !(rctx->flags & HASH_FLAGS_CPU))
898 return 0;
899
900 rctx->total = req->nbytes;
901 rctx->sg = req->src;
902 rctx->offset = 0;
903
904 if ((rctx->bufcnt + rctx->total < rctx->buflen)) {
905 stm32_hash_append_sg(rctx);
906 return 0;
907 }
908
909 ret = stm32_hash_enqueue(req, HASH_OP_UPDATE);
910
911 if (rctx->flags & HASH_FLAGS_FINUP)
912 return ret;
913
914 return 0;
915}
916
917static int stm32_hash_final(struct ahash_request *req)
918{
919 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
920
921 rctx->flags |= HASH_FLAGS_FINUP;
922
923 return stm32_hash_enqueue(req, HASH_OP_FINAL);
924}
925
926static int stm32_hash_finup(struct ahash_request *req)
927{
928 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
929 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
930 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
931 int err1, err2;
932
933 rctx->flags |= HASH_FLAGS_FINUP;
934
935 if (hdev->dma_lch && stm32_hash_dma_aligned_data(req))
936 rctx->flags &= ~HASH_FLAGS_CPU;
937
938 err1 = stm32_hash_update(req);
939
940 if (err1 == -EINPROGRESS || err1 == -EBUSY)
941 return err1;
942
943 /*
944 * final() has to be always called to cleanup resources
945 * even if update() failed, except EINPROGRESS
946 */
947 err2 = stm32_hash_final(req);
948
949 return err1 ?: err2;
950}
951
952static int stm32_hash_digest(struct ahash_request *req)
953{
954 return stm32_hash_init(req) ?: stm32_hash_finup(req);
955}
956
957static int stm32_hash_export(struct ahash_request *req, void *out)
958{
959 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
960 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
961 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
962 u32 *preg;
963 unsigned int i;
964
965 while (!(stm32_hash_read(hdev, HASH_SR) & HASH_SR_DATA_INPUT_READY))
966 cpu_relax();
967
968 rctx->hw_context = kmalloc(sizeof(u32) * (3 + HASH_CSR_REGISTER_NUMBER),
969 GFP_KERNEL);
970
971 preg = rctx->hw_context;
972
973 *preg++ = stm32_hash_read(hdev, HASH_IMR);
974 *preg++ = stm32_hash_read(hdev, HASH_STR);
975 *preg++ = stm32_hash_read(hdev, HASH_CR);
976 for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
977 *preg++ = stm32_hash_read(hdev, HASH_CSR(i));
978
979 memcpy(out, rctx, sizeof(*rctx));
980
981 return 0;
982}
983
984static int stm32_hash_import(struct ahash_request *req, const void *in)
985{
986 struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
987 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
988 struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
989 const u32 *preg = in;
990 u32 reg;
991 unsigned int i;
992
993 memcpy(rctx, in, sizeof(*rctx));
994
995 preg = rctx->hw_context;
996
997 stm32_hash_write(hdev, HASH_IMR, *preg++);
998 stm32_hash_write(hdev, HASH_STR, *preg++);
999 stm32_hash_write(hdev, HASH_CR, *preg);
1000 reg = *preg++ | HASH_CR_INIT;
1001 stm32_hash_write(hdev, HASH_CR, reg);
1002
1003 for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
1004 stm32_hash_write(hdev, HASH_CSR(i), *preg++);
1005
1006 kfree(rctx->hw_context);
1007
1008 return 0;
1009}
1010
1011static int stm32_hash_setkey(struct crypto_ahash *tfm,
1012 const u8 *key, unsigned int keylen)
1013{
1014 struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1015
1016 if (keylen <= HASH_MAX_KEY_SIZE) {
1017 memcpy(ctx->key, key, keylen);
1018 ctx->keylen = keylen;
1019 } else {
1020 return -ENOMEM;
1021 }
1022
1023 return 0;
1024}
1025
1026static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm,
1027 const char *algs_hmac_name)
1028{
1029 struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1030
1031 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1032 sizeof(struct stm32_hash_request_ctx));
1033
1034 ctx->keylen = 0;
1035
1036 if (algs_hmac_name)
1037 ctx->flags |= HASH_FLAGS_HMAC;
1038
1039 return 0;
1040}
1041
1042static int stm32_hash_cra_init(struct crypto_tfm *tfm)
1043{
1044 return stm32_hash_cra_init_algs(tfm, NULL);
1045}
1046
1047static int stm32_hash_cra_md5_init(struct crypto_tfm *tfm)
1048{
1049 return stm32_hash_cra_init_algs(tfm, "md5");
1050}
1051
1052static int stm32_hash_cra_sha1_init(struct crypto_tfm *tfm)
1053{
1054 return stm32_hash_cra_init_algs(tfm, "sha1");
1055}
1056
1057static int stm32_hash_cra_sha224_init(struct crypto_tfm *tfm)
1058{
1059 return stm32_hash_cra_init_algs(tfm, "sha224");
1060}
1061
1062static int stm32_hash_cra_sha256_init(struct crypto_tfm *tfm)
1063{
1064 return stm32_hash_cra_init_algs(tfm, "sha256");
1065}
1066
1067static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id)
1068{
1069 struct stm32_hash_dev *hdev = dev_id;
1070 int err;
1071
1072 if (HASH_FLAGS_CPU & hdev->flags) {
1073 if (HASH_FLAGS_OUTPUT_READY & hdev->flags) {
1074 hdev->flags &= ~HASH_FLAGS_OUTPUT_READY;
1075 goto finish;
1076 }
1077 } else if (HASH_FLAGS_DMA_READY & hdev->flags) {
1078 if (HASH_FLAGS_DMA_ACTIVE & hdev->flags) {
1079 hdev->flags &= ~HASH_FLAGS_DMA_ACTIVE;
1080 goto finish;
1081 }
1082 }
1083
1084 return IRQ_HANDLED;
1085
1086finish:
1087 /*Finish current request */
1088 stm32_hash_finish_req(hdev->req, err);
1089
1090 return IRQ_HANDLED;
1091}
1092
1093static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id)
1094{
1095 struct stm32_hash_dev *hdev = dev_id;
1096 u32 reg;
1097
1098 reg = stm32_hash_read(hdev, HASH_SR);
1099 if (reg & HASH_SR_OUTPUT_READY) {
1100 reg &= ~HASH_SR_OUTPUT_READY;
1101 stm32_hash_write(hdev, HASH_SR, reg);
1102 hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1103 return IRQ_WAKE_THREAD;
1104 }
1105
1106 return IRQ_NONE;
1107}
1108
1109static struct ahash_alg algs_md5_sha1[] = {
1110 {
1111 .init = stm32_hash_init,
1112 .update = stm32_hash_update,
1113 .final = stm32_hash_final,
1114 .finup = stm32_hash_finup,
1115 .digest = stm32_hash_digest,
1116 .export = stm32_hash_export,
1117 .import = stm32_hash_import,
1118 .halg = {
1119 .digestsize = MD5_DIGEST_SIZE,
1120 .statesize = sizeof(struct stm32_hash_request_ctx),
1121 .base = {
1122 .cra_name = "md5",
1123 .cra_driver_name = "stm32-md5",
1124 .cra_priority = 200,
1125 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1126 CRYPTO_ALG_ASYNC |
1127 CRYPTO_ALG_KERN_DRIVER_ONLY,
1128 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1129 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1130 .cra_alignmask = 3,
1131 .cra_init = stm32_hash_cra_init,
1132 .cra_module = THIS_MODULE,
1133 }
1134 }
1135 },
1136 {
1137 .init = stm32_hash_init,
1138 .update = stm32_hash_update,
1139 .final = stm32_hash_final,
1140 .finup = stm32_hash_finup,
1141 .digest = stm32_hash_digest,
1142 .export = stm32_hash_export,
1143 .import = stm32_hash_import,
1144 .setkey = stm32_hash_setkey,
1145 .halg = {
1146 .digestsize = MD5_DIGEST_SIZE,
1147 .statesize = sizeof(struct stm32_hash_request_ctx),
1148 .base = {
1149 .cra_name = "hmac(md5)",
1150 .cra_driver_name = "stm32-hmac-md5",
1151 .cra_priority = 200,
1152 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1153 CRYPTO_ALG_ASYNC |
1154 CRYPTO_ALG_KERN_DRIVER_ONLY,
1155 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1156 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1157 .cra_alignmask = 3,
1158 .cra_init = stm32_hash_cra_md5_init,
1159 .cra_module = THIS_MODULE,
1160 }
1161 }
1162 },
1163 {
1164 .init = stm32_hash_init,
1165 .update = stm32_hash_update,
1166 .final = stm32_hash_final,
1167 .finup = stm32_hash_finup,
1168 .digest = stm32_hash_digest,
1169 .export = stm32_hash_export,
1170 .import = stm32_hash_import,
1171 .halg = {
1172 .digestsize = SHA1_DIGEST_SIZE,
1173 .statesize = sizeof(struct stm32_hash_request_ctx),
1174 .base = {
1175 .cra_name = "sha1",
1176 .cra_driver_name = "stm32-sha1",
1177 .cra_priority = 200,
1178 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1179 CRYPTO_ALG_ASYNC |
1180 CRYPTO_ALG_KERN_DRIVER_ONLY,
1181 .cra_blocksize = SHA1_BLOCK_SIZE,
1182 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1183 .cra_alignmask = 3,
1184 .cra_init = stm32_hash_cra_init,
1185 .cra_module = THIS_MODULE,
1186 }
1187 }
1188 },
1189 {
1190 .init = stm32_hash_init,
1191 .update = stm32_hash_update,
1192 .final = stm32_hash_final,
1193 .finup = stm32_hash_finup,
1194 .digest = stm32_hash_digest,
1195 .export = stm32_hash_export,
1196 .import = stm32_hash_import,
1197 .setkey = stm32_hash_setkey,
1198 .halg = {
1199 .digestsize = SHA1_DIGEST_SIZE,
1200 .statesize = sizeof(struct stm32_hash_request_ctx),
1201 .base = {
1202 .cra_name = "hmac(sha1)",
1203 .cra_driver_name = "stm32-hmac-sha1",
1204 .cra_priority = 200,
1205 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1206 CRYPTO_ALG_ASYNC |
1207 CRYPTO_ALG_KERN_DRIVER_ONLY,
1208 .cra_blocksize = SHA1_BLOCK_SIZE,
1209 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1210 .cra_alignmask = 3,
1211 .cra_init = stm32_hash_cra_sha1_init,
1212 .cra_module = THIS_MODULE,
1213 }
1214 }
1215 },
1216};
1217
1218static struct ahash_alg algs_sha224_sha256[] = {
1219 {
1220 .init = stm32_hash_init,
1221 .update = stm32_hash_update,
1222 .final = stm32_hash_final,
1223 .finup = stm32_hash_finup,
1224 .digest = stm32_hash_digest,
1225 .export = stm32_hash_export,
1226 .import = stm32_hash_import,
1227 .halg = {
1228 .digestsize = SHA224_DIGEST_SIZE,
1229 .statesize = sizeof(struct stm32_hash_request_ctx),
1230 .base = {
1231 .cra_name = "sha224",
1232 .cra_driver_name = "stm32-sha224",
1233 .cra_priority = 200,
1234 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1235 CRYPTO_ALG_ASYNC |
1236 CRYPTO_ALG_KERN_DRIVER_ONLY,
1237 .cra_blocksize = SHA224_BLOCK_SIZE,
1238 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1239 .cra_alignmask = 3,
1240 .cra_init = stm32_hash_cra_init,
1241 .cra_module = THIS_MODULE,
1242 }
1243 }
1244 },
1245 {
1246 .init = stm32_hash_init,
1247 .update = stm32_hash_update,
1248 .final = stm32_hash_final,
1249 .finup = stm32_hash_finup,
1250 .digest = stm32_hash_digest,
1251 .setkey = stm32_hash_setkey,
1252 .export = stm32_hash_export,
1253 .import = stm32_hash_import,
1254 .halg = {
1255 .digestsize = SHA224_DIGEST_SIZE,
1256 .statesize = sizeof(struct stm32_hash_request_ctx),
1257 .base = {
1258 .cra_name = "hmac(sha224)",
1259 .cra_driver_name = "stm32-hmac-sha224",
1260 .cra_priority = 200,
1261 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1262 CRYPTO_ALG_ASYNC |
1263 CRYPTO_ALG_KERN_DRIVER_ONLY,
1264 .cra_blocksize = SHA224_BLOCK_SIZE,
1265 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1266 .cra_alignmask = 3,
1267 .cra_init = stm32_hash_cra_sha224_init,
1268 .cra_module = THIS_MODULE,
1269 }
1270 }
1271 },
1272 {
1273 .init = stm32_hash_init,
1274 .update = stm32_hash_update,
1275 .final = stm32_hash_final,
1276 .finup = stm32_hash_finup,
1277 .digest = stm32_hash_digest,
1278 .export = stm32_hash_export,
1279 .import = stm32_hash_import,
1280 .halg = {
1281 .digestsize = SHA256_DIGEST_SIZE,
1282 .statesize = sizeof(struct stm32_hash_request_ctx),
1283 .base = {
1284 .cra_name = "sha256",
1285 .cra_driver_name = "stm32-sha256",
1286 .cra_priority = 200,
1287 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1288 CRYPTO_ALG_ASYNC |
1289 CRYPTO_ALG_KERN_DRIVER_ONLY,
1290 .cra_blocksize = SHA256_BLOCK_SIZE,
1291 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1292 .cra_alignmask = 3,
1293 .cra_init = stm32_hash_cra_init,
1294 .cra_module = THIS_MODULE,
1295 }
1296 }
1297 },
1298 {
1299 .init = stm32_hash_init,
1300 .update = stm32_hash_update,
1301 .final = stm32_hash_final,
1302 .finup = stm32_hash_finup,
1303 .digest = stm32_hash_digest,
1304 .export = stm32_hash_export,
1305 .import = stm32_hash_import,
1306 .setkey = stm32_hash_setkey,
1307 .halg = {
1308 .digestsize = SHA256_DIGEST_SIZE,
1309 .statesize = sizeof(struct stm32_hash_request_ctx),
1310 .base = {
1311 .cra_name = "hmac(sha256)",
1312 .cra_driver_name = "stm32-hmac-sha256",
1313 .cra_priority = 200,
1314 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1315 CRYPTO_ALG_ASYNC |
1316 CRYPTO_ALG_KERN_DRIVER_ONLY,
1317 .cra_blocksize = SHA256_BLOCK_SIZE,
1318 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1319 .cra_alignmask = 3,
1320 .cra_init = stm32_hash_cra_sha256_init,
1321 .cra_module = THIS_MODULE,
1322 }
1323 }
1324 },
1325};
1326
1327static int stm32_hash_register_algs(struct stm32_hash_dev *hdev)
1328{
1329 unsigned int i, j;
1330 int err;
1331
1332 for (i = 0; i < hdev->pdata->algs_info_size; i++) {
1333 for (j = 0; j < hdev->pdata->algs_info[i].size; j++) {
1334 err = crypto_register_ahash(
1335 &hdev->pdata->algs_info[i].algs_list[j]);
1336 if (err)
1337 goto err_algs;
1338 }
1339 }
1340
1341 return 0;
1342err_algs:
1343 dev_err(hdev->dev, "Algo %d : %d failed\n", i, j);
1344 for (; i--; ) {
1345 for (; j--;)
1346 crypto_unregister_ahash(
1347 &hdev->pdata->algs_info[i].algs_list[j]);
1348 }
1349
1350 return err;
1351}
1352
1353static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev)
1354{
1355 unsigned int i, j;
1356
1357 for (i = 0; i < hdev->pdata->algs_info_size; i++) {
1358 for (j = 0; j < hdev->pdata->algs_info[i].size; j++)
1359 crypto_unregister_ahash(
1360 &hdev->pdata->algs_info[i].algs_list[j]);
1361 }
1362
1363 return 0;
1364}
1365
1366static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = {
1367 {
1368 .algs_list = algs_md5_sha1,
1369 .size = ARRAY_SIZE(algs_md5_sha1),
1370 },
1371};
1372
1373static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = {
1374 .algs_info = stm32_hash_algs_info_stm32f4,
1375 .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f4),
1376};
1377
1378static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = {
1379 {
1380 .algs_list = algs_md5_sha1,
1381 .size = ARRAY_SIZE(algs_md5_sha1),
1382 },
1383 {
1384 .algs_list = algs_sha224_sha256,
1385 .size = ARRAY_SIZE(algs_sha224_sha256),
1386 },
1387};
1388
1389static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = {
1390 .algs_info = stm32_hash_algs_info_stm32f7,
1391 .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f7),
1392};
1393
1394static const struct of_device_id stm32_hash_of_match[] = {
1395 {
1396 .compatible = "st,stm32f456-hash",
1397 .data = &stm32_hash_pdata_stm32f4,
1398 },
1399 {
1400 .compatible = "st,stm32f756-hash",
1401 .data = &stm32_hash_pdata_stm32f7,
1402 },
1403 {},
1404};
1405
1406MODULE_DEVICE_TABLE(of, stm32_hash_of_match);
1407
1408static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev,
1409 struct device *dev)
1410{
1411 const struct of_device_id *match;
1412 int err;
1413
1414 match = of_match_device(stm32_hash_of_match, dev);
1415 if (!match) {
1416 dev_err(dev, "no compatible OF match\n");
1417 return -EINVAL;
1418 }
1419
1420 err = of_property_read_u32(dev->of_node, "dma-maxburst",
1421 &hdev->dma_maxburst);
1422
1423 hdev->pdata = match->data;
1424
1425 return err;
1426}
1427
1428static int stm32_hash_probe(struct platform_device *pdev)
1429{
1430 struct stm32_hash_dev *hdev;
1431 struct device *dev = &pdev->dev;
1432 struct resource *res;
1433 int ret, irq;
1434
1435 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
1436 if (!hdev)
1437 return -ENOMEM;
1438
1439 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1440 hdev->io_base = devm_ioremap_resource(dev, res);
1441 if (IS_ERR(hdev->io_base))
1442 return PTR_ERR(hdev->io_base);
1443
1444 hdev->phys_base = res->start;
1445
1446 ret = stm32_hash_get_of_match(hdev, dev);
1447 if (ret)
1448 return ret;
1449
1450 irq = platform_get_irq(pdev, 0);
1451 if (irq < 0) {
1452 dev_err(dev, "Cannot get IRQ resource\n");
1453 return irq;
1454 }
1455
1456 ret = devm_request_threaded_irq(dev, irq, stm32_hash_irq_handler,
1457 stm32_hash_irq_thread, IRQF_ONESHOT,
1458 dev_name(dev), hdev);
1459 if (ret) {
1460 dev_err(dev, "Cannot grab IRQ\n");
1461 return ret;
1462 }
1463
1464 hdev->clk = devm_clk_get(&pdev->dev, NULL);
1465 if (IS_ERR(hdev->clk)) {
1466 dev_err(dev, "failed to get clock for hash (%lu)\n",
1467 PTR_ERR(hdev->clk));
1468 return PTR_ERR(hdev->clk);
1469 }
1470
1471 ret = clk_prepare_enable(hdev->clk);
1472 if (ret) {
1473 dev_err(dev, "failed to enable hash clock (%d)\n", ret);
1474 return ret;
1475 }
1476
1477 hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
1478 if (!IS_ERR(hdev->rst)) {
1479 reset_control_assert(hdev->rst);
1480 udelay(2);
1481 reset_control_deassert(hdev->rst);
1482 }
1483
1484 hdev->dev = dev;
1485
1486 platform_set_drvdata(pdev, hdev);
1487
1488 ret = stm32_hash_dma_init(hdev);
1489 if (ret)
1490 dev_dbg(dev, "DMA mode not available\n");
1491
1492 spin_lock(&stm32_hash.lock);
1493 list_add_tail(&hdev->list, &stm32_hash.dev_list);
1494 spin_unlock(&stm32_hash.lock);
1495
1496 /* Initialize crypto engine */
1497 hdev->engine = crypto_engine_alloc_init(dev, 1);
1498 if (!hdev->engine) {
1499 ret = -ENOMEM;
1500 goto err_engine;
1501 }
1502
1503 hdev->engine->prepare_hash_request = stm32_hash_prepare_req;
1504 hdev->engine->hash_one_request = stm32_hash_one_request;
1505
1506 ret = crypto_engine_start(hdev->engine);
1507 if (ret)
1508 goto err_engine_start;
1509
1510 hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR);
1511
1512 /* Register algos */
1513 ret = stm32_hash_register_algs(hdev);
1514 if (ret)
1515 goto err_algs;
1516
1517 dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n",
1518 stm32_hash_read(hdev, HASH_VER), hdev->dma_mode);
1519
1520 return 0;
1521
1522err_algs:
1523err_engine_start:
1524 crypto_engine_exit(hdev->engine);
1525err_engine:
1526 spin_lock(&stm32_hash.lock);
1527 list_del(&hdev->list);
1528 spin_unlock(&stm32_hash.lock);
1529
1530 if (hdev->dma_lch)
1531 dma_release_channel(hdev->dma_lch);
1532
1533 clk_disable_unprepare(hdev->clk);
1534
1535 return ret;
1536}
1537
1538static int stm32_hash_remove(struct platform_device *pdev)
1539{
1540 static struct stm32_hash_dev *hdev;
1541
1542 hdev = platform_get_drvdata(pdev);
1543 if (!hdev)
1544 return -ENODEV;
1545
1546 stm32_hash_unregister_algs(hdev);
1547
1548 crypto_engine_exit(hdev->engine);
1549
1550 spin_lock(&stm32_hash.lock);
1551 list_del(&hdev->list);
1552 spin_unlock(&stm32_hash.lock);
1553
1554 if (hdev->dma_lch)
1555 dma_release_channel(hdev->dma_lch);
1556
1557 clk_disable_unprepare(hdev->clk);
1558
1559 return 0;
1560}
1561
1562static struct platform_driver stm32_hash_driver = {
1563 .probe = stm32_hash_probe,
1564 .remove = stm32_hash_remove,
1565 .driver = {
1566 .name = "stm32-hash",
1567 .of_match_table = stm32_hash_of_match,
1568 }
1569};
1570
1571module_platform_driver(stm32_hash_driver);
1572
1573MODULE_DESCRIPTION("STM32 SHA1/224/256 & MD5 (HMAC) hw accelerator driver");
1574MODULE_AUTHOR("Lionel Debieve <lionel.debieve@st.com>");
1575MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c
index ec83b1e6bfe8..090582baecfe 100644
--- a/drivers/crypto/stm32/stm32_crc32.c
+++ b/drivers/crypto/stm32/stm32_crc32.c
@@ -107,12 +107,12 @@ static int stm32_crc_init(struct shash_desc *desc)
107 spin_unlock_bh(&crc_list.lock); 107 spin_unlock_bh(&crc_list.lock);
108 108
109 /* Reset, set key, poly and configure in bit reverse mode */ 109 /* Reset, set key, poly and configure in bit reverse mode */
110 writel(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT); 110 writel_relaxed(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT);
111 writel(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL); 111 writel_relaxed(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL);
112 writel(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR); 112 writel_relaxed(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR);
113 113
114 /* Store partial result */ 114 /* Store partial result */
115 ctx->partial = readl(ctx->crc->regs + CRC_DR); 115 ctx->partial = readl_relaxed(ctx->crc->regs + CRC_DR);
116 ctx->crc->nb_pending_bytes = 0; 116 ctx->crc->nb_pending_bytes = 0;
117 117
118 return 0; 118 return 0;
@@ -135,7 +135,8 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
135 135
136 if (crc->nb_pending_bytes == sizeof(u32)) { 136 if (crc->nb_pending_bytes == sizeof(u32)) {
137 /* Process completed pending data */ 137 /* Process completed pending data */
138 writel(*(u32 *)crc->pending_data, crc->regs + CRC_DR); 138 writel_relaxed(*(u32 *)crc->pending_data,
139 crc->regs + CRC_DR);
139 crc->nb_pending_bytes = 0; 140 crc->nb_pending_bytes = 0;
140 } 141 }
141 } 142 }
@@ -143,10 +144,10 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
143 d32 = (u32 *)d8; 144 d32 = (u32 *)d8;
144 for (i = 0; i < length >> 2; i++) 145 for (i = 0; i < length >> 2; i++)
145 /* Process 32 bits data */ 146 /* Process 32 bits data */
146 writel(*(d32++), crc->regs + CRC_DR); 147 writel_relaxed(*(d32++), crc->regs + CRC_DR);
147 148
148 /* Store partial result */ 149 /* Store partial result */
149 ctx->partial = readl(crc->regs + CRC_DR); 150 ctx->partial = readl_relaxed(crc->regs + CRC_DR);
150 151
151 /* Check for pending data (non 32 bits) */ 152 /* Check for pending data (non 32 bits) */
152 length &= 3; 153 length &= 3;
@@ -295,7 +296,7 @@ static int stm32_crc_remove(struct platform_device *pdev)
295 list_del(&crc->list); 296 list_del(&crc->list);
296 spin_unlock(&crc_list.lock); 297 spin_unlock(&crc_list.lock);
297 298
298 crypto_unregister_shash(algs); 299 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
299 300
300 clk_disable_unprepare(crc->clk); 301 clk_disable_unprepare(crc->clk);
301 302
diff --git a/drivers/crypto/sunxi-ss/Makefile b/drivers/crypto/sunxi-ss/Makefile
index 8f4c7a273141..ccb893219079 100644
--- a/drivers/crypto/sunxi-ss/Makefile
+++ b/drivers/crypto/sunxi-ss/Makefile
@@ -1,2 +1,3 @@
1obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o 1obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o
2sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o 2sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o
3sun4i-ss-$(CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG) += sun4i-ss-prng.o
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
index 02ad8256e900..1547cbe13dc2 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
@@ -213,6 +213,23 @@ static struct sun4i_ss_alg_template ss_algs[] = {
213 } 213 }
214 } 214 }
215}, 215},
216#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
217{
218 .type = CRYPTO_ALG_TYPE_RNG,
219 .alg.rng = {
220 .base = {
221 .cra_name = "stdrng",
222 .cra_driver_name = "sun4i_ss_rng",
223 .cra_priority = 300,
224 .cra_ctxsize = 0,
225 .cra_module = THIS_MODULE,
226 },
227 .generate = sun4i_ss_prng_generate,
228 .seed = sun4i_ss_prng_seed,
229 .seedsize = SS_SEED_LEN / BITS_PER_BYTE,
230 }
231},
232#endif
216}; 233};
217 234
218static int sun4i_ss_probe(struct platform_device *pdev) 235static int sun4i_ss_probe(struct platform_device *pdev)
@@ -355,6 +372,13 @@ static int sun4i_ss_probe(struct platform_device *pdev)
355 goto error_alg; 372 goto error_alg;
356 } 373 }
357 break; 374 break;
375 case CRYPTO_ALG_TYPE_RNG:
376 err = crypto_register_rng(&ss_algs[i].alg.rng);
377 if (err) {
378 dev_err(ss->dev, "Fail to register %s\n",
379 ss_algs[i].alg.rng.base.cra_name);
380 }
381 break;
358 } 382 }
359 } 383 }
360 platform_set_drvdata(pdev, ss); 384 platform_set_drvdata(pdev, ss);
@@ -369,6 +393,9 @@ error_alg:
369 case CRYPTO_ALG_TYPE_AHASH: 393 case CRYPTO_ALG_TYPE_AHASH:
370 crypto_unregister_ahash(&ss_algs[i].alg.hash); 394 crypto_unregister_ahash(&ss_algs[i].alg.hash);
371 break; 395 break;
396 case CRYPTO_ALG_TYPE_RNG:
397 crypto_unregister_rng(&ss_algs[i].alg.rng);
398 break;
372 } 399 }
373 } 400 }
374 if (ss->reset) 401 if (ss->reset)
@@ -393,6 +420,9 @@ static int sun4i_ss_remove(struct platform_device *pdev)
393 case CRYPTO_ALG_TYPE_AHASH: 420 case CRYPTO_ALG_TYPE_AHASH:
394 crypto_unregister_ahash(&ss_algs[i].alg.hash); 421 crypto_unregister_ahash(&ss_algs[i].alg.hash);
395 break; 422 break;
423 case CRYPTO_ALG_TYPE_RNG:
424 crypto_unregister_rng(&ss_algs[i].alg.rng);
425 break;
396 } 426 }
397 } 427 }
398 428
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-prng.c b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
new file mode 100644
index 000000000000..0d01d1624252
--- /dev/null
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
@@ -0,0 +1,56 @@
1#include "sun4i-ss.h"
2
3int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed,
4 unsigned int slen)
5{
6 struct sun4i_ss_alg_template *algt;
7 struct rng_alg *alg = crypto_rng_alg(tfm);
8
9 algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng);
10 memcpy(algt->ss->seed, seed, slen);
11
12 return 0;
13}
14
15int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
16 unsigned int slen, u8 *dst, unsigned int dlen)
17{
18 struct sun4i_ss_alg_template *algt;
19 struct rng_alg *alg = crypto_rng_alg(tfm);
20 int i;
21 u32 v;
22 u32 *data = (u32 *)dst;
23 const u32 mode = SS_OP_PRNG | SS_PRNG_CONTINUE | SS_ENABLED;
24 size_t len;
25 struct sun4i_ss_ctx *ss;
26 unsigned int todo = (dlen / 4) * 4;
27
28 algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng);
29 ss = algt->ss;
30
31 spin_lock(&ss->slock);
32
33 writel(mode, ss->base + SS_CTL);
34
35 while (todo > 0) {
36 /* write the seed */
37 for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++)
38 writel(ss->seed[i], ss->base + SS_KEY0 + i * 4);
39
40 /* Read the random data */
41 len = min_t(size_t, SS_DATA_LEN / BITS_PER_BYTE, todo);
42 readsl(ss->base + SS_TXFIFO, data, len / 4);
43 data += len / 4;
44 todo -= len;
45
46 /* Update the seed */
47 for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) {
48 v = readl(ss->base + SS_KEY0 + i * 4);
49 ss->seed[i] = v;
50 }
51 }
52
53 writel(0, ss->base + SS_CTL);
54 spin_unlock(&ss->slock);
55 return dlen;
56}
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h
index a0e1efc1cb2a..f3ac90692ac6 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss.h
+++ b/drivers/crypto/sunxi-ss/sun4i-ss.h
@@ -32,6 +32,7 @@
32#include <crypto/aes.h> 32#include <crypto/aes.h>
33#include <crypto/des.h> 33#include <crypto/des.h>
34#include <crypto/internal/rng.h> 34#include <crypto/internal/rng.h>
35#include <crypto/rng.h>
35 36
36#define SS_CTL 0x00 37#define SS_CTL 0x00
37#define SS_KEY0 0x04 38#define SS_KEY0 0x04
@@ -127,6 +128,9 @@
127#define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) 128#define SS_RXFIFO_EMP_INT_ENABLE (1 << 2)
128#define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) 129#define SS_TXFIFO_AVA_INT_ENABLE (1 << 0)
129 130
131#define SS_SEED_LEN 192
132#define SS_DATA_LEN 160
133
130struct sun4i_ss_ctx { 134struct sun4i_ss_ctx {
131 void __iomem *base; 135 void __iomem *base;
132 int irq; 136 int irq;
@@ -136,6 +140,9 @@ struct sun4i_ss_ctx {
136 struct device *dev; 140 struct device *dev;
137 struct resource *res; 141 struct resource *res;
138 spinlock_t slock; /* control the use of the device */ 142 spinlock_t slock; /* control the use of the device */
143#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
144 u32 seed[SS_SEED_LEN / BITS_PER_LONG];
145#endif
139}; 146};
140 147
141struct sun4i_ss_alg_template { 148struct sun4i_ss_alg_template {
@@ -144,6 +151,7 @@ struct sun4i_ss_alg_template {
144 union { 151 union {
145 struct skcipher_alg crypto; 152 struct skcipher_alg crypto;
146 struct ahash_alg hash; 153 struct ahash_alg hash;
154 struct rng_alg rng;
147 } alg; 155 } alg;
148 struct sun4i_ss_ctx *ss; 156 struct sun4i_ss_ctx *ss;
149}; 157};
@@ -201,3 +209,6 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
201 unsigned int keylen); 209 unsigned int keylen);
202int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, 210int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
203 unsigned int keylen); 211 unsigned int keylen);
212int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
213 unsigned int slen, u8 *dst, unsigned int dlen);
214int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c
index 49defda4e03d..5035b0dc1e40 100644
--- a/drivers/crypto/virtio/virtio_crypto_algs.c
+++ b/drivers/crypto/virtio/virtio_crypto_algs.c
@@ -27,12 +27,68 @@
27#include <uapi/linux/virtio_crypto.h> 27#include <uapi/linux/virtio_crypto.h>
28#include "virtio_crypto_common.h" 28#include "virtio_crypto_common.h"
29 29
30
31struct virtio_crypto_ablkcipher_ctx {
32 struct virtio_crypto *vcrypto;
33 struct crypto_tfm *tfm;
34
35 struct virtio_crypto_sym_session_info enc_sess_info;
36 struct virtio_crypto_sym_session_info dec_sess_info;
37};
38
39struct virtio_crypto_sym_request {
40 struct virtio_crypto_request base;
41
42 /* Cipher or aead */
43 uint32_t type;
44 struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
45 struct ablkcipher_request *ablkcipher_req;
46 uint8_t *iv;
47 /* Encryption? */
48 bool encrypt;
49};
50
30/* 51/*
31 * The algs_lock protects the below global virtio_crypto_active_devs 52 * The algs_lock protects the below global virtio_crypto_active_devs
32 * and crypto algorithms registion. 53 * and crypto algorithms registion.
33 */ 54 */
34static DEFINE_MUTEX(algs_lock); 55static DEFINE_MUTEX(algs_lock);
35static unsigned int virtio_crypto_active_devs; 56static unsigned int virtio_crypto_active_devs;
57static void virtio_crypto_ablkcipher_finalize_req(
58 struct virtio_crypto_sym_request *vc_sym_req,
59 struct ablkcipher_request *req,
60 int err);
61
62static void virtio_crypto_dataq_sym_callback
63 (struct virtio_crypto_request *vc_req, int len)
64{
65 struct virtio_crypto_sym_request *vc_sym_req =
66 container_of(vc_req, struct virtio_crypto_sym_request, base);
67 struct ablkcipher_request *ablk_req;
68 int error;
69
70 /* Finish the encrypt or decrypt process */
71 if (vc_sym_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
72 switch (vc_req->status) {
73 case VIRTIO_CRYPTO_OK:
74 error = 0;
75 break;
76 case VIRTIO_CRYPTO_INVSESS:
77 case VIRTIO_CRYPTO_ERR:
78 error = -EINVAL;
79 break;
80 case VIRTIO_CRYPTO_BADMSG:
81 error = -EBADMSG;
82 break;
83 default:
84 error = -EIO;
85 break;
86 }
87 ablk_req = vc_sym_req->ablkcipher_req;
88 virtio_crypto_ablkcipher_finalize_req(vc_sym_req,
89 ablk_req, error);
90 }
91}
36 92
37static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg) 93static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg)
38{ 94{
@@ -286,13 +342,14 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
286} 342}
287 343
288static int 344static int
289__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req, 345__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
290 struct ablkcipher_request *req, 346 struct ablkcipher_request *req,
291 struct data_queue *data_vq) 347 struct data_queue *data_vq)
292{ 348{
293 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 349 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
350 struct virtio_crypto_ablkcipher_ctx *ctx = vc_sym_req->ablkcipher_ctx;
351 struct virtio_crypto_request *vc_req = &vc_sym_req->base;
294 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 352 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
295 struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx;
296 struct virtio_crypto *vcrypto = ctx->vcrypto; 353 struct virtio_crypto *vcrypto = ctx->vcrypto;
297 struct virtio_crypto_op_data_req *req_data; 354 struct virtio_crypto_op_data_req *req_data;
298 int src_nents, dst_nents; 355 int src_nents, dst_nents;
@@ -326,9 +383,9 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
326 } 383 }
327 384
328 vc_req->req_data = req_data; 385 vc_req->req_data = req_data;
329 vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER; 386 vc_sym_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
330 /* Head of operation */ 387 /* Head of operation */
331 if (vc_req->encrypt) { 388 if (vc_sym_req->encrypt) {
332 req_data->header.session_id = 389 req_data->header.session_id =
333 cpu_to_le64(ctx->enc_sess_info.session_id); 390 cpu_to_le64(ctx->enc_sess_info.session_id);
334 req_data->header.opcode = 391 req_data->header.opcode =
@@ -383,7 +440,7 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
383 memcpy(iv, req->info, ivsize); 440 memcpy(iv, req->info, ivsize);
384 sg_init_one(&iv_sg, iv, ivsize); 441 sg_init_one(&iv_sg, iv, ivsize);
385 sgs[num_out++] = &iv_sg; 442 sgs[num_out++] = &iv_sg;
386 vc_req->iv = iv; 443 vc_sym_req->iv = iv;
387 444
388 /* Source data */ 445 /* Source data */
389 for (i = 0; i < src_nents; i++) 446 for (i = 0; i < src_nents; i++)
@@ -421,15 +478,18 @@ static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
421{ 478{
422 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); 479 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
423 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm); 480 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
424 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); 481 struct virtio_crypto_sym_request *vc_sym_req =
482 ablkcipher_request_ctx(req);
483 struct virtio_crypto_request *vc_req = &vc_sym_req->base;
425 struct virtio_crypto *vcrypto = ctx->vcrypto; 484 struct virtio_crypto *vcrypto = ctx->vcrypto;
426 /* Use the first data virtqueue as default */ 485 /* Use the first data virtqueue as default */
427 struct data_queue *data_vq = &vcrypto->data_vq[0]; 486 struct data_queue *data_vq = &vcrypto->data_vq[0];
428 487
429 vc_req->ablkcipher_ctx = ctx;
430 vc_req->ablkcipher_req = req;
431 vc_req->encrypt = true;
432 vc_req->dataq = data_vq; 488 vc_req->dataq = data_vq;
489 vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
490 vc_sym_req->ablkcipher_ctx = ctx;
491 vc_sym_req->ablkcipher_req = req;
492 vc_sym_req->encrypt = true;
433 493
434 return crypto_transfer_cipher_request_to_engine(data_vq->engine, req); 494 return crypto_transfer_cipher_request_to_engine(data_vq->engine, req);
435} 495}
@@ -438,16 +498,18 @@ static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
438{ 498{
439 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); 499 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
440 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm); 500 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
441 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); 501 struct virtio_crypto_sym_request *vc_sym_req =
502 ablkcipher_request_ctx(req);
503 struct virtio_crypto_request *vc_req = &vc_sym_req->base;
442 struct virtio_crypto *vcrypto = ctx->vcrypto; 504 struct virtio_crypto *vcrypto = ctx->vcrypto;
443 /* Use the first data virtqueue as default */ 505 /* Use the first data virtqueue as default */
444 struct data_queue *data_vq = &vcrypto->data_vq[0]; 506 struct data_queue *data_vq = &vcrypto->data_vq[0];
445 507
446 vc_req->ablkcipher_ctx = ctx;
447 vc_req->ablkcipher_req = req;
448
449 vc_req->encrypt = false;
450 vc_req->dataq = data_vq; 508 vc_req->dataq = data_vq;
509 vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
510 vc_sym_req->ablkcipher_ctx = ctx;
511 vc_sym_req->ablkcipher_req = req;
512 vc_sym_req->encrypt = false;
451 513
452 return crypto_transfer_cipher_request_to_engine(data_vq->engine, req); 514 return crypto_transfer_cipher_request_to_engine(data_vq->engine, req);
453} 515}
@@ -456,7 +518,7 @@ static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm)
456{ 518{
457 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 519 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
458 520
459 tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_request); 521 tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_sym_request);
460 ctx->tfm = tfm; 522 ctx->tfm = tfm;
461 523
462 return 0; 524 return 0;
@@ -479,11 +541,13 @@ int virtio_crypto_ablkcipher_crypt_req(
479 struct crypto_engine *engine, 541 struct crypto_engine *engine,
480 struct ablkcipher_request *req) 542 struct ablkcipher_request *req)
481{ 543{
482 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); 544 struct virtio_crypto_sym_request *vc_sym_req =
545 ablkcipher_request_ctx(req);
546 struct virtio_crypto_request *vc_req = &vc_sym_req->base;
483 struct data_queue *data_vq = vc_req->dataq; 547 struct data_queue *data_vq = vc_req->dataq;
484 int ret; 548 int ret;
485 549
486 ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq); 550 ret = __virtio_crypto_ablkcipher_do_req(vc_sym_req, req, data_vq);
487 if (ret < 0) 551 if (ret < 0)
488 return ret; 552 return ret;
489 553
@@ -492,14 +556,15 @@ int virtio_crypto_ablkcipher_crypt_req(
492 return 0; 556 return 0;
493} 557}
494 558
495void virtio_crypto_ablkcipher_finalize_req( 559static void virtio_crypto_ablkcipher_finalize_req(
496 struct virtio_crypto_request *vc_req, 560 struct virtio_crypto_sym_request *vc_sym_req,
497 struct ablkcipher_request *req, 561 struct ablkcipher_request *req,
498 int err) 562 int err)
499{ 563{
500 crypto_finalize_cipher_request(vc_req->dataq->engine, req, err); 564 crypto_finalize_cipher_request(vc_sym_req->base.dataq->engine,
501 565 req, err);
502 virtcrypto_clear_request(vc_req); 566 kzfree(vc_sym_req->iv);
567 virtcrypto_clear_request(&vc_sym_req->base);
503} 568}
504 569
505static struct crypto_alg virtio_crypto_algs[] = { { 570static struct crypto_alg virtio_crypto_algs[] = { {
diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h
index da6d8c0ea407..e976539a05d9 100644
--- a/drivers/crypto/virtio/virtio_crypto_common.h
+++ b/drivers/crypto/virtio/virtio_crypto_common.h
@@ -83,26 +83,16 @@ struct virtio_crypto_sym_session_info {
83 __u64 session_id; 83 __u64 session_id;
84}; 84};
85 85
86struct virtio_crypto_ablkcipher_ctx { 86struct virtio_crypto_request;
87 struct virtio_crypto *vcrypto; 87typedef void (*virtio_crypto_data_callback)
88 struct crypto_tfm *tfm; 88 (struct virtio_crypto_request *vc_req, int len);
89
90 struct virtio_crypto_sym_session_info enc_sess_info;
91 struct virtio_crypto_sym_session_info dec_sess_info;
92};
93 89
94struct virtio_crypto_request { 90struct virtio_crypto_request {
95 /* Cipher or aead */
96 uint32_t type;
97 uint8_t status; 91 uint8_t status;
98 struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
99 struct ablkcipher_request *ablkcipher_req;
100 struct virtio_crypto_op_data_req *req_data; 92 struct virtio_crypto_op_data_req *req_data;
101 struct scatterlist **sgs; 93 struct scatterlist **sgs;
102 uint8_t *iv;
103 /* Encryption? */
104 bool encrypt;
105 struct data_queue *dataq; 94 struct data_queue *dataq;
95 virtio_crypto_data_callback alg_cb;
106}; 96};
107 97
108int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev); 98int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev);
@@ -119,10 +109,6 @@ void virtcrypto_dev_stop(struct virtio_crypto *vcrypto);
119int virtio_crypto_ablkcipher_crypt_req( 109int virtio_crypto_ablkcipher_crypt_req(
120 struct crypto_engine *engine, 110 struct crypto_engine *engine,
121 struct ablkcipher_request *req); 111 struct ablkcipher_request *req);
122void virtio_crypto_ablkcipher_finalize_req(
123 struct virtio_crypto_request *vc_req,
124 struct ablkcipher_request *req,
125 int err);
126 112
127void 113void
128virtcrypto_clear_request(struct virtio_crypto_request *vc_req); 114virtcrypto_clear_request(struct virtio_crypto_request *vc_req);
diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c
index a111cd72797b..ff1410a32c2b 100644
--- a/drivers/crypto/virtio/virtio_crypto_core.c
+++ b/drivers/crypto/virtio/virtio_crypto_core.c
@@ -29,7 +29,6 @@ void
29virtcrypto_clear_request(struct virtio_crypto_request *vc_req) 29virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
30{ 30{
31 if (vc_req) { 31 if (vc_req) {
32 kzfree(vc_req->iv);
33 kzfree(vc_req->req_data); 32 kzfree(vc_req->req_data);
34 kfree(vc_req->sgs); 33 kfree(vc_req->sgs);
35 } 34 }
@@ -41,40 +40,18 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq)
41 struct virtio_crypto_request *vc_req; 40 struct virtio_crypto_request *vc_req;
42 unsigned long flags; 41 unsigned long flags;
43 unsigned int len; 42 unsigned int len;
44 struct ablkcipher_request *ablk_req;
45 int error;
46 unsigned int qid = vq->index; 43 unsigned int qid = vq->index;
47 44
48 spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags); 45 spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags);
49 do { 46 do {
50 virtqueue_disable_cb(vq); 47 virtqueue_disable_cb(vq);
51 while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { 48 while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
52 if (vc_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 49 spin_unlock_irqrestore(
53 switch (vc_req->status) { 50 &vcrypto->data_vq[qid].lock, flags);
54 case VIRTIO_CRYPTO_OK: 51 if (vc_req->alg_cb)
55 error = 0; 52 vc_req->alg_cb(vc_req, len);
56 break; 53 spin_lock_irqsave(
57 case VIRTIO_CRYPTO_INVSESS: 54 &vcrypto->data_vq[qid].lock, flags);
58 case VIRTIO_CRYPTO_ERR:
59 error = -EINVAL;
60 break;
61 case VIRTIO_CRYPTO_BADMSG:
62 error = -EBADMSG;
63 break;
64 default:
65 error = -EIO;
66 break;
67 }
68 ablk_req = vc_req->ablkcipher_req;
69
70 spin_unlock_irqrestore(
71 &vcrypto->data_vq[qid].lock, flags);
72 /* Finish the encrypt or decrypt process */
73 virtio_crypto_ablkcipher_finalize_req(vc_req,
74 ablk_req, error);
75 spin_lock_irqsave(
76 &vcrypto->data_vq[qid].lock, flags);
77 }
78 } 55 }
79 } while (!virtqueue_enable_cb(vq)); 56 } while (!virtqueue_enable_cb(vq));
80 spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags); 57 spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags);
@@ -270,7 +247,7 @@ static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
270 247
271 return -EPERM; 248 return -EPERM;
272 } 249 }
273 dev_info(&vcrypto->vdev->dev, "Accelerator is ready\n"); 250 dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n");
274 } else { 251 } else {
275 virtcrypto_dev_stop(vcrypto); 252 virtcrypto_dev_stop(vcrypto);
276 dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n"); 253 dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 9c26d9e8dbea..17d84217dd76 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -104,8 +104,7 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
104 pagefault_enable(); 104 pagefault_enable();
105 preempt_enable(); 105 preempt_enable();
106 106
107 crypto_xor(keystream, src, nbytes); 107 crypto_xor_cpy(dst, keystream, src, nbytes);
108 memcpy(dst, keystream, nbytes);
109 crypto_inc(ctrblk, AES_BLOCK_SIZE); 108 crypto_inc(ctrblk, AES_BLOCK_SIZE);
110} 109}
111 110
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index cdf6b1e12460..fa17e5452796 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -758,9 +758,8 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc,
758 int i, r; 758 int i, r;
759 759
760 /* xor whitening with sector number */ 760 /* xor whitening with sector number */
761 memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE); 761 crypto_xor_cpy(buf, tcw->whitening, (u8 *)&sector, 8);
762 crypto_xor(buf, (u8 *)&sector, 8); 762 crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)&sector, 8);
763 crypto_xor(&buf[8], (u8 *)&sector, 8);
764 763
765 /* calculate crc32 for every 32bit part and xor it */ 764 /* calculate crc32 for every 32bit part and xor it */
766 desc->tfm = tcw->crc32_tfm; 765 desc->tfm = tcw->crc32_tfm;
@@ -805,10 +804,10 @@ static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
805 } 804 }
806 805
807 /* Calculate IV */ 806 /* Calculate IV */
808 memcpy(iv, tcw->iv_seed, cc->iv_size); 807 crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)&sector, 8);
809 crypto_xor(iv, (u8 *)&sector, 8);
810 if (cc->iv_size > 8) 808 if (cc->iv_size > 8)
811 crypto_xor(&iv[8], (u8 *)&sector, cc->iv_size - 8); 809 crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)&sector,
810 cc->iv_size - 8);
812 811
813 return r; 812 return r;
814} 813}
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 436c4c2683c7..e3cebf640c00 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -192,7 +192,7 @@ static inline unsigned int crypto_queue_len(struct crypto_queue *queue)
192} 192}
193 193
194void crypto_inc(u8 *a, unsigned int size); 194void crypto_inc(u8 *a, unsigned int size);
195void __crypto_xor(u8 *dst, const u8 *src, unsigned int size); 195void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int size);
196 196
197static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int size) 197static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
198{ 198{
@@ -207,7 +207,26 @@ static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
207 size -= sizeof(unsigned long); 207 size -= sizeof(unsigned long);
208 } 208 }
209 } else { 209 } else {
210 __crypto_xor(dst, src, size); 210 __crypto_xor(dst, dst, src, size);
211 }
212}
213
214static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2,
215 unsigned int size)
216{
217 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
218 __builtin_constant_p(size) &&
219 (size % sizeof(unsigned long)) == 0) {
220 unsigned long *d = (unsigned long *)dst;
221 unsigned long *s1 = (unsigned long *)src1;
222 unsigned long *s2 = (unsigned long *)src2;
223
224 while (size > 0) {
225 *d++ = *s1++ ^ *s2++;
226 size -= sizeof(unsigned long);
227 }
228 } else {
229 __crypto_xor(dst, src1, src2, size);
211 } 230 }
212} 231}
213 232
diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h
index e2b9c6fe2714..75ec9c662268 100644
--- a/include/crypto/if_alg.h
+++ b/include/crypto/if_alg.h
@@ -20,6 +20,9 @@
20#include <linux/types.h> 20#include <linux/types.h>
21#include <net/sock.h> 21#include <net/sock.h>
22 22
23#include <crypto/aead.h>
24#include <crypto/skcipher.h>
25
23#define ALG_MAX_PAGES 16 26#define ALG_MAX_PAGES 16
24 27
25struct crypto_async_request; 28struct crypto_async_request;
@@ -68,6 +71,99 @@ struct af_alg_sgl {
68 unsigned int npages; 71 unsigned int npages;
69}; 72};
70 73
74/* TX SGL entry */
75struct af_alg_tsgl {
76 struct list_head list;
77 unsigned int cur; /* Last processed SG entry */
78 struct scatterlist sg[0]; /* Array of SGs forming the SGL */
79};
80
81#define MAX_SGL_ENTS ((4096 - sizeof(struct af_alg_tsgl)) / \
82 sizeof(struct scatterlist) - 1)
83
84/* RX SGL entry */
85struct af_alg_rsgl {
86 struct af_alg_sgl sgl;
87 struct list_head list;
88 size_t sg_num_bytes; /* Bytes of data in that SGL */
89};
90
91/**
92 * struct af_alg_async_req - definition of crypto request
93 * @iocb: IOCB for AIO operations
94 * @sk: Socket the request is associated with
95 * @first_rsgl: First RX SG
96 * @last_rsgl: Pointer to last RX SG
97 * @rsgl_list: Track RX SGs
98 * @tsgl: Private, per request TX SGL of buffers to process
99 * @tsgl_entries: Number of entries in priv. TX SGL
100 * @outlen: Number of output bytes generated by crypto op
101 * @areqlen: Length of this data structure
102 * @cra_u: Cipher request
103 */
104struct af_alg_async_req {
105 struct kiocb *iocb;
106 struct sock *sk;
107
108 struct af_alg_rsgl first_rsgl;
109 struct af_alg_rsgl *last_rsgl;
110 struct list_head rsgl_list;
111
112 struct scatterlist *tsgl;
113 unsigned int tsgl_entries;
114
115 unsigned int outlen;
116 unsigned int areqlen;
117
118 union {
119 struct aead_request aead_req;
120 struct skcipher_request skcipher_req;
121 } cra_u;
122
123 /* req ctx trails this struct */
124};
125
126/**
127 * struct af_alg_ctx - definition of the crypto context
128 *
129 * The crypto context tracks the input data during the lifetime of an AF_ALG
130 * socket.
131 *
132 * @tsgl_list: Link to TX SGL
133 * @iv: IV for cipher operation
134 * @aead_assoclen: Length of AAD for AEAD cipher operations
135 * @completion: Work queue for synchronous operation
136 * @used: TX bytes sent to kernel. This variable is used to
137 * ensure that user space cannot cause the kernel
138 * to allocate too much memory in sendmsg operation.
139 * @rcvused: Total RX bytes to be filled by kernel. This variable
140 * is used to ensure user space cannot cause the kernel
141 * to allocate too much memory in a recvmsg operation.
142 * @more: More data to be expected from user space?
143 * @merge: Shall new data from user space be merged into existing
144 * SG?
145 * @enc: Cryptographic operation to be performed when
146 * recvmsg is invoked.
147 * @len: Length of memory allocated for this data structure.
148 */
149struct af_alg_ctx {
150 struct list_head tsgl_list;
151
152 void *iv;
153 size_t aead_assoclen;
154
155 struct af_alg_completion completion;
156
157 size_t used;
158 size_t rcvused;
159
160 bool more;
161 bool merge;
162 bool enc;
163
164 unsigned int len;
165};
166
71int af_alg_register_type(const struct af_alg_type *type); 167int af_alg_register_type(const struct af_alg_type *type);
72int af_alg_unregister_type(const struct af_alg_type *type); 168int af_alg_unregister_type(const struct af_alg_type *type);
73 169
@@ -94,4 +190,78 @@ static inline void af_alg_init_completion(struct af_alg_completion *completion)
94 init_completion(&completion->completion); 190 init_completion(&completion->completion);
95} 191}
96 192
193/**
194 * Size of available buffer for sending data from user space to kernel.
195 *
196 * @sk socket of connection to user space
197 * @return number of bytes still available
198 */
199static inline int af_alg_sndbuf(struct sock *sk)
200{
201 struct alg_sock *ask = alg_sk(sk);
202 struct af_alg_ctx *ctx = ask->private;
203
204 return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
205 ctx->used, 0);
206}
207
208/**
209 * Can the send buffer still be written to?
210 *
211 * @sk socket of connection to user space
212 * @return true => writable, false => not writable
213 */
214static inline bool af_alg_writable(struct sock *sk)
215{
216 return PAGE_SIZE <= af_alg_sndbuf(sk);
217}
218
219/**
220 * Size of available buffer used by kernel for the RX user space operation.
221 *
222 * @sk socket of connection to user space
223 * @return number of bytes still available
224 */
225static inline int af_alg_rcvbuf(struct sock *sk)
226{
227 struct alg_sock *ask = alg_sk(sk);
228 struct af_alg_ctx *ctx = ask->private;
229
230 return max_t(int, max_t(int, sk->sk_rcvbuf & PAGE_MASK, PAGE_SIZE) -
231 ctx->rcvused, 0);
232}
233
234/**
235 * Can the RX buffer still be written to?
236 *
237 * @sk socket of connection to user space
238 * @return true => writable, false => not writable
239 */
240static inline bool af_alg_readable(struct sock *sk)
241{
242 return PAGE_SIZE <= af_alg_rcvbuf(sk);
243}
244
245int af_alg_alloc_tsgl(struct sock *sk);
246unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset);
247void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst,
248 size_t dst_offset);
249void af_alg_free_areq_sgls(struct af_alg_async_req *areq);
250int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags);
251void af_alg_wmem_wakeup(struct sock *sk);
252int af_alg_wait_for_data(struct sock *sk, unsigned flags);
253void af_alg_data_wakeup(struct sock *sk);
254int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
255 unsigned int ivsize);
256ssize_t af_alg_sendpage(struct socket *sock, struct page *page,
257 int offset, size_t size, int flags);
258void af_alg_async_cb(struct crypto_async_request *_req, int err);
259unsigned int af_alg_poll(struct file *file, struct socket *sock,
260 poll_table *wait);
261struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk,
262 unsigned int areqlen);
263int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags,
264 struct af_alg_async_req *areq, size_t maxsize,
265 size_t *outlen);
266
97#endif /* _CRYPTO_IF_ALG_H */ 267#endif /* _CRYPTO_IF_ALG_H */
diff --git a/include/crypto/internal/akcipher.h b/include/crypto/internal/akcipher.h
index 479a0078f0f7..805686ba2be4 100644
--- a/include/crypto/internal/akcipher.h
+++ b/include/crypto/internal/akcipher.h
@@ -38,6 +38,12 @@ static inline void *akcipher_request_ctx(struct akcipher_request *req)
38 return req->__ctx; 38 return req->__ctx;
39} 39}
40 40
41static inline void akcipher_set_reqsize(struct crypto_akcipher *akcipher,
42 unsigned int reqsize)
43{
44 crypto_akcipher_alg(akcipher)->reqsize = reqsize;
45}
46
41static inline void *akcipher_tfm_ctx(struct crypto_akcipher *tfm) 47static inline void *akcipher_tfm_ctx(struct crypto_akcipher *tfm)
42{ 48{
43 return tfm->base.__crt_ctx; 49 return tfm->base.__crt_ctx;
diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h
index f6d9af3efa45..f0b44c16e88f 100644
--- a/include/crypto/internal/hash.h
+++ b/include/crypto/internal/hash.h
@@ -76,6 +76,8 @@ static inline int crypto_ahash_walk_last(struct crypto_hash_walk *walk)
76 76
77int crypto_register_ahash(struct ahash_alg *alg); 77int crypto_register_ahash(struct ahash_alg *alg);
78int crypto_unregister_ahash(struct ahash_alg *alg); 78int crypto_unregister_ahash(struct ahash_alg *alg);
79int crypto_register_ahashes(struct ahash_alg *algs, int count);
80void crypto_unregister_ahashes(struct ahash_alg *algs, int count);
79int ahash_register_instance(struct crypto_template *tmpl, 81int ahash_register_instance(struct crypto_template *tmpl,
80 struct ahash_instance *inst); 82 struct ahash_instance *inst);
81void ahash_free_instance(struct crypto_instance *inst); 83void ahash_free_instance(struct crypto_instance *inst);
diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h
index 2133d17b7156..1bde0a6514fa 100644
--- a/include/crypto/kpp.h
+++ b/include/crypto/kpp.h
@@ -145,6 +145,16 @@ static inline struct crypto_kpp *crypto_kpp_reqtfm(struct kpp_request *req)
145 return __crypto_kpp_tfm(req->base.tfm); 145 return __crypto_kpp_tfm(req->base.tfm);
146} 146}
147 147
148static inline u32 crypto_kpp_get_flags(struct crypto_kpp *tfm)
149{
150 return crypto_tfm_get_flags(crypto_kpp_tfm(tfm));
151}
152
153static inline void crypto_kpp_set_flags(struct crypto_kpp *tfm, u32 flags)
154{
155 crypto_tfm_set_flags(crypto_kpp_tfm(tfm), flags);
156}
157
148/** 158/**
149 * crypto_free_kpp() - free KPP tfm handle 159 * crypto_free_kpp() - free KPP tfm handle
150 * 160 *
diff --git a/include/linux/ccp.h b/include/linux/ccp.h
index 3285c944194a..7e9c991c95e0 100644
--- a/include/linux/ccp.h
+++ b/include/linux/ccp.h
@@ -1,7 +1,7 @@
1/* 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver 2 * AMD Cryptographic Coprocessor (CCP) driver
3 * 3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 * 5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com>
@@ -20,12 +20,10 @@
20#include <crypto/aes.h> 20#include <crypto/aes.h>
21#include <crypto/sha.h> 21#include <crypto/sha.h>
22 22
23
24struct ccp_device; 23struct ccp_device;
25struct ccp_cmd; 24struct ccp_cmd;
26 25
27#if defined(CONFIG_CRYPTO_DEV_CCP_DD) || \ 26#if defined(CONFIG_CRYPTO_DEV_SP_CCP)
28 defined(CONFIG_CRYPTO_DEV_CCP_DD_MODULE)
29 27
30/** 28/**
31 * ccp_present - check if a CCP device is present 29 * ccp_present - check if a CCP device is present
@@ -71,7 +69,7 @@ unsigned int ccp_version(void);
71 */ 69 */
72int ccp_enqueue_cmd(struct ccp_cmd *cmd); 70int ccp_enqueue_cmd(struct ccp_cmd *cmd);
73 71
74#else /* CONFIG_CRYPTO_DEV_CCP_DD is not enabled */ 72#else /* CONFIG_CRYPTO_DEV_CCP_SP_DEV is not enabled */
75 73
76static inline int ccp_present(void) 74static inline int ccp_present(void)
77{ 75{
@@ -88,7 +86,7 @@ static inline int ccp_enqueue_cmd(struct ccp_cmd *cmd)
88 return -ENODEV; 86 return -ENODEV;
89} 87}
90 88
91#endif /* CONFIG_CRYPTO_DEV_CCP_DD */ 89#endif /* CONFIG_CRYPTO_DEV_SP_CCP */
92 90
93 91
94/***** AES engine *****/ 92/***** AES engine *****/
@@ -231,6 +229,7 @@ enum ccp_xts_aes_unit_size {
231 * AES operation the new IV overwrites the old IV. 229 * AES operation the new IV overwrites the old IV.
232 */ 230 */
233struct ccp_xts_aes_engine { 231struct ccp_xts_aes_engine {
232 enum ccp_aes_type type;
234 enum ccp_aes_action action; 233 enum ccp_aes_action action;
235 enum ccp_xts_aes_unit_size unit_size; 234 enum ccp_xts_aes_unit_size unit_size;
236 235
diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h
index 93336502af08..57fd45ab7af1 100644
--- a/lib/mpi/longlong.h
+++ b/lib/mpi/longlong.h
@@ -176,8 +176,8 @@ extern UDItype __udiv_qrnnd(UDItype *, UDItype, UDItype, UDItype);
176#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ 176#define add_ssaaaa(sh, sl, ah, al, bh, bl) \
177 __asm__ ("adds %1, %4, %5\n" \ 177 __asm__ ("adds %1, %4, %5\n" \
178 "adc %0, %2, %3" \ 178 "adc %0, %2, %3" \
179 : "=r" ((USItype)(sh)), \ 179 : "=r" (sh), \
180 "=&r" ((USItype)(sl)) \ 180 "=&r" (sl) \
181 : "%r" ((USItype)(ah)), \ 181 : "%r" ((USItype)(ah)), \
182 "rI" ((USItype)(bh)), \ 182 "rI" ((USItype)(bh)), \
183 "%r" ((USItype)(al)), \ 183 "%r" ((USItype)(al)), \
@@ -185,15 +185,15 @@ extern UDItype __udiv_qrnnd(UDItype *, UDItype, UDItype, UDItype);
185#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ 185#define sub_ddmmss(sh, sl, ah, al, bh, bl) \
186 __asm__ ("subs %1, %4, %5\n" \ 186 __asm__ ("subs %1, %4, %5\n" \
187 "sbc %0, %2, %3" \ 187 "sbc %0, %2, %3" \
188 : "=r" ((USItype)(sh)), \ 188 : "=r" (sh), \
189 "=&r" ((USItype)(sl)) \ 189 "=&r" (sl) \
190 : "r" ((USItype)(ah)), \ 190 : "r" ((USItype)(ah)), \
191 "rI" ((USItype)(bh)), \ 191 "rI" ((USItype)(bh)), \
192 "r" ((USItype)(al)), \ 192 "r" ((USItype)(al)), \
193 "rI" ((USItype)(bl))) 193 "rI" ((USItype)(bl)))
194#if defined __ARM_ARCH_2__ || defined __ARM_ARCH_3__ 194#if defined __ARM_ARCH_2__ || defined __ARM_ARCH_3__
195#define umul_ppmm(xh, xl, a, b) \ 195#define umul_ppmm(xh, xl, a, b) \
196 __asm__ ("%@ Inlined umul_ppmm\n" \ 196 __asm__ ("@ Inlined umul_ppmm\n" \
197 "mov %|r0, %2, lsr #16 @ AAAA\n" \ 197 "mov %|r0, %2, lsr #16 @ AAAA\n" \
198 "mov %|r2, %3, lsr #16 @ BBBB\n" \ 198 "mov %|r2, %3, lsr #16 @ BBBB\n" \
199 "bic %|r1, %2, %|r0, lsl #16 @ aaaa\n" \ 199 "bic %|r1, %2, %|r0, lsl #16 @ aaaa\n" \
@@ -206,19 +206,19 @@ extern UDItype __udiv_qrnnd(UDItype *, UDItype, UDItype, UDItype);
206 "addcs %|r2, %|r2, #65536\n" \ 206 "addcs %|r2, %|r2, #65536\n" \
207 "adds %1, %|r1, %|r0, lsl #16\n" \ 207 "adds %1, %|r1, %|r0, lsl #16\n" \
208 "adc %0, %|r2, %|r0, lsr #16" \ 208 "adc %0, %|r2, %|r0, lsr #16" \
209 : "=&r" ((USItype)(xh)), \ 209 : "=&r" (xh), \
210 "=r" ((USItype)(xl)) \ 210 "=r" (xl) \
211 : "r" ((USItype)(a)), \ 211 : "r" ((USItype)(a)), \
212 "r" ((USItype)(b)) \ 212 "r" ((USItype)(b)) \
213 : "r0", "r1", "r2") 213 : "r0", "r1", "r2")
214#else 214#else
215#define umul_ppmm(xh, xl, a, b) \ 215#define umul_ppmm(xh, xl, a, b) \
216 __asm__ ("%@ Inlined umul_ppmm\n" \ 216 __asm__ ("@ Inlined umul_ppmm\n" \
217 "umull %r1, %r0, %r2, %r3" \ 217 "umull %1, %0, %2, %3" \
218 : "=&r" ((USItype)(xh)), \ 218 : "=&r" (xh), \
219 "=&r" ((USItype)(xl)) \ 219 "=&r" (xl) \
220 : "r" ((USItype)(a)), \ 220 : "r" ((USItype)(a)), \
221 "r" ((USItype)(b)) \ 221 "r" ((USItype)(b)) \
222 : "r0", "r1") 222 : "r0", "r1")
223#endif 223#endif
224#define UMUL_TIME 20 224#define UMUL_TIME 20
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-18 22:32:00 -0500