summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-02-23 12:54:19 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2017-02-23 12:54:19 -0500
commit5bcbe22ca47da04cda3a858cef67f55b550c1d13 (patch)
tree49bd61e32eb2d652085a49182436322a3e0e9840
parent1db934a5b77a9e37c4742c704fde6af233187a98 (diff)
parent12cb3a1c4184f891d965d1f39f8cfcc9ef617647 (diff)
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu: "API: - Try to catch hash output overrun in testmgr - Introduce walksize attribute for batched walking - Make crypto_xor() and crypto_inc() alignment agnostic Algorithms: - Add time-invariant AES algorithm - Add standalone CBCMAC algorithm Drivers: - Add NEON acclerated chacha20 on ARM/ARM64 - Expose AES-CTR as synchronous skcipher on ARM64 - Add scalar AES implementation on ARM64 - Improve scalar AES implementation on ARM - Improve NEON AES implementation on ARM/ARM64 - Merge CRC32 and PMULL instruction based drivers on ARM64 - Add NEON acclerated CBCMAC/CMAC/XCBC AES on ARM64 - Add IPsec AUTHENC implementation in atmel - Add Support for Octeon-tx CPT Engine - Add Broadcom SPU driver - Add MediaTek driver" * 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (142 commits) crypto: xts - Add ECB dependency crypto: cavium - switch to pci_alloc_irq_vectors crypto: cavium - switch to pci_alloc_irq_vectors crypto: cavium - remove dead MSI-X related define crypto: brcm - Avoid double free in ahash_finup() crypto: cavium - fix Kconfig dependencies crypto: cavium - cpt_bind_vq_to_grp could return an error code crypto: doc - fix typo hwrng: omap - update Kconfig help description crypto: ccm - drop unnecessary minimum 32-bit alignment crypto: ccm - honour alignmask of subordinate MAC cipher crypto: caam - fix state buffer DMA (un)mapping crypto: caam - abstract ahash request double buffering crypto: caam - fix error path for ctx_dma mapping failure crypto: caam - fix DMA API leaks for multiple setkey() calls crypto: caam - don't dma_map key for hash algorithms crypto: caam - use dma_map_sg() return code crypto: caam - replace sg_count() with sg_nents_for_len() crypto: caam - check sg_count() return value crypto: caam - fix HW S/G in ablkcipher_giv_edesc_alloc() ..
Diffstat
-rw-r--r--Documentation/crypto/api-digest.rst2
-rw-r--r--Documentation/crypto/api-skcipher.rst2
-rw-r--r--Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt22
-rw-r--r--Documentation/devicetree/bindings/crypto/mediatek-crypto.txt27
-rw-r--r--MAINTAINERS7
-rw-r--r--arch/arm/crypto/Kconfig27
-rw-r--r--arch/arm/crypto/Makefile11
-rw-r--r--arch/arm/crypto/aes-armv4.S1089
-rw-r--r--arch/arm/crypto/aes-ce-core.S84
-rw-r--r--arch/arm/crypto/aes-ce-glue.c15
-rw-r--r--arch/arm/crypto/aes-cipher-core.S179
-rw-r--r--arch/arm/crypto/aes-cipher-glue.c74
-rw-r--r--arch/arm/crypto/aes-neonbs-core.S1023
-rw-r--r--arch/arm/crypto/aes-neonbs-glue.c406
-rw-r--r--arch/arm/crypto/aes_glue.c98
-rw-r--r--arch/arm/crypto/aes_glue.h19
-rw-r--r--arch/arm/crypto/aesbs-core.S_shipped2548
-rw-r--r--arch/arm/crypto/aesbs-glue.c367
-rw-r--r--arch/arm/crypto/bsaes-armv7.pl2471
-rw-r--r--arch/arm/crypto/chacha20-neon-core.S523
-rw-r--r--arch/arm/crypto/chacha20-neon-glue.c127
-rw-r--r--arch/arm64/configs/defconfig1
-rw-r--r--arch/arm64/crypto/Kconfig24
-rw-r--r--arch/arm64/crypto/Makefile13
-rw-r--r--arch/arm64/crypto/aes-ce-ccm-glue.c1
-rw-r--r--arch/arm64/crypto/aes-cipher-core.S110
-rw-r--r--arch/arm64/crypto/aes-cipher-glue.c69
-rw-r--r--arch/arm64/crypto/aes-glue.c281
-rw-r--r--arch/arm64/crypto/aes-modes.S37
-rw-r--r--arch/arm64/crypto/aes-neon.S235
-rw-r--r--arch/arm64/crypto/aes-neonbs-core.S972
-rw-r--r--arch/arm64/crypto/aes-neonbs-glue.c439
-rw-r--r--arch/arm64/crypto/chacha20-neon-core.S450
-rw-r--r--arch/arm64/crypto/chacha20-neon-glue.c126
-rw-r--r--arch/arm64/crypto/crc32-arm64.c290
-rw-r--r--arch/arm64/crypto/crc32-ce-glue.c49
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S37
-rw-r--r--arch/x86/crypto/aesni-intel_avx-x86_64.S32
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c12
-rw-r--r--arch/x86/crypto/camellia-aesni-avx-asm_64.S5
-rw-r--r--arch/x86/crypto/camellia-aesni-avx2-asm_64.S12
-rw-r--r--arch/x86/crypto/cast5-avx-x86_64-asm_64.S14
-rw-r--r--arch/x86/crypto/cast6-avx-x86_64-asm_64.S12
-rw-r--r--arch/x86/crypto/chacha20-avx2-x86_64.S9
-rw-r--r--arch/x86/crypto/chacha20-ssse3-x86_64.S7
-rw-r--r--arch/x86/crypto/chacha20_glue.c70
-rw-r--r--arch/x86/crypto/crc32c-pcl-intel-asm_64.S2
-rw-r--r--arch/x86/crypto/crct10dif-pcl-asm_64.S14
-rw-r--r--arch/x86/crypto/des3_ede-asm_64.S2
-rw-r--r--arch/x86/crypto/ghash-clmulni-intel_asm.S3
-rw-r--r--arch/x86/crypto/poly1305-avx2-x86_64.S6
-rw-r--r--arch/x86/crypto/poly1305-sse2-x86_64.S6
-rw-r--r--arch/x86/crypto/serpent-avx-x86_64-asm_64.S5
-rw-r--r--arch/x86/crypto/serpent-avx2-asm_64.S9
-rw-r--r--arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S6
-rw-r--r--arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S3
-rw-r--r--arch/x86/crypto/sha1-mb/sha1_x8_avx2.S15
-rw-r--r--arch/x86/crypto/sha1_ni_asm.S8
-rw-r--r--arch/x86/crypto/sha256-avx-asm.S9
-rw-r--r--arch/x86/crypto/sha256-avx2-asm.S9
-rw-r--r--arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S6
-rw-r--r--arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S3
-rw-r--r--arch/x86/crypto/sha256-mb/sha256_x8_avx2.S7
-rw-r--r--arch/x86/crypto/sha256-ssse3-asm.S8
-rw-r--r--arch/x86/crypto/sha256_ni_asm.S4
-rw-r--r--arch/x86/crypto/sha512-avx-asm.S9
-rw-r--r--arch/x86/crypto/sha512-avx2-asm.S10
-rw-r--r--arch/x86/crypto/sha512-mb/sha512_mb.c64
-rw-r--r--arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S10
-rw-r--r--arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S4
-rw-r--r--arch/x86/crypto/sha512-mb/sha512_x4_avx2.S4
-rw-r--r--arch/x86/crypto/sha512-ssse3-asm.S9
-rw-r--r--arch/x86/crypto/twofish-avx-x86_64-asm_64.S6
-rw-r--r--crypto/Kconfig19
-rw-r--r--crypto/Makefile3
-rw-r--r--crypto/ablkcipher.c5
-rw-r--r--crypto/acompress.c3
-rw-r--r--crypto/aead.c3
-rw-r--r--crypto/aes_generic.c64
-rw-r--r--crypto/aes_ti.c375
-rw-r--r--crypto/ahash.c3
-rw-r--r--crypto/akcipher.c3
-rw-r--r--crypto/algapi.c68
-rw-r--r--crypto/algif_hash.c2
-rw-r--r--crypto/blkcipher.c7
-rw-r--r--crypto/cbc.c3
-rw-r--r--crypto/ccm.c386
-rw-r--r--crypto/chacha20_generic.c73
-rw-r--r--crypto/cmac.c3
-rw-r--r--crypto/ctr.c2
-rw-r--r--crypto/cts.c8
-rw-r--r--crypto/kpp.c3
-rw-r--r--crypto/pcbc.c6
-rw-r--r--crypto/rng.c3
-rw-r--r--crypto/scompress.c3
-rw-r--r--crypto/seqiv.c2
-rw-r--r--crypto/shash.c9
-rw-r--r--crypto/skcipher.c23
-rw-r--r--crypto/tcrypt.c6
-rw-r--r--crypto/testmgr.c1055
-rw-r--r--crypto/testmgr.h330
-rw-r--r--drivers/char/hw_random/Kconfig4
-rw-r--r--drivers/char/hw_random/cavium-rng-vf.c6
-rw-r--r--drivers/char/hw_random/core.c64
-rw-r--r--drivers/char/hw_random/n2-drv.c204
-rw-r--r--drivers/char/hw_random/n2rng.h51
-rw-r--r--drivers/crypto/Kconfig53
-rw-r--r--drivers/crypto/Makefile17
-rw-r--r--drivers/crypto/atmel-aes-regs.h16
-rw-r--r--drivers/crypto/atmel-aes.c455
-rw-r--r--drivers/crypto/atmel-authenc.h64
-rw-r--r--drivers/crypto/atmel-sha-regs.h20
-rw-r--r--drivers/crypto/atmel-sha.c1433
-rw-r--r--drivers/crypto/atmel-tdes.c14
-rw-r--r--drivers/crypto/bcm/Makefile15
-rw-r--r--drivers/crypto/bcm/cipher.c4963
-rw-r--r--drivers/crypto/bcm/cipher.h483
-rw-r--r--drivers/crypto/bcm/spu.c1251
-rw-r--r--drivers/crypto/bcm/spu.h287
-rw-r--r--drivers/crypto/bcm/spu2.c1401
-rw-r--r--drivers/crypto/bcm/spu2.h228
-rw-r--r--drivers/crypto/bcm/spum.h174
-rw-r--r--drivers/crypto/bcm/util.c581
-rw-r--r--drivers/crypto/bcm/util.h116
-rw-r--r--drivers/crypto/bfin_crc.c6
-rw-r--r--drivers/crypto/bfin_crc.h1
-rw-r--r--drivers/crypto/caam/caamalg.c589
-rw-r--r--drivers/crypto/caam/caamhash.c268
-rw-r--r--drivers/crypto/caam/ctrl.c33
-rw-r--r--drivers/crypto/caam/error.c2
-rw-r--r--drivers/crypto/caam/jr.c19
-rw-r--r--drivers/crypto/caam/sg_sw_sec4.h11
-rw-r--r--drivers/crypto/cavium/cpt/Kconfig17
-rw-r--r--drivers/crypto/cavium/cpt/Makefile3
-rw-r--r--drivers/crypto/cavium/cpt/cpt_common.h156
-rw-r--r--drivers/crypto/cavium/cpt/cpt_hw_types.h658
-rw-r--r--drivers/crypto/cavium/cpt/cptpf.h64
-rw-r--r--drivers/crypto/cavium/cpt/cptpf_main.c670
-rw-r--r--drivers/crypto/cavium/cpt/cptpf_mbox.c163
-rw-r--r--drivers/crypto/cavium/cpt/cptvf.h132
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_algs.c444
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_algs.h113
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_main.c863
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_mbox.c211
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_reqmanager.c593
-rw-r--r--drivers/crypto/cavium/cpt/request_manager.h147
-rw-r--r--drivers/crypto/ccp/ccp-dev-v5.c15
-rw-r--r--drivers/crypto/ccp/ccp-dev.h1
-rw-r--r--drivers/crypto/ccp/ccp-ops.c150
-rwxr-xr-x[-rw-r--r--]drivers/crypto/chelsio/chcr_algo.c49
-rw-r--r--drivers/crypto/chelsio/chcr_algo.h9
-rw-r--r--drivers/crypto/chelsio/chcr_core.c11
-rw-r--r--drivers/crypto/chelsio/chcr_core.h1
-rw-r--r--drivers/crypto/chelsio/chcr_crypto.h2
-rw-r--r--drivers/crypto/img-hash.c4
-rw-r--r--drivers/crypto/mediatek/Makefile2
-rw-r--r--drivers/crypto/mediatek/mtk-aes.c1299
-rw-r--r--drivers/crypto/mediatek/mtk-platform.c604
-rw-r--r--drivers/crypto/mediatek/mtk-platform.h231
-rw-r--r--drivers/crypto/mediatek/mtk-regs.h194
-rw-r--r--drivers/crypto/mediatek/mtk-sha.c1435
-rw-r--r--drivers/crypto/picoxcell_crypto.c28
-rw-r--r--drivers/crypto/qat/qat_c3xxx/adf_drv.c2
-rw-r--r--drivers/crypto/qat/qat_c3xxxvf/adf_drv.c2
-rw-r--r--drivers/crypto/qat/qat_c62x/adf_drv.c2
-rw-r--r--drivers/crypto/qat/qat_c62xvf/adf_drv.c2
-rw-r--r--drivers/crypto/qat/qat_common/adf_cfg_common.h1
-rw-r--r--drivers/crypto/qat/qat_common/adf_common_drv.h4
-rw-r--r--drivers/crypto/qat/qat_common/adf_dev_mgr.c2
-rw-r--r--drivers/crypto/qat/qat_common/adf_init.c28
-rw-r--r--drivers/crypto/qat/qat_common/adf_sriov.c4
-rw-r--r--drivers/crypto/qat/qat_common/adf_vf_isr.c4
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_drv.c2
-rw-r--r--drivers/crypto/qat/qat_dh895xccvf/adf_drv.c2
-rw-r--r--drivers/crypto/virtio/Kconfig1
-rw-r--r--drivers/crypto/virtio/virtio_crypto_algs.c54
-rw-r--r--drivers/crypto/virtio/virtio_crypto_common.h16
-rw-r--r--drivers/crypto/virtio/virtio_crypto_core.c74
-rw-r--r--drivers/crypto/vmx/aes_ctr.c6
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h8
-rw-r--r--include/crypto/algapi.h20
-rw-r--r--include/crypto/chacha20.h6
-rw-r--r--include/crypto/hash.h18
-rw-r--r--include/crypto/internal/skcipher.h2
-rw-r--r--include/crypto/skcipher.h34
-rw-r--r--include/linux/compiler-gcc.h1
-rw-r--r--include/linux/miscdevice.h1
187 files changed, 26959 insertions, 9574 deletions
diff --git a/Documentation/crypto/api-digest.rst b/Documentation/crypto/api-digest.rst
index 07356fa99200..7a1e670d6ce1 100644
--- a/Documentation/crypto/api-digest.rst
+++ b/Documentation/crypto/api-digest.rst
@@ -14,7 +14,7 @@ Asynchronous Message Digest API
14 :doc: Asynchronous Message Digest API 14 :doc: Asynchronous Message Digest API
15 15
16.. kernel-doc:: include/crypto/hash.h 16.. kernel-doc:: include/crypto/hash.h
17 :functions: crypto_alloc_ahash crypto_free_ahash crypto_ahash_init crypto_ahash_digestsize crypto_ahash_reqtfm crypto_ahash_reqsize crypto_ahash_setkey crypto_ahash_finup crypto_ahash_final crypto_ahash_digest crypto_ahash_export crypto_ahash_import 17 :functions: crypto_alloc_ahash crypto_free_ahash crypto_ahash_init crypto_ahash_digestsize crypto_ahash_reqtfm crypto_ahash_reqsize crypto_ahash_statesize crypto_ahash_setkey crypto_ahash_finup crypto_ahash_final crypto_ahash_digest crypto_ahash_export crypto_ahash_import
18 18
19Asynchronous Hash Request Handle 19Asynchronous Hash Request Handle
20-------------------------------- 20--------------------------------
diff --git a/Documentation/crypto/api-skcipher.rst b/Documentation/crypto/api-skcipher.rst
index b20028a361a9..4eec4a93f7e3 100644
--- a/Documentation/crypto/api-skcipher.rst
+++ b/Documentation/crypto/api-skcipher.rst
@@ -59,4 +59,4 @@ Synchronous Block Cipher API - Deprecated
59 :doc: Synchronous Block Cipher API 59 :doc: Synchronous Block Cipher API
60 60
61.. kernel-doc:: include/linux/crypto.h 61.. kernel-doc:: include/linux/crypto.h
62 :functions: crypto_alloc_blkcipher rypto_free_blkcipher crypto_has_blkcipher crypto_blkcipher_name crypto_blkcipher_ivsize crypto_blkcipher_blocksize crypto_blkcipher_setkey crypto_blkcipher_encrypt crypto_blkcipher_encrypt_iv crypto_blkcipher_decrypt crypto_blkcipher_decrypt_iv crypto_blkcipher_set_iv crypto_blkcipher_get_iv 62 :functions: crypto_alloc_blkcipher crypto_free_blkcipher crypto_has_blkcipher crypto_blkcipher_name crypto_blkcipher_ivsize crypto_blkcipher_blocksize crypto_blkcipher_setkey crypto_blkcipher_encrypt crypto_blkcipher_encrypt_iv crypto_blkcipher_decrypt crypto_blkcipher_decrypt_iv crypto_blkcipher_set_iv crypto_blkcipher_get_iv
diff --git a/Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt b/Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt
new file mode 100644
index 000000000000..29b6007568eb
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/brcm,spu-crypto.txt
@@ -0,0 +1,22 @@
1The Broadcom Secure Processing Unit (SPU) hardware supports symmetric
2cryptographic offload for Broadcom SoCs. A SoC may have multiple SPU hardware
3blocks.
4
5Required properties:
6- compatible: Should be one of the following:
7 brcm,spum-crypto - for devices with SPU-M hardware
8 brcm,spu2-crypto - for devices with SPU2 hardware
9 brcm,spu2-v2-crypto - for devices with enhanced SPU2 hardware features like SHA3
10 and Rabin Fingerprint support
11 brcm,spum-nsp-crypto - for the Northstar Plus variant of the SPU-M hardware
12
13- reg: Should contain SPU registers location and length.
14- mboxes: The mailbox channel to be used to communicate with the SPU.
15 Mailbox channels correspond to DMA rings on the device.
16
17Example:
18 crypto@612d0000 {
19 compatible = "brcm,spum-crypto";
20 reg = <0 0x612d0000 0 0x900>;
21 mboxes = <&pdc0 0>;
22 };
diff --git a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt
new file mode 100644
index 000000000000..c204725e5873
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt
@@ -0,0 +1,27 @@
1MediaTek cryptographic accelerators
2
3Required properties:
4- compatible: Should be "mediatek,eip97-crypto"
5- reg: Address and length of the register set for the device
6- interrupts: Should contain the five crypto engines interrupts in numeric
7 order. These are global system and four descriptor rings.
8- clocks: the clock used by the core
9- clock-names: the names of the clock listed in the clocks property. These are
10 "ethif", "cryp"
11- power-domains: Must contain a reference to the PM domain.
12
13
14Example:
15 crypto: crypto@1b240000 {
16 compatible = "mediatek,eip97-crypto";
17 reg = <0 0x1b240000 0 0x20000>;
18 interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_LOW>,
19 <GIC_SPI 83 IRQ_TYPE_LEVEL_LOW>,
20 <GIC_SPI 84 IRQ_TYPE_LEVEL_LOW>,
21 <GIC_SPI 91 IRQ_TYPE_LEVEL_LOW>,
22 <GIC_SPI 97 IRQ_TYPE_LEVEL_LOW>;
23 clocks = <&topckgen CLK_TOP_ETHIF_SEL>,
24 <&ethsys CLK_ETHSYS_CRYPTO>;
25 clock-names = "ethif","cryp";
26 power-domains = <&scpsys MT2701_POWER_DOMAIN_ETH>;
27 };
diff --git a/MAINTAINERS b/MAINTAINERS
index 5a5fa41ac961..24fef3773d67 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3031,6 +3031,13 @@ W: http://www.cavium.com
3031S: Supported 3031S: Supported
3032F: drivers/net/ethernet/cavium/liquidio/ 3032F: drivers/net/ethernet/cavium/liquidio/
3033 3033
3034CAVIUM OCTEON-TX CRYPTO DRIVER
3035M: George Cherian <george.cherian@cavium.com>
3036L: linux-crypto@vger.kernel.org
3037W: http://www.cavium.com
3038S: Supported
3039F: drivers/crypto/cavium/cpt/
3040
3034CC2520 IEEE-802.15.4 RADIO DRIVER 3041CC2520 IEEE-802.15.4 RADIO DRIVER
3035M: Varka Bhadram <varkabhadram@gmail.com> 3042M: Varka Bhadram <varkabhadram@gmail.com>
3036L: linux-wpan@vger.kernel.org 3043L: linux-wpan@vger.kernel.org
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 13f1b4c289d4..a8fce93137fb 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -62,35 +62,18 @@ config CRYPTO_SHA512_ARM
62 using optimized ARM assembler and NEON, when available. 62 using optimized ARM assembler and NEON, when available.
63 63
64config CRYPTO_AES_ARM 64config CRYPTO_AES_ARM
65 tristate "AES cipher algorithms (ARM-asm)" 65 tristate "Scalar AES cipher for ARM"
66 depends on ARM
67 select CRYPTO_ALGAPI 66 select CRYPTO_ALGAPI
68 select CRYPTO_AES 67 select CRYPTO_AES
69 help 68 help
70 Use optimized AES assembler routines for ARM platforms. 69 Use optimized AES assembler routines for ARM platforms.
71 70
72 AES cipher algorithms (FIPS-197). AES uses the Rijndael
73 algorithm.
74
75 Rijndael appears to be consistently a very good performer in
76 both hardware and software across a wide range of computing
77 environments regardless of its use in feedback or non-feedback
78 modes. Its key setup time is excellent, and its key agility is
79 good. Rijndael's very low memory requirements make it very well
80 suited for restricted-space environments, in which it also
81 demonstrates excellent performance. Rijndael's operations are
82 among the easiest to defend against power and timing attacks.
83
84 The AES specifies three key sizes: 128, 192 and 256 bits
85
86 See <http://csrc.nist.gov/encryption/aes/> for more information.
87
88config CRYPTO_AES_ARM_BS 71config CRYPTO_AES_ARM_BS
89 tristate "Bit sliced AES using NEON instructions" 72 tristate "Bit sliced AES using NEON instructions"
90 depends on KERNEL_MODE_NEON 73 depends on KERNEL_MODE_NEON
91 select CRYPTO_AES_ARM
92 select CRYPTO_BLKCIPHER 74 select CRYPTO_BLKCIPHER
93 select CRYPTO_SIMD 75 select CRYPTO_SIMD
76 select CRYPTO_AES_ARM
94 help 77 help
95 Use a faster and more secure NEON based implementation of AES in CBC, 78 Use a faster and more secure NEON based implementation of AES in CBC,
96 CTR and XTS modes 79 CTR and XTS modes
@@ -130,4 +113,10 @@ config CRYPTO_CRC32_ARM_CE
130 depends on KERNEL_MODE_NEON && CRC32 113 depends on KERNEL_MODE_NEON && CRC32
131 select CRYPTO_HASH 114 select CRYPTO_HASH
132 115
116config CRYPTO_CHACHA20_NEON
117 tristate "NEON accelerated ChaCha20 symmetric cipher"
118 depends on KERNEL_MODE_NEON
119 select CRYPTO_BLKCIPHER
120 select CRYPTO_CHACHA20
121
133endif 122endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index b578a1820ab1..1822c4697278 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
8obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o 8obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
9obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o 9obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
10obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o 10obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o
11obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
11 12
12ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o 13ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
13ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o 14ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
@@ -26,8 +27,8 @@ $(warning $(ce-obj-y) $(ce-obj-m))
26endif 27endif
27endif 28endif
28 29
29aes-arm-y := aes-armv4.o aes_glue.o 30aes-arm-y := aes-cipher-core.o aes-cipher-glue.o
30aes-arm-bs-y := aesbs-core.o aesbs-glue.o 31aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
31sha1-arm-y := sha1-armv4-large.o sha1_glue.o 32sha1-arm-y := sha1-armv4-large.o sha1_glue.o
32sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o 33sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
33sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o 34sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
@@ -40,17 +41,15 @@ aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
40ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o 41ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
41crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o 42crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
42crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o 43crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
44chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
43 45
44quiet_cmd_perl = PERL $@ 46quiet_cmd_perl = PERL $@
45 cmd_perl = $(PERL) $(<) > $(@) 47 cmd_perl = $(PERL) $(<) > $(@)
46 48
47$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl
48 $(call cmd,perl)
49
50$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl 49$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl
51 $(call cmd,perl) 50 $(call cmd,perl)
52 51
53$(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl 52$(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl
54 $(call cmd,perl) 53 $(call cmd,perl)
55 54
56.PRECIOUS: $(obj)/aesbs-core.S $(obj)/sha256-core.S $(obj)/sha512-core.S 55.PRECIOUS: $(obj)/sha256-core.S $(obj)/sha512-core.S
diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S
deleted file mode 100644
index ebb9761fb572..000000000000
--- a/arch/arm/crypto/aes-armv4.S
+++ /dev/null
@@ -1,1089 +0,0 @@
1#define __ARM_ARCH__ __LINUX_ARM_ARCH__
2@ ====================================================================
3@ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
4@ project. The module is, however, dual licensed under OpenSSL and
5@ CRYPTOGAMS licenses depending on where you obtain it. For further
6@ details see http://www.openssl.org/~appro/cryptogams/.
7@ ====================================================================
8
9@ AES for ARMv4
10
11@ January 2007.
12@
13@ Code uses single 1K S-box and is >2 times faster than code generated
14@ by gcc-3.4.1. This is thanks to unique feature of ARMv4 ISA, which
15@ allows to merge logical or arithmetic operation with shift or rotate
16@ in one instruction and emit combined result every cycle. The module
17@ is endian-neutral. The performance is ~42 cycles/byte for 128-bit
18@ key [on single-issue Xscale PXA250 core].
19
20@ May 2007.
21@
22@ AES_set_[en|de]crypt_key is added.
23
24@ July 2010.
25@
26@ Rescheduling for dual-issue pipeline resulted in 12% improvement on
27@ Cortex A8 core and ~25 cycles per byte processed with 128-bit key.
28
29@ February 2011.
30@
31@ Profiler-assisted and platform-specific optimization resulted in 16%
32@ improvement on Cortex A8 core and ~21.5 cycles per byte.
33
34@ A little glue here to select the correct code below for the ARM CPU
35@ that is being targetted.
36
37#include <linux/linkage.h>
38#include <asm/assembler.h>
39
40.text
41
42.type AES_Te,%object
43.align 5
44AES_Te:
45.word 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d
46.word 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554
47.word 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d
48.word 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a
49.word 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87
50.word 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b
51.word 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea
52.word 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b
53.word 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a
54.word 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f
55.word 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108
56.word 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f
57.word 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e
58.word 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5
59.word 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d
60.word 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f
61.word 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e
62.word 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb
63.word 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce
64.word 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497
65.word 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c
66.word 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed
67.word 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b
68.word 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a
69.word 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16
70.word 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594
71.word 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81
72.word 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3
73.word 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a
74.word 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504
75.word 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163
76.word 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d
77.word 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f
78.word 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739
79.word 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47
80.word 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395
81.word 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f
82.word 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883
83.word 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c
84.word 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76
85.word 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e
86.word 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4
87.word 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6
88.word 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b
89.word 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7
90.word 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0
91.word 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25
92.word 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818
93.word 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72
94.word 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651
95.word 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21
96.word 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85
97.word 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa
98.word 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12
99.word 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0
100.word 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9
101.word 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133
102.word 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7
103.word 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920
104.word 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a
105.word 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17
106.word 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8
107.word 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11
108.word 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a
109@ Te4[256]
110.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
111.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
112.byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
113.byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
114.byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
115.byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
116.byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
117.byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
118.byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
119.byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
120.byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
121.byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
122.byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
123.byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
124.byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
125.byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
126.byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
127.byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
128.byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
129.byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
130.byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
131.byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
132.byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
133.byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
134.byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
135.byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
136.byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
137.byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
138.byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
139.byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
140.byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
141.byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
142@ rcon[]
143.word 0x01000000, 0x02000000, 0x04000000, 0x08000000
144.word 0x10000000, 0x20000000, 0x40000000, 0x80000000
145.word 0x1B000000, 0x36000000, 0, 0, 0, 0, 0, 0
146.size AES_Te,.-AES_Te
147
148@ void AES_encrypt(const unsigned char *in, unsigned char *out,
149@ const AES_KEY *key) {
150.align 5
151ENTRY(AES_encrypt)
152 adr r3,AES_encrypt
153 stmdb sp!,{r1,r4-r12,lr}
154 mov r12,r0 @ inp
155 mov r11,r2
156 sub r10,r3,#AES_encrypt-AES_Te @ Te
157#if __ARM_ARCH__<7
158 ldrb r0,[r12,#3] @ load input data in endian-neutral
159 ldrb r4,[r12,#2] @ manner...
160 ldrb r5,[r12,#1]
161 ldrb r6,[r12,#0]
162 orr r0,r0,r4,lsl#8
163 ldrb r1,[r12,#7]
164 orr r0,r0,r5,lsl#16
165 ldrb r4,[r12,#6]
166 orr r0,r0,r6,lsl#24
167 ldrb r5,[r12,#5]
168 ldrb r6,[r12,#4]
169 orr r1,r1,r4,lsl#8
170 ldrb r2,[r12,#11]
171 orr r1,r1,r5,lsl#16
172 ldrb r4,[r12,#10]
173 orr r1,r1,r6,lsl#24
174 ldrb r5,[r12,#9]
175 ldrb r6,[r12,#8]
176 orr r2,r2,r4,lsl#8
177 ldrb r3,[r12,#15]
178 orr r2,r2,r5,lsl#16
179 ldrb r4,[r12,#14]
180 orr r2,r2,r6,lsl#24
181 ldrb r5,[r12,#13]
182 ldrb r6,[r12,#12]
183 orr r3,r3,r4,lsl#8
184 orr r3,r3,r5,lsl#16
185 orr r3,r3,r6,lsl#24
186#else
187 ldr r0,[r12,#0]
188 ldr r1,[r12,#4]
189 ldr r2,[r12,#8]
190 ldr r3,[r12,#12]
191#ifdef __ARMEL__
192 rev r0,r0
193 rev r1,r1
194 rev r2,r2
195 rev r3,r3
196#endif
197#endif
198 bl _armv4_AES_encrypt
199
200 ldr r12,[sp],#4 @ pop out
201#if __ARM_ARCH__>=7
202#ifdef __ARMEL__
203 rev r0,r0
204 rev r1,r1
205 rev r2,r2
206 rev r3,r3
207#endif
208 str r0,[r12,#0]
209 str r1,[r12,#4]
210 str r2,[r12,#8]
211 str r3,[r12,#12]
212#else
213 mov r4,r0,lsr#24 @ write output in endian-neutral
214 mov r5,r0,lsr#16 @ manner...
215 mov r6,r0,lsr#8
216 strb r4,[r12,#0]
217 strb r5,[r12,#1]
218 mov r4,r1,lsr#24
219 strb r6,[r12,#2]
220 mov r5,r1,lsr#16
221 strb r0,[r12,#3]
222 mov r6,r1,lsr#8
223 strb r4,[r12,#4]
224 strb r5,[r12,#5]
225 mov r4,r2,lsr#24
226 strb r6,[r12,#6]
227 mov r5,r2,lsr#16
228 strb r1,[r12,#7]
229 mov r6,r2,lsr#8
230 strb r4,[r12,#8]
231 strb r5,[r12,#9]
232 mov r4,r3,lsr#24
233 strb r6,[r12,#10]
234 mov r5,r3,lsr#16
235 strb r2,[r12,#11]
236 mov r6,r3,lsr#8
237 strb r4,[r12,#12]
238 strb r5,[r12,#13]
239 strb r6,[r12,#14]
240 strb r3,[r12,#15]
241#endif
242 ldmia sp!,{r4-r12,pc}
243ENDPROC(AES_encrypt)
244
245.type _armv4_AES_encrypt,%function
246.align 2
247_armv4_AES_encrypt:
248 str lr,[sp,#-4]! @ push lr
249 ldmia r11!,{r4-r7}
250 eor r0,r0,r4
251 ldr r12,[r11,#240-16]
252 eor r1,r1,r5
253 eor r2,r2,r6
254 eor r3,r3,r7
255 sub r12,r12,#1
256 mov lr,#255
257
258 and r7,lr,r0
259 and r8,lr,r0,lsr#8
260 and r9,lr,r0,lsr#16
261 mov r0,r0,lsr#24
262.Lenc_loop:
263 ldr r4,[r10,r7,lsl#2] @ Te3[s0>>0]
264 and r7,lr,r1,lsr#16 @ i0
265 ldr r5,[r10,r8,lsl#2] @ Te2[s0>>8]
266 and r8,lr,r1
267 ldr r6,[r10,r9,lsl#2] @ Te1[s0>>16]
268 and r9,lr,r1,lsr#8
269 ldr r0,[r10,r0,lsl#2] @ Te0[s0>>24]
270 mov r1,r1,lsr#24
271
272 ldr r7,[r10,r7,lsl#2] @ Te1[s1>>16]
273 ldr r8,[r10,r8,lsl#2] @ Te3[s1>>0]
274 ldr r9,[r10,r9,lsl#2] @ Te2[s1>>8]
275 eor r0,r0,r7,ror#8
276 ldr r1,[r10,r1,lsl#2] @ Te0[s1>>24]
277 and r7,lr,r2,lsr#8 @ i0
278 eor r5,r5,r8,ror#8
279 and r8,lr,r2,lsr#16 @ i1
280 eor r6,r6,r9,ror#8
281 and r9,lr,r2
282 ldr r7,[r10,r7,lsl#2] @ Te2[s2>>8]
283 eor r1,r1,r4,ror#24
284 ldr r8,[r10,r8,lsl#2] @ Te1[s2>>16]
285 mov r2,r2,lsr#24
286
287 ldr r9,[r10,r9,lsl#2] @ Te3[s2>>0]
288 eor r0,r0,r7,ror#16
289 ldr r2,[r10,r2,lsl#2] @ Te0[s2>>24]
290 and r7,lr,r3 @ i0
291 eor r1,r1,r8,ror#8
292 and r8,lr,r3,lsr#8 @ i1
293 eor r6,r6,r9,ror#16
294 and r9,lr,r3,lsr#16 @ i2
295 ldr r7,[r10,r7,lsl#2] @ Te3[s3>>0]
296 eor r2,r2,r5,ror#16
297 ldr r8,[r10,r8,lsl#2] @ Te2[s3>>8]
298 mov r3,r3,lsr#24
299
300 ldr r9,[r10,r9,lsl#2] @ Te1[s3>>16]
301 eor r0,r0,r7,ror#24
302 ldr r7,[r11],#16
303 eor r1,r1,r8,ror#16
304 ldr r3,[r10,r3,lsl#2] @ Te0[s3>>24]
305 eor r2,r2,r9,ror#8
306 ldr r4,[r11,#-12]
307 eor r3,r3,r6,ror#8
308
309 ldr r5,[r11,#-8]
310 eor r0,r0,r7
311 ldr r6,[r11,#-4]
312 and r7,lr,r0
313 eor r1,r1,r4
314 and r8,lr,r0,lsr#8
315 eor r2,r2,r5
316 and r9,lr,r0,lsr#16
317 eor r3,r3,r6
318 mov r0,r0,lsr#24
319
320 subs r12,r12,#1
321 bne .Lenc_loop
322
323 add r10,r10,#2
324
325 ldrb r4,[r10,r7,lsl#2] @ Te4[s0>>0]
326 and r7,lr,r1,lsr#16 @ i0
327 ldrb r5,[r10,r8,lsl#2] @ Te4[s0>>8]
328 and r8,lr,r1
329 ldrb r6,[r10,r9,lsl#2] @ Te4[s0>>16]
330 and r9,lr,r1,lsr#8
331 ldrb r0,[r10,r0,lsl#2] @ Te4[s0>>24]
332 mov r1,r1,lsr#24
333
334 ldrb r7,[r10,r7,lsl#2] @ Te4[s1>>16]
335 ldrb r8,[r10,r8,lsl#2] @ Te4[s1>>0]
336 ldrb r9,[r10,r9,lsl#2] @ Te4[s1>>8]
337 eor r0,r7,r0,lsl#8
338 ldrb r1,[r10,r1,lsl#2] @ Te4[s1>>24]
339 and r7,lr,r2,lsr#8 @ i0
340 eor r5,r8,r5,lsl#8
341 and r8,lr,r2,lsr#16 @ i1
342 eor r6,r9,r6,lsl#8
343 and r9,lr,r2
344 ldrb r7,[r10,r7,lsl#2] @ Te4[s2>>8]
345 eor r1,r4,r1,lsl#24
346 ldrb r8,[r10,r8,lsl#2] @ Te4[s2>>16]
347 mov r2,r2,lsr#24
348
349 ldrb r9,[r10,r9,lsl#2] @ Te4[s2>>0]
350 eor r0,r7,r0,lsl#8
351 ldrb r2,[r10,r2,lsl#2] @ Te4[s2>>24]
352 and r7,lr,r3 @ i0
353 eor r1,r1,r8,lsl#16
354 and r8,lr,r3,lsr#8 @ i1
355 eor r6,r9,r6,lsl#8
356 and r9,lr,r3,lsr#16 @ i2
357 ldrb r7,[r10,r7,lsl#2] @ Te4[s3>>0]
358 eor r2,r5,r2,lsl#24
359 ldrb r8,[r10,r8,lsl#2] @ Te4[s3>>8]
360 mov r3,r3,lsr#24
361
362 ldrb r9,[r10,r9,lsl#2] @ Te4[s3>>16]
363 eor r0,r7,r0,lsl#8
364 ldr r7,[r11,#0]
365 ldrb r3,[r10,r3,lsl#2] @ Te4[s3>>24]
366 eor r1,r1,r8,lsl#8
367 ldr r4,[r11,#4]
368 eor r2,r2,r9,lsl#16
369 ldr r5,[r11,#8]
370 eor r3,r6,r3,lsl#24
371 ldr r6,[r11,#12]
372
373 eor r0,r0,r7
374 eor r1,r1,r4
375 eor r2,r2,r5
376 eor r3,r3,r6
377
378 sub r10,r10,#2
379 ldr pc,[sp],#4 @ pop and return
380.size _armv4_AES_encrypt,.-_armv4_AES_encrypt
381
382.align 5
383ENTRY(private_AES_set_encrypt_key)
384_armv4_AES_set_encrypt_key:
385 adr r3,_armv4_AES_set_encrypt_key
386 teq r0,#0
387 moveq r0,#-1
388 beq .Labrt
389 teq r2,#0
390 moveq r0,#-1
391 beq .Labrt
392
393 teq r1,#128
394 beq .Lok
395 teq r1,#192
396 beq .Lok
397 teq r1,#256
398 movne r0,#-1
399 bne .Labrt
400
401.Lok: stmdb sp!,{r4-r12,lr}
402 sub r10,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024 @ Te4
403
404 mov r12,r0 @ inp
405 mov lr,r1 @ bits
406 mov r11,r2 @ key
407
408#if __ARM_ARCH__<7
409 ldrb r0,[r12,#3] @ load input data in endian-neutral
410 ldrb r4,[r12,#2] @ manner...
411 ldrb r5,[r12,#1]
412 ldrb r6,[r12,#0]
413 orr r0,r0,r4,lsl#8
414 ldrb r1,[r12,#7]
415 orr r0,r0,r5,lsl#16
416 ldrb r4,[r12,#6]
417 orr r0,r0,r6,lsl#24
418 ldrb r5,[r12,#5]
419 ldrb r6,[r12,#4]
420 orr r1,r1,r4,lsl#8
421 ldrb r2,[r12,#11]
422 orr r1,r1,r5,lsl#16
423 ldrb r4,[r12,#10]
424 orr r1,r1,r6,lsl#24
425 ldrb r5,[r12,#9]
426 ldrb r6,[r12,#8]
427 orr r2,r2,r4,lsl#8
428 ldrb r3,[r12,#15]
429 orr r2,r2,r5,lsl#16
430 ldrb r4,[r12,#14]
431 orr r2,r2,r6,lsl#24
432 ldrb r5,[r12,#13]
433 ldrb r6,[r12,#12]
434 orr r3,r3,r4,lsl#8
435 str r0,[r11],#16
436 orr r3,r3,r5,lsl#16
437 str r1,[r11,#-12]
438 orr r3,r3,r6,lsl#24
439 str r2,[r11,#-8]
440 str r3,[r11,#-4]
441#else
442 ldr r0,[r12,#0]
443 ldr r1,[r12,#4]
444 ldr r2,[r12,#8]
445 ldr r3,[r12,#12]
446#ifdef __ARMEL__
447 rev r0,r0
448 rev r1,r1
449 rev r2,r2
450 rev r3,r3
451#endif
452 str r0,[r11],#16
453 str r1,[r11,#-12]
454 str r2,[r11,#-8]
455 str r3,[r11,#-4]
456#endif
457
458 teq lr,#128
459 bne .Lnot128
460 mov r12,#10
461 str r12,[r11,#240-16]
462 add r6,r10,#256 @ rcon
463 mov lr,#255
464
465.L128_loop:
466 and r5,lr,r3,lsr#24
467 and r7,lr,r3,lsr#16
468 ldrb r5,[r10,r5]
469 and r8,lr,r3,lsr#8
470 ldrb r7,[r10,r7]
471 and r9,lr,r3
472 ldrb r8,[r10,r8]
473 orr r5,r5,r7,lsl#24
474 ldrb r9,[r10,r9]
475 orr r5,r5,r8,lsl#16
476 ldr r4,[r6],#4 @ rcon[i++]
477 orr r5,r5,r9,lsl#8
478 eor r5,r5,r4
479 eor r0,r0,r5 @ rk[4]=rk[0]^...
480 eor r1,r1,r0 @ rk[5]=rk[1]^rk[4]
481 str r0,[r11],#16
482 eor r2,r2,r1 @ rk[6]=rk[2]^rk[5]
483 str r1,[r11,#-12]
484 eor r3,r3,r2 @ rk[7]=rk[3]^rk[6]
485 str r2,[r11,#-8]
486 subs r12,r12,#1
487 str r3,[r11,#-4]
488 bne .L128_loop
489 sub r2,r11,#176
490 b .Ldone
491
492.Lnot128:
493#if __ARM_ARCH__<7
494 ldrb r8,[r12,#19]
495 ldrb r4,[r12,#18]
496 ldrb r5,[r12,#17]
497 ldrb r6,[r12,#16]
498 orr r8,r8,r4,lsl#8
499 ldrb r9,[r12,#23]
500 orr r8,r8,r5,lsl#16
501 ldrb r4,[r12,#22]
502 orr r8,r8,r6,lsl#24
503 ldrb r5,[r12,#21]
504 ldrb r6,[r12,#20]
505 orr r9,r9,r4,lsl#8
506 orr r9,r9,r5,lsl#16
507 str r8,[r11],#8
508 orr r9,r9,r6,lsl#24
509 str r9,[r11,#-4]
510#else
511 ldr r8,[r12,#16]
512 ldr r9,[r12,#20]
513#ifdef __ARMEL__
514 rev r8,r8
515 rev r9,r9
516#endif
517 str r8,[r11],#8
518 str r9,[r11,#-4]
519#endif
520
521 teq lr,#192
522 bne .Lnot192
523 mov r12,#12
524 str r12,[r11,#240-24]
525 add r6,r10,#256 @ rcon
526 mov lr,#255
527 mov r12,#8
528
529.L192_loop:
530 and r5,lr,r9,lsr#24
531 and r7,lr,r9,lsr#16
532 ldrb r5,[r10,r5]
533 and r8,lr,r9,lsr#8
534 ldrb r7,[r10,r7]
535 and r9,lr,r9
536 ldrb r8,[r10,r8]
537 orr r5,r5,r7,lsl#24
538 ldrb r9,[r10,r9]
539 orr r5,r5,r8,lsl#16
540 ldr r4,[r6],#4 @ rcon[i++]
541 orr r5,r5,r9,lsl#8
542 eor r9,r5,r4
543 eor r0,r0,r9 @ rk[6]=rk[0]^...
544 eor r1,r1,r0 @ rk[7]=rk[1]^rk[6]
545 str r0,[r11],#24
546 eor r2,r2,r1 @ rk[8]=rk[2]^rk[7]
547 str r1,[r11,#-20]
548 eor r3,r3,r2 @ rk[9]=rk[3]^rk[8]
549 str r2,[r11,#-16]
550 subs r12,r12,#1
551 str r3,[r11,#-12]
552 subeq r2,r11,#216
553 beq .Ldone
554
555 ldr r7,[r11,#-32]
556 ldr r8,[r11,#-28]
557 eor r7,r7,r3 @ rk[10]=rk[4]^rk[9]
558 eor r9,r8,r7 @ rk[11]=rk[5]^rk[10]
559 str r7,[r11,#-8]
560 str r9,[r11,#-4]
561 b .L192_loop
562
563.Lnot192:
564#if __ARM_ARCH__<7
565 ldrb r8,[r12,#27]
566 ldrb r4,[r12,#26]
567 ldrb r5,[r12,#25]
568 ldrb r6,[r12,#24]
569 orr r8,r8,r4,lsl#8
570 ldrb r9,[r12,#31]
571 orr r8,r8,r5,lsl#16
572 ldrb r4,[r12,#30]
573 orr r8,r8,r6,lsl#24
574 ldrb r5,[r12,#29]
575 ldrb r6,[r12,#28]
576 orr r9,r9,r4,lsl#8
577 orr r9,r9,r5,lsl#16
578 str r8,[r11],#8
579 orr r9,r9,r6,lsl#24
580 str r9,[r11,#-4]
581#else
582 ldr r8,[r12,#24]
583 ldr r9,[r12,#28]
584#ifdef __ARMEL__
585 rev r8,r8
586 rev r9,r9
587#endif
588 str r8,[r11],#8
589 str r9,[r11,#-4]
590#endif
591
592 mov r12,#14
593 str r12,[r11,#240-32]
594 add r6,r10,#256 @ rcon
595 mov lr,#255
596 mov r12,#7
597
598.L256_loop:
599 and r5,lr,r9,lsr#24
600 and r7,lr,r9,lsr#16
601 ldrb r5,[r10,r5]
602 and r8,lr,r9,lsr#8
603 ldrb r7,[r10,r7]
604 and r9,lr,r9
605 ldrb r8,[r10,r8]
606 orr r5,r5,r7,lsl#24
607 ldrb r9,[r10,r9]
608 orr r5,r5,r8,lsl#16
609 ldr r4,[r6],#4 @ rcon[i++]
610 orr r5,r5,r9,lsl#8
611 eor r9,r5,r4
612 eor r0,r0,r9 @ rk[8]=rk[0]^...
613 eor r1,r1,r0 @ rk[9]=rk[1]^rk[8]
614 str r0,[r11],#32
615 eor r2,r2,r1 @ rk[10]=rk[2]^rk[9]
616 str r1,[r11,#-28]
617 eor r3,r3,r2 @ rk[11]=rk[3]^rk[10]
618 str r2,[r11,#-24]
619 subs r12,r12,#1
620 str r3,[r11,#-20]
621 subeq r2,r11,#256
622 beq .Ldone
623
624 and r5,lr,r3
625 and r7,lr,r3,lsr#8
626 ldrb r5,[r10,r5]
627 and r8,lr,r3,lsr#16
628 ldrb r7,[r10,r7]
629 and r9,lr,r3,lsr#24
630 ldrb r8,[r10,r8]
631 orr r5,r5,r7,lsl#8
632 ldrb r9,[r10,r9]
633 orr r5,r5,r8,lsl#16
634 ldr r4,[r11,#-48]
635 orr r5,r5,r9,lsl#24
636
637 ldr r7,[r11,#-44]
638 ldr r8,[r11,#-40]
639 eor r4,r4,r5 @ rk[12]=rk[4]^...
640 ldr r9,[r11,#-36]
641 eor r7,r7,r4 @ rk[13]=rk[5]^rk[12]
642 str r4,[r11,#-16]
643 eor r8,r8,r7 @ rk[14]=rk[6]^rk[13]
644 str r7,[r11,#-12]
645 eor r9,r9,r8 @ rk[15]=rk[7]^rk[14]
646 str r8,[r11,#-8]
647 str r9,[r11,#-4]
648 b .L256_loop
649
650.Ldone: mov r0,#0
651 ldmia sp!,{r4-r12,lr}
652.Labrt: ret lr
653ENDPROC(private_AES_set_encrypt_key)
654
655.align 5
656ENTRY(private_AES_set_decrypt_key)
657 str lr,[sp,#-4]! @ push lr
658#if 0
659 @ kernel does both of these in setkey so optimise this bit out by
660 @ expecting the key to already have the enc_key work done (see aes_glue.c)
661 bl _armv4_AES_set_encrypt_key
662#else
663 mov r0,#0
664#endif
665 teq r0,#0
666 ldrne lr,[sp],#4 @ pop lr
667 bne .Labrt
668
669 stmdb sp!,{r4-r12}
670
671 ldr r12,[r2,#240] @ AES_set_encrypt_key preserves r2,
672 mov r11,r2 @ which is AES_KEY *key
673 mov r7,r2
674 add r8,r2,r12,lsl#4
675
676.Linv: ldr r0,[r7]
677 ldr r1,[r7,#4]
678 ldr r2,[r7,#8]
679 ldr r3,[r7,#12]
680 ldr r4,[r8]
681 ldr r5,[r8,#4]
682 ldr r6,[r8,#8]
683 ldr r9,[r8,#12]
684 str r0,[r8],#-16
685 str r1,[r8,#16+4]
686 str r2,[r8,#16+8]
687 str r3,[r8,#16+12]
688 str r4,[r7],#16
689 str r5,[r7,#-12]
690 str r6,[r7,#-8]
691 str r9,[r7,#-4]
692 teq r7,r8
693 bne .Linv
694 ldr r0,[r11,#16]! @ prefetch tp1
695 mov r7,#0x80
696 mov r8,#0x1b
697 orr r7,r7,#0x8000
698 orr r8,r8,#0x1b00
699 orr r7,r7,r7,lsl#16
700 orr r8,r8,r8,lsl#16
701 sub r12,r12,#1
702 mvn r9,r7
703 mov r12,r12,lsl#2 @ (rounds-1)*4
704
705.Lmix: and r4,r0,r7
706 and r1,r0,r9
707 sub r4,r4,r4,lsr#7
708 and r4,r4,r8
709 eor r1,r4,r1,lsl#1 @ tp2
710
711 and r4,r1,r7
712 and r2,r1,r9
713 sub r4,r4,r4,lsr#7
714 and r4,r4,r8
715 eor r2,r4,r2,lsl#1 @ tp4
716
717 and r4,r2,r7
718 and r3,r2,r9
719 sub r4,r4,r4,lsr#7
720 and r4,r4,r8
721 eor r3,r4,r3,lsl#1 @ tp8
722
723 eor r4,r1,r2
724 eor r5,r0,r3 @ tp9
725 eor r4,r4,r3 @ tpe
726 eor r4,r4,r1,ror#24
727 eor r4,r4,r5,ror#24 @ ^= ROTATE(tpb=tp9^tp2,8)
728 eor r4,r4,r2,ror#16
729 eor r4,r4,r5,ror#16 @ ^= ROTATE(tpd=tp9^tp4,16)
730 eor r4,r4,r5,ror#8 @ ^= ROTATE(tp9,24)
731
732 ldr r0,[r11,#4] @ prefetch tp1
733 str r4,[r11],#4
734 subs r12,r12,#1
735 bne .Lmix
736
737 mov r0,#0
738 ldmia sp!,{r4-r12,pc}
739ENDPROC(private_AES_set_decrypt_key)
740
741.type AES_Td,%object
742.align 5
743AES_Td:
744.word 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96
745.word 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393
746.word 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25
747.word 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f
748.word 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1
749.word 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6
750.word 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da
751.word 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844
752.word 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd
753.word 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4
754.word 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45
755.word 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94
756.word 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7
757.word 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a
758.word 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5
759.word 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c
760.word 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1
761.word 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a
762.word 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75
763.word 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051
764.word 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46
765.word 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff
766.word 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77
767.word 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb
768.word 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000
769.word 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e
770.word 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927
771.word 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a
772.word 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e
773.word 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16
774.word 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d
775.word 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8
776.word 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd
777.word 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34
778.word 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163
779.word 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120
780.word 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d
781.word 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0
782.word 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422
783.word 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef
784.word 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36
785.word 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4
786.word 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662
787.word 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5
788.word 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3
789.word 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b
790.word 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8
791.word 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6
792.word 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6
793.word 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0
794.word 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815
795.word 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f
796.word 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df
797.word 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f
798.word 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e
799.word 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713
800.word 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89
801.word 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c
802.word 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf
803.word 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86
804.word 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f
805.word 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541
806.word 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190
807.word 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742
808@ Td4[256]
809.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
810.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
811.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
812.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
813.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
814.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
815.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
816.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
817.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
818.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
819.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
820.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
821.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
822.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
823.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
824.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
825.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
826.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
827.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
828.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
829.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
830.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
831.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
832.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
833.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
834.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
835.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
836.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
837.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
838.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
839.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
840.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
841.size AES_Td,.-AES_Td
842
843@ void AES_decrypt(const unsigned char *in, unsigned char *out,
844@ const AES_KEY *key) {
845.align 5
846ENTRY(AES_decrypt)
847 adr r3,AES_decrypt
848 stmdb sp!,{r1,r4-r12,lr}
849 mov r12,r0 @ inp
850 mov r11,r2
851 sub r10,r3,#AES_decrypt-AES_Td @ Td
852#if __ARM_ARCH__<7
853 ldrb r0,[r12,#3] @ load input data in endian-neutral
854 ldrb r4,[r12,#2] @ manner...
855 ldrb r5,[r12,#1]
856 ldrb r6,[r12,#0]
857 orr r0,r0,r4,lsl#8
858 ldrb r1,[r12,#7]
859 orr r0,r0,r5,lsl#16
860 ldrb r4,[r12,#6]
861 orr r0,r0,r6,lsl#24
862 ldrb r5,[r12,#5]
863 ldrb r6,[r12,#4]
864 orr r1,r1,r4,lsl#8
865 ldrb r2,[r12,#11]
866 orr r1,r1,r5,lsl#16
867 ldrb r4,[r12,#10]
868 orr r1,r1,r6,lsl#24
869 ldrb r5,[r12,#9]
870 ldrb r6,[r12,#8]
871 orr r2,r2,r4,lsl#8
872 ldrb r3,[r12,#15]
873 orr r2,r2,r5,lsl#16
874 ldrb r4,[r12,#14]
875 orr r2,r2,r6,lsl#24
876 ldrb r5,[r12,#13]
877 ldrb r6,[r12,#12]
878 orr r3,r3,r4,lsl#8
879 orr r3,r3,r5,lsl#16
880 orr r3,r3,r6,lsl#24
881#else
882 ldr r0,[r12,#0]
883 ldr r1,[r12,#4]
884 ldr r2,[r12,#8]
885 ldr r3,[r12,#12]
886#ifdef __ARMEL__
887 rev r0,r0
888 rev r1,r1
889 rev r2,r2
890 rev r3,r3
891#endif
892#endif
893 bl _armv4_AES_decrypt
894
895 ldr r12,[sp],#4 @ pop out
896#if __ARM_ARCH__>=7
897#ifdef __ARMEL__
898 rev r0,r0
899 rev r1,r1
900 rev r2,r2
901 rev r3,r3
902#endif
903 str r0,[r12,#0]
904 str r1,[r12,#4]
905 str r2,[r12,#8]
906 str r3,[r12,#12]
907#else
908 mov r4,r0,lsr#24 @ write output in endian-neutral
909 mov r5,r0,lsr#16 @ manner...
910 mov r6,r0,lsr#8
911 strb r4,[r12,#0]
912 strb r5,[r12,#1]
913 mov r4,r1,lsr#24
914 strb r6,[r12,#2]
915 mov r5,r1,lsr#16
916 strb r0,[r12,#3]
917 mov r6,r1,lsr#8
918 strb r4,[r12,#4]
919 strb r5,[r12,#5]
920 mov r4,r2,lsr#24
921 strb r6,[r12,#6]
922 mov r5,r2,lsr#16
923 strb r1,[r12,#7]
924 mov r6,r2,lsr#8
925 strb r4,[r12,#8]
926 strb r5,[r12,#9]
927 mov r4,r3,lsr#24
928 strb r6,[r12,#10]
929 mov r5,r3,lsr#16
930 strb r2,[r12,#11]
931 mov r6,r3,lsr#8
932 strb r4,[r12,#12]
933 strb r5,[r12,#13]
934 strb r6,[r12,#14]
935 strb r3,[r12,#15]
936#endif
937 ldmia sp!,{r4-r12,pc}
938ENDPROC(AES_decrypt)
939
940.type _armv4_AES_decrypt,%function
941.align 2
942_armv4_AES_decrypt:
943 str lr,[sp,#-4]! @ push lr
944 ldmia r11!,{r4-r7}
945 eor r0,r0,r4
946 ldr r12,[r11,#240-16]
947 eor r1,r1,r5
948 eor r2,r2,r6
949 eor r3,r3,r7
950 sub r12,r12,#1
951 mov lr,#255
952
953 and r7,lr,r0,lsr#16
954 and r8,lr,r0,lsr#8
955 and r9,lr,r0
956 mov r0,r0,lsr#24
957.Ldec_loop:
958 ldr r4,[r10,r7,lsl#2] @ Td1[s0>>16]
959 and r7,lr,r1 @ i0
960 ldr r5,[r10,r8,lsl#2] @ Td2[s0>>8]
961 and r8,lr,r1,lsr#16
962 ldr r6,[r10,r9,lsl#2] @ Td3[s0>>0]
963 and r9,lr,r1,lsr#8
964 ldr r0,[r10,r0,lsl#2] @ Td0[s0>>24]
965 mov r1,r1,lsr#24
966
967 ldr r7,[r10,r7,lsl#2] @ Td3[s1>>0]
968 ldr r8,[r10,r8,lsl#2] @ Td1[s1>>16]
969 ldr r9,[r10,r9,lsl#2] @ Td2[s1>>8]
970 eor r0,r0,r7,ror#24
971 ldr r1,[r10,r1,lsl#2] @ Td0[s1>>24]
972 and r7,lr,r2,lsr#8 @ i0
973 eor r5,r8,r5,ror#8
974 and r8,lr,r2 @ i1
975 eor r6,r9,r6,ror#8
976 and r9,lr,r2,lsr#16
977 ldr r7,[r10,r7,lsl#2] @ Td2[s2>>8]
978 eor r1,r1,r4,ror#8
979 ldr r8,[r10,r8,lsl#2] @ Td3[s2>>0]
980 mov r2,r2,lsr#24
981
982 ldr r9,[r10,r9,lsl#2] @ Td1[s2>>16]
983 eor r0,r0,r7,ror#16
984 ldr r2,[r10,r2,lsl#2] @ Td0[s2>>24]
985 and r7,lr,r3,lsr#16 @ i0
986 eor r1,r1,r8,ror#24
987 and r8,lr,r3,lsr#8 @ i1
988 eor r6,r9,r6,ror#8
989 and r9,lr,r3 @ i2
990 ldr r7,[r10,r7,lsl#2] @ Td1[s3>>16]
991 eor r2,r2,r5,ror#8
992 ldr r8,[r10,r8,lsl#2] @ Td2[s3>>8]
993 mov r3,r3,lsr#24
994
995 ldr r9,[r10,r9,lsl#2] @ Td3[s3>>0]
996 eor r0,r0,r7,ror#8
997 ldr r7,[r11],#16
998 eor r1,r1,r8,ror#16
999 ldr r3,[r10,r3,lsl#2] @ Td0[s3>>24]
1000 eor r2,r2,r9,ror#24
1001
1002 ldr r4,[r11,#-12]
1003 eor r0,r0,r7
1004 ldr r5,[r11,#-8]
1005 eor r3,r3,r6,ror#8
1006 ldr r6,[r11,#-4]
1007 and r7,lr,r0,lsr#16
1008 eor r1,r1,r4
1009 and r8,lr,r0,lsr#8
1010 eor r2,r2,r5
1011 and r9,lr,r0
1012 eor r3,r3,r6
1013 mov r0,r0,lsr#24
1014
1015 subs r12,r12,#1
1016 bne .Ldec_loop
1017
1018 add r10,r10,#1024
1019
1020 ldr r5,[r10,#0] @ prefetch Td4
1021 ldr r6,[r10,#32]
1022 ldr r4,[r10,#64]
1023 ldr r5,[r10,#96]
1024 ldr r6,[r10,#128]
1025 ldr r4,[r10,#160]
1026 ldr r5,[r10,#192]
1027 ldr r6,[r10,#224]
1028
1029 ldrb r0,[r10,r0] @ Td4[s0>>24]
1030 ldrb r4,[r10,r7] @ Td4[s0>>16]
1031 and r7,lr,r1 @ i0
1032 ldrb r5,[r10,r8] @ Td4[s0>>8]
1033 and r8,lr,r1,lsr#16
1034 ldrb r6,[r10,r9] @ Td4[s0>>0]
1035 and r9,lr,r1,lsr#8
1036
1037 ldrb r7,[r10,r7] @ Td4[s1>>0]
1038 ARM( ldrb r1,[r10,r1,lsr#24] ) @ Td4[s1>>24]
1039 THUMB( add r1,r10,r1,lsr#24 ) @ Td4[s1>>24]
1040 THUMB( ldrb r1,[r1] )
1041 ldrb r8,[r10,r8] @ Td4[s1>>16]
1042 eor r0,r7,r0,lsl#24
1043 ldrb r9,[r10,r9] @ Td4[s1>>8]
1044 eor r1,r4,r1,lsl#8
1045 and r7,lr,r2,lsr#8 @ i0
1046 eor r5,r5,r8,lsl#8
1047 and r8,lr,r2 @ i1
1048 ldrb r7,[r10,r7] @ Td4[s2>>8]
1049 eor r6,r6,r9,lsl#8
1050 ldrb r8,[r10,r8] @ Td4[s2>>0]
1051 and r9,lr,r2,lsr#16
1052
1053 ARM( ldrb r2,[r10,r2,lsr#24] ) @ Td4[s2>>24]
1054 THUMB( add r2,r10,r2,lsr#24 ) @ Td4[s2>>24]
1055 THUMB( ldrb r2,[r2] )
1056 eor r0,r0,r7,lsl#8
1057 ldrb r9,[r10,r9] @ Td4[s2>>16]
1058 eor r1,r8,r1,lsl#16
1059 and r7,lr,r3,lsr#16 @ i0
1060 eor r2,r5,r2,lsl#16
1061 and r8,lr,r3,lsr#8 @ i1
1062 ldrb r7,[r10,r7] @ Td4[s3>>16]
1063 eor r6,r6,r9,lsl#16
1064 ldrb r8,[r10,r8] @ Td4[s3>>8]
1065 and r9,lr,r3 @ i2
1066
1067 ldrb r9,[r10,r9] @ Td4[s3>>0]
1068 ARM( ldrb r3,[r10,r3,lsr#24] ) @ Td4[s3>>24]
1069 THUMB( add r3,r10,r3,lsr#24 ) @ Td4[s3>>24]
1070 THUMB( ldrb r3,[r3] )
1071 eor r0,r0,r7,lsl#16
1072 ldr r7,[r11,#0]
1073 eor r1,r1,r8,lsl#8
1074 ldr r4,[r11,#4]
1075 eor r2,r9,r2,lsl#8
1076 ldr r5,[r11,#8]
1077 eor r3,r6,r3,lsl#24
1078 ldr r6,[r11,#12]
1079
1080 eor r0,r0,r7
1081 eor r1,r1,r4
1082 eor r2,r2,r5
1083 eor r3,r3,r6
1084
1085 sub r10,r10,#1024
1086 ldr pc,[sp],#4 @ pop and return
1087.size _armv4_AES_decrypt,.-_armv4_AES_decrypt
1088.asciz "AES for ARMv4, CRYPTOGAMS by <appro@openssl.org>"
1089.align 2
diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S
index 987aa632c9f0..ba8e6a32fdc9 100644
--- a/arch/arm/crypto/aes-ce-core.S
+++ b/arch/arm/crypto/aes-ce-core.S
@@ -169,19 +169,19 @@ ENTRY(ce_aes_ecb_encrypt)
169.Lecbencloop3x: 169.Lecbencloop3x:
170 subs r4, r4, #3 170 subs r4, r4, #3
171 bmi .Lecbenc1x 171 bmi .Lecbenc1x
172 vld1.8 {q0-q1}, [r1, :64]! 172 vld1.8 {q0-q1}, [r1]!
173 vld1.8 {q2}, [r1, :64]! 173 vld1.8 {q2}, [r1]!
174 bl aes_encrypt_3x 174 bl aes_encrypt_3x
175 vst1.8 {q0-q1}, [r0, :64]! 175 vst1.8 {q0-q1}, [r0]!
176 vst1.8 {q2}, [r0, :64]! 176 vst1.8 {q2}, [r0]!
177 b .Lecbencloop3x 177 b .Lecbencloop3x
178.Lecbenc1x: 178.Lecbenc1x:
179 adds r4, r4, #3 179 adds r4, r4, #3
180 beq .Lecbencout 180 beq .Lecbencout
181.Lecbencloop: 181.Lecbencloop:
182 vld1.8 {q0}, [r1, :64]! 182 vld1.8 {q0}, [r1]!
183 bl aes_encrypt 183 bl aes_encrypt
184 vst1.8 {q0}, [r0, :64]! 184 vst1.8 {q0}, [r0]!
185 subs r4, r4, #1 185 subs r4, r4, #1
186 bne .Lecbencloop 186 bne .Lecbencloop
187.Lecbencout: 187.Lecbencout:
@@ -195,19 +195,19 @@ ENTRY(ce_aes_ecb_decrypt)
195.Lecbdecloop3x: 195.Lecbdecloop3x:
196 subs r4, r4, #3 196 subs r4, r4, #3
197 bmi .Lecbdec1x 197 bmi .Lecbdec1x
198 vld1.8 {q0-q1}, [r1, :64]! 198 vld1.8 {q0-q1}, [r1]!
199 vld1.8 {q2}, [r1, :64]! 199 vld1.8 {q2}, [r1]!
200 bl aes_decrypt_3x 200 bl aes_decrypt_3x
201 vst1.8 {q0-q1}, [r0, :64]! 201 vst1.8 {q0-q1}, [r0]!
202 vst1.8 {q2}, [r0, :64]! 202 vst1.8 {q2}, [r0]!
203 b .Lecbdecloop3x 203 b .Lecbdecloop3x
204.Lecbdec1x: 204.Lecbdec1x:
205 adds r4, r4, #3 205 adds r4, r4, #3
206 beq .Lecbdecout 206 beq .Lecbdecout
207.Lecbdecloop: 207.Lecbdecloop:
208 vld1.8 {q0}, [r1, :64]! 208 vld1.8 {q0}, [r1]!
209 bl aes_decrypt 209 bl aes_decrypt
210 vst1.8 {q0}, [r0, :64]! 210 vst1.8 {q0}, [r0]!
211 subs r4, r4, #1 211 subs r4, r4, #1
212 bne .Lecbdecloop 212 bne .Lecbdecloop
213.Lecbdecout: 213.Lecbdecout:
@@ -226,10 +226,10 @@ ENTRY(ce_aes_cbc_encrypt)
226 vld1.8 {q0}, [r5] 226 vld1.8 {q0}, [r5]
227 prepare_key r2, r3 227 prepare_key r2, r3
228.Lcbcencloop: 228.Lcbcencloop:
229 vld1.8 {q1}, [r1, :64]! @ get next pt block 229 vld1.8 {q1}, [r1]! @ get next pt block
230 veor q0, q0, q1 @ ..and xor with iv 230 veor q0, q0, q1 @ ..and xor with iv
231 bl aes_encrypt 231 bl aes_encrypt
232 vst1.8 {q0}, [r0, :64]! 232 vst1.8 {q0}, [r0]!
233 subs r4, r4, #1 233 subs r4, r4, #1
234 bne .Lcbcencloop 234 bne .Lcbcencloop
235 vst1.8 {q0}, [r5] 235 vst1.8 {q0}, [r5]
@@ -244,8 +244,8 @@ ENTRY(ce_aes_cbc_decrypt)
244.Lcbcdecloop3x: 244.Lcbcdecloop3x:
245 subs r4, r4, #3 245 subs r4, r4, #3
246 bmi .Lcbcdec1x 246 bmi .Lcbcdec1x
247 vld1.8 {q0-q1}, [r1, :64]! 247 vld1.8 {q0-q1}, [r1]!
248 vld1.8 {q2}, [r1, :64]! 248 vld1.8 {q2}, [r1]!
249 vmov q3, q0 249 vmov q3, q0
250 vmov q4, q1 250 vmov q4, q1
251 vmov q5, q2 251 vmov q5, q2
@@ -254,19 +254,19 @@ ENTRY(ce_aes_cbc_decrypt)
254 veor q1, q1, q3 254 veor q1, q1, q3
255 veor q2, q2, q4 255 veor q2, q2, q4
256 vmov q6, q5 256 vmov q6, q5
257 vst1.8 {q0-q1}, [r0, :64]! 257 vst1.8 {q0-q1}, [r0]!
258 vst1.8 {q2}, [r0, :64]! 258 vst1.8 {q2}, [r0]!
259 b .Lcbcdecloop3x 259 b .Lcbcdecloop3x
260.Lcbcdec1x: 260.Lcbcdec1x:
261 adds r4, r4, #3 261 adds r4, r4, #3
262 beq .Lcbcdecout 262 beq .Lcbcdecout
263 vmov q15, q14 @ preserve last round key 263 vmov q15, q14 @ preserve last round key
264.Lcbcdecloop: 264.Lcbcdecloop:
265 vld1.8 {q0}, [r1, :64]! @ get next ct block 265 vld1.8 {q0}, [r1]! @ get next ct block
266 veor q14, q15, q6 @ combine prev ct with last key 266 veor q14, q15, q6 @ combine prev ct with last key
267 vmov q6, q0 267 vmov q6, q0
268 bl aes_decrypt 268 bl aes_decrypt
269 vst1.8 {q0}, [r0, :64]! 269 vst1.8 {q0}, [r0]!
270 subs r4, r4, #1 270 subs r4, r4, #1
271 bne .Lcbcdecloop 271 bne .Lcbcdecloop
272.Lcbcdecout: 272.Lcbcdecout:
@@ -300,15 +300,15 @@ ENTRY(ce_aes_ctr_encrypt)
300 rev ip, r6 300 rev ip, r6
301 add r6, r6, #1 301 add r6, r6, #1
302 vmov s11, ip 302 vmov s11, ip
303 vld1.8 {q3-q4}, [r1, :64]! 303 vld1.8 {q3-q4}, [r1]!
304 vld1.8 {q5}, [r1, :64]! 304 vld1.8 {q5}, [r1]!
305 bl aes_encrypt_3x 305 bl aes_encrypt_3x
306 veor q0, q0, q3 306 veor q0, q0, q3
307 veor q1, q1, q4 307 veor q1, q1, q4
308 veor q2, q2, q5 308 veor q2, q2, q5
309 rev ip, r6 309 rev ip, r6
310 vst1.8 {q0-q1}, [r0, :64]! 310 vst1.8 {q0-q1}, [r0]!
311 vst1.8 {q2}, [r0, :64]! 311 vst1.8 {q2}, [r0]!
312 vmov s27, ip 312 vmov s27, ip
313 b .Lctrloop3x 313 b .Lctrloop3x
314.Lctr1x: 314.Lctr1x:
@@ -318,10 +318,10 @@ ENTRY(ce_aes_ctr_encrypt)
318 vmov q0, q6 318 vmov q0, q6
319 bl aes_encrypt 319 bl aes_encrypt
320 subs r4, r4, #1 320 subs r4, r4, #1
321 bmi .Lctrhalfblock @ blocks < 0 means 1/2 block 321 bmi .Lctrtailblock @ blocks < 0 means tail block
322 vld1.8 {q3}, [r1, :64]! 322 vld1.8 {q3}, [r1]!
323 veor q3, q0, q3 323 veor q3, q0, q3
324 vst1.8 {q3}, [r0, :64]! 324 vst1.8 {q3}, [r0]!
325 325
326 adds r6, r6, #1 @ increment BE ctr 326 adds r6, r6, #1 @ increment BE ctr
327 rev ip, r6 327 rev ip, r6
@@ -333,10 +333,8 @@ ENTRY(ce_aes_ctr_encrypt)
333 vst1.8 {q6}, [r5] 333 vst1.8 {q6}, [r5]
334 pop {r4-r6, pc} 334 pop {r4-r6, pc}
335 335
336.Lctrhalfblock: 336.Lctrtailblock:
337 vld1.8 {d1}, [r1, :64] 337 vst1.8 {q0}, [r0, :64] @ return just the key stream
338 veor d0, d0, d1
339 vst1.8 {d0}, [r0, :64]
340 pop {r4-r6, pc} 338 pop {r4-r6, pc}
341 339
342.Lctrcarry: 340.Lctrcarry:
@@ -405,8 +403,8 @@ ENTRY(ce_aes_xts_encrypt)
405.Lxtsenc3x: 403.Lxtsenc3x:
406 subs r4, r4, #3 404 subs r4, r4, #3
407 bmi .Lxtsenc1x 405 bmi .Lxtsenc1x
408 vld1.8 {q0-q1}, [r1, :64]! @ get 3 pt blocks 406 vld1.8 {q0-q1}, [r1]! @ get 3 pt blocks
409 vld1.8 {q2}, [r1, :64]! 407 vld1.8 {q2}, [r1]!
410 next_tweak q4, q3, q7, q6 408 next_tweak q4, q3, q7, q6
411 veor q0, q0, q3 409 veor q0, q0, q3
412 next_tweak q5, q4, q7, q6 410 next_tweak q5, q4, q7, q6
@@ -416,8 +414,8 @@ ENTRY(ce_aes_xts_encrypt)
416 veor q0, q0, q3 414 veor q0, q0, q3
417 veor q1, q1, q4 415 veor q1, q1, q4
418 veor q2, q2, q5 416 veor q2, q2, q5
419 vst1.8 {q0-q1}, [r0, :64]! @ write 3 ct blocks 417 vst1.8 {q0-q1}, [r0]! @ write 3 ct blocks
420 vst1.8 {q2}, [r0, :64]! 418 vst1.8 {q2}, [r0]!
421 vmov q3, q5 419 vmov q3, q5
422 teq r4, #0 420 teq r4, #0
423 beq .Lxtsencout 421 beq .Lxtsencout
@@ -426,11 +424,11 @@ ENTRY(ce_aes_xts_encrypt)
426 adds r4, r4, #3 424 adds r4, r4, #3
427 beq .Lxtsencout 425 beq .Lxtsencout
428.Lxtsencloop: 426.Lxtsencloop:
429 vld1.8 {q0}, [r1, :64]! 427 vld1.8 {q0}, [r1]!
430 veor q0, q0, q3 428 veor q0, q0, q3
431 bl aes_encrypt 429 bl aes_encrypt
432 veor q0, q0, q3 430 veor q0, q0, q3
433 vst1.8 {q0}, [r0, :64]! 431 vst1.8 {q0}, [r0]!
434 subs r4, r4, #1 432 subs r4, r4, #1
435 beq .Lxtsencout 433 beq .Lxtsencout
436 next_tweak q3, q3, q7, q6 434 next_tweak q3, q3, q7, q6
@@ -456,8 +454,8 @@ ENTRY(ce_aes_xts_decrypt)
456.Lxtsdec3x: 454.Lxtsdec3x:
457 subs r4, r4, #3 455 subs r4, r4, #3
458 bmi .Lxtsdec1x 456 bmi .Lxtsdec1x
459 vld1.8 {q0-q1}, [r1, :64]! @ get 3 ct blocks 457 vld1.8 {q0-q1}, [r1]! @ get 3 ct blocks
460 vld1.8 {q2}, [r1, :64]! 458 vld1.8 {q2}, [r1]!
461 next_tweak q4, q3, q7, q6 459 next_tweak q4, q3, q7, q6
462 veor q0, q0, q3 460 veor q0, q0, q3
463 next_tweak q5, q4, q7, q6 461 next_tweak q5, q4, q7, q6
@@ -467,8 +465,8 @@ ENTRY(ce_aes_xts_decrypt)
467 veor q0, q0, q3 465 veor q0, q0, q3
468 veor q1, q1, q4 466 veor q1, q1, q4
469 veor q2, q2, q5 467 veor q2, q2, q5
470 vst1.8 {q0-q1}, [r0, :64]! @ write 3 pt blocks 468 vst1.8 {q0-q1}, [r0]! @ write 3 pt blocks
471 vst1.8 {q2}, [r0, :64]! 469 vst1.8 {q2}, [r0]!
472 vmov q3, q5 470 vmov q3, q5
473 teq r4, #0 471 teq r4, #0
474 beq .Lxtsdecout 472 beq .Lxtsdecout
@@ -477,12 +475,12 @@ ENTRY(ce_aes_xts_decrypt)
477 adds r4, r4, #3 475 adds r4, r4, #3
478 beq .Lxtsdecout 476 beq .Lxtsdecout
479.Lxtsdecloop: 477.Lxtsdecloop:
480 vld1.8 {q0}, [r1, :64]! 478 vld1.8 {q0}, [r1]!
481 veor q0, q0, q3 479 veor q0, q0, q3
482 add ip, r2, #32 @ 3rd round key 480 add ip, r2, #32 @ 3rd round key
483 bl aes_decrypt 481 bl aes_decrypt
484 veor q0, q0, q3 482 veor q0, q0, q3
485 vst1.8 {q0}, [r0, :64]! 483 vst1.8 {q0}, [r0]!
486 subs r4, r4, #1 484 subs r4, r4, #1
487 beq .Lxtsdecout 485 beq .Lxtsdecout
488 next_tweak q3, q3, q7, q6 486 next_tweak q3, q3, q7, q6
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 8857531915bf..883b84d828c5 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -278,14 +278,15 @@ static int ctr_encrypt(struct skcipher_request *req)
278 u8 *tsrc = walk.src.virt.addr; 278 u8 *tsrc = walk.src.virt.addr;
279 279
280 /* 280 /*
281 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need 281 * Tell aes_ctr_encrypt() to process a tail block.
282 * to tell aes_ctr_encrypt() to only read half a block.
283 */ 282 */
284 blocks = (nbytes <= 8) ? -1 : 1; 283 blocks = -1;
285 284
286 ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, 285 ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc,
287 num_rounds(ctx), blocks, walk.iv); 286 num_rounds(ctx), blocks, walk.iv);
288 memcpy(tdst, tail, nbytes); 287 if (tdst != tsrc)
288 memcpy(tdst, tsrc, nbytes);
289 crypto_xor(tdst, tail, nbytes);
289 err = skcipher_walk_done(&walk, 0); 290 err = skcipher_walk_done(&walk, 0);
290 } 291 }
291 kernel_neon_end(); 292 kernel_neon_end();
@@ -345,7 +346,6 @@ static struct skcipher_alg aes_algs[] = { {
345 .cra_flags = CRYPTO_ALG_INTERNAL, 346 .cra_flags = CRYPTO_ALG_INTERNAL,
346 .cra_blocksize = AES_BLOCK_SIZE, 347 .cra_blocksize = AES_BLOCK_SIZE,
347 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 348 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
348 .cra_alignmask = 7,
349 .cra_module = THIS_MODULE, 349 .cra_module = THIS_MODULE,
350 }, 350 },
351 .min_keysize = AES_MIN_KEY_SIZE, 351 .min_keysize = AES_MIN_KEY_SIZE,
@@ -361,7 +361,6 @@ static struct skcipher_alg aes_algs[] = { {
361 .cra_flags = CRYPTO_ALG_INTERNAL, 361 .cra_flags = CRYPTO_ALG_INTERNAL,
362 .cra_blocksize = AES_BLOCK_SIZE, 362 .cra_blocksize = AES_BLOCK_SIZE,
363 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 363 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
364 .cra_alignmask = 7,
365 .cra_module = THIS_MODULE, 364 .cra_module = THIS_MODULE,
366 }, 365 },
367 .min_keysize = AES_MIN_KEY_SIZE, 366 .min_keysize = AES_MIN_KEY_SIZE,
@@ -378,7 +377,6 @@ static struct skcipher_alg aes_algs[] = { {
378 .cra_flags = CRYPTO_ALG_INTERNAL, 377 .cra_flags = CRYPTO_ALG_INTERNAL,
379 .cra_blocksize = 1, 378 .cra_blocksize = 1,
380 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 379 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
381 .cra_alignmask = 7,
382 .cra_module = THIS_MODULE, 380 .cra_module = THIS_MODULE,
383 }, 381 },
384 .min_keysize = AES_MIN_KEY_SIZE, 382 .min_keysize = AES_MIN_KEY_SIZE,
@@ -396,7 +394,6 @@ static struct skcipher_alg aes_algs[] = { {
396 .cra_flags = CRYPTO_ALG_INTERNAL, 394 .cra_flags = CRYPTO_ALG_INTERNAL,
397 .cra_blocksize = AES_BLOCK_SIZE, 395 .cra_blocksize = AES_BLOCK_SIZE,
398 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 396 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
399 .cra_alignmask = 7,
400 .cra_module = THIS_MODULE, 397 .cra_module = THIS_MODULE,
401 }, 398 },
402 .min_keysize = 2 * AES_MIN_KEY_SIZE, 399 .min_keysize = 2 * AES_MIN_KEY_SIZE,
diff --git a/arch/arm/crypto/aes-cipher-core.S b/arch/arm/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..c817a86c4ca8
--- /dev/null
+++ b/arch/arm/crypto/aes-cipher-core.S
@@ -0,0 +1,179 @@
1/*
2 * Scalar AES core transform
3 *
4 * Copyright (C) 2017 Linaro Ltd.
5 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/linkage.h>
13
14 .text
15 .align 5
16
17 rk .req r0
18 rounds .req r1
19 in .req r2
20 out .req r3
21 ttab .req ip
22
23 t0 .req lr
24 t1 .req r2
25 t2 .req r3
26
27 .macro __select, out, in, idx
28 .if __LINUX_ARM_ARCH__ < 7
29 and \out, \in, #0xff << (8 * \idx)
30 .else
31 ubfx \out, \in, #(8 * \idx), #8
32 .endif
33 .endm
34
35 .macro __load, out, in, idx
36 .if __LINUX_ARM_ARCH__ < 7 && \idx > 0
37 ldr \out, [ttab, \in, lsr #(8 * \idx) - 2]
38 .else
39 ldr \out, [ttab, \in, lsl #2]
40 .endif
41 .endm
42
43 .macro __hround, out0, out1, in0, in1, in2, in3, t3, t4, enc
44 __select \out0, \in0, 0
45 __select t0, \in1, 1
46 __load \out0, \out0, 0
47 __load t0, t0, 1
48
49 .if \enc
50 __select \out1, \in1, 0
51 __select t1, \in2, 1
52 .else
53 __select \out1, \in3, 0
54 __select t1, \in0, 1
55 .endif
56 __load \out1, \out1, 0
57 __select t2, \in2, 2
58 __load t1, t1, 1
59 __load t2, t2, 2
60
61 eor \out0, \out0, t0, ror #24
62
63 __select t0, \in3, 3
64 .if \enc
65 __select \t3, \in3, 2
66 __select \t4, \in0, 3
67 .else
68 __select \t3, \in1, 2
69 __select \t4, \in2, 3
70 .endif
71 __load \t3, \t3, 2
72 __load t0, t0, 3
73 __load \t4, \t4, 3
74
75 eor \out1, \out1, t1, ror #24
76 eor \out0, \out0, t2, ror #16
77 ldm rk!, {t1, t2}
78 eor \out1, \out1, \t3, ror #16
79 eor \out0, \out0, t0, ror #8
80 eor \out1, \out1, \t4, ror #8
81 eor \out0, \out0, t1
82 eor \out1, \out1, t2
83 .endm
84
85 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
86 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
87 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
88 .endm
89
90 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
91 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
92 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
93 .endm
94
95 .macro __rev, out, in
96 .if __LINUX_ARM_ARCH__ < 6
97 lsl t0, \in, #24
98 and t1, \in, #0xff00
99 and t2, \in, #0xff0000
100 orr \out, t0, \in, lsr #24
101 orr \out, \out, t1, lsl #8
102 orr \out, \out, t2, lsr #8
103 .else
104 rev \out, \in
105 .endif
106 .endm
107
108 .macro __adrl, out, sym, c
109 .if __LINUX_ARM_ARCH__ < 7
110 ldr\c \out, =\sym
111 .else
112 movw\c \out, #:lower16:\sym
113 movt\c \out, #:upper16:\sym
114 .endif
115 .endm
116
117 .macro do_crypt, round, ttab, ltab
118 push {r3-r11, lr}
119
120 ldr r4, [in]
121 ldr r5, [in, #4]
122 ldr r6, [in, #8]
123 ldr r7, [in, #12]
124
125 ldm rk!, {r8-r11}
126
127#ifdef CONFIG_CPU_BIG_ENDIAN
128 __rev r4, r4
129 __rev r5, r5
130 __rev r6, r6
131 __rev r7, r7
132#endif
133
134 eor r4, r4, r8
135 eor r5, r5, r9
136 eor r6, r6, r10
137 eor r7, r7, r11
138
139 __adrl ttab, \ttab
140
141 tst rounds, #2
142 bne 1f
143
1440: \round r8, r9, r10, r11, r4, r5, r6, r7
145 \round r4, r5, r6, r7, r8, r9, r10, r11
146
1471: subs rounds, rounds, #4
148 \round r8, r9, r10, r11, r4, r5, r6, r7
149 __adrl ttab, \ltab, ls
150 \round r4, r5, r6, r7, r8, r9, r10, r11
151 bhi 0b
152
153#ifdef CONFIG_CPU_BIG_ENDIAN
154 __rev r4, r4
155 __rev r5, r5
156 __rev r6, r6
157 __rev r7, r7
158#endif
159
160 ldr out, [sp]
161
162 str r4, [out]
163 str r5, [out, #4]
164 str r6, [out, #8]
165 str r7, [out, #12]
166
167 pop {r3-r11, pc}
168
169 .align 3
170 .ltorg
171 .endm
172
173ENTRY(__aes_arm_encrypt)
174 do_crypt fround, crypto_ft_tab, crypto_fl_tab
175ENDPROC(__aes_arm_encrypt)
176
177ENTRY(__aes_arm_decrypt)
178 do_crypt iround, crypto_it_tab, crypto_il_tab
179ENDPROC(__aes_arm_decrypt)
diff --git a/arch/arm/crypto/aes-cipher-glue.c b/arch/arm/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..c222f6e072ad
--- /dev/null
+++ b/arch/arm/crypto/aes-cipher-glue.c
@@ -0,0 +1,74 @@
1/*
2 * Scalar AES core transform
3 *
4 * Copyright (C) 2017 Linaro Ltd.
5 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <crypto/aes.h>
13#include <linux/crypto.h>
14#include <linux/module.h>
15
16asmlinkage void __aes_arm_encrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
17EXPORT_SYMBOL(__aes_arm_encrypt);
18
19asmlinkage void __aes_arm_decrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
20EXPORT_SYMBOL(__aes_arm_decrypt);
21
22static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
23{
24 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
25 int rounds = 6 + ctx->key_length / 4;
26
27 __aes_arm_encrypt(ctx->key_enc, rounds, in, out);
28}
29
30static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
31{
32 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
33 int rounds = 6 + ctx->key_length / 4;
34
35 __aes_arm_decrypt(ctx->key_dec, rounds, in, out);
36}
37
38static struct crypto_alg aes_alg = {
39 .cra_name = "aes",
40 .cra_driver_name = "aes-arm",
41 .cra_priority = 200,
42 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
43 .cra_blocksize = AES_BLOCK_SIZE,
44 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
45 .cra_module = THIS_MODULE,
46
47 .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
48 .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
49 .cra_cipher.cia_setkey = crypto_aes_set_key,
50 .cra_cipher.cia_encrypt = aes_encrypt,
51 .cra_cipher.cia_decrypt = aes_decrypt,
52
53#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
54 .cra_alignmask = 3,
55#endif
56};
57
58static int __init aes_init(void)
59{
60 return crypto_register_alg(&aes_alg);
61}
62
63static void __exit aes_fini(void)
64{
65 crypto_unregister_alg(&aes_alg);
66}
67
68module_init(aes_init);
69module_exit(aes_fini);
70
71MODULE_DESCRIPTION("Scalar AES cipher for ARM");
72MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
73MODULE_LICENSE("GPL v2");
74MODULE_ALIAS_CRYPTO("aes");
diff --git a/arch/arm/crypto/aes-neonbs-core.S b/arch/arm/crypto/aes-neonbs-core.S
new file mode 100644
index 000000000000..2b625c6d4712
--- /dev/null
+++ b/arch/arm/crypto/aes-neonbs-core.S
@@ -0,0 +1,1023 @@
1/*
2 * Bit sliced AES using NEON instructions
3 *
4 * Copyright (C) 2017 Linaro Ltd.
5 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12/*
13 * The algorithm implemented here is described in detail by the paper
14 * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
15 * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
16 *
17 * This implementation is based primarily on the OpenSSL implementation
18 * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
19 */
20
21#include <linux/linkage.h>
22#include <asm/assembler.h>
23
24 .text
25 .fpu neon
26
27 rounds .req ip
28 bskey .req r4
29
30 q0l .req d0
31 q0h .req d1
32 q1l .req d2
33 q1h .req d3
34 q2l .req d4
35 q2h .req d5
36 q3l .req d6
37 q3h .req d7
38 q4l .req d8
39 q4h .req d9
40 q5l .req d10
41 q5h .req d11
42 q6l .req d12
43 q6h .req d13
44 q7l .req d14
45 q7h .req d15
46 q8l .req d16
47 q8h .req d17
48 q9l .req d18
49 q9h .req d19
50 q10l .req d20
51 q10h .req d21
52 q11l .req d22
53 q11h .req d23
54 q12l .req d24
55 q12h .req d25
56 q13l .req d26
57 q13h .req d27
58 q14l .req d28
59 q14h .req d29
60 q15l .req d30
61 q15h .req d31
62
63 .macro __tbl, out, tbl, in, tmp
64 .ifc \out, \tbl
65 .ifb \tmp
66 .error __tbl needs temp register if out == tbl
67 .endif
68 vmov \tmp, \out
69 .endif
70 vtbl.8 \out\()l, {\tbl}, \in\()l
71 .ifc \out, \tbl
72 vtbl.8 \out\()h, {\tmp}, \in\()h
73 .else
74 vtbl.8 \out\()h, {\tbl}, \in\()h
75 .endif
76 .endm
77
78 .macro __ldr, out, sym
79 vldr \out\()l, \sym
80 vldr \out\()h, \sym + 8
81 .endm
82
83 .macro __adr, reg, lbl
84 adr \reg, \lbl
85THUMB( orr \reg, \reg, #1 )
86 .endm
87
88 .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
89 veor \b2, \b2, \b1
90 veor \b5, \b5, \b6
91 veor \b3, \b3, \b0
92 veor \b6, \b6, \b2
93 veor \b5, \b5, \b0
94 veor \b6, \b6, \b3
95 veor \b3, \b3, \b7
96 veor \b7, \b7, \b5
97 veor \b3, \b3, \b4
98 veor \b4, \b4, \b5
99 veor \b2, \b2, \b7
100 veor \b3, \b3, \b1
101 veor \b1, \b1, \b5
102 .endm
103
104 .macro out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
105 veor \b0, \b0, \b6
106 veor \b1, \b1, \b4
107 veor \b4, \b4, \b6
108 veor \b2, \b2, \b0
109 veor \b6, \b6, \b1
110 veor \b1, \b1, \b5
111 veor \b5, \b5, \b3
112 veor \b3, \b3, \b7
113 veor \b7, \b7, \b5
114 veor \b2, \b2, \b5
115 veor \b4, \b4, \b7
116 .endm
117
118 .macro inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
119 veor \b1, \b1, \b7
120 veor \b4, \b4, \b7
121 veor \b7, \b7, \b5
122 veor \b1, \b1, \b3
123 veor \b2, \b2, \b5
124 veor \b3, \b3, \b7
125 veor \b6, \b6, \b1
126 veor \b2, \b2, \b0
127 veor \b5, \b5, \b3
128 veor \b4, \b4, \b6
129 veor \b0, \b0, \b6
130 veor \b1, \b1, \b4
131 .endm
132
133 .macro inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
134 veor \b1, \b1, \b5
135 veor \b2, \b2, \b7
136 veor \b3, \b3, \b1
137 veor \b4, \b4, \b5
138 veor \b7, \b7, \b5
139 veor \b3, \b3, \b4
140 veor \b5, \b5, \b0
141 veor \b3, \b3, \b7
142 veor \b6, \b6, \b2
143 veor \b2, \b2, \b1
144 veor \b6, \b6, \b3
145 veor \b3, \b3, \b0
146 veor \b5, \b5, \b6
147 .endm
148
149 .macro mul_gf4, x0, x1, y0, y1, t0, t1
150 veor \t0, \y0, \y1
151 vand \t0, \t0, \x0
152 veor \x0, \x0, \x1
153 vand \t1, \x1, \y0
154 vand \x0, \x0, \y1
155 veor \x1, \t1, \t0
156 veor \x0, \x0, \t1
157 .endm
158
159 .macro mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
160 veor \t0, \y0, \y1
161 veor \t1, \y2, \y3
162 vand \t0, \t0, \x0
163 vand \t1, \t1, \x2
164 veor \x0, \x0, \x1
165 veor \x2, \x2, \x3
166 vand \x1, \x1, \y0
167 vand \x3, \x3, \y2
168 vand \x0, \x0, \y1
169 vand \x2, \x2, \y3
170 veor \x1, \x1, \x0
171 veor \x2, \x2, \x3
172 veor \x0, \x0, \t0
173 veor \x3, \x3, \t1
174 .endm
175
176 .macro mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
177 y0, y1, y2, y3, t0, t1, t2, t3
178 veor \t0, \x0, \x2
179 veor \t1, \x1, \x3
180 mul_gf4 \x0, \x1, \y0, \y1, \t2, \t3
181 veor \y0, \y0, \y2
182 veor \y1, \y1, \y3
183 mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
184 veor \x0, \x0, \t0
185 veor \x2, \x2, \t0
186 veor \x1, \x1, \t1
187 veor \x3, \x3, \t1
188 veor \t0, \x4, \x6
189 veor \t1, \x5, \x7
190 mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
191 veor \y0, \y0, \y2
192 veor \y1, \y1, \y3
193 mul_gf4 \x4, \x5, \y0, \y1, \t2, \t3
194 veor \x4, \x4, \t0
195 veor \x6, \x6, \t0
196 veor \x5, \x5, \t1
197 veor \x7, \x7, \t1
198 .endm
199
200 .macro inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
201 t0, t1, t2, t3, s0, s1, s2, s3
202 veor \t3, \x4, \x6
203 veor \t0, \x5, \x7
204 veor \t1, \x1, \x3
205 veor \s1, \x7, \x6
206 veor \s0, \x0, \x2
207 veor \s3, \t3, \t0
208 vorr \t2, \t0, \t1
209 vand \s2, \t3, \s0
210 vorr \t3, \t3, \s0
211 veor \s0, \s0, \t1
212 vand \t0, \t0, \t1
213 veor \t1, \x3, \x2
214 vand \s3, \s3, \s0
215 vand \s1, \s1, \t1
216 veor \t1, \x4, \x5
217 veor \s0, \x1, \x0
218 veor \t3, \t3, \s1
219 veor \t2, \t2, \s1
220 vand \s1, \t1, \s0
221 vorr \t1, \t1, \s0
222 veor \t3, \t3, \s3
223 veor \t0, \t0, \s1
224 veor \t2, \t2, \s2
225 veor \t1, \t1, \s3
226 veor \t0, \t0, \s2
227 vand \s0, \x7, \x3
228 veor \t1, \t1, \s2
229 vand \s1, \x6, \x2
230 vand \s2, \x5, \x1
231 vorr \s3, \x4, \x0
232 veor \t3, \t3, \s0
233 veor \t1, \t1, \s2
234 veor \s0, \t0, \s3
235 veor \t2, \t2, \s1
236 vand \s2, \t3, \t1
237 veor \s1, \t2, \s2
238 veor \s3, \s0, \s2
239 vbsl \s1, \t1, \s0
240 vmvn \t0, \s0
241 vbsl \s0, \s1, \s3
242 vbsl \t0, \s1, \s3
243 vbsl \s3, \t3, \t2
244 veor \t3, \t3, \t2
245 vand \s2, \s0, \s3
246 veor \t1, \t1, \t0
247 veor \s2, \s2, \t3
248 mul_gf16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
249 \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
250 .endm
251
252 .macro sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
253 t0, t1, t2, t3, s0, s1, s2, s3
254 in_bs_ch \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
255 inv_gf256 \b6, \b5, \b0, \b3, \b7, \b1, \b4, \b2, \
256 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
257 out_bs_ch \b7, \b1, \b4, \b2, \b6, \b5, \b0, \b3
258 .endm
259
260 .macro inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
261 t0, t1, t2, t3, s0, s1, s2, s3
262 inv_in_bs_ch \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
263 inv_gf256 \b5, \b1, \b2, \b6, \b3, \b7, \b0, \b4, \
264 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
265 inv_out_bs_ch \b3, \b7, \b0, \b4, \b5, \b1, \b2, \b6
266 .endm
267
268 .macro shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, \
269 t0, t1, t2, t3, mask
270 vld1.8 {\t0-\t1}, [bskey, :256]!
271 veor \t0, \t0, \x0
272 vld1.8 {\t2-\t3}, [bskey, :256]!
273 veor \t1, \t1, \x1
274 __tbl \x0, \t0, \mask
275 veor \t2, \t2, \x2
276 __tbl \x1, \t1, \mask
277 vld1.8 {\t0-\t1}, [bskey, :256]!
278 veor \t3, \t3, \x3
279 __tbl \x2, \t2, \mask
280 __tbl \x3, \t3, \mask
281 vld1.8 {\t2-\t3}, [bskey, :256]!
282 veor \t0, \t0, \x4
283 veor \t1, \t1, \x5
284 __tbl \x4, \t0, \mask
285 veor \t2, \t2, \x6
286 __tbl \x5, \t1, \mask
287 veor \t3, \t3, \x7
288 __tbl \x6, \t2, \mask
289 __tbl \x7, \t3, \mask
290 .endm
291
292 .macro inv_shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, \
293 t0, t1, t2, t3, mask
294 __tbl \x0, \x0, \mask, \t0
295 __tbl \x1, \x1, \mask, \t1
296 __tbl \x2, \x2, \mask, \t2
297 __tbl \x3, \x3, \mask, \t3
298 __tbl \x4, \x4, \mask, \t0
299 __tbl \x5, \x5, \mask, \t1
300 __tbl \x6, \x6, \mask, \t2
301 __tbl \x7, \x7, \mask, \t3
302 .endm
303
304 .macro mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
305 t0, t1, t2, t3, t4, t5, t6, t7, inv
306 vext.8 \t0, \x0, \x0, #12
307 vext.8 \t1, \x1, \x1, #12
308 veor \x0, \x0, \t0
309 vext.8 \t2, \x2, \x2, #12
310 veor \x1, \x1, \t1
311 vext.8 \t3, \x3, \x3, #12
312 veor \x2, \x2, \t2
313 vext.8 \t4, \x4, \x4, #12
314 veor \x3, \x3, \t3
315 vext.8 \t5, \x5, \x5, #12
316 veor \x4, \x4, \t4
317 vext.8 \t6, \x6, \x6, #12
318 veor \x5, \x5, \t5
319 vext.8 \t7, \x7, \x7, #12
320 veor \x6, \x6, \t6
321 veor \t1, \t1, \x0
322 veor.8 \x7, \x7, \t7
323 vext.8 \x0, \x0, \x0, #8
324 veor \t2, \t2, \x1
325 veor \t0, \t0, \x7
326 veor \t1, \t1, \x7
327 vext.8 \x1, \x1, \x1, #8
328 veor \t5, \t5, \x4
329 veor \x0, \x0, \t0
330 veor \t6, \t6, \x5
331 veor \x1, \x1, \t1
332 vext.8 \t0, \x4, \x4, #8
333 veor \t4, \t4, \x3
334 vext.8 \t1, \x5, \x5, #8
335 veor \t7, \t7, \x6
336 vext.8 \x4, \x3, \x3, #8
337 veor \t3, \t3, \x2
338 vext.8 \x5, \x7, \x7, #8
339 veor \t4, \t4, \x7
340 vext.8 \x3, \x6, \x6, #8
341 veor \t3, \t3, \x7
342 vext.8 \x6, \x2, \x2, #8
343 veor \x7, \t1, \t5
344 .ifb \inv
345 veor \x2, \t0, \t4
346 veor \x4, \x4, \t3
347 veor \x5, \x5, \t7
348 veor \x3, \x3, \t6
349 veor \x6, \x6, \t2
350 .else
351 veor \t3, \t3, \x4
352 veor \x5, \x5, \t7
353 veor \x2, \x3, \t6
354 veor \x3, \t0, \t4
355 veor \x4, \x6, \t2
356 vmov \x6, \t3
357 .endif
358 .endm
359
360 .macro inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
361 t0, t1, t2, t3, t4, t5, t6, t7
362 vld1.8 {\t0-\t1}, [bskey, :256]!
363 veor \x0, \x0, \t0
364 vld1.8 {\t2-\t3}, [bskey, :256]!
365 veor \x1, \x1, \t1
366 vld1.8 {\t4-\t5}, [bskey, :256]!
367 veor \x2, \x2, \t2
368 vld1.8 {\t6-\t7}, [bskey, :256]
369 sub bskey, bskey, #224
370 veor \x3, \x3, \t3
371 veor \x4, \x4, \t4
372 veor \x5, \x5, \t5
373 veor \x6, \x6, \t6
374 veor \x7, \x7, \t7
375 vext.8 \t0, \x0, \x0, #8
376 vext.8 \t6, \x6, \x6, #8
377 vext.8 \t7, \x7, \x7, #8
378 veor \t0, \t0, \x0
379 vext.8 \t1, \x1, \x1, #8
380 veor \t6, \t6, \x6
381 vext.8 \t2, \x2, \x2, #8
382 veor \t7, \t7, \x7
383 vext.8 \t3, \x3, \x3, #8
384 veor \t1, \t1, \x1
385 vext.8 \t4, \x4, \x4, #8
386 veor \t2, \t2, \x2
387 vext.8 \t5, \x5, \x5, #8
388 veor \t3, \t3, \x3
389 veor \t4, \t4, \x4
390 veor \t5, \t5, \x5
391 veor \x0, \x0, \t6
392 veor \x1, \x1, \t6
393 veor \x2, \x2, \t0
394 veor \x4, \x4, \t2
395 veor \x3, \x3, \t1
396 veor \x1, \x1, \t7
397 veor \x2, \x2, \t7
398 veor \x4, \x4, \t6
399 veor \x5, \x5, \t3
400 veor \x3, \x3, \t6
401 veor \x6, \x6, \t4
402 veor \x4, \x4, \t7
403 veor \x5, \x5, \t7
404 veor \x7, \x7, \t5
405 mix_cols \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
406 \t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
407 .endm
408
409 .macro swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
410 vshr.u64 \t0, \b0, #\n
411 vshr.u64 \t1, \b1, #\n
412 veor \t0, \t0, \a0
413 veor \t1, \t1, \a1
414 vand \t0, \t0, \mask
415 vand \t1, \t1, \mask
416 veor \a0, \a0, \t0
417 vshl.s64 \t0, \t0, #\n
418 veor \a1, \a1, \t1
419 vshl.s64 \t1, \t1, #\n
420 veor \b0, \b0, \t0
421 veor \b1, \b1, \t1
422 .endm
423
424 .macro bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
425 vmov.i8 \t0, #0x55
426 vmov.i8 \t1, #0x33
427 swapmove_2x \x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
428 swapmove_2x \x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
429 vmov.i8 \t0, #0x0f
430 swapmove_2x \x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
431 swapmove_2x \x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
432 swapmove_2x \x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
433 swapmove_2x \x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
434 .endm
435
436 .align 4
437M0: .quad 0x02060a0e03070b0f, 0x0004080c0105090d
438
439 /*
440 * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
441 */
442ENTRY(aesbs_convert_key)
443 vld1.32 {q7}, [r1]! // load round 0 key
444 vld1.32 {q15}, [r1]! // load round 1 key
445
446 vmov.i8 q8, #0x01 // bit masks
447 vmov.i8 q9, #0x02
448 vmov.i8 q10, #0x04
449 vmov.i8 q11, #0x08
450 vmov.i8 q12, #0x10
451 vmov.i8 q13, #0x20
452 __ldr q14, M0
453
454 sub r2, r2, #1
455 vst1.8 {q7}, [r0, :128]! // save round 0 key
456
457.Lkey_loop:
458 __tbl q7, q15, q14
459 vmov.i8 q6, #0x40
460 vmov.i8 q15, #0x80
461
462 vtst.8 q0, q7, q8
463 vtst.8 q1, q7, q9
464 vtst.8 q2, q7, q10
465 vtst.8 q3, q7, q11
466 vtst.8 q4, q7, q12
467 vtst.8 q5, q7, q13
468 vtst.8 q6, q7, q6
469 vtst.8 q7, q7, q15
470 vld1.32 {q15}, [r1]! // load next round key
471 vmvn q0, q0
472 vmvn q1, q1
473 vmvn q5, q5
474 vmvn q6, q6
475
476 subs r2, r2, #1
477 vst1.8 {q0-q1}, [r0, :256]!
478 vst1.8 {q2-q3}, [r0, :256]!
479 vst1.8 {q4-q5}, [r0, :256]!
480 vst1.8 {q6-q7}, [r0, :256]!
481 bne .Lkey_loop
482
483 vmov.i8 q7, #0x63 // compose .L63
484 veor q15, q15, q7
485 vst1.8 {q15}, [r0, :128]
486 bx lr
487ENDPROC(aesbs_convert_key)
488
489 .align 4
490M0SR: .quad 0x0a0e02060f03070b, 0x0004080c05090d01
491
492aesbs_encrypt8:
493 vld1.8 {q9}, [bskey, :128]! // round 0 key
494 __ldr q8, M0SR
495
496 veor q10, q0, q9 // xor with round0 key
497 veor q11, q1, q9
498 __tbl q0, q10, q8
499 veor q12, q2, q9
500 __tbl q1, q11, q8
501 veor q13, q3, q9
502 __tbl q2, q12, q8
503 veor q14, q4, q9
504 __tbl q3, q13, q8
505 veor q15, q5, q9
506 __tbl q4, q14, q8
507 veor q10, q6, q9
508 __tbl q5, q15, q8
509 veor q11, q7, q9
510 __tbl q6, q10, q8
511 __tbl q7, q11, q8
512
513 bitslice q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11
514
515 sub rounds, rounds, #1
516 b .Lenc_sbox
517
518 .align 5
519SR: .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
520SRM0: .quad 0x0304090e00050a0f, 0x01060b0c0207080d
521
522.Lenc_last:
523 __ldr q12, SRM0
524.Lenc_loop:
525 shift_rows q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12
526.Lenc_sbox:
527 sbox q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, \
528 q13, q14, q15
529 subs rounds, rounds, #1
530 bcc .Lenc_done
531
532 mix_cols q0, q1, q4, q6, q3, q7, q2, q5, q8, q9, q10, q11, q12, \
533 q13, q14, q15
534
535 beq .Lenc_last
536 __ldr q12, SR
537 b .Lenc_loop
538
539.Lenc_done:
540 vld1.8 {q12}, [bskey, :128] // last round key
541
542 bitslice q0, q1, q4, q6, q3, q7, q2, q5, q8, q9, q10, q11
543
544 veor q0, q0, q12
545 veor q1, q1, q12
546 veor q4, q4, q12
547 veor q6, q6, q12
548 veor q3, q3, q12
549 veor q7, q7, q12
550 veor q2, q2, q12
551 veor q5, q5, q12
552 bx lr
553ENDPROC(aesbs_encrypt8)
554
555 .align 4
556M0ISR: .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
557
558aesbs_decrypt8:
559 add bskey, bskey, rounds, lsl #7
560 sub bskey, bskey, #112
561 vld1.8 {q9}, [bskey, :128] // round 0 key
562 sub bskey, bskey, #128
563 __ldr q8, M0ISR
564
565 veor q10, q0, q9 // xor with round0 key
566 veor q11, q1, q9
567 __tbl q0, q10, q8
568 veor q12, q2, q9
569 __tbl q1, q11, q8
570 veor q13, q3, q9
571 __tbl q2, q12, q8
572 veor q14, q4, q9
573 __tbl q3, q13, q8
574 veor q15, q5, q9
575 __tbl q4, q14, q8
576 veor q10, q6, q9
577 __tbl q5, q15, q8
578 veor q11, q7, q9
579 __tbl q6, q10, q8
580 __tbl q7, q11, q8
581
582 bitslice q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11
583
584 sub rounds, rounds, #1
585 b .Ldec_sbox
586
587 .align 5
588ISR: .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
589ISRM0: .quad 0x01040b0e0205080f, 0x0306090c00070a0d
590
591.Ldec_last:
592 __ldr q12, ISRM0
593.Ldec_loop:
594 inv_shift_rows q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12
595.Ldec_sbox:
596 inv_sbox q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, \
597 q13, q14, q15
598 subs rounds, rounds, #1
599 bcc .Ldec_done
600
601 inv_mix_cols q0, q1, q6, q4, q2, q7, q3, q5, q8, q9, q10, q11, q12, \
602 q13, q14, q15
603
604 beq .Ldec_last
605 __ldr q12, ISR
606 b .Ldec_loop
607
608.Ldec_done:
609 add bskey, bskey, #112
610 vld1.8 {q12}, [bskey, :128] // last round key
611
612 bitslice q0, q1, q6, q4, q2, q7, q3, q5, q8, q9, q10, q11
613
614 veor q0, q0, q12
615 veor q1, q1, q12
616 veor q6, q6, q12
617 veor q4, q4, q12
618 veor q2, q2, q12
619 veor q7, q7, q12
620 veor q3, q3, q12
621 veor q5, q5, q12
622 bx lr
623ENDPROC(aesbs_decrypt8)
624
625 /*
626 * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
627 * int blocks)
628 * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
629 * int blocks)
630 */
631 .macro __ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
632 push {r4-r6, lr}
633 ldr r5, [sp, #16] // number of blocks
634
63599: __adr ip, 0f
636 and lr, r5, #7
637 cmp r5, #8
638 sub ip, ip, lr, lsl #2
639 bxlt ip // computed goto if blocks < 8
640
641 vld1.8 {q0}, [r1]!
642 vld1.8 {q1}, [r1]!
643 vld1.8 {q2}, [r1]!
644 vld1.8 {q3}, [r1]!
645 vld1.8 {q4}, [r1]!
646 vld1.8 {q5}, [r1]!
647 vld1.8 {q6}, [r1]!
648 vld1.8 {q7}, [r1]!
649
6500: mov bskey, r2
651 mov rounds, r3
652 bl \do8
653
654 __adr ip, 1f
655 and lr, r5, #7
656 cmp r5, #8
657 sub ip, ip, lr, lsl #2
658 bxlt ip // computed goto if blocks < 8
659
660 vst1.8 {\o0}, [r0]!
661 vst1.8 {\o1}, [r0]!
662 vst1.8 {\o2}, [r0]!
663 vst1.8 {\o3}, [r0]!
664 vst1.8 {\o4}, [r0]!
665 vst1.8 {\o5}, [r0]!
666 vst1.8 {\o6}, [r0]!
667 vst1.8 {\o7}, [r0]!
668
6691: subs r5, r5, #8
670 bgt 99b
671
672 pop {r4-r6, pc}
673 .endm
674
675 .align 4
676ENTRY(aesbs_ecb_encrypt)
677 __ecb_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
678ENDPROC(aesbs_ecb_encrypt)
679
680 .align 4
681ENTRY(aesbs_ecb_decrypt)
682 __ecb_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
683ENDPROC(aesbs_ecb_decrypt)
684
685 /*
686 * aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
687 * int rounds, int blocks, u8 iv[])
688 */
689 .align 4
690ENTRY(aesbs_cbc_decrypt)
691 mov ip, sp
692 push {r4-r6, lr}
693 ldm ip, {r5-r6} // load args 4-5
694
69599: __adr ip, 0f
696 and lr, r5, #7
697 cmp r5, #8
698 sub ip, ip, lr, lsl #2
699 mov lr, r1
700 bxlt ip // computed goto if blocks < 8
701
702 vld1.8 {q0}, [lr]!
703 vld1.8 {q1}, [lr]!
704 vld1.8 {q2}, [lr]!
705 vld1.8 {q3}, [lr]!
706 vld1.8 {q4}, [lr]!
707 vld1.8 {q5}, [lr]!
708 vld1.8 {q6}, [lr]!
709 vld1.8 {q7}, [lr]
710
7110: mov bskey, r2
712 mov rounds, r3
713 bl aesbs_decrypt8
714
715 vld1.8 {q8}, [r6]
716 vmov q9, q8
717 vmov q10, q8
718 vmov q11, q8
719 vmov q12, q8
720 vmov q13, q8
721 vmov q14, q8
722 vmov q15, q8
723
724 __adr ip, 1f
725 and lr, r5, #7
726 cmp r5, #8
727 sub ip, ip, lr, lsl #2
728 bxlt ip // computed goto if blocks < 8
729
730 vld1.8 {q9}, [r1]!
731 vld1.8 {q10}, [r1]!
732 vld1.8 {q11}, [r1]!
733 vld1.8 {q12}, [r1]!
734 vld1.8 {q13}, [r1]!
735 vld1.8 {q14}, [r1]!
736 vld1.8 {q15}, [r1]!
737 W(nop)
738
7391: __adr ip, 2f
740 sub ip, ip, lr, lsl #3
741 bxlt ip // computed goto if blocks < 8
742
743 veor q0, q0, q8
744 vst1.8 {q0}, [r0]!
745 veor q1, q1, q9
746 vst1.8 {q1}, [r0]!
747 veor q6, q6, q10
748 vst1.8 {q6}, [r0]!
749 veor q4, q4, q11
750 vst1.8 {q4}, [r0]!
751 veor q2, q2, q12
752 vst1.8 {q2}, [r0]!
753 veor q7, q7, q13
754 vst1.8 {q7}, [r0]!
755 veor q3, q3, q14
756 vst1.8 {q3}, [r0]!
757 veor q5, q5, q15
758 vld1.8 {q8}, [r1]! // load next round's iv
7592: vst1.8 {q5}, [r0]!
760
761 subs r5, r5, #8
762 vst1.8 {q8}, [r6] // store next round's iv
763 bgt 99b
764
765 pop {r4-r6, pc}
766ENDPROC(aesbs_cbc_decrypt)
767
768 .macro next_ctr, q
769 vmov.32 \q\()h[1], r10
770 adds r10, r10, #1
771 vmov.32 \q\()h[0], r9
772 adcs r9, r9, #0
773 vmov.32 \q\()l[1], r8
774 adcs r8, r8, #0
775 vmov.32 \q\()l[0], r7
776 adc r7, r7, #0
777 vrev32.8 \q, \q
778 .endm
779
780 /*
781 * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
782 * int rounds, int blocks, u8 ctr[], u8 final[])
783 */
784ENTRY(aesbs_ctr_encrypt)
785 mov ip, sp
786 push {r4-r10, lr}
787
788 ldm ip, {r5-r7} // load args 4-6
789 teq r7, #0
790 addne r5, r5, #1 // one extra block if final != 0
791
792 vld1.8 {q0}, [r6] // load counter
793 vrev32.8 q1, q0
794 vmov r9, r10, d3
795 vmov r7, r8, d2
796
797 adds r10, r10, #1
798 adcs r9, r9, #0
799 adcs r8, r8, #0
800 adc r7, r7, #0
801
80299: vmov q1, q0
803 vmov q2, q0
804 vmov q3, q0
805 vmov q4, q0
806 vmov q5, q0
807 vmov q6, q0
808 vmov q7, q0
809
810 __adr ip, 0f
811 sub lr, r5, #1
812 and lr, lr, #7
813 cmp r5, #8
814 sub ip, ip, lr, lsl #5
815 sub ip, ip, lr, lsl #2
816 bxlt ip // computed goto if blocks < 8
817
818 next_ctr q1
819 next_ctr q2
820 next_ctr q3
821 next_ctr q4
822 next_ctr q5
823 next_ctr q6
824 next_ctr q7
825
8260: mov bskey, r2
827 mov rounds, r3
828 bl aesbs_encrypt8
829
830 __adr ip, 1f
831 and lr, r5, #7
832 cmp r5, #8
833 movgt r4, #0
834 ldrle r4, [sp, #40] // load final in the last round
835 sub ip, ip, lr, lsl #2
836 bxlt ip // computed goto if blocks < 8
837
838 vld1.8 {q8}, [r1]!
839 vld1.8 {q9}, [r1]!
840 vld1.8 {q10}, [r1]!
841 vld1.8 {q11}, [r1]!
842 vld1.8 {q12}, [r1]!
843 vld1.8 {q13}, [r1]!
844 vld1.8 {q14}, [r1]!
845 teq r4, #0 // skip last block if 'final'
8461: bne 2f
847 vld1.8 {q15}, [r1]!
848
8492: __adr ip, 3f
850 cmp r5, #8
851 sub ip, ip, lr, lsl #3
852 bxlt ip // computed goto if blocks < 8
853
854 veor q0, q0, q8
855 vst1.8 {q0}, [r0]!
856 veor q1, q1, q9
857 vst1.8 {q1}, [r0]!
858 veor q4, q4, q10
859 vst1.8 {q4}, [r0]!
860 veor q6, q6, q11
861 vst1.8 {q6}, [r0]!
862 veor q3, q3, q12
863 vst1.8 {q3}, [r0]!
864 veor q7, q7, q13
865 vst1.8 {q7}, [r0]!
866 veor q2, q2, q14
867 vst1.8 {q2}, [r0]!
868 teq r4, #0 // skip last block if 'final'
869 W(bne) 5f
8703: veor q5, q5, q15
871 vst1.8 {q5}, [r0]!
872
8734: next_ctr q0
874
875 subs r5, r5, #8
876 bgt 99b
877
878 vst1.8 {q0}, [r6]
879 pop {r4-r10, pc}
880
8815: vst1.8 {q5}, [r4]
882 b 4b
883ENDPROC(aesbs_ctr_encrypt)
884
885 .macro next_tweak, out, in, const, tmp
886 vshr.s64 \tmp, \in, #63
887 vand \tmp, \tmp, \const
888 vadd.u64 \out, \in, \in
889 vext.8 \tmp, \tmp, \tmp, #8
890 veor \out, \out, \tmp
891 .endm
892
893 .align 4
894.Lxts_mul_x:
895 .quad 1, 0x87
896
897 /*
898 * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
899 * int blocks, u8 iv[])
900 * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
901 * int blocks, u8 iv[])
902 */
903__xts_prepare8:
904 vld1.8 {q14}, [r7] // load iv
905 __ldr q15, .Lxts_mul_x // load tweak mask
906 vmov q12, q14
907
908 __adr ip, 0f
909 and r4, r6, #7
910 cmp r6, #8
911 sub ip, ip, r4, lsl #5
912 mov r4, sp
913 bxlt ip // computed goto if blocks < 8
914
915 vld1.8 {q0}, [r1]!
916 next_tweak q12, q14, q15, q13
917 veor q0, q0, q14
918 vst1.8 {q14}, [r4, :128]!
919
920 vld1.8 {q1}, [r1]!
921 next_tweak q14, q12, q15, q13
922 veor q1, q1, q12
923 vst1.8 {q12}, [r4, :128]!
924
925 vld1.8 {q2}, [r1]!
926 next_tweak q12, q14, q15, q13
927 veor q2, q2, q14
928 vst1.8 {q14}, [r4, :128]!
929
930 vld1.8 {q3}, [r1]!
931 next_tweak q14, q12, q15, q13
932 veor q3, q3, q12
933 vst1.8 {q12}, [r4, :128]!
934
935 vld1.8 {q4}, [r1]!
936 next_tweak q12, q14, q15, q13
937 veor q4, q4, q14
938 vst1.8 {q14}, [r4, :128]!
939
940 vld1.8 {q5}, [r1]!
941 next_tweak q14, q12, q15, q13
942 veor q5, q5, q12
943 vst1.8 {q12}, [r4, :128]!
944
945 vld1.8 {q6}, [r1]!
946 next_tweak q12, q14, q15, q13
947 veor q6, q6, q14
948 vst1.8 {q14}, [r4, :128]!
949
950 vld1.8 {q7}, [r1]!
951 next_tweak q14, q12, q15, q13
952 veor q7, q7, q12
953 vst1.8 {q12}, [r4, :128]
954
9550: vst1.8 {q14}, [r7] // store next iv
956 bx lr
957ENDPROC(__xts_prepare8)
958
959 .macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
960 push {r4-r8, lr}
961 mov r5, sp // preserve sp
962 ldrd r6, r7, [sp, #24] // get blocks and iv args
963 sub ip, sp, #128 // make room for 8x tweak
964 bic ip, ip, #0xf // align sp to 16 bytes
965 mov sp, ip
966
96799: bl __xts_prepare8
968
969 mov bskey, r2
970 mov rounds, r3
971 bl \do8
972
973 __adr ip, 0f
974 and lr, r6, #7
975 cmp r6, #8
976 sub ip, ip, lr, lsl #2
977 mov r4, sp
978 bxlt ip // computed goto if blocks < 8
979
980 vld1.8 {q8}, [r4, :128]!
981 vld1.8 {q9}, [r4, :128]!
982 vld1.8 {q10}, [r4, :128]!
983 vld1.8 {q11}, [r4, :128]!
984 vld1.8 {q12}, [r4, :128]!
985 vld1.8 {q13}, [r4, :128]!
986 vld1.8 {q14}, [r4, :128]!
987 vld1.8 {q15}, [r4, :128]
988
9890: __adr ip, 1f
990 sub ip, ip, lr, lsl #3
991 bxlt ip // computed goto if blocks < 8
992
993 veor \o0, \o0, q8
994 vst1.8 {\o0}, [r0]!
995 veor \o1, \o1, q9
996 vst1.8 {\o1}, [r0]!
997 veor \o2, \o2, q10
998 vst1.8 {\o2}, [r0]!
999 veor \o3, \o3, q11
1000 vst1.8 {\o3}, [r0]!
1001 veor \o4, \o4, q12
1002 vst1.8 {\o4}, [r0]!
1003 veor \o5, \o5, q13
1004 vst1.8 {\o5}, [r0]!
1005 veor \o6, \o6, q14
1006 vst1.8 {\o6}, [r0]!
1007 veor \o7, \o7, q15
1008 vst1.8 {\o7}, [r0]!
1009
10101: subs r6, r6, #8
1011 bgt 99b
1012
1013 mov sp, r5
1014 pop {r4-r8, pc}
1015 .endm
1016
1017ENTRY(aesbs_xts_encrypt)
1018 __xts_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
1019ENDPROC(aesbs_xts_encrypt)
1020
1021ENTRY(aesbs_xts_decrypt)
1022 __xts_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
1023ENDPROC(aesbs_xts_decrypt)
diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
new file mode 100644
index 000000000000..2920b96dbd36
--- /dev/null
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -0,0 +1,406 @@
1/*
2 * Bit sliced AES using NEON instructions
3 *
4 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <crypto/aes.h>
13#include <crypto/cbc.h>
14#include <crypto/internal/simd.h>
15#include <crypto/internal/skcipher.h>
16#include <crypto/xts.h>
17#include <linux/module.h>
18
19MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
20MODULE_LICENSE("GPL v2");
21
22MODULE_ALIAS_CRYPTO("ecb(aes)");
23MODULE_ALIAS_CRYPTO("cbc(aes)");
24MODULE_ALIAS_CRYPTO("ctr(aes)");
25MODULE_ALIAS_CRYPTO("xts(aes)");
26
27asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
28
29asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
30 int rounds, int blocks);
31asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
32 int rounds, int blocks);
33
34asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
35 int rounds, int blocks, u8 iv[]);
36
37asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
38 int rounds, int blocks, u8 ctr[], u8 final[]);
39
40asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
41 int rounds, int blocks, u8 iv[]);
42asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
43 int rounds, int blocks, u8 iv[]);
44
45asmlinkage void __aes_arm_encrypt(const u32 rk[], int rounds, const u8 in[],
46 u8 out[]);
47
48struct aesbs_ctx {
49 int rounds;
50 u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32] __aligned(AES_BLOCK_SIZE);
51};
52
53struct aesbs_cbc_ctx {
54 struct aesbs_ctx key;
55 u32 enc[AES_MAX_KEYLENGTH_U32];
56};
57
58struct aesbs_xts_ctx {
59 struct aesbs_ctx key;
60 u32 twkey[AES_MAX_KEYLENGTH_U32];
61};
62
63static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
64 unsigned int key_len)
65{
66 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
67 struct crypto_aes_ctx rk;
68 int err;
69
70 err = crypto_aes_expand_key(&rk, in_key, key_len);
71 if (err)
72 return err;
73
74 ctx->rounds = 6 + key_len / 4;
75
76 kernel_neon_begin();
77 aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
78 kernel_neon_end();
79
80 return 0;
81}
82
83static int __ecb_crypt(struct skcipher_request *req,
84 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
85 int rounds, int blocks))
86{
87 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
88 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
89 struct skcipher_walk walk;
90 int err;
91
92 err = skcipher_walk_virt(&walk, req, true);
93
94 kernel_neon_begin();
95 while (walk.nbytes >= AES_BLOCK_SIZE) {
96 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
97
98 if (walk.nbytes < walk.total)
99 blocks = round_down(blocks,
100 walk.stride / AES_BLOCK_SIZE);
101
102 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
103 ctx->rounds, blocks);
104 err = skcipher_walk_done(&walk,
105 walk.nbytes - blocks * AES_BLOCK_SIZE);
106 }
107 kernel_neon_end();
108
109 return err;
110}
111
112static int ecb_encrypt(struct skcipher_request *req)
113{
114 return __ecb_crypt(req, aesbs_ecb_encrypt);
115}
116
117static int ecb_decrypt(struct skcipher_request *req)
118{
119 return __ecb_crypt(req, aesbs_ecb_decrypt);
120}
121
122static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
123 unsigned int key_len)
124{
125 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
126 struct crypto_aes_ctx rk;
127 int err;
128
129 err = crypto_aes_expand_key(&rk, in_key, key_len);
130 if (err)
131 return err;
132
133 ctx->key.rounds = 6 + key_len / 4;
134
135 memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc));
136
137 kernel_neon_begin();
138 aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
139 kernel_neon_end();
140
141 return 0;
142}
143
144static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
145{
146 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
147
148 __aes_arm_encrypt(ctx->enc, ctx->key.rounds, src, dst);
149}
150
151static int cbc_encrypt(struct skcipher_request *req)
152{
153 return crypto_cbc_encrypt_walk(req, cbc_encrypt_one);
154}
155
156static int cbc_decrypt(struct skcipher_request *req)
157{
158 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
159 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
160 struct skcipher_walk walk;
161 int err;
162
163 err = skcipher_walk_virt(&walk, req, true);
164
165 kernel_neon_begin();
166 while (walk.nbytes >= AES_BLOCK_SIZE) {
167 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
168
169 if (walk.nbytes < walk.total)
170 blocks = round_down(blocks,
171 walk.stride / AES_BLOCK_SIZE);
172
173 aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
174 ctx->key.rk, ctx->key.rounds, blocks,
175 walk.iv);
176 err = skcipher_walk_done(&walk,
177 walk.nbytes - blocks * AES_BLOCK_SIZE);
178 }
179 kernel_neon_end();
180
181 return err;
182}
183
184static int ctr_encrypt(struct skcipher_request *req)
185{
186 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
187 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
188 struct skcipher_walk walk;
189 u8 buf[AES_BLOCK_SIZE];
190 int err;
191
192 err = skcipher_walk_virt(&walk, req, true);
193
194 kernel_neon_begin();
195 while (walk.nbytes > 0) {
196 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
197 u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL;
198
199 if (walk.nbytes < walk.total) {
200 blocks = round_down(blocks,
201 walk.stride / AES_BLOCK_SIZE);
202 final = NULL;
203 }
204
205 aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
206 ctx->rk, ctx->rounds, blocks, walk.iv, final);
207
208 if (final) {
209 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
210 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
211
212 if (dst != src)
213 memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
214 crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
215
216 err = skcipher_walk_done(&walk, 0);
217 break;
218 }
219 err = skcipher_walk_done(&walk,
220 walk.nbytes - blocks * AES_BLOCK_SIZE);
221 }
222 kernel_neon_end();
223
224 return err;
225}
226
227static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
228 unsigned int key_len)
229{
230 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
231 struct crypto_aes_ctx rk;
232 int err;
233
234 err = xts_verify_key(tfm, in_key, key_len);
235 if (err)
236 return err;
237
238 key_len /= 2;
239 err = crypto_aes_expand_key(&rk, in_key + key_len, key_len);
240 if (err)
241 return err;
242
243 memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey));
244
245 return aesbs_setkey(tfm, in_key, key_len);
246}
247
248static int __xts_crypt(struct skcipher_request *req,
249 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
250 int rounds, int blocks, u8 iv[]))
251{
252 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
253 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
254 struct skcipher_walk walk;
255 int err;
256
257 err = skcipher_walk_virt(&walk, req, true);
258
259 __aes_arm_encrypt(ctx->twkey, ctx->key.rounds, walk.iv, walk.iv);
260
261 kernel_neon_begin();
262 while (walk.nbytes >= AES_BLOCK_SIZE) {
263 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
264
265 if (walk.nbytes < walk.total)
266 blocks = round_down(blocks,
267 walk.stride / AES_BLOCK_SIZE);
268
269 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
270 ctx->key.rounds, blocks, walk.iv);
271 err = skcipher_walk_done(&walk,
272 walk.nbytes - blocks * AES_BLOCK_SIZE);
273 }
274 kernel_neon_end();
275
276 return err;
277}
278
279static int xts_encrypt(struct skcipher_request *req)
280{
281 return __xts_crypt(req, aesbs_xts_encrypt);
282}
283
284static int xts_decrypt(struct skcipher_request *req)
285{
286 return __xts_crypt(req, aesbs_xts_decrypt);
287}
288
289static struct skcipher_alg aes_algs[] = { {
290 .base.cra_name = "__ecb(aes)",
291 .base.cra_driver_name = "__ecb-aes-neonbs",
292 .base.cra_priority = 250,
293 .base.cra_blocksize = AES_BLOCK_SIZE,
294 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
295 .base.cra_module = THIS_MODULE,
296 .base.cra_flags = CRYPTO_ALG_INTERNAL,
297
298 .min_keysize = AES_MIN_KEY_SIZE,
299 .max_keysize = AES_MAX_KEY_SIZE,
300 .walksize = 8 * AES_BLOCK_SIZE,
301 .setkey = aesbs_setkey,
302 .encrypt = ecb_encrypt,
303 .decrypt = ecb_decrypt,
304}, {
305 .base.cra_name = "__cbc(aes)",
306 .base.cra_driver_name = "__cbc-aes-neonbs",
307 .base.cra_priority = 250,
308 .base.cra_blocksize = AES_BLOCK_SIZE,
309 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
310 .base.cra_module = THIS_MODULE,
311 .base.cra_flags = CRYPTO_ALG_INTERNAL,
312
313 .min_keysize = AES_MIN_KEY_SIZE,
314 .max_keysize = AES_MAX_KEY_SIZE,
315 .walksize = 8 * AES_BLOCK_SIZE,
316 .ivsize = AES_BLOCK_SIZE,
317 .setkey = aesbs_cbc_setkey,
318 .encrypt = cbc_encrypt,
319 .decrypt = cbc_decrypt,
320}, {
321 .base.cra_name = "__ctr(aes)",
322 .base.cra_driver_name = "__ctr-aes-neonbs",
323 .base.cra_priority = 250,
324 .base.cra_blocksize = 1,
325 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
326 .base.cra_module = THIS_MODULE,
327 .base.cra_flags = CRYPTO_ALG_INTERNAL,
328
329 .min_keysize = AES_MIN_KEY_SIZE,
330 .max_keysize = AES_MAX_KEY_SIZE,
331 .chunksize = AES_BLOCK_SIZE,
332 .walksize = 8 * AES_BLOCK_SIZE,
333 .ivsize = AES_BLOCK_SIZE,
334 .setkey = aesbs_setkey,
335 .encrypt = ctr_encrypt,
336 .decrypt = ctr_encrypt,
337}, {
338 .base.cra_name = "__xts(aes)",
339 .base.cra_driver_name = "__xts-aes-neonbs",
340 .base.cra_priority = 250,
341 .base.cra_blocksize = AES_BLOCK_SIZE,
342 .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
343 .base.cra_module = THIS_MODULE,
344 .base.cra_flags = CRYPTO_ALG_INTERNAL,
345
346 .min_keysize = 2 * AES_MIN_KEY_SIZE,
347 .max_keysize = 2 * AES_MAX_KEY_SIZE,
348 .walksize = 8 * AES_BLOCK_SIZE,
349 .ivsize = AES_BLOCK_SIZE,
350 .setkey = aesbs_xts_setkey,
351 .encrypt = xts_encrypt,
352 .decrypt = xts_decrypt,
353} };
354
355static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
356
357static void aes_exit(void)
358{
359 int i;
360
361 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
362 if (aes_simd_algs[i])
363 simd_skcipher_free(aes_simd_algs[i]);
364
365 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
366}
367
368static int __init aes_init(void)
369{
370 struct simd_skcipher_alg *simd;
371 const char *basename;
372 const char *algname;
373 const char *drvname;
374 int err;
375 int i;
376
377 if (!(elf_hwcap & HWCAP_NEON))
378 return -ENODEV;
379
380 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
381 if (err)
382 return err;
383
384 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
385 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
386 continue;
387
388 algname = aes_algs[i].base.cra_name + 2;
389 drvname = aes_algs[i].base.cra_driver_name + 2;
390 basename = aes_algs[i].base.cra_driver_name;
391 simd = simd_skcipher_create_compat(algname, drvname, basename);
392 err = PTR_ERR(simd);
393 if (IS_ERR(simd))
394 goto unregister_simds;
395
396 aes_simd_algs[i] = simd;
397 }
398 return 0;
399
400unregister_simds:
401 aes_exit();
402 return err;
403}
404
405module_init(aes_init);
406module_exit(aes_exit);
diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c
deleted file mode 100644
index 0409b8f89782..000000000000
--- a/arch/arm/crypto/aes_glue.c
+++ /dev/null
@@ -1,98 +0,0 @@
1/*
2 * Glue Code for the asm optimized version of the AES Cipher Algorithm
3 */
4
5#include <linux/module.h>
6#include <linux/crypto.h>
7#include <crypto/aes.h>
8
9#include "aes_glue.h"
10
11EXPORT_SYMBOL(AES_encrypt);
12EXPORT_SYMBOL(AES_decrypt);
13EXPORT_SYMBOL(private_AES_set_encrypt_key);
14EXPORT_SYMBOL(private_AES_set_decrypt_key);
15
16static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
17{
18 struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
19 AES_encrypt(src, dst, &ctx->enc_key);
20}
21
22static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
23{
24 struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
25 AES_decrypt(src, dst, &ctx->dec_key);
26}
27
28static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
29 unsigned int key_len)
30{
31 struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
32
33 switch (key_len) {
34 case AES_KEYSIZE_128:
35 key_len = 128;
36 break;
37 case AES_KEYSIZE_192:
38 key_len = 192;
39 break;
40 case AES_KEYSIZE_256:
41 key_len = 256;
42 break;
43 default:
44 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
45 return -EINVAL;
46 }
47
48 if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) {
49 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
50 return -EINVAL;
51 }
52 /* private_AES_set_decrypt_key expects an encryption key as input */
53 ctx->dec_key = ctx->enc_key;
54 if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) {
55 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
56 return -EINVAL;
57 }
58 return 0;
59}
60
61static struct crypto_alg aes_alg = {
62 .cra_name = "aes",
63 .cra_driver_name = "aes-asm",
64 .cra_priority = 200,
65 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
66 .cra_blocksize = AES_BLOCK_SIZE,
67 .cra_ctxsize = sizeof(struct AES_CTX),
68 .cra_module = THIS_MODULE,
69 .cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
70 .cra_u = {
71 .cipher = {
72 .cia_min_keysize = AES_MIN_KEY_SIZE,
73 .cia_max_keysize = AES_MAX_KEY_SIZE,
74 .cia_setkey = aes_set_key,
75 .cia_encrypt = aes_encrypt,
76 .cia_decrypt = aes_decrypt
77 }
78 }
79};
80
81static int __init aes_init(void)
82{
83 return crypto_register_alg(&aes_alg);
84}
85
86static void __exit aes_fini(void)
87{
88 crypto_unregister_alg(&aes_alg);
89}
90
91module_init(aes_init);
92module_exit(aes_fini);
93
94MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)");
95MODULE_LICENSE("GPL");
96MODULE_ALIAS_CRYPTO("aes");
97MODULE_ALIAS_CRYPTO("aes-asm");
98MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>");
diff --git a/arch/arm/crypto/aes_glue.h b/arch/arm/crypto/aes_glue.h
deleted file mode 100644
index cca3e51eb606..000000000000
--- a/arch/arm/crypto/aes_glue.h
+++ /dev/null
@@ -1,19 +0,0 @@
1
2#define AES_MAXNR 14
3
4struct AES_KEY {
5 unsigned int rd_key[4 * (AES_MAXNR + 1)];
6 int rounds;
7};
8
9struct AES_CTX {
10 struct AES_KEY enc_key;
11 struct AES_KEY dec_key;
12};
13
14asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct AES_KEY *ctx);
15asmlinkage void AES_decrypt(const u8 *in, u8 *out, struct AES_KEY *ctx);
16asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey,
17 const int bits, struct AES_KEY *key);
18asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey,
19 const int bits, struct AES_KEY *key);
diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped
deleted file mode 100644
index 1d1800f71c5b..000000000000
--- a/arch/arm/crypto/aesbs-core.S_shipped
+++ /dev/null
@@ -1,2548 +0,0 @@
1
2@ ====================================================================
3@ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
4@ project. The module is, however, dual licensed under OpenSSL and
5@ CRYPTOGAMS licenses depending on where you obtain it. For further
6@ details see http://www.openssl.org/~appro/cryptogams/.
7@
8@ Specific modes and adaptation for Linux kernel by Ard Biesheuvel
9@ <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
10@ granted.
11@ ====================================================================
12
13@ Bit-sliced AES for ARM NEON
14@
15@ February 2012.
16@
17@ This implementation is direct adaptation of bsaes-x86_64 module for
18@ ARM NEON. Except that this module is endian-neutral [in sense that
19@ it can be compiled for either endianness] by courtesy of vld1.8's
20@ neutrality. Initial version doesn't implement interface to OpenSSL,
21@ only low-level primitives and unsupported entry points, just enough
22@ to collect performance results, which for Cortex-A8 core are:
23@
24@ encrypt 19.5 cycles per byte processed with 128-bit key
25@ decrypt 22.1 cycles per byte processed with 128-bit key
26@ key conv. 440 cycles per 128-bit key/0.18 of 8x block
27@
28@ Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7,
29@ which is [much] worse than anticipated (for further details see
30@ http://www.openssl.org/~appro/Snapdragon-S4.html).
31@
32@ Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code
33@ manages in 20.0 cycles].
34@
35@ When comparing to x86_64 results keep in mind that NEON unit is
36@ [mostly] single-issue and thus can't [fully] benefit from
37@ instruction-level parallelism. And when comparing to aes-armv4
38@ results keep in mind key schedule conversion overhead (see
39@ bsaes-x86_64.pl for further details)...
40@
41@ <appro@openssl.org>
42
43@ April-August 2013
44@
45@ Add CBC, CTR and XTS subroutines, adapt for kernel use.
46@
47@ <ard.biesheuvel@linaro.org>
48
49#ifndef __KERNEL__
50# include "arm_arch.h"
51
52# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
53# define VFP_ABI_POP vldmia sp!,{d8-d15}
54# define VFP_ABI_FRAME 0x40
55#else
56# define VFP_ABI_PUSH
57# define VFP_ABI_POP
58# define VFP_ABI_FRAME 0
59# define BSAES_ASM_EXTENDED_KEY
60# define XTS_CHAIN_TWEAK
61# define __ARM_ARCH__ __LINUX_ARM_ARCH__
62# define __ARM_MAX_ARCH__ 7
63#endif
64
65#ifdef __thumb__
66# define adrl adr
67#endif
68
69#if __ARM_MAX_ARCH__>=7
70.arch armv7-a
71.fpu neon
72
73.text
74.syntax unified @ ARMv7-capable assembler is expected to handle this
75#ifdef __thumb2__
76.thumb
77#else
78.code 32
79#endif
80
81.type _bsaes_decrypt8,%function
82.align 4
83_bsaes_decrypt8:
84 adr r6,_bsaes_decrypt8
85 vldmia r4!, {q9} @ round 0 key
86 add r6,r6,#.LM0ISR-_bsaes_decrypt8
87
88 vldmia r6!, {q8} @ .LM0ISR
89 veor q10, q0, q9 @ xor with round0 key
90 veor q11, q1, q9
91 vtbl.8 d0, {q10}, d16
92 vtbl.8 d1, {q10}, d17
93 veor q12, q2, q9
94 vtbl.8 d2, {q11}, d16
95 vtbl.8 d3, {q11}, d17
96 veor q13, q3, q9
97 vtbl.8 d4, {q12}, d16
98 vtbl.8 d5, {q12}, d17
99 veor q14, q4, q9
100 vtbl.8 d6, {q13}, d16
101 vtbl.8 d7, {q13}, d17
102 veor q15, q5, q9
103 vtbl.8 d8, {q14}, d16
104 vtbl.8 d9, {q14}, d17
105 veor q10, q6, q9
106 vtbl.8 d10, {q15}, d16
107 vtbl.8 d11, {q15}, d17
108 veor q11, q7, q9
109 vtbl.8 d12, {q10}, d16
110 vtbl.8 d13, {q10}, d17
111 vtbl.8 d14, {q11}, d16
112 vtbl.8 d15, {q11}, d17
113 vmov.i8 q8,#0x55 @ compose .LBS0
114 vmov.i8 q9,#0x33 @ compose .LBS1
115 vshr.u64 q10, q6, #1
116 vshr.u64 q11, q4, #1
117 veor q10, q10, q7
118 veor q11, q11, q5
119 vand q10, q10, q8
120 vand q11, q11, q8
121 veor q7, q7, q10
122 vshl.u64 q10, q10, #1
123 veor q5, q5, q11
124 vshl.u64 q11, q11, #1
125 veor q6, q6, q10
126 veor q4, q4, q11
127 vshr.u64 q10, q2, #1
128 vshr.u64 q11, q0, #1
129 veor q10, q10, q3
130 veor q11, q11, q1
131 vand q10, q10, q8
132 vand q11, q11, q8
133 veor q3, q3, q10
134 vshl.u64 q10, q10, #1
135 veor q1, q1, q11
136 vshl.u64 q11, q11, #1
137 veor q2, q2, q10
138 veor q0, q0, q11
139 vmov.i8 q8,#0x0f @ compose .LBS2
140 vshr.u64 q10, q5, #2
141 vshr.u64 q11, q4, #2
142 veor q10, q10, q7
143 veor q11, q11, q6
144 vand q10, q10, q9
145 vand q11, q11, q9
146 veor q7, q7, q10
147 vshl.u64 q10, q10, #2
148 veor q6, q6, q11
149 vshl.u64 q11, q11, #2
150 veor q5, q5, q10
151 veor q4, q4, q11
152 vshr.u64 q10, q1, #2
153 vshr.u64 q11, q0, #2
154 veor q10, q10, q3
155 veor q11, q11, q2
156 vand q10, q10, q9
157 vand q11, q11, q9
158 veor q3, q3, q10
159 vshl.u64 q10, q10, #2
160 veor q2, q2, q11
161 vshl.u64 q11, q11, #2
162 veor q1, q1, q10
163 veor q0, q0, q11
164 vshr.u64 q10, q3, #4
165 vshr.u64 q11, q2, #4
166 veor q10, q10, q7
167 veor q11, q11, q6
168 vand q10, q10, q8
169 vand q11, q11, q8
170 veor q7, q7, q10
171 vshl.u64 q10, q10, #4
172 veor q6, q6, q11
173 vshl.u64 q11, q11, #4
174 veor q3, q3, q10
175 veor q2, q2, q11
176 vshr.u64 q10, q1, #4
177 vshr.u64 q11, q0, #4
178 veor q10, q10, q5
179 veor q11, q11, q4
180 vand q10, q10, q8
181 vand q11, q11, q8
182 veor q5, q5, q10
183 vshl.u64 q10, q10, #4
184 veor q4, q4, q11
185 vshl.u64 q11, q11, #4
186 veor q1, q1, q10
187 veor q0, q0, q11
188 sub r5,r5,#1
189 b .Ldec_sbox
190.align 4
191.Ldec_loop:
192 vldmia r4!, {q8-q11}
193 veor q8, q8, q0
194 veor q9, q9, q1
195 vtbl.8 d0, {q8}, d24
196 vtbl.8 d1, {q8}, d25
197 vldmia r4!, {q8}
198 veor q10, q10, q2
199 vtbl.8 d2, {q9}, d24
200 vtbl.8 d3, {q9}, d25
201 vldmia r4!, {q9}
202 veor q11, q11, q3
203 vtbl.8 d4, {q10}, d24
204 vtbl.8 d5, {q10}, d25
205 vldmia r4!, {q10}
206 vtbl.8 d6, {q11}, d24
207 vtbl.8 d7, {q11}, d25
208 vldmia r4!, {q11}
209 veor q8, q8, q4
210 veor q9, q9, q5
211 vtbl.8 d8, {q8}, d24
212 vtbl.8 d9, {q8}, d25
213 veor q10, q10, q6
214 vtbl.8 d10, {q9}, d24
215 vtbl.8 d11, {q9}, d25
216 veor q11, q11, q7
217 vtbl.8 d12, {q10}, d24
218 vtbl.8 d13, {q10}, d25
219 vtbl.8 d14, {q11}, d24
220 vtbl.8 d15, {q11}, d25
221.Ldec_sbox:
222 veor q1, q1, q4
223 veor q3, q3, q4
224
225 veor q4, q4, q7
226 veor q1, q1, q6
227 veor q2, q2, q7
228 veor q6, q6, q4
229
230 veor q0, q0, q1
231 veor q2, q2, q5
232 veor q7, q7, q6
233 veor q3, q3, q0
234 veor q5, q5, q0
235 veor q1, q1, q3
236 veor q11, q3, q0
237 veor q10, q7, q4
238 veor q9, q1, q6
239 veor q13, q4, q0
240 vmov q8, q10
241 veor q12, q5, q2
242
243 vorr q10, q10, q9
244 veor q15, q11, q8
245 vand q14, q11, q12
246 vorr q11, q11, q12
247 veor q12, q12, q9
248 vand q8, q8, q9
249 veor q9, q6, q2
250 vand q15, q15, q12
251 vand q13, q13, q9
252 veor q9, q3, q7
253 veor q12, q1, q5
254 veor q11, q11, q13
255 veor q10, q10, q13
256 vand q13, q9, q12
257 vorr q9, q9, q12
258 veor q11, q11, q15
259 veor q8, q8, q13
260 veor q10, q10, q14
261 veor q9, q9, q15
262 veor q8, q8, q14
263 vand q12, q4, q6
264 veor q9, q9, q14
265 vand q13, q0, q2
266 vand q14, q7, q1
267 vorr q15, q3, q5
268 veor q11, q11, q12
269 veor q9, q9, q14
270 veor q8, q8, q15
271 veor q10, q10, q13
272
273 @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3
274
275 @ new smaller inversion
276
277 vand q14, q11, q9
278 vmov q12, q8
279
280 veor q13, q10, q14
281 veor q15, q8, q14
282 veor q14, q8, q14 @ q14=q15
283
284 vbsl q13, q9, q8
285 vbsl q15, q11, q10
286 veor q11, q11, q10
287
288 vbsl q12, q13, q14
289 vbsl q8, q14, q13
290
291 vand q14, q12, q15
292 veor q9, q9, q8
293
294 veor q14, q14, q11
295 veor q12, q5, q2
296 veor q8, q1, q6
297 veor q10, q15, q14
298 vand q10, q10, q5
299 veor q5, q5, q1
300 vand q11, q1, q15
301 vand q5, q5, q14
302 veor q1, q11, q10
303 veor q5, q5, q11
304 veor q15, q15, q13
305 veor q14, q14, q9
306 veor q11, q15, q14
307 veor q10, q13, q9
308 vand q11, q11, q12
309 vand q10, q10, q2
310 veor q12, q12, q8
311 veor q2, q2, q6
312 vand q8, q8, q15
313 vand q6, q6, q13
314 vand q12, q12, q14
315 vand q2, q2, q9
316 veor q8, q8, q12
317 veor q2, q2, q6
318 veor q12, q12, q11
319 veor q6, q6, q10
320 veor q5, q5, q12
321 veor q2, q2, q12
322 veor q1, q1, q8
323 veor q6, q6, q8
324
325 veor q12, q3, q0
326 veor q8, q7, q4
327 veor q11, q15, q14
328 veor q10, q13, q9
329 vand q11, q11, q12
330 vand q10, q10, q0
331 veor q12, q12, q8
332 veor q0, q0, q4
333 vand q8, q8, q15
334 vand q4, q4, q13
335 vand q12, q12, q14
336 vand q0, q0, q9
337 veor q8, q8, q12
338 veor q0, q0, q4
339 veor q12, q12, q11
340 veor q4, q4, q10
341 veor q15, q15, q13
342 veor q14, q14, q9
343 veor q10, q15, q14
344 vand q10, q10, q3
345 veor q3, q3, q7
346 vand q11, q7, q15
347 vand q3, q3, q14
348 veor q7, q11, q10
349 veor q3, q3, q11
350 veor q3, q3, q12
351 veor q0, q0, q12
352 veor q7, q7, q8
353 veor q4, q4, q8
354 veor q1, q1, q7
355 veor q6, q6, q5
356
357 veor q4, q4, q1
358 veor q2, q2, q7
359 veor q5, q5, q7
360 veor q4, q4, q2
361 veor q7, q7, q0
362 veor q4, q4, q5
363 veor q3, q3, q6
364 veor q6, q6, q1
365 veor q3, q3, q4
366
367 veor q4, q4, q0
368 veor q7, q7, q3
369 subs r5,r5,#1
370 bcc .Ldec_done
371 @ multiplication by 0x05-0x00-0x04-0x00
372 vext.8 q8, q0, q0, #8
373 vext.8 q14, q3, q3, #8
374 vext.8 q15, q5, q5, #8
375 veor q8, q8, q0
376 vext.8 q9, q1, q1, #8
377 veor q14, q14, q3
378 vext.8 q10, q6, q6, #8
379 veor q15, q15, q5
380 vext.8 q11, q4, q4, #8
381 veor q9, q9, q1
382 vext.8 q12, q2, q2, #8
383 veor q10, q10, q6
384 vext.8 q13, q7, q7, #8
385 veor q11, q11, q4
386 veor q12, q12, q2
387 veor q13, q13, q7
388
389 veor q0, q0, q14
390 veor q1, q1, q14
391 veor q6, q6, q8
392 veor q2, q2, q10
393 veor q4, q4, q9
394 veor q1, q1, q15
395 veor q6, q6, q15
396 veor q2, q2, q14
397 veor q7, q7, q11
398 veor q4, q4, q14
399 veor q3, q3, q12
400 veor q2, q2, q15
401 veor q7, q7, q15
402 veor q5, q5, q13
403 vext.8 q8, q0, q0, #12 @ x0 <<< 32
404 vext.8 q9, q1, q1, #12
405 veor q0, q0, q8 @ x0 ^ (x0 <<< 32)
406 vext.8 q10, q6, q6, #12
407 veor q1, q1, q9
408 vext.8 q11, q4, q4, #12
409 veor q6, q6, q10
410 vext.8 q12, q2, q2, #12
411 veor q4, q4, q11
412 vext.8 q13, q7, q7, #12
413 veor q2, q2, q12
414 vext.8 q14, q3, q3, #12
415 veor q7, q7, q13
416 vext.8 q15, q5, q5, #12
417 veor q3, q3, q14
418
419 veor q9, q9, q0
420 veor q5, q5, q15
421 vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
422 veor q10, q10, q1
423 veor q8, q8, q5
424 veor q9, q9, q5
425 vext.8 q1, q1, q1, #8
426 veor q13, q13, q2
427 veor q0, q0, q8
428 veor q14, q14, q7
429 veor q1, q1, q9
430 vext.8 q8, q2, q2, #8
431 veor q12, q12, q4
432 vext.8 q9, q7, q7, #8
433 veor q15, q15, q3
434 vext.8 q2, q4, q4, #8
435 veor q11, q11, q6
436 vext.8 q7, q5, q5, #8
437 veor q12, q12, q5
438 vext.8 q4, q3, q3, #8
439 veor q11, q11, q5
440 vext.8 q3, q6, q6, #8
441 veor q5, q9, q13
442 veor q11, q11, q2
443 veor q7, q7, q15
444 veor q6, q4, q14
445 veor q4, q8, q12
446 veor q2, q3, q10
447 vmov q3, q11
448 @ vmov q5, q9
449 vldmia r6, {q12} @ .LISR
450 ite eq @ Thumb2 thing, sanity check in ARM
451 addeq r6,r6,#0x10
452 bne .Ldec_loop
453 vldmia r6, {q12} @ .LISRM0
454 b .Ldec_loop
455.align 4
456.Ldec_done:
457 vmov.i8 q8,#0x55 @ compose .LBS0
458 vmov.i8 q9,#0x33 @ compose .LBS1
459 vshr.u64 q10, q3, #1
460 vshr.u64 q11, q2, #1
461 veor q10, q10, q5
462 veor q11, q11, q7
463 vand q10, q10, q8
464 vand q11, q11, q8
465 veor q5, q5, q10
466 vshl.u64 q10, q10, #1
467 veor q7, q7, q11
468 vshl.u64 q11, q11, #1
469 veor q3, q3, q10
470 veor q2, q2, q11
471 vshr.u64 q10, q6, #1
472 vshr.u64 q11, q0, #1
473 veor q10, q10, q4
474 veor q11, q11, q1
475 vand q10, q10, q8
476 vand q11, q11, q8
477 veor q4, q4, q10
478 vshl.u64 q10, q10, #1
479 veor q1, q1, q11
480 vshl.u64 q11, q11, #1
481 veor q6, q6, q10
482 veor q0, q0, q11
483 vmov.i8 q8,#0x0f @ compose .LBS2
484 vshr.u64 q10, q7, #2
485 vshr.u64 q11, q2, #2
486 veor q10, q10, q5
487 veor q11, q11, q3
488 vand q10, q10, q9
489 vand q11, q11, q9
490 veor q5, q5, q10
491 vshl.u64 q10, q10, #2
492 veor q3, q3, q11
493 vshl.u64 q11, q11, #2
494 veor q7, q7, q10
495 veor q2, q2, q11
496 vshr.u64 q10, q1, #2
497 vshr.u64 q11, q0, #2
498 veor q10, q10, q4
499 veor q11, q11, q6
500 vand q10, q10, q9
501 vand q11, q11, q9
502 veor q4, q4, q10
503 vshl.u64 q10, q10, #2
504 veor q6, q6, q11
505 vshl.u64 q11, q11, #2
506 veor q1, q1, q10
507 veor q0, q0, q11
508 vshr.u64 q10, q4, #4
509 vshr.u64 q11, q6, #4
510 veor q10, q10, q5
511 veor q11, q11, q3
512 vand q10, q10, q8
513 vand q11, q11, q8
514 veor q5, q5, q10
515 vshl.u64 q10, q10, #4
516 veor q3, q3, q11
517 vshl.u64 q11, q11, #4
518 veor q4, q4, q10
519 veor q6, q6, q11
520 vshr.u64 q10, q1, #4
521 vshr.u64 q11, q0, #4
522 veor q10, q10, q7
523 veor q11, q11, q2
524 vand q10, q10, q8
525 vand q11, q11, q8
526 veor q7, q7, q10
527 vshl.u64 q10, q10, #4
528 veor q2, q2, q11
529 vshl.u64 q11, q11, #4
530 veor q1, q1, q10
531 veor q0, q0, q11
532 vldmia r4, {q8} @ last round key
533 veor q6, q6, q8
534 veor q4, q4, q8
535 veor q2, q2, q8
536 veor q7, q7, q8
537 veor q3, q3, q8
538 veor q5, q5, q8
539 veor q0, q0, q8
540 veor q1, q1, q8
541 bx lr
542.size _bsaes_decrypt8,.-_bsaes_decrypt8
543
544.type _bsaes_const,%object
545.align 6
546_bsaes_const:
547.LM0ISR: @ InvShiftRows constants
548 .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
549.LISR:
550 .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
551.LISRM0:
552 .quad 0x01040b0e0205080f, 0x0306090c00070a0d
553.LM0SR: @ ShiftRows constants
554 .quad 0x0a0e02060f03070b, 0x0004080c05090d01
555.LSR:
556 .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
557.LSRM0:
558 .quad 0x0304090e00050a0f, 0x01060b0c0207080d
559.LM0:
560 .quad 0x02060a0e03070b0f, 0x0004080c0105090d
561.LREVM0SR:
562 .quad 0x090d01050c000408, 0x03070b0f060a0e02
563.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro@openssl.org>"
564.align 6
565.size _bsaes_const,.-_bsaes_const
566
567.type _bsaes_encrypt8,%function
568.align 4
569_bsaes_encrypt8:
570 adr r6,_bsaes_encrypt8
571 vldmia r4!, {q9} @ round 0 key
572 sub r6,r6,#_bsaes_encrypt8-.LM0SR
573
574 vldmia r6!, {q8} @ .LM0SR
575_bsaes_encrypt8_alt:
576 veor q10, q0, q9 @ xor with round0 key
577 veor q11, q1, q9
578 vtbl.8 d0, {q10}, d16
579 vtbl.8 d1, {q10}, d17
580 veor q12, q2, q9
581 vtbl.8 d2, {q11}, d16
582 vtbl.8 d3, {q11}, d17
583 veor q13, q3, q9
584 vtbl.8 d4, {q12}, d16
585 vtbl.8 d5, {q12}, d17
586 veor q14, q4, q9
587 vtbl.8 d6, {q13}, d16
588 vtbl.8 d7, {q13}, d17
589 veor q15, q5, q9
590 vtbl.8 d8, {q14}, d16
591 vtbl.8 d9, {q14}, d17
592 veor q10, q6, q9
593 vtbl.8 d10, {q15}, d16
594 vtbl.8 d11, {q15}, d17
595 veor q11, q7, q9
596 vtbl.8 d12, {q10}, d16
597 vtbl.8 d13, {q10}, d17
598 vtbl.8 d14, {q11}, d16
599 vtbl.8 d15, {q11}, d17
600_bsaes_encrypt8_bitslice:
601 vmov.i8 q8,#0x55 @ compose .LBS0
602 vmov.i8 q9,#0x33 @ compose .LBS1
603 vshr.u64 q10, q6, #1
604 vshr.u64 q11, q4, #1
605 veor q10, q10, q7
606 veor q11, q11, q5
607 vand q10, q10, q8
608 vand q11, q11, q8
609 veor q7, q7, q10
610 vshl.u64 q10, q10, #1
611 veor q5, q5, q11
612 vshl.u64 q11, q11, #1
613 veor q6, q6, q10
614 veor q4, q4, q11
615 vshr.u64 q10, q2, #1
616 vshr.u64 q11, q0, #1
617 veor q10, q10, q3
618 veor q11, q11, q1
619 vand q10, q10, q8
620 vand q11, q11, q8
621 veor q3, q3, q10
622 vshl.u64 q10, q10, #1
623 veor q1, q1, q11
624 vshl.u64 q11, q11, #1
625 veor q2, q2, q10
626 veor q0, q0, q11
627 vmov.i8 q8,#0x0f @ compose .LBS2
628 vshr.u64 q10, q5, #2
629 vshr.u64 q11, q4, #2
630 veor q10, q10, q7
631 veor q11, q11, q6
632 vand q10, q10, q9
633 vand q11, q11, q9
634 veor q7, q7, q10
635 vshl.u64 q10, q10, #2
636 veor q6, q6, q11
637 vshl.u64 q11, q11, #2
638 veor q5, q5, q10
639 veor q4, q4, q11
640 vshr.u64 q10, q1, #2
641 vshr.u64 q11, q0, #2
642 veor q10, q10, q3
643 veor q11, q11, q2
644 vand q10, q10, q9
645 vand q11, q11, q9
646 veor q3, q3, q10
647 vshl.u64 q10, q10, #2
648 veor q2, q2, q11
649 vshl.u64 q11, q11, #2
650 veor q1, q1, q10
651 veor q0, q0, q11
652 vshr.u64 q10, q3, #4
653 vshr.u64 q11, q2, #4
654 veor q10, q10, q7
655 veor q11, q11, q6
656 vand q10, q10, q8
657 vand q11, q11, q8
658 veor q7, q7, q10
659 vshl.u64 q10, q10, #4
660 veor q6, q6, q11
661 vshl.u64 q11, q11, #4
662 veor q3, q3, q10
663 veor q2, q2, q11
664 vshr.u64 q10, q1, #4
665 vshr.u64 q11, q0, #4
666 veor q10, q10, q5
667 veor q11, q11, q4
668 vand q10, q10, q8
669 vand q11, q11, q8
670 veor q5, q5, q10
671 vshl.u64 q10, q10, #4
672 veor q4, q4, q11
673 vshl.u64 q11, q11, #4
674 veor q1, q1, q10
675 veor q0, q0, q11
676 sub r5,r5,#1
677 b .Lenc_sbox
678.align 4
679.Lenc_loop:
680 vldmia r4!, {q8-q11}
681 veor q8, q8, q0
682 veor q9, q9, q1
683 vtbl.8 d0, {q8}, d24
684 vtbl.8 d1, {q8}, d25
685 vldmia r4!, {q8}
686 veor q10, q10, q2
687 vtbl.8 d2, {q9}, d24
688 vtbl.8 d3, {q9}, d25
689 vldmia r4!, {q9}
690 veor q11, q11, q3
691 vtbl.8 d4, {q10}, d24
692 vtbl.8 d5, {q10}, d25
693 vldmia r4!, {q10}
694 vtbl.8 d6, {q11}, d24
695 vtbl.8 d7, {q11}, d25
696 vldmia r4!, {q11}
697 veor q8, q8, q4
698 veor q9, q9, q5
699 vtbl.8 d8, {q8}, d24
700 vtbl.8 d9, {q8}, d25
701 veor q10, q10, q6
702 vtbl.8 d10, {q9}, d24
703 vtbl.8 d11, {q9}, d25
704 veor q11, q11, q7
705 vtbl.8 d12, {q10}, d24
706 vtbl.8 d13, {q10}, d25
707 vtbl.8 d14, {q11}, d24
708 vtbl.8 d15, {q11}, d25
709.Lenc_sbox:
710 veor q2, q2, q1
711 veor q5, q5, q6
712 veor q3, q3, q0
713 veor q6, q6, q2
714 veor q5, q5, q0
715
716 veor q6, q6, q3
717 veor q3, q3, q7
718 veor q7, q7, q5
719 veor q3, q3, q4
720 veor q4, q4, q5
721
722 veor q2, q2, q7
723 veor q3, q3, q1
724 veor q1, q1, q5
725 veor q11, q7, q4
726 veor q10, q1, q2
727 veor q9, q5, q3
728 veor q13, q2, q4
729 vmov q8, q10
730 veor q12, q6, q0
731
732 vorr q10, q10, q9
733 veor q15, q11, q8
734 vand q14, q11, q12
735 vorr q11, q11, q12
736 veor q12, q12, q9
737 vand q8, q8, q9
738 veor q9, q3, q0
739 vand q15, q15, q12
740 vand q13, q13, q9
741 veor q9, q7, q1
742 veor q12, q5, q6
743 veor q11, q11, q13
744 veor q10, q10, q13
745 vand q13, q9, q12
746 vorr q9, q9, q12
747 veor q11, q11, q15
748 veor q8, q8, q13
749 veor q10, q10, q14
750 veor q9, q9, q15
751 veor q8, q8, q14
752 vand q12, q2, q3
753 veor q9, q9, q14
754 vand q13, q4, q0
755 vand q14, q1, q5
756 vorr q15, q7, q6
757 veor q11, q11, q12
758 veor q9, q9, q14
759 veor q8, q8, q15
760 veor q10, q10, q13
761
762 @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3
763
764 @ new smaller inversion
765
766 vand q14, q11, q9
767 vmov q12, q8
768
769 veor q13, q10, q14
770 veor q15, q8, q14
771 veor q14, q8, q14 @ q14=q15
772
773 vbsl q13, q9, q8
774 vbsl q15, q11, q10
775 veor q11, q11, q10
776
777 vbsl q12, q13, q14
778 vbsl q8, q14, q13
779
780 vand q14, q12, q15
781 veor q9, q9, q8
782
783 veor q14, q14, q11
784 veor q12, q6, q0
785 veor q8, q5, q3
786 veor q10, q15, q14
787 vand q10, q10, q6
788 veor q6, q6, q5
789 vand q11, q5, q15
790 vand q6, q6, q14
791 veor q5, q11, q10
792 veor q6, q6, q11
793 veor q15, q15, q13
794 veor q14, q14, q9
795 veor q11, q15, q14
796 veor q10, q13, q9
797 vand q11, q11, q12
798 vand q10, q10, q0
799 veor q12, q12, q8
800 veor q0, q0, q3
801 vand q8, q8, q15
802 vand q3, q3, q13
803 vand q12, q12, q14
804 vand q0, q0, q9
805 veor q8, q8, q12
806 veor q0, q0, q3
807 veor q12, q12, q11
808 veor q3, q3, q10
809 veor q6, q6, q12
810 veor q0, q0, q12
811 veor q5, q5, q8
812 veor q3, q3, q8
813
814 veor q12, q7, q4
815 veor q8, q1, q2
816 veor q11, q15, q14
817 veor q10, q13, q9
818 vand q11, q11, q12
819 vand q10, q10, q4
820 veor q12, q12, q8
821 veor q4, q4, q2
822 vand q8, q8, q15
823 vand q2, q2, q13
824 vand q12, q12, q14
825 vand q4, q4, q9
826 veor q8, q8, q12
827 veor q4, q4, q2
828 veor q12, q12, q11
829 veor q2, q2, q10
830 veor q15, q15, q13
831 veor q14, q14, q9
832 veor q10, q15, q14
833 vand q10, q10, q7
834 veor q7, q7, q1
835 vand q11, q1, q15
836 vand q7, q7, q14
837 veor q1, q11, q10
838 veor q7, q7, q11
839 veor q7, q7, q12
840 veor q4, q4, q12
841 veor q1, q1, q8
842 veor q2, q2, q8
843 veor q7, q7, q0
844 veor q1, q1, q6
845 veor q6, q6, q0
846 veor q4, q4, q7
847 veor q0, q0, q1
848
849 veor q1, q1, q5
850 veor q5, q5, q2
851 veor q2, q2, q3
852 veor q3, q3, q5
853 veor q4, q4, q5
854
855 veor q6, q6, q3
856 subs r5,r5,#1
857 bcc .Lenc_done
858 vext.8 q8, q0, q0, #12 @ x0 <<< 32
859 vext.8 q9, q1, q1, #12
860 veor q0, q0, q8 @ x0 ^ (x0 <<< 32)
861 vext.8 q10, q4, q4, #12
862 veor q1, q1, q9
863 vext.8 q11, q6, q6, #12
864 veor q4, q4, q10
865 vext.8 q12, q3, q3, #12
866 veor q6, q6, q11
867 vext.8 q13, q7, q7, #12
868 veor q3, q3, q12
869 vext.8 q14, q2, q2, #12
870 veor q7, q7, q13
871 vext.8 q15, q5, q5, #12
872 veor q2, q2, q14
873
874 veor q9, q9, q0
875 veor q5, q5, q15
876 vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
877 veor q10, q10, q1
878 veor q8, q8, q5
879 veor q9, q9, q5
880 vext.8 q1, q1, q1, #8
881 veor q13, q13, q3
882 veor q0, q0, q8
883 veor q14, q14, q7
884 veor q1, q1, q9
885 vext.8 q8, q3, q3, #8
886 veor q12, q12, q6
887 vext.8 q9, q7, q7, #8
888 veor q15, q15, q2
889 vext.8 q3, q6, q6, #8
890 veor q11, q11, q4
891 vext.8 q7, q5, q5, #8
892 veor q12, q12, q5
893 vext.8 q6, q2, q2, #8
894 veor q11, q11, q5
895 vext.8 q2, q4, q4, #8
896 veor q5, q9, q13
897 veor q4, q8, q12
898 veor q3, q3, q11
899 veor q7, q7, q15
900 veor q6, q6, q14
901 @ vmov q4, q8
902 veor q2, q2, q10
903 @ vmov q5, q9
904 vldmia r6, {q12} @ .LSR
905 ite eq @ Thumb2 thing, samity check in ARM
906 addeq r6,r6,#0x10
907 bne .Lenc_loop
908 vldmia r6, {q12} @ .LSRM0
909 b .Lenc_loop
910.align 4
911.Lenc_done:
912 vmov.i8 q8,#0x55 @ compose .LBS0
913 vmov.i8 q9,#0x33 @ compose .LBS1
914 vshr.u64 q10, q2, #1
915 vshr.u64 q11, q3, #1
916 veor q10, q10, q5
917 veor q11, q11, q7
918 vand q10, q10, q8
919 vand q11, q11, q8
920 veor q5, q5, q10
921 vshl.u64 q10, q10, #1
922 veor q7, q7, q11
923 vshl.u64 q11, q11, #1
924 veor q2, q2, q10
925 veor q3, q3, q11
926 vshr.u64 q10, q4, #1
927 vshr.u64 q11, q0, #1
928 veor q10, q10, q6
929 veor q11, q11, q1
930 vand q10, q10, q8
931 vand q11, q11, q8
932 veor q6, q6, q10
933 vshl.u64 q10, q10, #1
934 veor q1, q1, q11
935 vshl.u64 q11, q11, #1
936 veor q4, q4, q10
937 veor q0, q0, q11
938 vmov.i8 q8,#0x0f @ compose .LBS2
939 vshr.u64 q10, q7, #2
940 vshr.u64 q11, q3, #2
941 veor q10, q10, q5
942 veor q11, q11, q2
943 vand q10, q10, q9
944 vand q11, q11, q9
945 veor q5, q5, q10
946 vshl.u64 q10, q10, #2
947 veor q2, q2, q11
948 vshl.u64 q11, q11, #2
949 veor q7, q7, q10
950 veor q3, q3, q11
951 vshr.u64 q10, q1, #2
952 vshr.u64 q11, q0, #2
953 veor q10, q10, q6
954 veor q11, q11, q4
955 vand q10, q10, q9
956 vand q11, q11, q9
957 veor q6, q6, q10
958 vshl.u64 q10, q10, #2
959 veor q4, q4, q11
960 vshl.u64 q11, q11, #2
961 veor q1, q1, q10
962 veor q0, q0, q11
963 vshr.u64 q10, q6, #4
964 vshr.u64 q11, q4, #4
965 veor q10, q10, q5
966 veor q11, q11, q2
967 vand q10, q10, q8
968 vand q11, q11, q8
969 veor q5, q5, q10
970 vshl.u64 q10, q10, #4
971 veor q2, q2, q11
972 vshl.u64 q11, q11, #4
973 veor q6, q6, q10
974 veor q4, q4, q11
975 vshr.u64 q10, q1, #4
976 vshr.u64 q11, q0, #4
977 veor q10, q10, q7
978 veor q11, q11, q3
979 vand q10, q10, q8
980 vand q11, q11, q8
981 veor q7, q7, q10
982 vshl.u64 q10, q10, #4
983 veor q3, q3, q11
984 vshl.u64 q11, q11, #4
985 veor q1, q1, q10
986 veor q0, q0, q11
987 vldmia r4, {q8} @ last round key
988 veor q4, q4, q8
989 veor q6, q6, q8
990 veor q3, q3, q8
991 veor q7, q7, q8
992 veor q2, q2, q8
993 veor q5, q5, q8
994 veor q0, q0, q8
995 veor q1, q1, q8
996 bx lr
997.size _bsaes_encrypt8,.-_bsaes_encrypt8
998.type _bsaes_key_convert,%function
999.align 4
1000_bsaes_key_convert:
1001 adr r6,_bsaes_key_convert
1002 vld1.8 {q7}, [r4]! @ load round 0 key
1003 sub r6,r6,#_bsaes_key_convert-.LM0
1004 vld1.8 {q15}, [r4]! @ load round 1 key
1005
1006 vmov.i8 q8, #0x01 @ bit masks
1007 vmov.i8 q9, #0x02
1008 vmov.i8 q10, #0x04
1009 vmov.i8 q11, #0x08
1010 vmov.i8 q12, #0x10
1011 vmov.i8 q13, #0x20
1012 vldmia r6, {q14} @ .LM0
1013
1014#ifdef __ARMEL__
1015 vrev32.8 q7, q7
1016 vrev32.8 q15, q15
1017#endif
1018 sub r5,r5,#1
1019 vstmia r12!, {q7} @ save round 0 key
1020 b .Lkey_loop
1021
1022.align 4
1023.Lkey_loop:
1024 vtbl.8 d14,{q15},d28
1025 vtbl.8 d15,{q15},d29
1026 vmov.i8 q6, #0x40
1027 vmov.i8 q15, #0x80
1028
1029 vtst.8 q0, q7, q8
1030 vtst.8 q1, q7, q9
1031 vtst.8 q2, q7, q10
1032 vtst.8 q3, q7, q11
1033 vtst.8 q4, q7, q12
1034 vtst.8 q5, q7, q13
1035 vtst.8 q6, q7, q6
1036 vtst.8 q7, q7, q15
1037 vld1.8 {q15}, [r4]! @ load next round key
1038 vmvn q0, q0 @ "pnot"
1039 vmvn q1, q1
1040 vmvn q5, q5
1041 vmvn q6, q6
1042#ifdef __ARMEL__
1043 vrev32.8 q15, q15
1044#endif
1045 subs r5,r5,#1
1046 vstmia r12!,{q0-q7} @ write bit-sliced round key
1047 bne .Lkey_loop
1048
1049 vmov.i8 q7,#0x63 @ compose .L63
1050 @ don't save last round key
1051 bx lr
1052.size _bsaes_key_convert,.-_bsaes_key_convert
1053.extern AES_cbc_encrypt
1054.extern AES_decrypt
1055
1056.global bsaes_cbc_encrypt
1057.type bsaes_cbc_encrypt,%function
1058.align 5
1059bsaes_cbc_encrypt:
1060#ifndef __KERNEL__
1061 cmp r2, #128
1062#ifndef __thumb__
1063 blo AES_cbc_encrypt
1064#else
1065 bhs 1f
1066 b AES_cbc_encrypt
10671:
1068#endif
1069#endif
1070
1071 @ it is up to the caller to make sure we are called with enc == 0
1072
1073 mov ip, sp
1074 stmdb sp!, {r4-r10, lr}
1075 VFP_ABI_PUSH
1076 ldr r8, [ip] @ IV is 1st arg on the stack
1077 mov r2, r2, lsr#4 @ len in 16 byte blocks
1078 sub sp, #0x10 @ scratch space to carry over the IV
1079 mov r9, sp @ save sp
1080
1081 ldr r10, [r3, #240] @ get # of rounds
1082#ifndef BSAES_ASM_EXTENDED_KEY
1083 @ allocate the key schedule on the stack
1084 sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key
1085 add r12, #96 @ sifze of bit-slices key schedule
1086
1087 @ populate the key schedule
1088 mov r4, r3 @ pass key
1089 mov r5, r10 @ pass # of rounds
1090 mov sp, r12 @ sp is sp
1091 bl _bsaes_key_convert
1092 vldmia sp, {q6}
1093 vstmia r12, {q15} @ save last round key
1094 veor q7, q7, q6 @ fix up round 0 key
1095 vstmia sp, {q7}
1096#else
1097 ldr r12, [r3, #244]
1098 eors r12, #1
1099 beq 0f
1100
1101 @ populate the key schedule
1102 str r12, [r3, #244]
1103 mov r4, r3 @ pass key
1104 mov r5, r10 @ pass # of rounds
1105 add r12, r3, #248 @ pass key schedule
1106 bl _bsaes_key_convert
1107 add r4, r3, #248
1108 vldmia r4, {q6}
1109 vstmia r12, {q15} @ save last round key
1110 veor q7, q7, q6 @ fix up round 0 key
1111 vstmia r4, {q7}
1112
1113.align 2
11140:
1115#endif
1116
1117 vld1.8 {q15}, [r8] @ load IV
1118 b .Lcbc_dec_loop
1119
1120.align 4
1121.Lcbc_dec_loop:
1122 subs r2, r2, #0x8
1123 bmi .Lcbc_dec_loop_finish
1124
1125 vld1.8 {q0-q1}, [r0]! @ load input
1126 vld1.8 {q2-q3}, [r0]!
1127#ifndef BSAES_ASM_EXTENDED_KEY
1128 mov r4, sp @ pass the key
1129#else
1130 add r4, r3, #248
1131#endif
1132 vld1.8 {q4-q5}, [r0]!
1133 mov r5, r10
1134 vld1.8 {q6-q7}, [r0]
1135 sub r0, r0, #0x60
1136 vstmia r9, {q15} @ put aside IV
1137
1138 bl _bsaes_decrypt8
1139
1140 vldmia r9, {q14} @ reload IV
1141 vld1.8 {q8-q9}, [r0]! @ reload input
1142 veor q0, q0, q14 @ ^= IV
1143 vld1.8 {q10-q11}, [r0]!
1144 veor q1, q1, q8
1145 veor q6, q6, q9
1146 vld1.8 {q12-q13}, [r0]!
1147 veor q4, q4, q10
1148 veor q2, q2, q11
1149 vld1.8 {q14-q15}, [r0]!
1150 veor q7, q7, q12
1151 vst1.8 {q0-q1}, [r1]! @ write output
1152 veor q3, q3, q13
1153 vst1.8 {q6}, [r1]!
1154 veor q5, q5, q14
1155 vst1.8 {q4}, [r1]!
1156 vst1.8 {q2}, [r1]!
1157 vst1.8 {q7}, [r1]!
1158 vst1.8 {q3}, [r1]!
1159 vst1.8 {q5}, [r1]!
1160
1161 b .Lcbc_dec_loop
1162
1163.Lcbc_dec_loop_finish:
1164 adds r2, r2, #8
1165 beq .Lcbc_dec_done
1166
1167 vld1.8 {q0}, [r0]! @ load input
1168 cmp r2, #2
1169 blo .Lcbc_dec_one
1170 vld1.8 {q1}, [r0]!
1171#ifndef BSAES_ASM_EXTENDED_KEY
1172 mov r4, sp @ pass the key
1173#else
1174 add r4, r3, #248
1175#endif
1176 mov r5, r10
1177 vstmia r9, {q15} @ put aside IV
1178 beq .Lcbc_dec_two
1179 vld1.8 {q2}, [r0]!
1180 cmp r2, #4
1181 blo .Lcbc_dec_three
1182 vld1.8 {q3}, [r0]!
1183 beq .Lcbc_dec_four
1184 vld1.8 {q4}, [r0]!
1185 cmp r2, #6
1186 blo .Lcbc_dec_five
1187 vld1.8 {q5}, [r0]!
1188 beq .Lcbc_dec_six
1189 vld1.8 {q6}, [r0]!
1190 sub r0, r0, #0x70
1191
1192 bl _bsaes_decrypt8
1193
1194 vldmia r9, {q14} @ reload IV
1195 vld1.8 {q8-q9}, [r0]! @ reload input
1196 veor q0, q0, q14 @ ^= IV
1197 vld1.8 {q10-q11}, [r0]!
1198 veor q1, q1, q8
1199 veor q6, q6, q9
1200 vld1.8 {q12-q13}, [r0]!
1201 veor q4, q4, q10
1202 veor q2, q2, q11
1203 vld1.8 {q15}, [r0]!
1204 veor q7, q7, q12
1205 vst1.8 {q0-q1}, [r1]! @ write output
1206 veor q3, q3, q13
1207 vst1.8 {q6}, [r1]!
1208 vst1.8 {q4}, [r1]!
1209 vst1.8 {q2}, [r1]!
1210 vst1.8 {q7}, [r1]!
1211 vst1.8 {q3}, [r1]!
1212 b .Lcbc_dec_done
1213.align 4
1214.Lcbc_dec_six:
1215 sub r0, r0, #0x60
1216 bl _bsaes_decrypt8
1217 vldmia r9,{q14} @ reload IV
1218 vld1.8 {q8-q9}, [r0]! @ reload input
1219 veor q0, q0, q14 @ ^= IV
1220 vld1.8 {q10-q11}, [r0]!
1221 veor q1, q1, q8
1222 veor q6, q6, q9
1223 vld1.8 {q12}, [r0]!
1224 veor q4, q4, q10
1225 veor q2, q2, q11
1226 vld1.8 {q15}, [r0]!
1227 veor q7, q7, q12
1228 vst1.8 {q0-q1}, [r1]! @ write output
1229 vst1.8 {q6}, [r1]!
1230 vst1.8 {q4}, [r1]!
1231 vst1.8 {q2}, [r1]!
1232 vst1.8 {q7}, [r1]!
1233 b .Lcbc_dec_done
1234.align 4
1235.Lcbc_dec_five:
1236 sub r0, r0, #0x50
1237 bl _bsaes_decrypt8
1238 vldmia r9, {q14} @ reload IV
1239 vld1.8 {q8-q9}, [r0]! @ reload input
1240 veor q0, q0, q14 @ ^= IV
1241 vld1.8 {q10-q11}, [r0]!
1242 veor q1, q1, q8
1243 veor q6, q6, q9
1244 vld1.8 {q15}, [r0]!
1245 veor q4, q4, q10
1246 vst1.8 {q0-q1}, [r1]! @ write output
1247 veor q2, q2, q11
1248 vst1.8 {q6}, [r1]!
1249 vst1.8 {q4}, [r1]!
1250 vst1.8 {q2}, [r1]!
1251 b .Lcbc_dec_done
1252.align 4
1253.Lcbc_dec_four:
1254 sub r0, r0, #0x40
1255 bl _bsaes_decrypt8
1256 vldmia r9, {q14} @ reload IV
1257 vld1.8 {q8-q9}, [r0]! @ reload input
1258 veor q0, q0, q14 @ ^= IV
1259 vld1.8 {q10}, [r0]!
1260 veor q1, q1, q8
1261 veor q6, q6, q9
1262 vld1.8 {q15}, [r0]!
1263 veor q4, q4, q10
1264 vst1.8 {q0-q1}, [r1]! @ write output
1265 vst1.8 {q6}, [r1]!
1266 vst1.8 {q4}, [r1]!
1267 b .Lcbc_dec_done
1268.align 4
1269.Lcbc_dec_three:
1270 sub r0, r0, #0x30
1271 bl _bsaes_decrypt8
1272 vldmia r9, {q14} @ reload IV
1273 vld1.8 {q8-q9}, [r0]! @ reload input
1274 veor q0, q0, q14 @ ^= IV
1275 vld1.8 {q15}, [r0]!
1276 veor q1, q1, q8
1277 veor q6, q6, q9
1278 vst1.8 {q0-q1}, [r1]! @ write output
1279 vst1.8 {q6}, [r1]!
1280 b .Lcbc_dec_done
1281.align 4
1282.Lcbc_dec_two:
1283 sub r0, r0, #0x20
1284 bl _bsaes_decrypt8
1285 vldmia r9, {q14} @ reload IV
1286 vld1.8 {q8}, [r0]! @ reload input
1287 veor q0, q0, q14 @ ^= IV
1288 vld1.8 {q15}, [r0]! @ reload input
1289 veor q1, q1, q8
1290 vst1.8 {q0-q1}, [r1]! @ write output
1291 b .Lcbc_dec_done
1292.align 4
1293.Lcbc_dec_one:
1294 sub r0, r0, #0x10
1295 mov r10, r1 @ save original out pointer
1296 mov r1, r9 @ use the iv scratch space as out buffer
1297 mov r2, r3
1298 vmov q4,q15 @ just in case ensure that IV
1299 vmov q5,q0 @ and input are preserved
1300 bl AES_decrypt
1301 vld1.8 {q0}, [r9,:64] @ load result
1302 veor q0, q0, q4 @ ^= IV
1303 vmov q15, q5 @ q5 holds input
1304 vst1.8 {q0}, [r10] @ write output
1305
1306.Lcbc_dec_done:
1307#ifndef BSAES_ASM_EXTENDED_KEY
1308 vmov.i32 q0, #0
1309 vmov.i32 q1, #0
1310.Lcbc_dec_bzero: @ wipe key schedule [if any]
1311 vstmia sp!, {q0-q1}
1312 cmp sp, r9
1313 bne .Lcbc_dec_bzero
1314#endif
1315
1316 mov sp, r9
1317 add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb
1318 vst1.8 {q15}, [r8] @ return IV
1319 VFP_ABI_POP
1320 ldmia sp!, {r4-r10, pc}
1321.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
1322.extern AES_encrypt
1323.global bsaes_ctr32_encrypt_blocks
1324.type bsaes_ctr32_encrypt_blocks,%function
1325.align 5
1326bsaes_ctr32_encrypt_blocks:
1327 cmp r2, #8 @ use plain AES for
1328 blo .Lctr_enc_short @ small sizes
1329
1330 mov ip, sp
1331 stmdb sp!, {r4-r10, lr}
1332 VFP_ABI_PUSH
1333 ldr r8, [ip] @ ctr is 1st arg on the stack
1334 sub sp, sp, #0x10 @ scratch space to carry over the ctr
1335 mov r9, sp @ save sp
1336
1337 ldr r10, [r3, #240] @ get # of rounds
1338#ifndef BSAES_ASM_EXTENDED_KEY
1339 @ allocate the key schedule on the stack
1340 sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key
1341 add r12, #96 @ size of bit-sliced key schedule
1342
1343 @ populate the key schedule
1344 mov r4, r3 @ pass key
1345 mov r5, r10 @ pass # of rounds
1346 mov sp, r12 @ sp is sp
1347 bl _bsaes_key_convert
1348 veor q7,q7,q15 @ fix up last round key
1349 vstmia r12, {q7} @ save last round key
1350
1351 vld1.8 {q0}, [r8] @ load counter
1352 add r8, r6, #.LREVM0SR-.LM0 @ borrow r8
1353 vldmia sp, {q4} @ load round0 key
1354#else
1355 ldr r12, [r3, #244]
1356 eors r12, #1
1357 beq 0f
1358
1359 @ populate the key schedule
1360 str r12, [r3, #244]
1361 mov r4, r3 @ pass key
1362 mov r5, r10 @ pass # of rounds
1363 add r12, r3, #248 @ pass key schedule
1364 bl _bsaes_key_convert
1365 veor q7,q7,q15 @ fix up last round key
1366 vstmia r12, {q7} @ save last round key
1367
1368.align 2
13690: add r12, r3, #248
1370 vld1.8 {q0}, [r8] @ load counter
1371 adrl r8, .LREVM0SR @ borrow r8
1372 vldmia r12, {q4} @ load round0 key
1373 sub sp, #0x10 @ place for adjusted round0 key
1374#endif
1375
1376 vmov.i32 q8,#1 @ compose 1<<96
1377 veor q9,q9,q9
1378 vrev32.8 q0,q0
1379 vext.8 q8,q9,q8,#4
1380 vrev32.8 q4,q4
1381 vadd.u32 q9,q8,q8 @ compose 2<<96
1382 vstmia sp, {q4} @ save adjusted round0 key
1383 b .Lctr_enc_loop
1384
1385.align 4
1386.Lctr_enc_loop:
1387 vadd.u32 q10, q8, q9 @ compose 3<<96
1388 vadd.u32 q1, q0, q8 @ +1
1389 vadd.u32 q2, q0, q9 @ +2
1390 vadd.u32 q3, q0, q10 @ +3
1391 vadd.u32 q4, q1, q10
1392 vadd.u32 q5, q2, q10
1393 vadd.u32 q6, q3, q10
1394 vadd.u32 q7, q4, q10
1395 vadd.u32 q10, q5, q10 @ next counter
1396
1397 @ Borrow prologue from _bsaes_encrypt8 to use the opportunity
1398 @ to flip byte order in 32-bit counter
1399
1400 vldmia sp, {q9} @ load round0 key
1401#ifndef BSAES_ASM_EXTENDED_KEY
1402 add r4, sp, #0x10 @ pass next round key
1403#else
1404 add r4, r3, #264
1405#endif
1406 vldmia r8, {q8} @ .LREVM0SR
1407 mov r5, r10 @ pass rounds
1408 vstmia r9, {q10} @ save next counter
1409 sub r6, r8, #.LREVM0SR-.LSR @ pass constants
1410
1411 bl _bsaes_encrypt8_alt
1412
1413 subs r2, r2, #8
1414 blo .Lctr_enc_loop_done
1415
1416 vld1.8 {q8-q9}, [r0]! @ load input
1417 vld1.8 {q10-q11}, [r0]!
1418 veor q0, q8
1419 veor q1, q9
1420 vld1.8 {q12-q13}, [r0]!
1421 veor q4, q10
1422 veor q6, q11
1423 vld1.8 {q14-q15}, [r0]!
1424 veor q3, q12
1425 vst1.8 {q0-q1}, [r1]! @ write output
1426 veor q7, q13
1427 veor q2, q14
1428 vst1.8 {q4}, [r1]!
1429 veor q5, q15
1430 vst1.8 {q6}, [r1]!
1431 vmov.i32 q8, #1 @ compose 1<<96
1432 vst1.8 {q3}, [r1]!
1433 veor q9, q9, q9
1434 vst1.8 {q7}, [r1]!
1435 vext.8 q8, q9, q8, #4
1436 vst1.8 {q2}, [r1]!
1437 vadd.u32 q9,q8,q8 @ compose 2<<96
1438 vst1.8 {q5}, [r1]!
1439 vldmia r9, {q0} @ load counter
1440
1441 bne .Lctr_enc_loop
1442 b .Lctr_enc_done
1443
1444.align 4
1445.Lctr_enc_loop_done:
1446 add r2, r2, #8
1447 vld1.8 {q8}, [r0]! @ load input
1448 veor q0, q8
1449 vst1.8 {q0}, [r1]! @ write output
1450 cmp r2, #2
1451 blo .Lctr_enc_done
1452 vld1.8 {q9}, [r0]!
1453 veor q1, q9
1454 vst1.8 {q1}, [r1]!
1455 beq .Lctr_enc_done
1456 vld1.8 {q10}, [r0]!
1457 veor q4, q10
1458 vst1.8 {q4}, [r1]!
1459 cmp r2, #4
1460 blo .Lctr_enc_done
1461 vld1.8 {q11}, [r0]!
1462 veor q6, q11
1463 vst1.8 {q6}, [r1]!
1464 beq .Lctr_enc_done
1465 vld1.8 {q12}, [r0]!
1466 veor q3, q12
1467 vst1.8 {q3}, [r1]!
1468 cmp r2, #6
1469 blo .Lctr_enc_done
1470 vld1.8 {q13}, [r0]!
1471 veor q7, q13
1472 vst1.8 {q7}, [r1]!
1473 beq .Lctr_enc_done
1474 vld1.8 {q14}, [r0]
1475 veor q2, q14
1476 vst1.8 {q2}, [r1]!
1477
1478.Lctr_enc_done:
1479 vmov.i32 q0, #0
1480 vmov.i32 q1, #0
1481#ifndef BSAES_ASM_EXTENDED_KEY
1482.Lctr_enc_bzero: @ wipe key schedule [if any]
1483 vstmia sp!, {q0-q1}
1484 cmp sp, r9
1485 bne .Lctr_enc_bzero
1486#else
1487 vstmia sp, {q0-q1}
1488#endif
1489
1490 mov sp, r9
1491 add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb
1492 VFP_ABI_POP
1493 ldmia sp!, {r4-r10, pc} @ return
1494
1495.align 4
1496.Lctr_enc_short:
1497 ldr ip, [sp] @ ctr pointer is passed on stack
1498 stmdb sp!, {r4-r8, lr}
1499
1500 mov r4, r0 @ copy arguments
1501 mov r5, r1
1502 mov r6, r2
1503 mov r7, r3
1504 ldr r8, [ip, #12] @ load counter LSW
1505 vld1.8 {q1}, [ip] @ load whole counter value
1506#ifdef __ARMEL__
1507 rev r8, r8
1508#endif
1509 sub sp, sp, #0x10
1510 vst1.8 {q1}, [sp,:64] @ copy counter value
1511 sub sp, sp, #0x10
1512
1513.Lctr_enc_short_loop:
1514 add r0, sp, #0x10 @ input counter value
1515 mov r1, sp @ output on the stack
1516 mov r2, r7 @ key
1517
1518 bl AES_encrypt
1519
1520 vld1.8 {q0}, [r4]! @ load input
1521 vld1.8 {q1}, [sp,:64] @ load encrypted counter
1522 add r8, r8, #1
1523#ifdef __ARMEL__
1524 rev r0, r8
1525 str r0, [sp, #0x1c] @ next counter value
1526#else
1527 str r8, [sp, #0x1c] @ next counter value
1528#endif
1529 veor q0,q0,q1
1530 vst1.8 {q0}, [r5]! @ store output
1531 subs r6, r6, #1
1532 bne .Lctr_enc_short_loop
1533
1534 vmov.i32 q0, #0
1535 vmov.i32 q1, #0
1536 vstmia sp!, {q0-q1}
1537
1538 ldmia sp!, {r4-r8, pc}
1539.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
1540.globl bsaes_xts_encrypt
1541.type bsaes_xts_encrypt,%function
1542.align 4
1543bsaes_xts_encrypt:
1544 mov ip, sp
1545 stmdb sp!, {r4-r10, lr} @ 0x20
1546 VFP_ABI_PUSH
1547 mov r6, sp @ future r3
1548
1549 mov r7, r0
1550 mov r8, r1
1551 mov r9, r2
1552 mov r10, r3
1553
1554 sub r0, sp, #0x10 @ 0x10
1555 bic r0, #0xf @ align at 16 bytes
1556 mov sp, r0
1557
1558#ifdef XTS_CHAIN_TWEAK
1559 ldr r0, [ip] @ pointer to input tweak
1560#else
1561 @ generate initial tweak
1562 ldr r0, [ip, #4] @ iv[]
1563 mov r1, sp
1564 ldr r2, [ip, #0] @ key2
1565 bl AES_encrypt
1566 mov r0,sp @ pointer to initial tweak
1567#endif
1568
1569 ldr r1, [r10, #240] @ get # of rounds
1570 mov r3, r6
1571#ifndef BSAES_ASM_EXTENDED_KEY
1572 @ allocate the key schedule on the stack
1573 sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key
1574 @ add r12, #96 @ size of bit-sliced key schedule
1575 sub r12, #48 @ place for tweak[9]
1576
1577 @ populate the key schedule
1578 mov r4, r10 @ pass key
1579 mov r5, r1 @ pass # of rounds
1580 mov sp, r12
1581 add r12, #0x90 @ pass key schedule
1582 bl _bsaes_key_convert
1583 veor q7, q7, q15 @ fix up last round key
1584 vstmia r12, {q7} @ save last round key
1585#else
1586 ldr r12, [r10, #244]
1587 eors r12, #1
1588 beq 0f
1589
1590 str r12, [r10, #244]
1591 mov r4, r10 @ pass key
1592 mov r5, r1 @ pass # of rounds
1593 add r12, r10, #248 @ pass key schedule
1594 bl _bsaes_key_convert
1595 veor q7, q7, q15 @ fix up last round key
1596 vstmia r12, {q7}
1597
1598.align 2
15990: sub sp, #0x90 @ place for tweak[9]
1600#endif
1601
1602 vld1.8 {q8}, [r0] @ initial tweak
1603 adr r2, .Lxts_magic
1604
1605 subs r9, #0x80
1606 blo .Lxts_enc_short
1607 b .Lxts_enc_loop
1608
1609.align 4
1610.Lxts_enc_loop:
1611 vldmia r2, {q5} @ load XTS magic
1612 vshr.s64 q6, q8, #63
1613 mov r0, sp
1614 vand q6, q6, q5
1615 vadd.u64 q9, q8, q8
1616 vst1.64 {q8}, [r0,:128]!
1617 vswp d13,d12
1618 vshr.s64 q7, q9, #63
1619 veor q9, q9, q6
1620 vand q7, q7, q5
1621 vadd.u64 q10, q9, q9
1622 vst1.64 {q9}, [r0,:128]!
1623 vswp d15,d14
1624 vshr.s64 q6, q10, #63
1625 veor q10, q10, q7
1626 vand q6, q6, q5
1627 vld1.8 {q0}, [r7]!
1628 vadd.u64 q11, q10, q10
1629 vst1.64 {q10}, [r0,:128]!
1630 vswp d13,d12
1631 vshr.s64 q7, q11, #63
1632 veor q11, q11, q6
1633 vand q7, q7, q5
1634 vld1.8 {q1}, [r7]!
1635 veor q0, q0, q8
1636 vadd.u64 q12, q11, q11
1637 vst1.64 {q11}, [r0,:128]!
1638 vswp d15,d14
1639 vshr.s64 q6, q12, #63
1640 veor q12, q12, q7
1641 vand q6, q6, q5
1642 vld1.8 {q2}, [r7]!
1643 veor q1, q1, q9
1644 vadd.u64 q13, q12, q12
1645 vst1.64 {q12}, [r0,:128]!
1646 vswp d13,d12
1647 vshr.s64 q7, q13, #63
1648 veor q13, q13, q6
1649 vand q7, q7, q5
1650 vld1.8 {q3}, [r7]!
1651 veor q2, q2, q10
1652 vadd.u64 q14, q13, q13
1653 vst1.64 {q13}, [r0,:128]!
1654 vswp d15,d14
1655 vshr.s64 q6, q14, #63
1656 veor q14, q14, q7
1657 vand q6, q6, q5
1658 vld1.8 {q4}, [r7]!
1659 veor q3, q3, q11
1660 vadd.u64 q15, q14, q14
1661 vst1.64 {q14}, [r0,:128]!
1662 vswp d13,d12
1663 vshr.s64 q7, q15, #63
1664 veor q15, q15, q6
1665 vand q7, q7, q5
1666 vld1.8 {q5}, [r7]!
1667 veor q4, q4, q12
1668 vadd.u64 q8, q15, q15
1669 vst1.64 {q15}, [r0,:128]!
1670 vswp d15,d14
1671 veor q8, q8, q7
1672 vst1.64 {q8}, [r0,:128] @ next round tweak
1673
1674 vld1.8 {q6-q7}, [r7]!
1675 veor q5, q5, q13
1676#ifndef BSAES_ASM_EXTENDED_KEY
1677 add r4, sp, #0x90 @ pass key schedule
1678#else
1679 add r4, r10, #248 @ pass key schedule
1680#endif
1681 veor q6, q6, q14
1682 mov r5, r1 @ pass rounds
1683 veor q7, q7, q15
1684 mov r0, sp
1685
1686 bl _bsaes_encrypt8
1687
1688 vld1.64 {q8-q9}, [r0,:128]!
1689 vld1.64 {q10-q11}, [r0,:128]!
1690 veor q0, q0, q8
1691 vld1.64 {q12-q13}, [r0,:128]!
1692 veor q1, q1, q9
1693 veor q8, q4, q10
1694 vst1.8 {q0-q1}, [r8]!
1695 veor q9, q6, q11
1696 vld1.64 {q14-q15}, [r0,:128]!
1697 veor q10, q3, q12
1698 vst1.8 {q8-q9}, [r8]!
1699 veor q11, q7, q13
1700 veor q12, q2, q14
1701 vst1.8 {q10-q11}, [r8]!
1702 veor q13, q5, q15
1703 vst1.8 {q12-q13}, [r8]!
1704
1705 vld1.64 {q8}, [r0,:128] @ next round tweak
1706
1707 subs r9, #0x80
1708 bpl .Lxts_enc_loop
1709
1710.Lxts_enc_short:
1711 adds r9, #0x70
1712 bmi .Lxts_enc_done
1713
1714 vldmia r2, {q5} @ load XTS magic
1715 vshr.s64 q7, q8, #63
1716 mov r0, sp
1717 vand q7, q7, q5
1718 vadd.u64 q9, q8, q8
1719 vst1.64 {q8}, [r0,:128]!
1720 vswp d15,d14
1721 vshr.s64 q6, q9, #63
1722 veor q9, q9, q7
1723 vand q6, q6, q5
1724 vadd.u64 q10, q9, q9
1725 vst1.64 {q9}, [r0,:128]!
1726 vswp d13,d12
1727 vshr.s64 q7, q10, #63
1728 veor q10, q10, q6
1729 vand q7, q7, q5
1730 vld1.8 {q0}, [r7]!
1731 subs r9, #0x10
1732 bmi .Lxts_enc_1
1733 vadd.u64 q11, q10, q10
1734 vst1.64 {q10}, [r0,:128]!
1735 vswp d15,d14
1736 vshr.s64 q6, q11, #63
1737 veor q11, q11, q7
1738 vand q6, q6, q5
1739 vld1.8 {q1}, [r7]!
1740 subs r9, #0x10
1741 bmi .Lxts_enc_2
1742 veor q0, q0, q8
1743 vadd.u64 q12, q11, q11
1744 vst1.64 {q11}, [r0,:128]!
1745 vswp d13,d12
1746 vshr.s64 q7, q12, #63
1747 veor q12, q12, q6
1748 vand q7, q7, q5
1749 vld1.8 {q2}, [r7]!
1750 subs r9, #0x10
1751 bmi .Lxts_enc_3
1752 veor q1, q1, q9
1753 vadd.u64 q13, q12, q12
1754 vst1.64 {q12}, [r0,:128]!
1755 vswp d15,d14
1756 vshr.s64 q6, q13, #63
1757 veor q13, q13, q7
1758 vand q6, q6, q5
1759 vld1.8 {q3}, [r7]!
1760 subs r9, #0x10
1761 bmi .Lxts_enc_4
1762 veor q2, q2, q10
1763 vadd.u64 q14, q13, q13
1764 vst1.64 {q13}, [r0,:128]!
1765 vswp d13,d12
1766 vshr.s64 q7, q14, #63
1767 veor q14, q14, q6
1768 vand q7, q7, q5
1769 vld1.8 {q4}, [r7]!
1770 subs r9, #0x10
1771 bmi .Lxts_enc_5
1772 veor q3, q3, q11
1773 vadd.u64 q15, q14, q14
1774 vst1.64 {q14}, [r0,:128]!
1775 vswp d15,d14
1776 vshr.s64 q6, q15, #63
1777 veor q15, q15, q7
1778 vand q6, q6, q5
1779 vld1.8 {q5}, [r7]!
1780 subs r9, #0x10
1781 bmi .Lxts_enc_6
1782 veor q4, q4, q12
1783 sub r9, #0x10
1784 vst1.64 {q15}, [r0,:128] @ next round tweak
1785
1786 vld1.8 {q6}, [r7]!
1787 veor q5, q5, q13
1788#ifndef BSAES_ASM_EXTENDED_KEY
1789 add r4, sp, #0x90 @ pass key schedule
1790#else
1791 add r4, r10, #248 @ pass key schedule
1792#endif
1793 veor q6, q6, q14
1794 mov r5, r1 @ pass rounds
1795 mov r0, sp
1796
1797 bl _bsaes_encrypt8
1798
1799 vld1.64 {q8-q9}, [r0,:128]!
1800 vld1.64 {q10-q11}, [r0,:128]!
1801 veor q0, q0, q8
1802 vld1.64 {q12-q13}, [r0,:128]!
1803 veor q1, q1, q9
1804 veor q8, q4, q10
1805 vst1.8 {q0-q1}, [r8]!
1806 veor q9, q6, q11
1807 vld1.64 {q14}, [r0,:128]!
1808 veor q10, q3, q12
1809 vst1.8 {q8-q9}, [r8]!
1810 veor q11, q7, q13
1811 veor q12, q2, q14
1812 vst1.8 {q10-q11}, [r8]!
1813 vst1.8 {q12}, [r8]!
1814
1815 vld1.64 {q8}, [r0,:128] @ next round tweak
1816 b .Lxts_enc_done
1817.align 4
1818.Lxts_enc_6:
1819 vst1.64 {q14}, [r0,:128] @ next round tweak
1820
1821 veor q4, q4, q12
1822#ifndef BSAES_ASM_EXTENDED_KEY
1823 add r4, sp, #0x90 @ pass key schedule
1824#else
1825 add r4, r10, #248 @ pass key schedule
1826#endif
1827 veor q5, q5, q13
1828 mov r5, r1 @ pass rounds
1829 mov r0, sp
1830
1831 bl _bsaes_encrypt8
1832
1833 vld1.64 {q8-q9}, [r0,:128]!
1834 vld1.64 {q10-q11}, [r0,:128]!
1835 veor q0, q0, q8
1836 vld1.64 {q12-q13}, [r0,:128]!
1837 veor q1, q1, q9
1838 veor q8, q4, q10
1839 vst1.8 {q0-q1}, [r8]!
1840 veor q9, q6, q11
1841 veor q10, q3, q12
1842 vst1.8 {q8-q9}, [r8]!
1843 veor q11, q7, q13
1844 vst1.8 {q10-q11}, [r8]!
1845
1846 vld1.64 {q8}, [r0,:128] @ next round tweak
1847 b .Lxts_enc_done
1848
1849@ put this in range for both ARM and Thumb mode adr instructions
1850.align 5
1851.Lxts_magic:
1852 .quad 1, 0x87
1853
1854.align 5
1855.Lxts_enc_5:
1856 vst1.64 {q13}, [r0,:128] @ next round tweak
1857
1858 veor q3, q3, q11
1859#ifndef BSAES_ASM_EXTENDED_KEY
1860 add r4, sp, #0x90 @ pass key schedule
1861#else
1862 add r4, r10, #248 @ pass key schedule
1863#endif
1864 veor q4, q4, q12
1865 mov r5, r1 @ pass rounds
1866 mov r0, sp
1867
1868 bl _bsaes_encrypt8
1869
1870 vld1.64 {q8-q9}, [r0,:128]!
1871 vld1.64 {q10-q11}, [r0,:128]!
1872 veor q0, q0, q8
1873 vld1.64 {q12}, [r0,:128]!
1874 veor q1, q1, q9
1875 veor q8, q4, q10
1876 vst1.8 {q0-q1}, [r8]!
1877 veor q9, q6, q11
1878 veor q10, q3, q12
1879 vst1.8 {q8-q9}, [r8]!
1880 vst1.8 {q10}, [r8]!
1881
1882 vld1.64 {q8}, [r0,:128] @ next round tweak
1883 b .Lxts_enc_done
1884.align 4
1885.Lxts_enc_4:
1886 vst1.64 {q12}, [r0,:128] @ next round tweak
1887
1888 veor q2, q2, q10
1889#ifndef BSAES_ASM_EXTENDED_KEY
1890 add r4, sp, #0x90 @ pass key schedule
1891#else
1892 add r4, r10, #248 @ pass key schedule
1893#endif
1894 veor q3, q3, q11
1895 mov r5, r1 @ pass rounds
1896 mov r0, sp
1897
1898 bl _bsaes_encrypt8
1899
1900 vld1.64 {q8-q9}, [r0,:128]!
1901 vld1.64 {q10-q11}, [r0,:128]!
1902 veor q0, q0, q8
1903 veor q1, q1, q9
1904 veor q8, q4, q10
1905 vst1.8 {q0-q1}, [r8]!
1906 veor q9, q6, q11
1907 vst1.8 {q8-q9}, [r8]!
1908
1909 vld1.64 {q8}, [r0,:128] @ next round tweak
1910 b .Lxts_enc_done
1911.align 4
1912.Lxts_enc_3:
1913 vst1.64 {q11}, [r0,:128] @ next round tweak
1914
1915 veor q1, q1, q9
1916#ifndef BSAES_ASM_EXTENDED_KEY
1917 add r4, sp, #0x90 @ pass key schedule
1918#else
1919 add r4, r10, #248 @ pass key schedule
1920#endif
1921 veor q2, q2, q10
1922 mov r5, r1 @ pass rounds
1923 mov r0, sp
1924
1925 bl _bsaes_encrypt8
1926
1927 vld1.64 {q8-q9}, [r0,:128]!
1928 vld1.64 {q10}, [r0,:128]!
1929 veor q0, q0, q8
1930 veor q1, q1, q9
1931 veor q8, q4, q10
1932 vst1.8 {q0-q1}, [r8]!
1933 vst1.8 {q8}, [r8]!
1934
1935 vld1.64 {q8}, [r0,:128] @ next round tweak
1936 b .Lxts_enc_done
1937.align 4
1938.Lxts_enc_2:
1939 vst1.64 {q10}, [r0,:128] @ next round tweak
1940
1941 veor q0, q0, q8
1942#ifndef BSAES_ASM_EXTENDED_KEY
1943 add r4, sp, #0x90 @ pass key schedule
1944#else
1945 add r4, r10, #248 @ pass key schedule
1946#endif
1947 veor q1, q1, q9
1948 mov r5, r1 @ pass rounds
1949 mov r0, sp
1950
1951 bl _bsaes_encrypt8
1952
1953 vld1.64 {q8-q9}, [r0,:128]!
1954 veor q0, q0, q8
1955 veor q1, q1, q9
1956 vst1.8 {q0-q1}, [r8]!
1957
1958 vld1.64 {q8}, [r0,:128] @ next round tweak
1959 b .Lxts_enc_done
1960.align 4
1961.Lxts_enc_1:
1962 mov r0, sp
1963 veor q0, q8
1964 mov r1, sp
1965 vst1.8 {q0}, [sp,:128]
1966 mov r2, r10
1967 mov r4, r3 @ preserve fp
1968
1969 bl AES_encrypt
1970
1971 vld1.8 {q0}, [sp,:128]
1972 veor q0, q0, q8
1973 vst1.8 {q0}, [r8]!
1974 mov r3, r4
1975
1976 vmov q8, q9 @ next round tweak
1977
1978.Lxts_enc_done:
1979#ifndef XTS_CHAIN_TWEAK
1980 adds r9, #0x10
1981 beq .Lxts_enc_ret
1982 sub r6, r8, #0x10
1983
1984.Lxts_enc_steal:
1985 ldrb r0, [r7], #1
1986 ldrb r1, [r8, #-0x10]
1987 strb r0, [r8, #-0x10]
1988 strb r1, [r8], #1
1989
1990 subs r9, #1
1991 bhi .Lxts_enc_steal
1992
1993 vld1.8 {q0}, [r6]
1994 mov r0, sp
1995 veor q0, q0, q8
1996 mov r1, sp
1997 vst1.8 {q0}, [sp,:128]
1998 mov r2, r10
1999 mov r4, r3 @ preserve fp
2000
2001 bl AES_encrypt
2002
2003 vld1.8 {q0}, [sp,:128]
2004 veor q0, q0, q8
2005 vst1.8 {q0}, [r6]
2006 mov r3, r4
2007#endif
2008
2009.Lxts_enc_ret:
2010 bic r0, r3, #0xf
2011 vmov.i32 q0, #0
2012 vmov.i32 q1, #0
2013#ifdef XTS_CHAIN_TWEAK
2014 ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak
2015#endif
2016.Lxts_enc_bzero: @ wipe key schedule [if any]
2017 vstmia sp!, {q0-q1}
2018 cmp sp, r0
2019 bne .Lxts_enc_bzero
2020
2021 mov sp, r3
2022#ifdef XTS_CHAIN_TWEAK
2023 vst1.8 {q8}, [r1]
2024#endif
2025 VFP_ABI_POP
2026 ldmia sp!, {r4-r10, pc} @ return
2027
2028.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
2029
2030.globl bsaes_xts_decrypt
2031.type bsaes_xts_decrypt,%function
2032.align 4
2033bsaes_xts_decrypt:
2034 mov ip, sp
2035 stmdb sp!, {r4-r10, lr} @ 0x20
2036 VFP_ABI_PUSH
2037 mov r6, sp @ future r3
2038
2039 mov r7, r0
2040 mov r8, r1
2041 mov r9, r2
2042 mov r10, r3
2043
2044 sub r0, sp, #0x10 @ 0x10
2045 bic r0, #0xf @ align at 16 bytes
2046 mov sp, r0
2047
2048#ifdef XTS_CHAIN_TWEAK
2049 ldr r0, [ip] @ pointer to input tweak
2050#else
2051 @ generate initial tweak
2052 ldr r0, [ip, #4] @ iv[]
2053 mov r1, sp
2054 ldr r2, [ip, #0] @ key2
2055 bl AES_encrypt
2056 mov r0, sp @ pointer to initial tweak
2057#endif
2058
2059 ldr r1, [r10, #240] @ get # of rounds
2060 mov r3, r6
2061#ifndef BSAES_ASM_EXTENDED_KEY
2062 @ allocate the key schedule on the stack
2063 sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key
2064 @ add r12, #96 @ size of bit-sliced key schedule
2065 sub r12, #48 @ place for tweak[9]
2066
2067 @ populate the key schedule
2068 mov r4, r10 @ pass key
2069 mov r5, r1 @ pass # of rounds
2070 mov sp, r12
2071 add r12, #0x90 @ pass key schedule
2072 bl _bsaes_key_convert
2073 add r4, sp, #0x90
2074 vldmia r4, {q6}
2075 vstmia r12, {q15} @ save last round key
2076 veor q7, q7, q6 @ fix up round 0 key
2077 vstmia r4, {q7}
2078#else
2079 ldr r12, [r10, #244]
2080 eors r12, #1
2081 beq 0f
2082
2083 str r12, [r10, #244]
2084 mov r4, r10 @ pass key
2085 mov r5, r1 @ pass # of rounds
2086 add r12, r10, #248 @ pass key schedule
2087 bl _bsaes_key_convert
2088 add r4, r10, #248
2089 vldmia r4, {q6}
2090 vstmia r12, {q15} @ save last round key
2091 veor q7, q7, q6 @ fix up round 0 key
2092 vstmia r4, {q7}
2093
2094.align 2
20950: sub sp, #0x90 @ place for tweak[9]
2096#endif
2097 vld1.8 {q8}, [r0] @ initial tweak
2098 adr r2, .Lxts_magic
2099
2100#ifndef XTS_CHAIN_TWEAK
2101 tst r9, #0xf @ if not multiple of 16
2102 it ne @ Thumb2 thing, sanity check in ARM
2103 subne r9, #0x10 @ subtract another 16 bytes
2104#endif
2105 subs r9, #0x80
2106
2107 blo .Lxts_dec_short
2108 b .Lxts_dec_loop
2109
2110.align 4
2111.Lxts_dec_loop:
2112 vldmia r2, {q5} @ load XTS magic
2113 vshr.s64 q6, q8, #63
2114 mov r0, sp
2115 vand q6, q6, q5
2116 vadd.u64 q9, q8, q8
2117 vst1.64 {q8}, [r0,:128]!
2118 vswp d13,d12
2119 vshr.s64 q7, q9, #63
2120 veor q9, q9, q6
2121 vand q7, q7, q5
2122 vadd.u64 q10, q9, q9
2123 vst1.64 {q9}, [r0,:128]!
2124 vswp d15,d14
2125 vshr.s64 q6, q10, #63
2126 veor q10, q10, q7
2127 vand q6, q6, q5
2128 vld1.8 {q0}, [r7]!
2129 vadd.u64 q11, q10, q10
2130 vst1.64 {q10}, [r0,:128]!
2131 vswp d13,d12
2132 vshr.s64 q7, q11, #63
2133 veor q11, q11, q6
2134 vand q7, q7, q5
2135 vld1.8 {q1}, [r7]!
2136 veor q0, q0, q8
2137 vadd.u64 q12, q11, q11
2138 vst1.64 {q11}, [r0,:128]!
2139 vswp d15,d14
2140 vshr.s64 q6, q12, #63
2141 veor q12, q12, q7
2142 vand q6, q6, q5
2143 vld1.8 {q2}, [r7]!
2144 veor q1, q1, q9
2145 vadd.u64 q13, q12, q12
2146 vst1.64 {q12}, [r0,:128]!
2147 vswp d13,d12
2148 vshr.s64 q7, q13, #63
2149 veor q13, q13, q6
2150 vand q7, q7, q5
2151 vld1.8 {q3}, [r7]!
2152 veor q2, q2, q10
2153 vadd.u64 q14, q13, q13
2154 vst1.64 {q13}, [r0,:128]!
2155 vswp d15,d14
2156 vshr.s64 q6, q14, #63
2157 veor q14, q14, q7
2158 vand q6, q6, q5
2159 vld1.8 {q4}, [r7]!
2160 veor q3, q3, q11
2161 vadd.u64 q15, q14, q14
2162 vst1.64 {q14}, [r0,:128]!
2163 vswp d13,d12
2164 vshr.s64 q7, q15, #63
2165 veor q15, q15, q6
2166 vand q7, q7, q5
2167 vld1.8 {q5}, [r7]!
2168 veor q4, q4, q12
2169 vadd.u64 q8, q15, q15
2170 vst1.64 {q15}, [r0,:128]!
2171 vswp d15,d14
2172 veor q8, q8, q7
2173 vst1.64 {q8}, [r0,:128] @ next round tweak
2174
2175 vld1.8 {q6-q7}, [r7]!
2176 veor q5, q5, q13
2177#ifndef BSAES_ASM_EXTENDED_KEY
2178 add r4, sp, #0x90 @ pass key schedule
2179#else
2180 add r4, r10, #248 @ pass key schedule
2181#endif
2182 veor q6, q6, q14
2183 mov r5, r1 @ pass rounds
2184 veor q7, q7, q15
2185 mov r0, sp
2186
2187 bl _bsaes_decrypt8
2188
2189 vld1.64 {q8-q9}, [r0,:128]!
2190 vld1.64 {q10-q11}, [r0,:128]!
2191 veor q0, q0, q8
2192 vld1.64 {q12-q13}, [r0,:128]!
2193 veor q1, q1, q9
2194 veor q8, q6, q10
2195 vst1.8 {q0-q1}, [r8]!
2196 veor q9, q4, q11
2197 vld1.64 {q14-q15}, [r0,:128]!
2198 veor q10, q2, q12
2199 vst1.8 {q8-q9}, [r8]!
2200 veor q11, q7, q13
2201 veor q12, q3, q14
2202 vst1.8 {q10-q11}, [r8]!
2203 veor q13, q5, q15
2204 vst1.8 {q12-q13}, [r8]!
2205
2206 vld1.64 {q8}, [r0,:128] @ next round tweak
2207
2208 subs r9, #0x80
2209 bpl .Lxts_dec_loop
2210
2211.Lxts_dec_short:
2212 adds r9, #0x70
2213 bmi .Lxts_dec_done
2214
2215 vldmia r2, {q5} @ load XTS magic
2216 vshr.s64 q7, q8, #63
2217 mov r0, sp
2218 vand q7, q7, q5
2219 vadd.u64 q9, q8, q8
2220 vst1.64 {q8}, [r0,:128]!
2221 vswp d15,d14
2222 vshr.s64 q6, q9, #63
2223 veor q9, q9, q7
2224 vand q6, q6, q5
2225 vadd.u64 q10, q9, q9
2226 vst1.64 {q9}, [r0,:128]!
2227 vswp d13,d12
2228 vshr.s64 q7, q10, #63
2229 veor q10, q10, q6
2230 vand q7, q7, q5
2231 vld1.8 {q0}, [r7]!
2232 subs r9, #0x10
2233 bmi .Lxts_dec_1
2234 vadd.u64 q11, q10, q10
2235 vst1.64 {q10}, [r0,:128]!
2236 vswp d15,d14
2237 vshr.s64 q6, q11, #63
2238 veor q11, q11, q7
2239 vand q6, q6, q5
2240 vld1.8 {q1}, [r7]!
2241 subs r9, #0x10
2242 bmi .Lxts_dec_2
2243 veor q0, q0, q8
2244 vadd.u64 q12, q11, q11
2245 vst1.64 {q11}, [r0,:128]!
2246 vswp d13,d12
2247 vshr.s64 q7, q12, #63
2248 veor q12, q12, q6
2249 vand q7, q7, q5
2250 vld1.8 {q2}, [r7]!
2251 subs r9, #0x10
2252 bmi .Lxts_dec_3
2253 veor q1, q1, q9
2254 vadd.u64 q13, q12, q12
2255 vst1.64 {q12}, [r0,:128]!
2256 vswp d15,d14
2257 vshr.s64 q6, q13, #63
2258 veor q13, q13, q7
2259 vand q6, q6, q5
2260 vld1.8 {q3}, [r7]!
2261 subs r9, #0x10
2262 bmi .Lxts_dec_4
2263 veor q2, q2, q10
2264 vadd.u64 q14, q13, q13
2265 vst1.64 {q13}, [r0,:128]!
2266 vswp d13,d12
2267 vshr.s64 q7, q14, #63
2268 veor q14, q14, q6
2269 vand q7, q7, q5
2270 vld1.8 {q4}, [r7]!
2271 subs r9, #0x10
2272 bmi .Lxts_dec_5
2273 veor q3, q3, q11
2274 vadd.u64 q15, q14, q14
2275 vst1.64 {q14}, [r0,:128]!
2276 vswp d15,d14
2277 vshr.s64 q6, q15, #63
2278 veor q15, q15, q7
2279 vand q6, q6, q5
2280 vld1.8 {q5}, [r7]!
2281 subs r9, #0x10
2282 bmi .Lxts_dec_6
2283 veor q4, q4, q12
2284 sub r9, #0x10
2285 vst1.64 {q15}, [r0,:128] @ next round tweak
2286
2287 vld1.8 {q6}, [r7]!
2288 veor q5, q5, q13
2289#ifndef BSAES_ASM_EXTENDED_KEY
2290 add r4, sp, #0x90 @ pass key schedule
2291#else
2292 add r4, r10, #248 @ pass key schedule
2293#endif
2294 veor q6, q6, q14
2295 mov r5, r1 @ pass rounds
2296 mov r0, sp
2297
2298 bl _bsaes_decrypt8
2299
2300 vld1.64 {q8-q9}, [r0,:128]!
2301 vld1.64 {q10-q11}, [r0,:128]!
2302 veor q0, q0, q8
2303 vld1.64 {q12-q13}, [r0,:128]!
2304 veor q1, q1, q9
2305 veor q8, q6, q10
2306 vst1.8 {q0-q1}, [r8]!
2307 veor q9, q4, q11
2308 vld1.64 {q14}, [r0,:128]!
2309 veor q10, q2, q12
2310 vst1.8 {q8-q9}, [r8]!
2311 veor q11, q7, q13
2312 veor q12, q3, q14
2313 vst1.8 {q10-q11}, [r8]!
2314 vst1.8 {q12}, [r8]!
2315
2316 vld1.64 {q8}, [r0,:128] @ next round tweak
2317 b .Lxts_dec_done
2318.align 4
2319.Lxts_dec_6:
2320 vst1.64 {q14}, [r0,:128] @ next round tweak
2321
2322 veor q4, q4, q12
2323#ifndef BSAES_ASM_EXTENDED_KEY
2324 add r4, sp, #0x90 @ pass key schedule
2325#else
2326 add r4, r10, #248 @ pass key schedule
2327#endif
2328 veor q5, q5, q13
2329 mov r5, r1 @ pass rounds
2330 mov r0, sp
2331
2332 bl _bsaes_decrypt8
2333
2334 vld1.64 {q8-q9}, [r0,:128]!
2335 vld1.64 {q10-q11}, [r0,:128]!
2336 veor q0, q0, q8
2337 vld1.64 {q12-q13}, [r0,:128]!
2338 veor q1, q1, q9
2339 veor q8, q6, q10
2340 vst1.8 {q0-q1}, [r8]!
2341 veor q9, q4, q11
2342 veor q10, q2, q12
2343 vst1.8 {q8-q9}, [r8]!
2344 veor q11, q7, q13
2345 vst1.8 {q10-q11}, [r8]!
2346
2347 vld1.64 {q8}, [r0,:128] @ next round tweak
2348 b .Lxts_dec_done
2349.align 4
2350.Lxts_dec_5:
2351 vst1.64 {q13}, [r0,:128] @ next round tweak
2352
2353 veor q3, q3, q11
2354#ifndef BSAES_ASM_EXTENDED_KEY
2355 add r4, sp, #0x90 @ pass key schedule
2356#else
2357 add r4, r10, #248 @ pass key schedule
2358#endif
2359 veor q4, q4, q12
2360 mov r5, r1 @ pass rounds
2361 mov r0, sp
2362
2363 bl _bsaes_decrypt8
2364
2365 vld1.64 {q8-q9}, [r0,:128]!
2366 vld1.64 {q10-q11}, [r0,:128]!
2367 veor q0, q0, q8
2368 vld1.64 {q12}, [r0,:128]!
2369 veor q1, q1, q9
2370 veor q8, q6, q10
2371 vst1.8 {q0-q1}, [r8]!
2372 veor q9, q4, q11
2373 veor q10, q2, q12
2374 vst1.8 {q8-q9}, [r8]!
2375 vst1.8 {q10}, [r8]!
2376
2377 vld1.64 {q8}, [r0,:128] @ next round tweak
2378 b .Lxts_dec_done
2379.align 4
2380.Lxts_dec_4:
2381 vst1.64 {q12}, [r0,:128] @ next round tweak
2382
2383 veor q2, q2, q10
2384#ifndef BSAES_ASM_EXTENDED_KEY
2385 add r4, sp, #0x90 @ pass key schedule
2386#else
2387 add r4, r10, #248 @ pass key schedule
2388#endif
2389 veor q3, q3, q11
2390 mov r5, r1 @ pass rounds
2391 mov r0, sp
2392
2393 bl _bsaes_decrypt8
2394
2395 vld1.64 {q8-q9}, [r0,:128]!
2396 vld1.64 {q10-q11}, [r0,:128]!
2397 veor q0, q0, q8
2398 veor q1, q1, q9
2399 veor q8, q6, q10
2400 vst1.8 {q0-q1}, [r8]!
2401 veor q9, q4, q11
2402 vst1.8 {q8-q9}, [r8]!
2403
2404 vld1.64 {q8}, [r0,:128] @ next round tweak
2405 b .Lxts_dec_done
2406.align 4
2407.Lxts_dec_3:
2408 vst1.64 {q11}, [r0,:128] @ next round tweak
2409
2410 veor q1, q1, q9
2411#ifndef BSAES_ASM_EXTENDED_KEY
2412 add r4, sp, #0x90 @ pass key schedule
2413#else
2414 add r4, r10, #248 @ pass key schedule
2415#endif
2416 veor q2, q2, q10
2417 mov r5, r1 @ pass rounds
2418 mov r0, sp
2419
2420 bl _bsaes_decrypt8
2421
2422 vld1.64 {q8-q9}, [r0,:128]!
2423 vld1.64 {q10}, [r0,:128]!
2424 veor q0, q0, q8
2425 veor q1, q1, q9
2426 veor q8, q6, q10
2427 vst1.8 {q0-q1}, [r8]!
2428 vst1.8 {q8}, [r8]!
2429
2430 vld1.64 {q8}, [r0,:128] @ next round tweak
2431 b .Lxts_dec_done
2432.align 4
2433.Lxts_dec_2:
2434 vst1.64 {q10}, [r0,:128] @ next round tweak
2435
2436 veor q0, q0, q8
2437#ifndef BSAES_ASM_EXTENDED_KEY
2438 add r4, sp, #0x90 @ pass key schedule
2439#else
2440 add r4, r10, #248 @ pass key schedule
2441#endif
2442 veor q1, q1, q9
2443 mov r5, r1 @ pass rounds
2444 mov r0, sp
2445
2446 bl _bsaes_decrypt8
2447
2448 vld1.64 {q8-q9}, [r0,:128]!
2449 veor q0, q0, q8
2450 veor q1, q1, q9
2451 vst1.8 {q0-q1}, [r8]!
2452
2453 vld1.64 {q8}, [r0,:128] @ next round tweak
2454 b .Lxts_dec_done
2455.align 4
2456.Lxts_dec_1:
2457 mov r0, sp
2458 veor q0, q8
2459 mov r1, sp
2460 vst1.8 {q0}, [sp,:128]
2461 mov r2, r10
2462 mov r4, r3 @ preserve fp
2463 mov r5, r2 @ preserve magic
2464
2465 bl AES_decrypt
2466
2467 vld1.8 {q0}, [sp,:128]
2468 veor q0, q0, q8
2469 vst1.8 {q0}, [r8]!
2470 mov r3, r4
2471 mov r2, r5
2472
2473 vmov q8, q9 @ next round tweak
2474
2475.Lxts_dec_done:
2476#ifndef XTS_CHAIN_TWEAK
2477 adds r9, #0x10
2478 beq .Lxts_dec_ret
2479
2480 @ calculate one round of extra tweak for the stolen ciphertext
2481 vldmia r2, {q5}
2482 vshr.s64 q6, q8, #63
2483 vand q6, q6, q5
2484 vadd.u64 q9, q8, q8
2485 vswp d13,d12
2486 veor q9, q9, q6
2487
2488 @ perform the final decryption with the last tweak value
2489 vld1.8 {q0}, [r7]!
2490 mov r0, sp
2491 veor q0, q0, q9
2492 mov r1, sp
2493 vst1.8 {q0}, [sp,:128]
2494 mov r2, r10
2495 mov r4, r3 @ preserve fp
2496
2497 bl AES_decrypt
2498
2499 vld1.8 {q0}, [sp,:128]
2500 veor q0, q0, q9
2501 vst1.8 {q0}, [r8]
2502
2503 mov r6, r8
2504.Lxts_dec_steal:
2505 ldrb r1, [r8]
2506 ldrb r0, [r7], #1
2507 strb r1, [r8, #0x10]
2508 strb r0, [r8], #1
2509
2510 subs r9, #1
2511 bhi .Lxts_dec_steal
2512
2513 vld1.8 {q0}, [r6]
2514 mov r0, sp
2515 veor q0, q8
2516 mov r1, sp
2517 vst1.8 {q0}, [sp,:128]
2518 mov r2, r10
2519
2520 bl AES_decrypt
2521
2522 vld1.8 {q0}, [sp,:128]
2523 veor q0, q0, q8
2524 vst1.8 {q0}, [r6]
2525 mov r3, r4
2526#endif
2527
2528.Lxts_dec_ret:
2529 bic r0, r3, #0xf
2530 vmov.i32 q0, #0
2531 vmov.i32 q1, #0
2532#ifdef XTS_CHAIN_TWEAK
2533 ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak
2534#endif
2535.Lxts_dec_bzero: @ wipe key schedule [if any]
2536 vstmia sp!, {q0-q1}
2537 cmp sp, r0
2538 bne .Lxts_dec_bzero
2539
2540 mov sp, r3
2541#ifdef XTS_CHAIN_TWEAK
2542 vst1.8 {q8}, [r1]
2543#endif
2544 VFP_ABI_POP
2545 ldmia sp!, {r4-r10, pc} @ return
2546
2547.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
2548#endif
diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c
deleted file mode 100644
index d8e06de72ef3..000000000000
--- a/arch/arm/crypto/aesbs-glue.c
+++ /dev/null
@@ -1,367 +0,0 @@
1/*
2 * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES
3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <crypto/aes.h>
13#include <crypto/cbc.h>
14#include <crypto/internal/simd.h>
15#include <crypto/internal/skcipher.h>
16#include <linux/module.h>
17#include <crypto/xts.h>
18
19#include "aes_glue.h"
20
21#define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE)
22
23struct BS_KEY {
24 struct AES_KEY rk;
25 int converted;
26 u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE];
27} __aligned(8);
28
29asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in);
30asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in);
31
32asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes,
33 struct BS_KEY *key, u8 iv[]);
34
35asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks,
36 struct BS_KEY *key, u8 const iv[]);
37
38asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes,
39 struct BS_KEY *key, u8 tweak[]);
40
41asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes,
42 struct BS_KEY *key, u8 tweak[]);
43
44struct aesbs_cbc_ctx {
45 struct AES_KEY enc;
46 struct BS_KEY dec;
47};
48
49struct aesbs_ctr_ctx {
50 struct BS_KEY enc;
51};
52
53struct aesbs_xts_ctx {
54 struct BS_KEY enc;
55 struct BS_KEY dec;
56 struct AES_KEY twkey;
57};
58
59static int aesbs_cbc_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
60 unsigned int key_len)
61{
62 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
63 int bits = key_len * 8;
64
65 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) {
66 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
67 return -EINVAL;
68 }
69 ctx->dec.rk = ctx->enc;
70 private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
71 ctx->dec.converted = 0;
72 return 0;
73}
74
75static int aesbs_ctr_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
76 unsigned int key_len)
77{
78 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
79 int bits = key_len * 8;
80
81 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
82 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
83 return -EINVAL;
84 }
85 ctx->enc.converted = 0;
86 return 0;
87}
88
89static int aesbs_xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
90 unsigned int key_len)
91{
92 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
93 int bits = key_len * 4;
94 int err;
95
96 err = xts_verify_key(tfm, in_key, key_len);
97 if (err)
98 return err;
99
100 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
101 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
102 return -EINVAL;
103 }
104 ctx->dec.rk = ctx->enc.rk;
105 private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
106 private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey);
107 ctx->enc.converted = ctx->dec.converted = 0;
108 return 0;
109}
110
111static inline void aesbs_encrypt_one(struct crypto_skcipher *tfm,
112 const u8 *src, u8 *dst)
113{
114 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
115
116 AES_encrypt(src, dst, &ctx->enc);
117}
118
119static int aesbs_cbc_encrypt(struct skcipher_request *req)
120{
121 return crypto_cbc_encrypt_walk(req, aesbs_encrypt_one);
122}
123
124static inline void aesbs_decrypt_one(struct crypto_skcipher *tfm,
125 const u8 *src, u8 *dst)
126{
127 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
128
129 AES_decrypt(src, dst, &ctx->dec.rk);
130}
131
132static int aesbs_cbc_decrypt(struct skcipher_request *req)
133{
134 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
135 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
136 struct skcipher_walk walk;
137 unsigned int nbytes;
138 int err;
139
140 for (err = skcipher_walk_virt(&walk, req, false);
141 (nbytes = walk.nbytes); err = skcipher_walk_done(&walk, nbytes)) {
142 u32 blocks = nbytes / AES_BLOCK_SIZE;
143 u8 *dst = walk.dst.virt.addr;
144 u8 *src = walk.src.virt.addr;
145 u8 *iv = walk.iv;
146
147 if (blocks >= 8) {
148 kernel_neon_begin();
149 bsaes_cbc_encrypt(src, dst, nbytes, &ctx->dec, iv);
150 kernel_neon_end();
151 nbytes %= AES_BLOCK_SIZE;
152 continue;
153 }
154
155 nbytes = crypto_cbc_decrypt_blocks(&walk, tfm,
156 aesbs_decrypt_one);
157 }
158 return err;
159}
160
161static void inc_be128_ctr(__be32 ctr[], u32 addend)
162{
163 int i;
164
165 for (i = 3; i >= 0; i--, addend = 1) {
166 u32 n = be32_to_cpu(ctr[i]) + addend;
167
168 ctr[i] = cpu_to_be32(n);
169 if (n >= addend)
170 break;
171 }
172}
173
174static int aesbs_ctr_encrypt(struct skcipher_request *req)
175{
176 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
177 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
178 struct skcipher_walk walk;
179 u32 blocks;
180 int err;
181
182 err = skcipher_walk_virt(&walk, req, false);
183
184 while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) {
185 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
186 __be32 *ctr = (__be32 *)walk.iv;
187 u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]);
188
189 /* avoid 32 bit counter overflow in the NEON code */
190 if (unlikely(headroom < blocks)) {
191 blocks = headroom + 1;
192 tail = walk.nbytes - blocks * AES_BLOCK_SIZE;
193 }
194 kernel_neon_begin();
195 bsaes_ctr32_encrypt_blocks(walk.src.virt.addr,
196 walk.dst.virt.addr, blocks,
197 &ctx->enc, walk.iv);
198 kernel_neon_end();
199 inc_be128_ctr(ctr, blocks);
200
201 err = skcipher_walk_done(&walk, tail);
202 }
203 if (walk.nbytes) {
204 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
205 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
206 u8 ks[AES_BLOCK_SIZE];
207
208 AES_encrypt(walk.iv, ks, &ctx->enc.rk);
209 if (tdst != tsrc)
210 memcpy(tdst, tsrc, walk.nbytes);
211 crypto_xor(tdst, ks, walk.nbytes);
212 err = skcipher_walk_done(&walk, 0);
213 }
214 return err;
215}
216
217static int aesbs_xts_encrypt(struct skcipher_request *req)
218{
219 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
220 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
221 struct skcipher_walk walk;
222 int err;
223
224 err = skcipher_walk_virt(&walk, req, false);
225
226 /* generate the initial tweak */
227 AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
228
229 while (walk.nbytes) {
230 kernel_neon_begin();
231 bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
232 walk.nbytes, &ctx->enc, walk.iv);
233 kernel_neon_end();
234 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
235 }
236 return err;
237}
238
239static int aesbs_xts_decrypt(struct skcipher_request *req)
240{
241 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
242 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
243 struct skcipher_walk walk;
244 int err;
245
246 err = skcipher_walk_virt(&walk, req, false);
247
248 /* generate the initial tweak */
249 AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
250
251 while (walk.nbytes) {
252 kernel_neon_begin();
253 bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
254 walk.nbytes, &ctx->dec, walk.iv);
255 kernel_neon_end();
256 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
257 }
258 return err;
259}
260
261static struct skcipher_alg aesbs_algs[] = { {
262 .base = {
263 .cra_name = "__cbc(aes)",
264 .cra_driver_name = "__cbc-aes-neonbs",
265 .cra_priority = 300,
266 .cra_flags = CRYPTO_ALG_INTERNAL,
267 .cra_blocksize = AES_BLOCK_SIZE,
268 .cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
269 .cra_alignmask = 7,
270 .cra_module = THIS_MODULE,
271 },
272 .min_keysize = AES_MIN_KEY_SIZE,
273 .max_keysize = AES_MAX_KEY_SIZE,
274 .ivsize = AES_BLOCK_SIZE,
275 .setkey = aesbs_cbc_set_key,
276 .encrypt = aesbs_cbc_encrypt,
277 .decrypt = aesbs_cbc_decrypt,
278}, {
279 .base = {
280 .cra_name = "__ctr(aes)",
281 .cra_driver_name = "__ctr-aes-neonbs",
282 .cra_priority = 300,
283 .cra_flags = CRYPTO_ALG_INTERNAL,
284 .cra_blocksize = 1,
285 .cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
286 .cra_alignmask = 7,
287 .cra_module = THIS_MODULE,
288 },
289 .min_keysize = AES_MIN_KEY_SIZE,
290 .max_keysize = AES_MAX_KEY_SIZE,
291 .ivsize = AES_BLOCK_SIZE,
292 .chunksize = AES_BLOCK_SIZE,
293 .setkey = aesbs_ctr_set_key,
294 .encrypt = aesbs_ctr_encrypt,
295 .decrypt = aesbs_ctr_encrypt,
296}, {
297 .base = {
298 .cra_name = "__xts(aes)",
299 .cra_driver_name = "__xts-aes-neonbs",
300 .cra_priority = 300,
301 .cra_flags = CRYPTO_ALG_INTERNAL,
302 .cra_blocksize = AES_BLOCK_SIZE,
303 .cra_ctxsize = sizeof(struct aesbs_xts_ctx),
304 .cra_alignmask = 7,
305 .cra_module = THIS_MODULE,
306 },
307 .min_keysize = 2 * AES_MIN_KEY_SIZE,
308 .max_keysize = 2 * AES_MAX_KEY_SIZE,
309 .ivsize = AES_BLOCK_SIZE,
310 .setkey = aesbs_xts_set_key,
311 .encrypt = aesbs_xts_encrypt,
312 .decrypt = aesbs_xts_decrypt,
313} };
314
315struct simd_skcipher_alg *aesbs_simd_algs[ARRAY_SIZE(aesbs_algs)];
316
317static void aesbs_mod_exit(void)
318{
319 int i;
320
321 for (i = 0; i < ARRAY_SIZE(aesbs_simd_algs) && aesbs_simd_algs[i]; i++)
322 simd_skcipher_free(aesbs_simd_algs[i]);
323
324 crypto_unregister_skciphers(aesbs_algs, ARRAY_SIZE(aesbs_algs));
325}
326
327static int __init aesbs_mod_init(void)
328{
329 struct simd_skcipher_alg *simd;
330 const char *basename;
331 const char *algname;
332 const char *drvname;
333 int err;
334 int i;
335
336 if (!cpu_has_neon())
337 return -ENODEV;
338
339 err = crypto_register_skciphers(aesbs_algs, ARRAY_SIZE(aesbs_algs));
340 if (err)
341 return err;
342
343 for (i = 0; i < ARRAY_SIZE(aesbs_algs); i++) {
344 algname = aesbs_algs[i].base.cra_name + 2;
345 drvname = aesbs_algs[i].base.cra_driver_name + 2;
346 basename = aesbs_algs[i].base.cra_driver_name;
347 simd = simd_skcipher_create_compat(algname, drvname, basename);
348 err = PTR_ERR(simd);
349 if (IS_ERR(simd))
350 goto unregister_simds;
351
352 aesbs_simd_algs[i] = simd;
353 }
354
355 return 0;
356
357unregister_simds:
358 aesbs_mod_exit();
359 return err;
360}
361
362module_init(aesbs_mod_init);
363module_exit(aesbs_mod_exit);
364
365MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON");
366MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
367MODULE_LICENSE("GPL");
diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl
deleted file mode 100644
index a4d3856e7d24..000000000000
--- a/arch/arm/crypto/bsaes-armv7.pl
+++ /dev/null
@@ -1,2471 +0,0 @@
1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8#
9# Specific modes and adaptation for Linux kernel by Ard Biesheuvel
10# <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
11# granted.
12# ====================================================================
13
14# Bit-sliced AES for ARM NEON
15#
16# February 2012.
17#
18# This implementation is direct adaptation of bsaes-x86_64 module for
19# ARM NEON. Except that this module is endian-neutral [in sense that
20# it can be compiled for either endianness] by courtesy of vld1.8's
21# neutrality. Initial version doesn't implement interface to OpenSSL,
22# only low-level primitives and unsupported entry points, just enough
23# to collect performance results, which for Cortex-A8 core are:
24#
25# encrypt 19.5 cycles per byte processed with 128-bit key
26# decrypt 22.1 cycles per byte processed with 128-bit key
27# key conv. 440 cycles per 128-bit key/0.18 of 8x block
28#
29# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7,
30# which is [much] worse than anticipated (for further details see
31# http://www.openssl.org/~appro/Snapdragon-S4.html).
32#
33# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code
34# manages in 20.0 cycles].
35#
36# When comparing to x86_64 results keep in mind that NEON unit is
37# [mostly] single-issue and thus can't [fully] benefit from
38# instruction-level parallelism. And when comparing to aes-armv4
39# results keep in mind key schedule conversion overhead (see
40# bsaes-x86_64.pl for further details)...
41#
42# <appro@openssl.org>
43
44# April-August 2013
45#
46# Add CBC, CTR and XTS subroutines, adapt for kernel use.
47#
48# <ard.biesheuvel@linaro.org>
49
50while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
51open STDOUT,">$output";
52
53my ($inp,$out,$len,$key)=("r0","r1","r2","r3");
54my @XMM=map("q$_",(0..15));
55
56{
57my ($key,$rounds,$const)=("r4","r5","r6");
58
59sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
60sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
61
62sub Sbox {
63# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
64# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb
65my @b=@_[0..7];
66my @t=@_[8..11];
67my @s=@_[12..15];
68 &InBasisChange (@b);
69 &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s);
70 &OutBasisChange (@b[7,1,4,2,6,5,0,3]);
71}
72
73sub InBasisChange {
74# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
75# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
76my @b=@_[0..7];
77$code.=<<___;
78 veor @b[2], @b[2], @b[1]
79 veor @b[5], @b[5], @b[6]
80 veor @b[3], @b[3], @b[0]
81 veor @b[6], @b[6], @b[2]
82 veor @b[5], @b[5], @b[0]
83
84 veor @b[6], @b[6], @b[3]
85 veor @b[3], @b[3], @b[7]
86 veor @b[7], @b[7], @b[5]
87 veor @b[3], @b[3], @b[4]
88 veor @b[4], @b[4], @b[5]
89
90 veor @b[2], @b[2], @b[7]
91 veor @b[3], @b[3], @b[1]
92 veor @b[1], @b[1], @b[5]
93___
94}
95
96sub OutBasisChange {
97# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
98# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb
99my @b=@_[0..7];
100$code.=<<___;
101 veor @b[0], @b[0], @b[6]
102 veor @b[1], @b[1], @b[4]
103 veor @b[4], @b[4], @b[6]
104 veor @b[2], @b[2], @b[0]
105 veor @b[6], @b[6], @b[1]
106
107 veor @b[1], @b[1], @b[5]
108 veor @b[5], @b[5], @b[3]
109 veor @b[3], @b[3], @b[7]
110 veor @b[7], @b[7], @b[5]
111 veor @b[2], @b[2], @b[5]
112
113 veor @b[4], @b[4], @b[7]
114___
115}
116
117sub InvSbox {
118# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
119# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb
120my @b=@_[0..7];
121my @t=@_[8..11];
122my @s=@_[12..15];
123 &InvInBasisChange (@b);
124 &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s);
125 &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]);
126}
127
128sub InvInBasisChange { # OutBasisChange in reverse (with twist)
129my @b=@_[5,1,2,6,3,7,0,4];
130$code.=<<___
131 veor @b[1], @b[1], @b[7]
132 veor @b[4], @b[4], @b[7]
133
134 veor @b[7], @b[7], @b[5]
135 veor @b[1], @b[1], @b[3]
136 veor @b[2], @b[2], @b[5]
137 veor @b[3], @b[3], @b[7]
138
139 veor @b[6], @b[6], @b[1]
140 veor @b[2], @b[2], @b[0]
141 veor @b[5], @b[5], @b[3]
142 veor @b[4], @b[4], @b[6]
143 veor @b[0], @b[0], @b[6]
144 veor @b[1], @b[1], @b[4]
145___
146}
147
148sub InvOutBasisChange { # InBasisChange in reverse
149my @b=@_[2,5,7,3,6,1,0,4];
150$code.=<<___;
151 veor @b[1], @b[1], @b[5]
152 veor @b[2], @b[2], @b[7]
153
154 veor @b[3], @b[3], @b[1]
155 veor @b[4], @b[4], @b[5]
156 veor @b[7], @b[7], @b[5]
157 veor @b[3], @b[3], @b[4]
158 veor @b[5], @b[5], @b[0]
159 veor @b[3], @b[3], @b[7]
160 veor @b[6], @b[6], @b[2]
161 veor @b[2], @b[2], @b[1]
162 veor @b[6], @b[6], @b[3]
163
164 veor @b[3], @b[3], @b[0]
165 veor @b[5], @b[5], @b[6]
166___
167}
168
169sub Mul_GF4 {
170#;*************************************************************
171#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) *
172#;*************************************************************
173my ($x0,$x1,$y0,$y1,$t0,$t1)=@_;
174$code.=<<___;
175 veor $t0, $y0, $y1
176 vand $t0, $t0, $x0
177 veor $x0, $x0, $x1
178 vand $t1, $x1, $y0
179 vand $x0, $x0, $y1
180 veor $x1, $t1, $t0
181 veor $x0, $x0, $t1
182___
183}
184
185sub Mul_GF4_N { # not used, see next subroutine
186# multiply and scale by N
187my ($x0,$x1,$y0,$y1,$t0)=@_;
188$code.=<<___;
189 veor $t0, $y0, $y1
190 vand $t0, $t0, $x0
191 veor $x0, $x0, $x1
192 vand $x1, $x1, $y0
193 vand $x0, $x0, $y1
194 veor $x1, $x1, $x0
195 veor $x0, $x0, $t0
196___
197}
198
199sub Mul_GF4_N_GF4 {
200# interleaved Mul_GF4_N and Mul_GF4
201my ($x0,$x1,$y0,$y1,$t0,
202 $x2,$x3,$y2,$y3,$t1)=@_;
203$code.=<<___;
204 veor $t0, $y0, $y1
205 veor $t1, $y2, $y3
206 vand $t0, $t0, $x0
207 vand $t1, $t1, $x2
208 veor $x0, $x0, $x1
209 veor $x2, $x2, $x3
210 vand $x1, $x1, $y0
211 vand $x3, $x3, $y2
212 vand $x0, $x0, $y1
213 vand $x2, $x2, $y3
214 veor $x1, $x1, $x0
215 veor $x2, $x2, $x3
216 veor $x0, $x0, $t0
217 veor $x3, $x3, $t1
218___
219}
220sub Mul_GF16_2 {
221my @x=@_[0..7];
222my @y=@_[8..11];
223my @t=@_[12..15];
224$code.=<<___;
225 veor @t[0], @x[0], @x[2]
226 veor @t[1], @x[1], @x[3]
227___
228 &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]);
229$code.=<<___;
230 veor @y[0], @y[0], @y[2]
231 veor @y[1], @y[1], @y[3]
232___
233 Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3],
234 @x[2], @x[3], @y[2], @y[3], @t[2]);
235$code.=<<___;
236 veor @x[0], @x[0], @t[0]
237 veor @x[2], @x[2], @t[0]
238 veor @x[1], @x[1], @t[1]
239 veor @x[3], @x[3], @t[1]
240
241 veor @t[0], @x[4], @x[6]
242 veor @t[1], @x[5], @x[7]
243___
244 &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3],
245 @x[6], @x[7], @y[2], @y[3], @t[2]);
246$code.=<<___;
247 veor @y[0], @y[0], @y[2]
248 veor @y[1], @y[1], @y[3]
249___
250 &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]);
251$code.=<<___;
252 veor @x[4], @x[4], @t[0]
253 veor @x[6], @x[6], @t[0]
254 veor @x[5], @x[5], @t[1]
255 veor @x[7], @x[7], @t[1]
256___
257}
258sub Inv_GF256 {
259#;********************************************************************
260#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) *
261#;********************************************************************
262my @x=@_[0..7];
263my @t=@_[8..11];
264my @s=@_[12..15];
265# direct optimizations from hardware
266$code.=<<___;
267 veor @t[3], @x[4], @x[6]
268 veor @t[2], @x[5], @x[7]
269 veor @t[1], @x[1], @x[3]
270 veor @s[1], @x[7], @x[6]
271 vmov @t[0], @t[2]
272 veor @s[0], @x[0], @x[2]
273
274 vorr @t[2], @t[2], @t[1]
275 veor @s[3], @t[3], @t[0]
276 vand @s[2], @t[3], @s[0]
277 vorr @t[3], @t[3], @s[0]
278 veor @s[0], @s[0], @t[1]
279 vand @t[0], @t[0], @t[1]
280 veor @t[1], @x[3], @x[2]
281 vand @s[3], @s[3], @s[0]
282 vand @s[1], @s[1], @t[1]
283 veor @t[1], @x[4], @x[5]
284 veor @s[0], @x[1], @x[0]
285 veor @t[3], @t[3], @s[1]
286 veor @t[2], @t[2], @s[1]
287 vand @s[1], @t[1], @s[0]
288 vorr @t[1], @t[1], @s[0]
289 veor @t[3], @t[3], @s[3]
290 veor @t[0], @t[0], @s[1]
291 veor @t[2], @t[2], @s[2]
292 veor @t[1], @t[1], @s[3]
293 veor @t[0], @t[0], @s[2]
294 vand @s[0], @x[7], @x[3]
295 veor @t[1], @t[1], @s[2]
296 vand @s[1], @x[6], @x[2]
297 vand @s[2], @x[5], @x[1]
298 vorr @s[3], @x[4], @x[0]
299 veor @t[3], @t[3], @s[0]
300 veor @t[1], @t[1], @s[2]
301 veor @t[0], @t[0], @s[3]
302 veor @t[2], @t[2], @s[1]
303
304 @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
305
306 @ new smaller inversion
307
308 vand @s[2], @t[3], @t[1]
309 vmov @s[0], @t[0]
310
311 veor @s[1], @t[2], @s[2]
312 veor @s[3], @t[0], @s[2]
313 veor @s[2], @t[0], @s[2] @ @s[2]=@s[3]
314
315 vbsl @s[1], @t[1], @t[0]
316 vbsl @s[3], @t[3], @t[2]
317 veor @t[3], @t[3], @t[2]
318
319 vbsl @s[0], @s[1], @s[2]
320 vbsl @t[0], @s[2], @s[1]
321
322 vand @s[2], @s[0], @s[3]
323 veor @t[1], @t[1], @t[0]
324
325 veor @s[2], @s[2], @t[3]
326___
327# output in s3, s2, s1, t1
328
329# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3
330
331# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
332 &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]);
333
334### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb
335}
336
337# AES linear components
338
339sub ShiftRows {
340my @x=@_[0..7];
341my @t=@_[8..11];
342my $mask=pop;
343$code.=<<___;
344 vldmia $key!, {@t[0]-@t[3]}
345 veor @t[0], @t[0], @x[0]
346 veor @t[1], @t[1], @x[1]
347 vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)`
348 vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)`
349 vldmia $key!, {@t[0]}
350 veor @t[2], @t[2], @x[2]
351 vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)`
352 vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)`
353 vldmia $key!, {@t[1]}
354 veor @t[3], @t[3], @x[3]
355 vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)`
356 vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)`
357 vldmia $key!, {@t[2]}
358 vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)`
359 vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)`
360 vldmia $key!, {@t[3]}
361 veor @t[0], @t[0], @x[4]
362 veor @t[1], @t[1], @x[5]
363 vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)`
364 vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)`
365 veor @t[2], @t[2], @x[6]
366 vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)`
367 vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)`
368 veor @t[3], @t[3], @x[7]
369 vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)`
370 vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)`
371 vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)`
372 vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)`
373___
374}
375
376sub MixColumns {
377# modified to emit output in order suitable for feeding back to aesenc[last]
378my @x=@_[0..7];
379my @t=@_[8..15];
380my $inv=@_[16]; # optional
381$code.=<<___;
382 vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32
383 vext.8 @t[1], @x[1], @x[1], #12
384 veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32)
385 vext.8 @t[2], @x[2], @x[2], #12
386 veor @x[1], @x[1], @t[1]
387 vext.8 @t[3], @x[3], @x[3], #12
388 veor @x[2], @x[2], @t[2]
389 vext.8 @t[4], @x[4], @x[4], #12
390 veor @x[3], @x[3], @t[3]
391 vext.8 @t[5], @x[5], @x[5], #12
392 veor @x[4], @x[4], @t[4]
393 vext.8 @t[6], @x[6], @x[6], #12
394 veor @x[5], @x[5], @t[5]
395 vext.8 @t[7], @x[7], @x[7], #12
396 veor @x[6], @x[6], @t[6]
397
398 veor @t[1], @t[1], @x[0]
399 veor @x[7], @x[7], @t[7]
400 vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
401 veor @t[2], @t[2], @x[1]
402 veor @t[0], @t[0], @x[7]
403 veor @t[1], @t[1], @x[7]
404 vext.8 @x[1], @x[1], @x[1], #8
405 veor @t[5], @t[5], @x[4]
406 veor @x[0], @x[0], @t[0]
407 veor @t[6], @t[6], @x[5]
408 veor @x[1], @x[1], @t[1]
409 vext.8 @t[0], @x[4], @x[4], #8
410 veor @t[4], @t[4], @x[3]
411 vext.8 @t[1], @x[5], @x[5], #8
412 veor @t[7], @t[7], @x[6]
413 vext.8 @x[4], @x[3], @x[3], #8
414 veor @t[3], @t[3], @x[2]
415 vext.8 @x[5], @x[7], @x[7], #8
416 veor @t[4], @t[4], @x[7]
417 vext.8 @x[3], @x[6], @x[6], #8
418 veor @t[3], @t[3], @x[7]
419 vext.8 @x[6], @x[2], @x[2], #8
420 veor @x[7], @t[1], @t[5]
421___
422$code.=<<___ if (!$inv);
423 veor @x[2], @t[0], @t[4]
424 veor @x[4], @x[4], @t[3]
425 veor @x[5], @x[5], @t[7]
426 veor @x[3], @x[3], @t[6]
427 @ vmov @x[2], @t[0]
428 veor @x[6], @x[6], @t[2]
429 @ vmov @x[7], @t[1]
430___
431$code.=<<___ if ($inv);
432 veor @t[3], @t[3], @x[4]
433 veor @x[5], @x[5], @t[7]
434 veor @x[2], @x[3], @t[6]
435 veor @x[3], @t[0], @t[4]
436 veor @x[4], @x[6], @t[2]
437 vmov @x[6], @t[3]
438 @ vmov @x[7], @t[1]
439___
440}
441
442sub InvMixColumns_orig {
443my @x=@_[0..7];
444my @t=@_[8..15];
445
446$code.=<<___;
447 @ multiplication by 0x0e
448 vext.8 @t[7], @x[7], @x[7], #12
449 vmov @t[2], @x[2]
450 veor @x[2], @x[2], @x[5] @ 2 5
451 veor @x[7], @x[7], @x[5] @ 7 5
452 vext.8 @t[0], @x[0], @x[0], #12
453 vmov @t[5], @x[5]
454 veor @x[5], @x[5], @x[0] @ 5 0 [1]
455 veor @x[0], @x[0], @x[1] @ 0 1
456 vext.8 @t[1], @x[1], @x[1], #12
457 veor @x[1], @x[1], @x[2] @ 1 25
458 veor @x[0], @x[0], @x[6] @ 01 6 [2]
459 vext.8 @t[3], @x[3], @x[3], #12
460 veor @x[1], @x[1], @x[3] @ 125 3 [4]
461 veor @x[2], @x[2], @x[0] @ 25 016 [3]
462 veor @x[3], @x[3], @x[7] @ 3 75
463 veor @x[7], @x[7], @x[6] @ 75 6 [0]
464 vext.8 @t[6], @x[6], @x[6], #12
465 vmov @t[4], @x[4]
466 veor @x[6], @x[6], @x[4] @ 6 4
467 veor @x[4], @x[4], @x[3] @ 4 375 [6]
468 veor @x[3], @x[3], @x[7] @ 375 756=36
469 veor @x[6], @x[6], @t[5] @ 64 5 [7]
470 veor @x[3], @x[3], @t[2] @ 36 2
471 vext.8 @t[5], @t[5], @t[5], #12
472 veor @x[3], @x[3], @t[4] @ 362 4 [5]
473___
474 my @y = @x[7,5,0,2,1,3,4,6];
475$code.=<<___;
476 @ multiplication by 0x0b
477 veor @y[1], @y[1], @y[0]
478 veor @y[0], @y[0], @t[0]
479 vext.8 @t[2], @t[2], @t[2], #12
480 veor @y[1], @y[1], @t[1]
481 veor @y[0], @y[0], @t[5]
482 vext.8 @t[4], @t[4], @t[4], #12
483 veor @y[1], @y[1], @t[6]
484 veor @y[0], @y[0], @t[7]
485 veor @t[7], @t[7], @t[6] @ clobber t[7]
486
487 veor @y[3], @y[3], @t[0]
488 veor @y[1], @y[1], @y[0]
489 vext.8 @t[0], @t[0], @t[0], #12
490 veor @y[2], @y[2], @t[1]
491 veor @y[4], @y[4], @t[1]
492 vext.8 @t[1], @t[1], @t[1], #12
493 veor @y[2], @y[2], @t[2]
494 veor @y[3], @y[3], @t[2]
495 veor @y[5], @y[5], @t[2]
496 veor @y[2], @y[2], @t[7]
497 vext.8 @t[2], @t[2], @t[2], #12
498 veor @y[3], @y[3], @t[3]
499 veor @y[6], @y[6], @t[3]
500 veor @y[4], @y[4], @t[3]
501 veor @y[7], @y[7], @t[4]
502 vext.8 @t[3], @t[3], @t[3], #12
503 veor @y[5], @y[5], @t[4]
504 veor @y[7], @y[7], @t[7]
505 veor @t[7], @t[7], @t[5] @ clobber t[7] even more
506 veor @y[3], @y[3], @t[5]
507 veor @y[4], @y[4], @t[4]
508
509 veor @y[5], @y[5], @t[7]
510 vext.8 @t[4], @t[4], @t[4], #12
511 veor @y[6], @y[6], @t[7]
512 veor @y[4], @y[4], @t[7]
513
514 veor @t[7], @t[7], @t[5]
515 vext.8 @t[5], @t[5], @t[5], #12
516
517 @ multiplication by 0x0d
518 veor @y[4], @y[4], @y[7]
519 veor @t[7], @t[7], @t[6] @ restore t[7]
520 veor @y[7], @y[7], @t[4]
521 vext.8 @t[6], @t[6], @t[6], #12
522 veor @y[2], @y[2], @t[0]
523 veor @y[7], @y[7], @t[5]
524 vext.8 @t[7], @t[7], @t[7], #12
525 veor @y[2], @y[2], @t[2]
526
527 veor @y[3], @y[3], @y[1]
528 veor @y[1], @y[1], @t[1]
529 veor @y[0], @y[0], @t[0]
530 veor @y[3], @y[3], @t[0]
531 veor @y[1], @y[1], @t[5]
532 veor @y[0], @y[0], @t[5]
533 vext.8 @t[0], @t[0], @t[0], #12
534 veor @y[1], @y[1], @t[7]
535 veor @y[0], @y[0], @t[6]
536 veor @y[3], @y[3], @y[1]
537 veor @y[4], @y[4], @t[1]
538 vext.8 @t[1], @t[1], @t[1], #12
539
540 veor @y[7], @y[7], @t[7]
541 veor @y[4], @y[4], @t[2]
542 veor @y[5], @y[5], @t[2]
543 veor @y[2], @y[2], @t[6]
544 veor @t[6], @t[6], @t[3] @ clobber t[6]
545 vext.8 @t[2], @t[2], @t[2], #12
546 veor @y[4], @y[4], @y[7]
547 veor @y[3], @y[3], @t[6]
548
549 veor @y[6], @y[6], @t[6]
550 veor @y[5], @y[5], @t[5]
551 vext.8 @t[5], @t[5], @t[5], #12
552 veor @y[6], @y[6], @t[4]
553 vext.8 @t[4], @t[4], @t[4], #12
554 veor @y[5], @y[5], @t[6]
555 veor @y[6], @y[6], @t[7]
556 vext.8 @t[7], @t[7], @t[7], #12
557 veor @t[6], @t[6], @t[3] @ restore t[6]
558 vext.8 @t[3], @t[3], @t[3], #12
559
560 @ multiplication by 0x09
561 veor @y[4], @y[4], @y[1]
562 veor @t[1], @t[1], @y[1] @ t[1]=y[1]
563 veor @t[0], @t[0], @t[5] @ clobber t[0]
564 vext.8 @t[6], @t[6], @t[6], #12
565 veor @t[1], @t[1], @t[5]
566 veor @y[3], @y[3], @t[0]
567 veor @t[0], @t[0], @y[0] @ t[0]=y[0]
568 veor @t[1], @t[1], @t[6]
569 veor @t[6], @t[6], @t[7] @ clobber t[6]
570 veor @y[4], @y[4], @t[1]
571 veor @y[7], @y[7], @t[4]
572 veor @y[6], @y[6], @t[3]
573 veor @y[5], @y[5], @t[2]
574 veor @t[4], @t[4], @y[4] @ t[4]=y[4]
575 veor @t[3], @t[3], @y[3] @ t[3]=y[3]
576 veor @t[5], @t[5], @y[5] @ t[5]=y[5]
577 veor @t[2], @t[2], @y[2] @ t[2]=y[2]
578 veor @t[3], @t[3], @t[7]
579 veor @XMM[5], @t[5], @t[6]
580 veor @XMM[6], @t[6], @y[6] @ t[6]=y[6]
581 veor @XMM[2], @t[2], @t[6]
582 veor @XMM[7], @t[7], @y[7] @ t[7]=y[7]
583
584 vmov @XMM[0], @t[0]
585 vmov @XMM[1], @t[1]
586 @ vmov @XMM[2], @t[2]
587 vmov @XMM[3], @t[3]
588 vmov @XMM[4], @t[4]
589 @ vmov @XMM[5], @t[5]
590 @ vmov @XMM[6], @t[6]
591 @ vmov @XMM[7], @t[7]
592___
593}
594
595sub InvMixColumns {
596my @x=@_[0..7];
597my @t=@_[8..15];
598
599# Thanks to Jussi Kivilinna for providing pointer to
600#
601# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 |
602# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 |
603# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 |
604# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 |
605
606$code.=<<___;
607 @ multiplication by 0x05-0x00-0x04-0x00
608 vext.8 @t[0], @x[0], @x[0], #8
609 vext.8 @t[6], @x[6], @x[6], #8
610 vext.8 @t[7], @x[7], @x[7], #8
611 veor @t[0], @t[0], @x[0]
612 vext.8 @t[1], @x[1], @x[1], #8
613 veor @t[6], @t[6], @x[6]
614 vext.8 @t[2], @x[2], @x[2], #8
615 veor @t[7], @t[7], @x[7]
616 vext.8 @t[3], @x[3], @x[3], #8
617 veor @t[1], @t[1], @x[1]
618 vext.8 @t[4], @x[4], @x[4], #8
619 veor @t[2], @t[2], @x[2]
620 vext.8 @t[5], @x[5], @x[5], #8
621 veor @t[3], @t[3], @x[3]
622 veor @t[4], @t[4], @x[4]
623 veor @t[5], @t[5], @x[5]
624
625 veor @x[0], @x[0], @t[6]
626 veor @x[1], @x[1], @t[6]
627 veor @x[2], @x[2], @t[0]
628 veor @x[4], @x[4], @t[2]
629 veor @x[3], @x[3], @t[1]
630 veor @x[1], @x[1], @t[7]
631 veor @x[2], @x[2], @t[7]
632 veor @x[4], @x[4], @t[6]
633 veor @x[5], @x[5], @t[3]
634 veor @x[3], @x[3], @t[6]
635 veor @x[6], @x[6], @t[4]
636 veor @x[4], @x[4], @t[7]
637 veor @x[5], @x[5], @t[7]
638 veor @x[7], @x[7], @t[5]
639___
640 &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6
641}
642
643sub swapmove {
644my ($a,$b,$n,$mask,$t)=@_;
645$code.=<<___;
646 vshr.u64 $t, $b, #$n
647 veor $t, $t, $a
648 vand $t, $t, $mask
649 veor $a, $a, $t
650 vshl.u64 $t, $t, #$n
651 veor $b, $b, $t
652___
653}
654sub swapmove2x {
655my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_;
656$code.=<<___;
657 vshr.u64 $t0, $b0, #$n
658 vshr.u64 $t1, $b1, #$n
659 veor $t0, $t0, $a0
660 veor $t1, $t1, $a1
661 vand $t0, $t0, $mask
662 vand $t1, $t1, $mask
663 veor $a0, $a0, $t0
664 vshl.u64 $t0, $t0, #$n
665 veor $a1, $a1, $t1
666 vshl.u64 $t1, $t1, #$n
667 veor $b0, $b0, $t0
668 veor $b1, $b1, $t1
669___
670}
671
672sub bitslice {
673my @x=reverse(@_[0..7]);
674my ($t0,$t1,$t2,$t3)=@_[8..11];
675$code.=<<___;
676 vmov.i8 $t0,#0x55 @ compose .LBS0
677 vmov.i8 $t1,#0x33 @ compose .LBS1
678___
679 &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3);
680 &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3);
681$code.=<<___;
682 vmov.i8 $t0,#0x0f @ compose .LBS2
683___
684 &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3);
685 &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3);
686
687 &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3);
688 &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3);
689}
690
691$code.=<<___;
692#ifndef __KERNEL__
693# include "arm_arch.h"
694
695# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
696# define VFP_ABI_POP vldmia sp!,{d8-d15}
697# define VFP_ABI_FRAME 0x40
698#else
699# define VFP_ABI_PUSH
700# define VFP_ABI_POP
701# define VFP_ABI_FRAME 0
702# define BSAES_ASM_EXTENDED_KEY
703# define XTS_CHAIN_TWEAK
704# define __ARM_ARCH__ __LINUX_ARM_ARCH__
705# define __ARM_MAX_ARCH__ 7
706#endif
707
708#ifdef __thumb__
709# define adrl adr
710#endif
711
712#if __ARM_MAX_ARCH__>=7
713.arch armv7-a
714.fpu neon
715
716.text
717.syntax unified @ ARMv7-capable assembler is expected to handle this
718#ifdef __thumb2__
719.thumb
720#else
721.code 32
722#endif
723
724.type _bsaes_decrypt8,%function
725.align 4
726_bsaes_decrypt8:
727 adr $const,_bsaes_decrypt8
728 vldmia $key!, {@XMM[9]} @ round 0 key
729 add $const,$const,#.LM0ISR-_bsaes_decrypt8
730
731 vldmia $const!, {@XMM[8]} @ .LM0ISR
732 veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key
733 veor @XMM[11], @XMM[1], @XMM[9]
734 vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])`
735 vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])`
736 veor @XMM[12], @XMM[2], @XMM[9]
737 vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])`
738 vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])`
739 veor @XMM[13], @XMM[3], @XMM[9]
740 vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])`
741 vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])`
742 veor @XMM[14], @XMM[4], @XMM[9]
743 vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])`
744 vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])`
745 veor @XMM[15], @XMM[5], @XMM[9]
746 vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])`
747 vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])`
748 veor @XMM[10], @XMM[6], @XMM[9]
749 vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])`
750 vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])`
751 veor @XMM[11], @XMM[7], @XMM[9]
752 vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])`
753 vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])`
754 vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])`
755 vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])`
756___
757 &bitslice (@XMM[0..7, 8..11]);
758$code.=<<___;
759 sub $rounds,$rounds,#1
760 b .Ldec_sbox
761.align 4
762.Ldec_loop:
763___
764 &ShiftRows (@XMM[0..7, 8..12]);
765$code.=".Ldec_sbox:\n";
766 &InvSbox (@XMM[0..7, 8..15]);
767$code.=<<___;
768 subs $rounds,$rounds,#1
769 bcc .Ldec_done
770___
771 &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]);
772$code.=<<___;
773 vldmia $const, {@XMM[12]} @ .LISR
774 ite eq @ Thumb2 thing, sanity check in ARM
775 addeq $const,$const,#0x10
776 bne .Ldec_loop
777 vldmia $const, {@XMM[12]} @ .LISRM0
778 b .Ldec_loop
779.align 4
780.Ldec_done:
781___
782 &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]);
783$code.=<<___;
784 vldmia $key, {@XMM[8]} @ last round key
785 veor @XMM[6], @XMM[6], @XMM[8]
786 veor @XMM[4], @XMM[4], @XMM[8]
787 veor @XMM[2], @XMM[2], @XMM[8]
788 veor @XMM[7], @XMM[7], @XMM[8]
789 veor @XMM[3], @XMM[3], @XMM[8]
790 veor @XMM[5], @XMM[5], @XMM[8]
791 veor @XMM[0], @XMM[0], @XMM[8]
792 veor @XMM[1], @XMM[1], @XMM[8]
793 bx lr
794.size _bsaes_decrypt8,.-_bsaes_decrypt8
795
796.type _bsaes_const,%object
797.align 6
798_bsaes_const:
799.LM0ISR: @ InvShiftRows constants
800 .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
801.LISR:
802 .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
803.LISRM0:
804 .quad 0x01040b0e0205080f, 0x0306090c00070a0d
805.LM0SR: @ ShiftRows constants
806 .quad 0x0a0e02060f03070b, 0x0004080c05090d01
807.LSR:
808 .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
809.LSRM0:
810 .quad 0x0304090e00050a0f, 0x01060b0c0207080d
811.LM0:
812 .quad 0x02060a0e03070b0f, 0x0004080c0105090d
813.LREVM0SR:
814 .quad 0x090d01050c000408, 0x03070b0f060a0e02
815.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro\@openssl.org>"
816.align 6
817.size _bsaes_const,.-_bsaes_const
818
819.type _bsaes_encrypt8,%function
820.align 4
821_bsaes_encrypt8:
822 adr $const,_bsaes_encrypt8
823 vldmia $key!, {@XMM[9]} @ round 0 key
824 sub $const,$const,#_bsaes_encrypt8-.LM0SR
825
826 vldmia $const!, {@XMM[8]} @ .LM0SR
827_bsaes_encrypt8_alt:
828 veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key
829 veor @XMM[11], @XMM[1], @XMM[9]
830 vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])`
831 vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])`
832 veor @XMM[12], @XMM[2], @XMM[9]
833 vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])`
834 vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])`
835 veor @XMM[13], @XMM[3], @XMM[9]
836 vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])`
837 vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])`
838 veor @XMM[14], @XMM[4], @XMM[9]
839 vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])`
840 vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])`
841 veor @XMM[15], @XMM[5], @XMM[9]
842 vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])`
843 vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])`
844 veor @XMM[10], @XMM[6], @XMM[9]
845 vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])`
846 vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])`
847 veor @XMM[11], @XMM[7], @XMM[9]
848 vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])`
849 vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])`
850 vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])`
851 vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])`
852_bsaes_encrypt8_bitslice:
853___
854 &bitslice (@XMM[0..7, 8..11]);
855$code.=<<___;
856 sub $rounds,$rounds,#1
857 b .Lenc_sbox
858.align 4
859.Lenc_loop:
860___
861 &ShiftRows (@XMM[0..7, 8..12]);
862$code.=".Lenc_sbox:\n";
863 &Sbox (@XMM[0..7, 8..15]);
864$code.=<<___;
865 subs $rounds,$rounds,#1
866 bcc .Lenc_done
867___
868 &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]);
869$code.=<<___;
870 vldmia $const, {@XMM[12]} @ .LSR
871 ite eq @ Thumb2 thing, samity check in ARM
872 addeq $const,$const,#0x10
873 bne .Lenc_loop
874 vldmia $const, {@XMM[12]} @ .LSRM0
875 b .Lenc_loop
876.align 4
877.Lenc_done:
878___
879 # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb
880 &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]);
881$code.=<<___;
882 vldmia $key, {@XMM[8]} @ last round key
883 veor @XMM[4], @XMM[4], @XMM[8]
884 veor @XMM[6], @XMM[6], @XMM[8]
885 veor @XMM[3], @XMM[3], @XMM[8]
886 veor @XMM[7], @XMM[7], @XMM[8]
887 veor @XMM[2], @XMM[2], @XMM[8]
888 veor @XMM[5], @XMM[5], @XMM[8]
889 veor @XMM[0], @XMM[0], @XMM[8]
890 veor @XMM[1], @XMM[1], @XMM[8]
891 bx lr
892.size _bsaes_encrypt8,.-_bsaes_encrypt8
893___
894}
895{
896my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6");
897
898sub bitslice_key {
899my @x=reverse(@_[0..7]);
900my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12];
901
902 &swapmove (@x[0,1],1,$bs0,$t2,$t3);
903$code.=<<___;
904 @ &swapmove(@x[2,3],1,$t0,$t2,$t3);
905 vmov @x[2], @x[0]
906 vmov @x[3], @x[1]
907___
908 #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3);
909
910 &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3);
911$code.=<<___;
912 @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3);
913 vmov @x[4], @x[0]
914 vmov @x[6], @x[2]
915 vmov @x[5], @x[1]
916 vmov @x[7], @x[3]
917___
918 &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3);
919 &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3);
920}
921
922$code.=<<___;
923.type _bsaes_key_convert,%function
924.align 4
925_bsaes_key_convert:
926 adr $const,_bsaes_key_convert
927 vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key
928 sub $const,$const,#_bsaes_key_convert-.LM0
929 vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key
930
931 vmov.i8 @XMM[8], #0x01 @ bit masks
932 vmov.i8 @XMM[9], #0x02
933 vmov.i8 @XMM[10], #0x04
934 vmov.i8 @XMM[11], #0x08
935 vmov.i8 @XMM[12], #0x10
936 vmov.i8 @XMM[13], #0x20
937 vldmia $const, {@XMM[14]} @ .LM0
938
939#ifdef __ARMEL__
940 vrev32.8 @XMM[7], @XMM[7]
941 vrev32.8 @XMM[15], @XMM[15]
942#endif
943 sub $rounds,$rounds,#1
944 vstmia $out!, {@XMM[7]} @ save round 0 key
945 b .Lkey_loop
946
947.align 4
948.Lkey_loop:
949 vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])`
950 vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])`
951 vmov.i8 @XMM[6], #0x40
952 vmov.i8 @XMM[15], #0x80
953
954 vtst.8 @XMM[0], @XMM[7], @XMM[8]
955 vtst.8 @XMM[1], @XMM[7], @XMM[9]
956 vtst.8 @XMM[2], @XMM[7], @XMM[10]
957 vtst.8 @XMM[3], @XMM[7], @XMM[11]
958 vtst.8 @XMM[4], @XMM[7], @XMM[12]
959 vtst.8 @XMM[5], @XMM[7], @XMM[13]
960 vtst.8 @XMM[6], @XMM[7], @XMM[6]
961 vtst.8 @XMM[7], @XMM[7], @XMM[15]
962 vld1.8 {@XMM[15]}, [$inp]! @ load next round key
963 vmvn @XMM[0], @XMM[0] @ "pnot"
964 vmvn @XMM[1], @XMM[1]
965 vmvn @XMM[5], @XMM[5]
966 vmvn @XMM[6], @XMM[6]
967#ifdef __ARMEL__
968 vrev32.8 @XMM[15], @XMM[15]
969#endif
970 subs $rounds,$rounds,#1
971 vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key
972 bne .Lkey_loop
973
974 vmov.i8 @XMM[7],#0x63 @ compose .L63
975 @ don't save last round key
976 bx lr
977.size _bsaes_key_convert,.-_bsaes_key_convert
978___
979}
980
981if (0) { # following four functions are unsupported interface
982 # used for benchmarking...
983$code.=<<___;
984.globl bsaes_enc_key_convert
985.type bsaes_enc_key_convert,%function
986.align 4
987bsaes_enc_key_convert:
988 stmdb sp!,{r4-r6,lr}
989 vstmdb sp!,{d8-d15} @ ABI specification says so
990
991 ldr r5,[$inp,#240] @ pass rounds
992 mov r4,$inp @ pass key
993 mov r12,$out @ pass key schedule
994 bl _bsaes_key_convert
995 veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
996 vstmia r12, {@XMM[7]} @ save last round key
997
998 vldmia sp!,{d8-d15}
999 ldmia sp!,{r4-r6,pc}
1000.size bsaes_enc_key_convert,.-bsaes_enc_key_convert
1001
1002.globl bsaes_encrypt_128
1003.type bsaes_encrypt_128,%function
1004.align 4
1005bsaes_encrypt_128:
1006 stmdb sp!,{r4-r6,lr}
1007 vstmdb sp!,{d8-d15} @ ABI specification says so
1008.Lenc128_loop:
1009 vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
1010 vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
1011 mov r4,$key @ pass the key
1012 vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
1013 mov r5,#10 @ pass rounds
1014 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
1015
1016 bl _bsaes_encrypt8
1017
1018 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1019 vst1.8 {@XMM[4]}, [$out]!
1020 vst1.8 {@XMM[6]}, [$out]!
1021 vst1.8 {@XMM[3]}, [$out]!
1022 vst1.8 {@XMM[7]}, [$out]!
1023 vst1.8 {@XMM[2]}, [$out]!
1024 subs $len,$len,#0x80
1025 vst1.8 {@XMM[5]}, [$out]!
1026 bhi .Lenc128_loop
1027
1028 vldmia sp!,{d8-d15}
1029 ldmia sp!,{r4-r6,pc}
1030.size bsaes_encrypt_128,.-bsaes_encrypt_128
1031
1032.globl bsaes_dec_key_convert
1033.type bsaes_dec_key_convert,%function
1034.align 4
1035bsaes_dec_key_convert:
1036 stmdb sp!,{r4-r6,lr}
1037 vstmdb sp!,{d8-d15} @ ABI specification says so
1038
1039 ldr r5,[$inp,#240] @ pass rounds
1040 mov r4,$inp @ pass key
1041 mov r12,$out @ pass key schedule
1042 bl _bsaes_key_convert
1043 vldmia $out, {@XMM[6]}
1044 vstmia r12, {@XMM[15]} @ save last round key
1045 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
1046 vstmia $out, {@XMM[7]}
1047
1048 vldmia sp!,{d8-d15}
1049 ldmia sp!,{r4-r6,pc}
1050.size bsaes_dec_key_convert,.-bsaes_dec_key_convert
1051
1052.globl bsaes_decrypt_128
1053.type bsaes_decrypt_128,%function
1054.align 4
1055bsaes_decrypt_128:
1056 stmdb sp!,{r4-r6,lr}
1057 vstmdb sp!,{d8-d15} @ ABI specification says so
1058.Ldec128_loop:
1059 vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
1060 vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
1061 mov r4,$key @ pass the key
1062 vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
1063 mov r5,#10 @ pass rounds
1064 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
1065
1066 bl _bsaes_decrypt8
1067
1068 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1069 vst1.8 {@XMM[6]}, [$out]!
1070 vst1.8 {@XMM[4]}, [$out]!
1071 vst1.8 {@XMM[2]}, [$out]!
1072 vst1.8 {@XMM[7]}, [$out]!
1073 vst1.8 {@XMM[3]}, [$out]!
1074 subs $len,$len,#0x80
1075 vst1.8 {@XMM[5]}, [$out]!
1076 bhi .Ldec128_loop
1077
1078 vldmia sp!,{d8-d15}
1079 ldmia sp!,{r4-r6,pc}
1080.size bsaes_decrypt_128,.-bsaes_decrypt_128
1081___
1082}
1083{
1084my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10));
1085my ($keysched)=("sp");
1086
1087$code.=<<___;
1088.extern AES_cbc_encrypt
1089.extern AES_decrypt
1090
1091.global bsaes_cbc_encrypt
1092.type bsaes_cbc_encrypt,%function
1093.align 5
1094bsaes_cbc_encrypt:
1095#ifndef __KERNEL__
1096 cmp $len, #128
1097#ifndef __thumb__
1098 blo AES_cbc_encrypt
1099#else
1100 bhs 1f
1101 b AES_cbc_encrypt
11021:
1103#endif
1104#endif
1105
1106 @ it is up to the caller to make sure we are called with enc == 0
1107
1108 mov ip, sp
1109 stmdb sp!, {r4-r10, lr}
1110 VFP_ABI_PUSH
1111 ldr $ivp, [ip] @ IV is 1st arg on the stack
1112 mov $len, $len, lsr#4 @ len in 16 byte blocks
1113 sub sp, #0x10 @ scratch space to carry over the IV
1114 mov $fp, sp @ save sp
1115
1116 ldr $rounds, [$key, #240] @ get # of rounds
1117#ifndef BSAES_ASM_EXTENDED_KEY
1118 @ allocate the key schedule on the stack
1119 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
1120 add r12, #`128-32` @ sifze of bit-slices key schedule
1121
1122 @ populate the key schedule
1123 mov r4, $key @ pass key
1124 mov r5, $rounds @ pass # of rounds
1125 mov sp, r12 @ sp is $keysched
1126 bl _bsaes_key_convert
1127 vldmia $keysched, {@XMM[6]}
1128 vstmia r12, {@XMM[15]} @ save last round key
1129 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
1130 vstmia $keysched, {@XMM[7]}
1131#else
1132 ldr r12, [$key, #244]
1133 eors r12, #1
1134 beq 0f
1135
1136 @ populate the key schedule
1137 str r12, [$key, #244]
1138 mov r4, $key @ pass key
1139 mov r5, $rounds @ pass # of rounds
1140 add r12, $key, #248 @ pass key schedule
1141 bl _bsaes_key_convert
1142 add r4, $key, #248
1143 vldmia r4, {@XMM[6]}
1144 vstmia r12, {@XMM[15]} @ save last round key
1145 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
1146 vstmia r4, {@XMM[7]}
1147
1148.align 2
11490:
1150#endif
1151
1152 vld1.8 {@XMM[15]}, [$ivp] @ load IV
1153 b .Lcbc_dec_loop
1154
1155.align 4
1156.Lcbc_dec_loop:
1157 subs $len, $len, #0x8
1158 bmi .Lcbc_dec_loop_finish
1159
1160 vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
1161 vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
1162#ifndef BSAES_ASM_EXTENDED_KEY
1163 mov r4, $keysched @ pass the key
1164#else
1165 add r4, $key, #248
1166#endif
1167 vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
1168 mov r5, $rounds
1169 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]
1170 sub $inp, $inp, #0x60
1171 vstmia $fp, {@XMM[15]} @ put aside IV
1172
1173 bl _bsaes_decrypt8
1174
1175 vldmia $fp, {@XMM[14]} @ reload IV
1176 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1177 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1178 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1179 veor @XMM[1], @XMM[1], @XMM[8]
1180 veor @XMM[6], @XMM[6], @XMM[9]
1181 vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
1182 veor @XMM[4], @XMM[4], @XMM[10]
1183 veor @XMM[2], @XMM[2], @XMM[11]
1184 vld1.8 {@XMM[14]-@XMM[15]}, [$inp]!
1185 veor @XMM[7], @XMM[7], @XMM[12]
1186 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1187 veor @XMM[3], @XMM[3], @XMM[13]
1188 vst1.8 {@XMM[6]}, [$out]!
1189 veor @XMM[5], @XMM[5], @XMM[14]
1190 vst1.8 {@XMM[4]}, [$out]!
1191 vst1.8 {@XMM[2]}, [$out]!
1192 vst1.8 {@XMM[7]}, [$out]!
1193 vst1.8 {@XMM[3]}, [$out]!
1194 vst1.8 {@XMM[5]}, [$out]!
1195
1196 b .Lcbc_dec_loop
1197
1198.Lcbc_dec_loop_finish:
1199 adds $len, $len, #8
1200 beq .Lcbc_dec_done
1201
1202 vld1.8 {@XMM[0]}, [$inp]! @ load input
1203 cmp $len, #2
1204 blo .Lcbc_dec_one
1205 vld1.8 {@XMM[1]}, [$inp]!
1206#ifndef BSAES_ASM_EXTENDED_KEY
1207 mov r4, $keysched @ pass the key
1208#else
1209 add r4, $key, #248
1210#endif
1211 mov r5, $rounds
1212 vstmia $fp, {@XMM[15]} @ put aside IV
1213 beq .Lcbc_dec_two
1214 vld1.8 {@XMM[2]}, [$inp]!
1215 cmp $len, #4
1216 blo .Lcbc_dec_three
1217 vld1.8 {@XMM[3]}, [$inp]!
1218 beq .Lcbc_dec_four
1219 vld1.8 {@XMM[4]}, [$inp]!
1220 cmp $len, #6
1221 blo .Lcbc_dec_five
1222 vld1.8 {@XMM[5]}, [$inp]!
1223 beq .Lcbc_dec_six
1224 vld1.8 {@XMM[6]}, [$inp]!
1225 sub $inp, $inp, #0x70
1226
1227 bl _bsaes_decrypt8
1228
1229 vldmia $fp, {@XMM[14]} @ reload IV
1230 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1231 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1232 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1233 veor @XMM[1], @XMM[1], @XMM[8]
1234 veor @XMM[6], @XMM[6], @XMM[9]
1235 vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
1236 veor @XMM[4], @XMM[4], @XMM[10]
1237 veor @XMM[2], @XMM[2], @XMM[11]
1238 vld1.8 {@XMM[15]}, [$inp]!
1239 veor @XMM[7], @XMM[7], @XMM[12]
1240 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1241 veor @XMM[3], @XMM[3], @XMM[13]
1242 vst1.8 {@XMM[6]}, [$out]!
1243 vst1.8 {@XMM[4]}, [$out]!
1244 vst1.8 {@XMM[2]}, [$out]!
1245 vst1.8 {@XMM[7]}, [$out]!
1246 vst1.8 {@XMM[3]}, [$out]!
1247 b .Lcbc_dec_done
1248.align 4
1249.Lcbc_dec_six:
1250 sub $inp, $inp, #0x60
1251 bl _bsaes_decrypt8
1252 vldmia $fp,{@XMM[14]} @ reload IV
1253 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1254 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1255 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1256 veor @XMM[1], @XMM[1], @XMM[8]
1257 veor @XMM[6], @XMM[6], @XMM[9]
1258 vld1.8 {@XMM[12]}, [$inp]!
1259 veor @XMM[4], @XMM[4], @XMM[10]
1260 veor @XMM[2], @XMM[2], @XMM[11]
1261 vld1.8 {@XMM[15]}, [$inp]!
1262 veor @XMM[7], @XMM[7], @XMM[12]
1263 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1264 vst1.8 {@XMM[6]}, [$out]!
1265 vst1.8 {@XMM[4]}, [$out]!
1266 vst1.8 {@XMM[2]}, [$out]!
1267 vst1.8 {@XMM[7]}, [$out]!
1268 b .Lcbc_dec_done
1269.align 4
1270.Lcbc_dec_five:
1271 sub $inp, $inp, #0x50
1272 bl _bsaes_decrypt8
1273 vldmia $fp, {@XMM[14]} @ reload IV
1274 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1275 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1276 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1277 veor @XMM[1], @XMM[1], @XMM[8]
1278 veor @XMM[6], @XMM[6], @XMM[9]
1279 vld1.8 {@XMM[15]}, [$inp]!
1280 veor @XMM[4], @XMM[4], @XMM[10]
1281 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1282 veor @XMM[2], @XMM[2], @XMM[11]
1283 vst1.8 {@XMM[6]}, [$out]!
1284 vst1.8 {@XMM[4]}, [$out]!
1285 vst1.8 {@XMM[2]}, [$out]!
1286 b .Lcbc_dec_done
1287.align 4
1288.Lcbc_dec_four:
1289 sub $inp, $inp, #0x40
1290 bl _bsaes_decrypt8
1291 vldmia $fp, {@XMM[14]} @ reload IV
1292 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1293 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1294 vld1.8 {@XMM[10]}, [$inp]!
1295 veor @XMM[1], @XMM[1], @XMM[8]
1296 veor @XMM[6], @XMM[6], @XMM[9]
1297 vld1.8 {@XMM[15]}, [$inp]!
1298 veor @XMM[4], @XMM[4], @XMM[10]
1299 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1300 vst1.8 {@XMM[6]}, [$out]!
1301 vst1.8 {@XMM[4]}, [$out]!
1302 b .Lcbc_dec_done
1303.align 4
1304.Lcbc_dec_three:
1305 sub $inp, $inp, #0x30
1306 bl _bsaes_decrypt8
1307 vldmia $fp, {@XMM[14]} @ reload IV
1308 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
1309 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1310 vld1.8 {@XMM[15]}, [$inp]!
1311 veor @XMM[1], @XMM[1], @XMM[8]
1312 veor @XMM[6], @XMM[6], @XMM[9]
1313 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1314 vst1.8 {@XMM[6]}, [$out]!
1315 b .Lcbc_dec_done
1316.align 4
1317.Lcbc_dec_two:
1318 sub $inp, $inp, #0x20
1319 bl _bsaes_decrypt8
1320 vldmia $fp, {@XMM[14]} @ reload IV
1321 vld1.8 {@XMM[8]}, [$inp]! @ reload input
1322 veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
1323 vld1.8 {@XMM[15]}, [$inp]! @ reload input
1324 veor @XMM[1], @XMM[1], @XMM[8]
1325 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1326 b .Lcbc_dec_done
1327.align 4
1328.Lcbc_dec_one:
1329 sub $inp, $inp, #0x10
1330 mov $rounds, $out @ save original out pointer
1331 mov $out, $fp @ use the iv scratch space as out buffer
1332 mov r2, $key
1333 vmov @XMM[4],@XMM[15] @ just in case ensure that IV
1334 vmov @XMM[5],@XMM[0] @ and input are preserved
1335 bl AES_decrypt
1336 vld1.8 {@XMM[0]}, [$fp,:64] @ load result
1337 veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV
1338 vmov @XMM[15], @XMM[5] @ @XMM[5] holds input
1339 vst1.8 {@XMM[0]}, [$rounds] @ write output
1340
1341.Lcbc_dec_done:
1342#ifndef BSAES_ASM_EXTENDED_KEY
1343 vmov.i32 q0, #0
1344 vmov.i32 q1, #0
1345.Lcbc_dec_bzero: @ wipe key schedule [if any]
1346 vstmia $keysched!, {q0-q1}
1347 cmp $keysched, $fp
1348 bne .Lcbc_dec_bzero
1349#endif
1350
1351 mov sp, $fp
1352 add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb
1353 vst1.8 {@XMM[15]}, [$ivp] @ return IV
1354 VFP_ABI_POP
1355 ldmia sp!, {r4-r10, pc}
1356.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
1357___
1358}
1359{
1360my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10)));
1361my $const = "r6"; # shared with _bsaes_encrypt8_alt
1362my $keysched = "sp";
1363
1364$code.=<<___;
1365.extern AES_encrypt
1366.global bsaes_ctr32_encrypt_blocks
1367.type bsaes_ctr32_encrypt_blocks,%function
1368.align 5
1369bsaes_ctr32_encrypt_blocks:
1370 cmp $len, #8 @ use plain AES for
1371 blo .Lctr_enc_short @ small sizes
1372
1373 mov ip, sp
1374 stmdb sp!, {r4-r10, lr}
1375 VFP_ABI_PUSH
1376 ldr $ctr, [ip] @ ctr is 1st arg on the stack
1377 sub sp, sp, #0x10 @ scratch space to carry over the ctr
1378 mov $fp, sp @ save sp
1379
1380 ldr $rounds, [$key, #240] @ get # of rounds
1381#ifndef BSAES_ASM_EXTENDED_KEY
1382 @ allocate the key schedule on the stack
1383 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
1384 add r12, #`128-32` @ size of bit-sliced key schedule
1385
1386 @ populate the key schedule
1387 mov r4, $key @ pass key
1388 mov r5, $rounds @ pass # of rounds
1389 mov sp, r12 @ sp is $keysched
1390 bl _bsaes_key_convert
1391 veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
1392 vstmia r12, {@XMM[7]} @ save last round key
1393
1394 vld1.8 {@XMM[0]}, [$ctr] @ load counter
1395 add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr
1396 vldmia $keysched, {@XMM[4]} @ load round0 key
1397#else
1398 ldr r12, [$key, #244]
1399 eors r12, #1
1400 beq 0f
1401
1402 @ populate the key schedule
1403 str r12, [$key, #244]
1404 mov r4, $key @ pass key
1405 mov r5, $rounds @ pass # of rounds
1406 add r12, $key, #248 @ pass key schedule
1407 bl _bsaes_key_convert
1408 veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
1409 vstmia r12, {@XMM[7]} @ save last round key
1410
1411.align 2
14120: add r12, $key, #248
1413 vld1.8 {@XMM[0]}, [$ctr] @ load counter
1414 adrl $ctr, .LREVM0SR @ borrow $ctr
1415 vldmia r12, {@XMM[4]} @ load round0 key
1416 sub sp, #0x10 @ place for adjusted round0 key
1417#endif
1418
1419 vmov.i32 @XMM[8],#1 @ compose 1<<96
1420 veor @XMM[9],@XMM[9],@XMM[9]
1421 vrev32.8 @XMM[0],@XMM[0]
1422 vext.8 @XMM[8],@XMM[9],@XMM[8],#4
1423 vrev32.8 @XMM[4],@XMM[4]
1424 vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96
1425 vstmia $keysched, {@XMM[4]} @ save adjusted round0 key
1426 b .Lctr_enc_loop
1427
1428.align 4
1429.Lctr_enc_loop:
1430 vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96
1431 vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1
1432 vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2
1433 vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3
1434 vadd.u32 @XMM[4], @XMM[1], @XMM[10]
1435 vadd.u32 @XMM[5], @XMM[2], @XMM[10]
1436 vadd.u32 @XMM[6], @XMM[3], @XMM[10]
1437 vadd.u32 @XMM[7], @XMM[4], @XMM[10]
1438 vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter
1439
1440 @ Borrow prologue from _bsaes_encrypt8 to use the opportunity
1441 @ to flip byte order in 32-bit counter
1442
1443 vldmia $keysched, {@XMM[9]} @ load round0 key
1444#ifndef BSAES_ASM_EXTENDED_KEY
1445 add r4, $keysched, #0x10 @ pass next round key
1446#else
1447 add r4, $key, #`248+16`
1448#endif
1449 vldmia $ctr, {@XMM[8]} @ .LREVM0SR
1450 mov r5, $rounds @ pass rounds
1451 vstmia $fp, {@XMM[10]} @ save next counter
1452 sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants
1453
1454 bl _bsaes_encrypt8_alt
1455
1456 subs $len, $len, #8
1457 blo .Lctr_enc_loop_done
1458
1459 vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input
1460 vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
1461 veor @XMM[0], @XMM[8]
1462 veor @XMM[1], @XMM[9]
1463 vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
1464 veor @XMM[4], @XMM[10]
1465 veor @XMM[6], @XMM[11]
1466 vld1.8 {@XMM[14]-@XMM[15]}, [$inp]!
1467 veor @XMM[3], @XMM[12]
1468 vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
1469 veor @XMM[7], @XMM[13]
1470 veor @XMM[2], @XMM[14]
1471 vst1.8 {@XMM[4]}, [$out]!
1472 veor @XMM[5], @XMM[15]
1473 vst1.8 {@XMM[6]}, [$out]!
1474 vmov.i32 @XMM[8], #1 @ compose 1<<96
1475 vst1.8 {@XMM[3]}, [$out]!
1476 veor @XMM[9], @XMM[9], @XMM[9]
1477 vst1.8 {@XMM[7]}, [$out]!
1478 vext.8 @XMM[8], @XMM[9], @XMM[8], #4
1479 vst1.8 {@XMM[2]}, [$out]!
1480 vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96
1481 vst1.8 {@XMM[5]}, [$out]!
1482 vldmia $fp, {@XMM[0]} @ load counter
1483
1484 bne .Lctr_enc_loop
1485 b .Lctr_enc_done
1486
1487.align 4
1488.Lctr_enc_loop_done:
1489 add $len, $len, #8
1490 vld1.8 {@XMM[8]}, [$inp]! @ load input
1491 veor @XMM[0], @XMM[8]
1492 vst1.8 {@XMM[0]}, [$out]! @ write output
1493 cmp $len, #2
1494 blo .Lctr_enc_done
1495 vld1.8 {@XMM[9]}, [$inp]!
1496 veor @XMM[1], @XMM[9]
1497 vst1.8 {@XMM[1]}, [$out]!
1498 beq .Lctr_enc_done
1499 vld1.8 {@XMM[10]}, [$inp]!
1500 veor @XMM[4], @XMM[10]
1501 vst1.8 {@XMM[4]}, [$out]!
1502 cmp $len, #4
1503 blo .Lctr_enc_done
1504 vld1.8 {@XMM[11]}, [$inp]!
1505 veor @XMM[6], @XMM[11]
1506 vst1.8 {@XMM[6]}, [$out]!
1507 beq .Lctr_enc_done
1508 vld1.8 {@XMM[12]}, [$inp]!
1509 veor @XMM[3], @XMM[12]
1510 vst1.8 {@XMM[3]}, [$out]!
1511 cmp $len, #6
1512 blo .Lctr_enc_done
1513 vld1.8 {@XMM[13]}, [$inp]!
1514 veor @XMM[7], @XMM[13]
1515 vst1.8 {@XMM[7]}, [$out]!
1516 beq .Lctr_enc_done
1517 vld1.8 {@XMM[14]}, [$inp]
1518 veor @XMM[2], @XMM[14]
1519 vst1.8 {@XMM[2]}, [$out]!
1520
1521.Lctr_enc_done:
1522 vmov.i32 q0, #0
1523 vmov.i32 q1, #0
1524#ifndef BSAES_ASM_EXTENDED_KEY
1525.Lctr_enc_bzero: @ wipe key schedule [if any]
1526 vstmia $keysched!, {q0-q1}
1527 cmp $keysched, $fp
1528 bne .Lctr_enc_bzero
1529#else
1530 vstmia $keysched, {q0-q1}
1531#endif
1532
1533 mov sp, $fp
1534 add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb
1535 VFP_ABI_POP
1536 ldmia sp!, {r4-r10, pc} @ return
1537
1538.align 4
1539.Lctr_enc_short:
1540 ldr ip, [sp] @ ctr pointer is passed on stack
1541 stmdb sp!, {r4-r8, lr}
1542
1543 mov r4, $inp @ copy arguments
1544 mov r5, $out
1545 mov r6, $len
1546 mov r7, $key
1547 ldr r8, [ip, #12] @ load counter LSW
1548 vld1.8 {@XMM[1]}, [ip] @ load whole counter value
1549#ifdef __ARMEL__
1550 rev r8, r8
1551#endif
1552 sub sp, sp, #0x10
1553 vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value
1554 sub sp, sp, #0x10
1555
1556.Lctr_enc_short_loop:
1557 add r0, sp, #0x10 @ input counter value
1558 mov r1, sp @ output on the stack
1559 mov r2, r7 @ key
1560
1561 bl AES_encrypt
1562
1563 vld1.8 {@XMM[0]}, [r4]! @ load input
1564 vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter
1565 add r8, r8, #1
1566#ifdef __ARMEL__
1567 rev r0, r8
1568 str r0, [sp, #0x1c] @ next counter value
1569#else
1570 str r8, [sp, #0x1c] @ next counter value
1571#endif
1572 veor @XMM[0],@XMM[0],@XMM[1]
1573 vst1.8 {@XMM[0]}, [r5]! @ store output
1574 subs r6, r6, #1
1575 bne .Lctr_enc_short_loop
1576
1577 vmov.i32 q0, #0
1578 vmov.i32 q1, #0
1579 vstmia sp!, {q0-q1}
1580
1581 ldmia sp!, {r4-r8, pc}
1582.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
1583___
1584}
1585{
1586######################################################################
1587# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len,
1588# const AES_KEY *key1, const AES_KEY *key2,
1589# const unsigned char iv[16]);
1590#
1591my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3)));
1592my $const="r6"; # returned by _bsaes_key_convert
1593my $twmask=@XMM[5];
1594my @T=@XMM[6..7];
1595
1596$code.=<<___;
1597.globl bsaes_xts_encrypt
1598.type bsaes_xts_encrypt,%function
1599.align 4
1600bsaes_xts_encrypt:
1601 mov ip, sp
1602 stmdb sp!, {r4-r10, lr} @ 0x20
1603 VFP_ABI_PUSH
1604 mov r6, sp @ future $fp
1605
1606 mov $inp, r0
1607 mov $out, r1
1608 mov $len, r2
1609 mov $key, r3
1610
1611 sub r0, sp, #0x10 @ 0x10
1612 bic r0, #0xf @ align at 16 bytes
1613 mov sp, r0
1614
1615#ifdef XTS_CHAIN_TWEAK
1616 ldr r0, [ip] @ pointer to input tweak
1617#else
1618 @ generate initial tweak
1619 ldr r0, [ip, #4] @ iv[]
1620 mov r1, sp
1621 ldr r2, [ip, #0] @ key2
1622 bl AES_encrypt
1623 mov r0,sp @ pointer to initial tweak
1624#endif
1625
1626 ldr $rounds, [$key, #240] @ get # of rounds
1627 mov $fp, r6
1628#ifndef BSAES_ASM_EXTENDED_KEY
1629 @ allocate the key schedule on the stack
1630 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
1631 @ add r12, #`128-32` @ size of bit-sliced key schedule
1632 sub r12, #`32+16` @ place for tweak[9]
1633
1634 @ populate the key schedule
1635 mov r4, $key @ pass key
1636 mov r5, $rounds @ pass # of rounds
1637 mov sp, r12
1638 add r12, #0x90 @ pass key schedule
1639 bl _bsaes_key_convert
1640 veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key
1641 vstmia r12, {@XMM[7]} @ save last round key
1642#else
1643 ldr r12, [$key, #244]
1644 eors r12, #1
1645 beq 0f
1646
1647 str r12, [$key, #244]
1648 mov r4, $key @ pass key
1649 mov r5, $rounds @ pass # of rounds
1650 add r12, $key, #248 @ pass key schedule
1651 bl _bsaes_key_convert
1652 veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key
1653 vstmia r12, {@XMM[7]}
1654
1655.align 2
16560: sub sp, #0x90 @ place for tweak[9]
1657#endif
1658
1659 vld1.8 {@XMM[8]}, [r0] @ initial tweak
1660 adr $magic, .Lxts_magic
1661
1662 subs $len, #0x80
1663 blo .Lxts_enc_short
1664 b .Lxts_enc_loop
1665
1666.align 4
1667.Lxts_enc_loop:
1668 vldmia $magic, {$twmask} @ load XTS magic
1669 vshr.s64 @T[0], @XMM[8], #63
1670 mov r0, sp
1671 vand @T[0], @T[0], $twmask
1672___
1673for($i=9;$i<16;$i++) {
1674$code.=<<___;
1675 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
1676 vst1.64 {@XMM[$i-1]}, [r0,:128]!
1677 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
1678 vshr.s64 @T[1], @XMM[$i], #63
1679 veor @XMM[$i], @XMM[$i], @T[0]
1680 vand @T[1], @T[1], $twmask
1681___
1682 @T=reverse(@T);
1683
1684$code.=<<___ if ($i>=10);
1685 vld1.8 {@XMM[$i-10]}, [$inp]!
1686___
1687$code.=<<___ if ($i>=11);
1688 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
1689___
1690}
1691$code.=<<___;
1692 vadd.u64 @XMM[8], @XMM[15], @XMM[15]
1693 vst1.64 {@XMM[15]}, [r0,:128]!
1694 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
1695 veor @XMM[8], @XMM[8], @T[0]
1696 vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1697
1698 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
1699 veor @XMM[5], @XMM[5], @XMM[13]
1700#ifndef BSAES_ASM_EXTENDED_KEY
1701 add r4, sp, #0x90 @ pass key schedule
1702#else
1703 add r4, $key, #248 @ pass key schedule
1704#endif
1705 veor @XMM[6], @XMM[6], @XMM[14]
1706 mov r5, $rounds @ pass rounds
1707 veor @XMM[7], @XMM[7], @XMM[15]
1708 mov r0, sp
1709
1710 bl _bsaes_encrypt8
1711
1712 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1713 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1714 veor @XMM[0], @XMM[0], @XMM[ 8]
1715 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
1716 veor @XMM[1], @XMM[1], @XMM[ 9]
1717 veor @XMM[8], @XMM[4], @XMM[10]
1718 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1719 veor @XMM[9], @XMM[6], @XMM[11]
1720 vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]!
1721 veor @XMM[10], @XMM[3], @XMM[12]
1722 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1723 veor @XMM[11], @XMM[7], @XMM[13]
1724 veor @XMM[12], @XMM[2], @XMM[14]
1725 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
1726 veor @XMM[13], @XMM[5], @XMM[15]
1727 vst1.8 {@XMM[12]-@XMM[13]}, [$out]!
1728
1729 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1730
1731 subs $len, #0x80
1732 bpl .Lxts_enc_loop
1733
1734.Lxts_enc_short:
1735 adds $len, #0x70
1736 bmi .Lxts_enc_done
1737
1738 vldmia $magic, {$twmask} @ load XTS magic
1739 vshr.s64 @T[0], @XMM[8], #63
1740 mov r0, sp
1741 vand @T[0], @T[0], $twmask
1742___
1743for($i=9;$i<16;$i++) {
1744$code.=<<___;
1745 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
1746 vst1.64 {@XMM[$i-1]}, [r0,:128]!
1747 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
1748 vshr.s64 @T[1], @XMM[$i], #63
1749 veor @XMM[$i], @XMM[$i], @T[0]
1750 vand @T[1], @T[1], $twmask
1751___
1752 @T=reverse(@T);
1753
1754$code.=<<___ if ($i>=10);
1755 vld1.8 {@XMM[$i-10]}, [$inp]!
1756 subs $len, #0x10
1757 bmi .Lxts_enc_`$i-9`
1758___
1759$code.=<<___ if ($i>=11);
1760 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
1761___
1762}
1763$code.=<<___;
1764 sub $len, #0x10
1765 vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak
1766
1767 vld1.8 {@XMM[6]}, [$inp]!
1768 veor @XMM[5], @XMM[5], @XMM[13]
1769#ifndef BSAES_ASM_EXTENDED_KEY
1770 add r4, sp, #0x90 @ pass key schedule
1771#else
1772 add r4, $key, #248 @ pass key schedule
1773#endif
1774 veor @XMM[6], @XMM[6], @XMM[14]
1775 mov r5, $rounds @ pass rounds
1776 mov r0, sp
1777
1778 bl _bsaes_encrypt8
1779
1780 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1781 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1782 veor @XMM[0], @XMM[0], @XMM[ 8]
1783 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
1784 veor @XMM[1], @XMM[1], @XMM[ 9]
1785 veor @XMM[8], @XMM[4], @XMM[10]
1786 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1787 veor @XMM[9], @XMM[6], @XMM[11]
1788 vld1.64 {@XMM[14]}, [r0,:128]!
1789 veor @XMM[10], @XMM[3], @XMM[12]
1790 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1791 veor @XMM[11], @XMM[7], @XMM[13]
1792 veor @XMM[12], @XMM[2], @XMM[14]
1793 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
1794 vst1.8 {@XMM[12]}, [$out]!
1795
1796 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1797 b .Lxts_enc_done
1798.align 4
1799.Lxts_enc_6:
1800 vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak
1801
1802 veor @XMM[4], @XMM[4], @XMM[12]
1803#ifndef BSAES_ASM_EXTENDED_KEY
1804 add r4, sp, #0x90 @ pass key schedule
1805#else
1806 add r4, $key, #248 @ pass key schedule
1807#endif
1808 veor @XMM[5], @XMM[5], @XMM[13]
1809 mov r5, $rounds @ pass rounds
1810 mov r0, sp
1811
1812 bl _bsaes_encrypt8
1813
1814 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1815 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1816 veor @XMM[0], @XMM[0], @XMM[ 8]
1817 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
1818 veor @XMM[1], @XMM[1], @XMM[ 9]
1819 veor @XMM[8], @XMM[4], @XMM[10]
1820 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1821 veor @XMM[9], @XMM[6], @XMM[11]
1822 veor @XMM[10], @XMM[3], @XMM[12]
1823 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1824 veor @XMM[11], @XMM[7], @XMM[13]
1825 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
1826
1827 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1828 b .Lxts_enc_done
1829
1830@ put this in range for both ARM and Thumb mode adr instructions
1831.align 5
1832.Lxts_magic:
1833 .quad 1, 0x87
1834
1835.align 5
1836.Lxts_enc_5:
1837 vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak
1838
1839 veor @XMM[3], @XMM[3], @XMM[11]
1840#ifndef BSAES_ASM_EXTENDED_KEY
1841 add r4, sp, #0x90 @ pass key schedule
1842#else
1843 add r4, $key, #248 @ pass key schedule
1844#endif
1845 veor @XMM[4], @XMM[4], @XMM[12]
1846 mov r5, $rounds @ pass rounds
1847 mov r0, sp
1848
1849 bl _bsaes_encrypt8
1850
1851 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1852 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1853 veor @XMM[0], @XMM[0], @XMM[ 8]
1854 vld1.64 {@XMM[12]}, [r0,:128]!
1855 veor @XMM[1], @XMM[1], @XMM[ 9]
1856 veor @XMM[8], @XMM[4], @XMM[10]
1857 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1858 veor @XMM[9], @XMM[6], @XMM[11]
1859 veor @XMM[10], @XMM[3], @XMM[12]
1860 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1861 vst1.8 {@XMM[10]}, [$out]!
1862
1863 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1864 b .Lxts_enc_done
1865.align 4
1866.Lxts_enc_4:
1867 vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak
1868
1869 veor @XMM[2], @XMM[2], @XMM[10]
1870#ifndef BSAES_ASM_EXTENDED_KEY
1871 add r4, sp, #0x90 @ pass key schedule
1872#else
1873 add r4, $key, #248 @ pass key schedule
1874#endif
1875 veor @XMM[3], @XMM[3], @XMM[11]
1876 mov r5, $rounds @ pass rounds
1877 mov r0, sp
1878
1879 bl _bsaes_encrypt8
1880
1881 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
1882 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
1883 veor @XMM[0], @XMM[0], @XMM[ 8]
1884 veor @XMM[1], @XMM[1], @XMM[ 9]
1885 veor @XMM[8], @XMM[4], @XMM[10]
1886 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1887 veor @XMM[9], @XMM[6], @XMM[11]
1888 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
1889
1890 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1891 b .Lxts_enc_done
1892.align 4
1893.Lxts_enc_3:
1894 vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak
1895
1896 veor @XMM[1], @XMM[1], @XMM[9]
1897#ifndef BSAES_ASM_EXTENDED_KEY
1898 add r4, sp, #0x90 @ pass key schedule
1899#else
1900 add r4, $key, #248 @ pass key schedule
1901#endif
1902 veor @XMM[2], @XMM[2], @XMM[10]
1903 mov r5, $rounds @ pass rounds
1904 mov r0, sp
1905
1906 bl _bsaes_encrypt8
1907
1908 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
1909 vld1.64 {@XMM[10]}, [r0,:128]!
1910 veor @XMM[0], @XMM[0], @XMM[ 8]
1911 veor @XMM[1], @XMM[1], @XMM[ 9]
1912 veor @XMM[8], @XMM[4], @XMM[10]
1913 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1914 vst1.8 {@XMM[8]}, [$out]!
1915
1916 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1917 b .Lxts_enc_done
1918.align 4
1919.Lxts_enc_2:
1920 vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak
1921
1922 veor @XMM[0], @XMM[0], @XMM[8]
1923#ifndef BSAES_ASM_EXTENDED_KEY
1924 add r4, sp, #0x90 @ pass key schedule
1925#else
1926 add r4, $key, #248 @ pass key schedule
1927#endif
1928 veor @XMM[1], @XMM[1], @XMM[9]
1929 mov r5, $rounds @ pass rounds
1930 mov r0, sp
1931
1932 bl _bsaes_encrypt8
1933
1934 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
1935 veor @XMM[0], @XMM[0], @XMM[ 8]
1936 veor @XMM[1], @XMM[1], @XMM[ 9]
1937 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
1938
1939 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
1940 b .Lxts_enc_done
1941.align 4
1942.Lxts_enc_1:
1943 mov r0, sp
1944 veor @XMM[0], @XMM[8]
1945 mov r1, sp
1946 vst1.8 {@XMM[0]}, [sp,:128]
1947 mov r2, $key
1948 mov r4, $fp @ preserve fp
1949
1950 bl AES_encrypt
1951
1952 vld1.8 {@XMM[0]}, [sp,:128]
1953 veor @XMM[0], @XMM[0], @XMM[8]
1954 vst1.8 {@XMM[0]}, [$out]!
1955 mov $fp, r4
1956
1957 vmov @XMM[8], @XMM[9] @ next round tweak
1958
1959.Lxts_enc_done:
1960#ifndef XTS_CHAIN_TWEAK
1961 adds $len, #0x10
1962 beq .Lxts_enc_ret
1963 sub r6, $out, #0x10
1964
1965.Lxts_enc_steal:
1966 ldrb r0, [$inp], #1
1967 ldrb r1, [$out, #-0x10]
1968 strb r0, [$out, #-0x10]
1969 strb r1, [$out], #1
1970
1971 subs $len, #1
1972 bhi .Lxts_enc_steal
1973
1974 vld1.8 {@XMM[0]}, [r6]
1975 mov r0, sp
1976 veor @XMM[0], @XMM[0], @XMM[8]
1977 mov r1, sp
1978 vst1.8 {@XMM[0]}, [sp,:128]
1979 mov r2, $key
1980 mov r4, $fp @ preserve fp
1981
1982 bl AES_encrypt
1983
1984 vld1.8 {@XMM[0]}, [sp,:128]
1985 veor @XMM[0], @XMM[0], @XMM[8]
1986 vst1.8 {@XMM[0]}, [r6]
1987 mov $fp, r4
1988#endif
1989
1990.Lxts_enc_ret:
1991 bic r0, $fp, #0xf
1992 vmov.i32 q0, #0
1993 vmov.i32 q1, #0
1994#ifdef XTS_CHAIN_TWEAK
1995 ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak
1996#endif
1997.Lxts_enc_bzero: @ wipe key schedule [if any]
1998 vstmia sp!, {q0-q1}
1999 cmp sp, r0
2000 bne .Lxts_enc_bzero
2001
2002 mov sp, $fp
2003#ifdef XTS_CHAIN_TWEAK
2004 vst1.8 {@XMM[8]}, [r1]
2005#endif
2006 VFP_ABI_POP
2007 ldmia sp!, {r4-r10, pc} @ return
2008
2009.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
2010
2011.globl bsaes_xts_decrypt
2012.type bsaes_xts_decrypt,%function
2013.align 4
2014bsaes_xts_decrypt:
2015 mov ip, sp
2016 stmdb sp!, {r4-r10, lr} @ 0x20
2017 VFP_ABI_PUSH
2018 mov r6, sp @ future $fp
2019
2020 mov $inp, r0
2021 mov $out, r1
2022 mov $len, r2
2023 mov $key, r3
2024
2025 sub r0, sp, #0x10 @ 0x10
2026 bic r0, #0xf @ align at 16 bytes
2027 mov sp, r0
2028
2029#ifdef XTS_CHAIN_TWEAK
2030 ldr r0, [ip] @ pointer to input tweak
2031#else
2032 @ generate initial tweak
2033 ldr r0, [ip, #4] @ iv[]
2034 mov r1, sp
2035 ldr r2, [ip, #0] @ key2
2036 bl AES_encrypt
2037 mov r0, sp @ pointer to initial tweak
2038#endif
2039
2040 ldr $rounds, [$key, #240] @ get # of rounds
2041 mov $fp, r6
2042#ifndef BSAES_ASM_EXTENDED_KEY
2043 @ allocate the key schedule on the stack
2044 sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
2045 @ add r12, #`128-32` @ size of bit-sliced key schedule
2046 sub r12, #`32+16` @ place for tweak[9]
2047
2048 @ populate the key schedule
2049 mov r4, $key @ pass key
2050 mov r5, $rounds @ pass # of rounds
2051 mov sp, r12
2052 add r12, #0x90 @ pass key schedule
2053 bl _bsaes_key_convert
2054 add r4, sp, #0x90
2055 vldmia r4, {@XMM[6]}
2056 vstmia r12, {@XMM[15]} @ save last round key
2057 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
2058 vstmia r4, {@XMM[7]}
2059#else
2060 ldr r12, [$key, #244]
2061 eors r12, #1
2062 beq 0f
2063
2064 str r12, [$key, #244]
2065 mov r4, $key @ pass key
2066 mov r5, $rounds @ pass # of rounds
2067 add r12, $key, #248 @ pass key schedule
2068 bl _bsaes_key_convert
2069 add r4, $key, #248
2070 vldmia r4, {@XMM[6]}
2071 vstmia r12, {@XMM[15]} @ save last round key
2072 veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
2073 vstmia r4, {@XMM[7]}
2074
2075.align 2
20760: sub sp, #0x90 @ place for tweak[9]
2077#endif
2078 vld1.8 {@XMM[8]}, [r0] @ initial tweak
2079 adr $magic, .Lxts_magic
2080
2081#ifndef XTS_CHAIN_TWEAK
2082 tst $len, #0xf @ if not multiple of 16
2083 it ne @ Thumb2 thing, sanity check in ARM
2084 subne $len, #0x10 @ subtract another 16 bytes
2085#endif
2086 subs $len, #0x80
2087
2088 blo .Lxts_dec_short
2089 b .Lxts_dec_loop
2090
2091.align 4
2092.Lxts_dec_loop:
2093 vldmia $magic, {$twmask} @ load XTS magic
2094 vshr.s64 @T[0], @XMM[8], #63
2095 mov r0, sp
2096 vand @T[0], @T[0], $twmask
2097___
2098for($i=9;$i<16;$i++) {
2099$code.=<<___;
2100 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
2101 vst1.64 {@XMM[$i-1]}, [r0,:128]!
2102 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
2103 vshr.s64 @T[1], @XMM[$i], #63
2104 veor @XMM[$i], @XMM[$i], @T[0]
2105 vand @T[1], @T[1], $twmask
2106___
2107 @T=reverse(@T);
2108
2109$code.=<<___ if ($i>=10);
2110 vld1.8 {@XMM[$i-10]}, [$inp]!
2111___
2112$code.=<<___ if ($i>=11);
2113 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
2114___
2115}
2116$code.=<<___;
2117 vadd.u64 @XMM[8], @XMM[15], @XMM[15]
2118 vst1.64 {@XMM[15]}, [r0,:128]!
2119 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
2120 veor @XMM[8], @XMM[8], @T[0]
2121 vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2122
2123 vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
2124 veor @XMM[5], @XMM[5], @XMM[13]
2125#ifndef BSAES_ASM_EXTENDED_KEY
2126 add r4, sp, #0x90 @ pass key schedule
2127#else
2128 add r4, $key, #248 @ pass key schedule
2129#endif
2130 veor @XMM[6], @XMM[6], @XMM[14]
2131 mov r5, $rounds @ pass rounds
2132 veor @XMM[7], @XMM[7], @XMM[15]
2133 mov r0, sp
2134
2135 bl _bsaes_decrypt8
2136
2137 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2138 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2139 veor @XMM[0], @XMM[0], @XMM[ 8]
2140 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
2141 veor @XMM[1], @XMM[1], @XMM[ 9]
2142 veor @XMM[8], @XMM[6], @XMM[10]
2143 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2144 veor @XMM[9], @XMM[4], @XMM[11]
2145 vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]!
2146 veor @XMM[10], @XMM[2], @XMM[12]
2147 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2148 veor @XMM[11], @XMM[7], @XMM[13]
2149 veor @XMM[12], @XMM[3], @XMM[14]
2150 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
2151 veor @XMM[13], @XMM[5], @XMM[15]
2152 vst1.8 {@XMM[12]-@XMM[13]}, [$out]!
2153
2154 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2155
2156 subs $len, #0x80
2157 bpl .Lxts_dec_loop
2158
2159.Lxts_dec_short:
2160 adds $len, #0x70
2161 bmi .Lxts_dec_done
2162
2163 vldmia $magic, {$twmask} @ load XTS magic
2164 vshr.s64 @T[0], @XMM[8], #63
2165 mov r0, sp
2166 vand @T[0], @T[0], $twmask
2167___
2168for($i=9;$i<16;$i++) {
2169$code.=<<___;
2170 vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
2171 vst1.64 {@XMM[$i-1]}, [r0,:128]!
2172 vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
2173 vshr.s64 @T[1], @XMM[$i], #63
2174 veor @XMM[$i], @XMM[$i], @T[0]
2175 vand @T[1], @T[1], $twmask
2176___
2177 @T=reverse(@T);
2178
2179$code.=<<___ if ($i>=10);
2180 vld1.8 {@XMM[$i-10]}, [$inp]!
2181 subs $len, #0x10
2182 bmi .Lxts_dec_`$i-9`
2183___
2184$code.=<<___ if ($i>=11);
2185 veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
2186___
2187}
2188$code.=<<___;
2189 sub $len, #0x10
2190 vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak
2191
2192 vld1.8 {@XMM[6]}, [$inp]!
2193 veor @XMM[5], @XMM[5], @XMM[13]
2194#ifndef BSAES_ASM_EXTENDED_KEY
2195 add r4, sp, #0x90 @ pass key schedule
2196#else
2197 add r4, $key, #248 @ pass key schedule
2198#endif
2199 veor @XMM[6], @XMM[6], @XMM[14]
2200 mov r5, $rounds @ pass rounds
2201 mov r0, sp
2202
2203 bl _bsaes_decrypt8
2204
2205 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2206 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2207 veor @XMM[0], @XMM[0], @XMM[ 8]
2208 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
2209 veor @XMM[1], @XMM[1], @XMM[ 9]
2210 veor @XMM[8], @XMM[6], @XMM[10]
2211 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2212 veor @XMM[9], @XMM[4], @XMM[11]
2213 vld1.64 {@XMM[14]}, [r0,:128]!
2214 veor @XMM[10], @XMM[2], @XMM[12]
2215 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2216 veor @XMM[11], @XMM[7], @XMM[13]
2217 veor @XMM[12], @XMM[3], @XMM[14]
2218 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
2219 vst1.8 {@XMM[12]}, [$out]!
2220
2221 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2222 b .Lxts_dec_done
2223.align 4
2224.Lxts_dec_6:
2225 vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak
2226
2227 veor @XMM[4], @XMM[4], @XMM[12]
2228#ifndef BSAES_ASM_EXTENDED_KEY
2229 add r4, sp, #0x90 @ pass key schedule
2230#else
2231 add r4, $key, #248 @ pass key schedule
2232#endif
2233 veor @XMM[5], @XMM[5], @XMM[13]
2234 mov r5, $rounds @ pass rounds
2235 mov r0, sp
2236
2237 bl _bsaes_decrypt8
2238
2239 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2240 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2241 veor @XMM[0], @XMM[0], @XMM[ 8]
2242 vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
2243 veor @XMM[1], @XMM[1], @XMM[ 9]
2244 veor @XMM[8], @XMM[6], @XMM[10]
2245 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2246 veor @XMM[9], @XMM[4], @XMM[11]
2247 veor @XMM[10], @XMM[2], @XMM[12]
2248 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2249 veor @XMM[11], @XMM[7], @XMM[13]
2250 vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
2251
2252 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2253 b .Lxts_dec_done
2254.align 4
2255.Lxts_dec_5:
2256 vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak
2257
2258 veor @XMM[3], @XMM[3], @XMM[11]
2259#ifndef BSAES_ASM_EXTENDED_KEY
2260 add r4, sp, #0x90 @ pass key schedule
2261#else
2262 add r4, $key, #248 @ pass key schedule
2263#endif
2264 veor @XMM[4], @XMM[4], @XMM[12]
2265 mov r5, $rounds @ pass rounds
2266 mov r0, sp
2267
2268 bl _bsaes_decrypt8
2269
2270 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2271 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2272 veor @XMM[0], @XMM[0], @XMM[ 8]
2273 vld1.64 {@XMM[12]}, [r0,:128]!
2274 veor @XMM[1], @XMM[1], @XMM[ 9]
2275 veor @XMM[8], @XMM[6], @XMM[10]
2276 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2277 veor @XMM[9], @XMM[4], @XMM[11]
2278 veor @XMM[10], @XMM[2], @XMM[12]
2279 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2280 vst1.8 {@XMM[10]}, [$out]!
2281
2282 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2283 b .Lxts_dec_done
2284.align 4
2285.Lxts_dec_4:
2286 vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak
2287
2288 veor @XMM[2], @XMM[2], @XMM[10]
2289#ifndef BSAES_ASM_EXTENDED_KEY
2290 add r4, sp, #0x90 @ pass key schedule
2291#else
2292 add r4, $key, #248 @ pass key schedule
2293#endif
2294 veor @XMM[3], @XMM[3], @XMM[11]
2295 mov r5, $rounds @ pass rounds
2296 mov r0, sp
2297
2298 bl _bsaes_decrypt8
2299
2300 vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
2301 vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
2302 veor @XMM[0], @XMM[0], @XMM[ 8]
2303 veor @XMM[1], @XMM[1], @XMM[ 9]
2304 veor @XMM[8], @XMM[6], @XMM[10]
2305 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2306 veor @XMM[9], @XMM[4], @XMM[11]
2307 vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
2308
2309 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2310 b .Lxts_dec_done
2311.align 4
2312.Lxts_dec_3:
2313 vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak
2314
2315 veor @XMM[1], @XMM[1], @XMM[9]
2316#ifndef BSAES_ASM_EXTENDED_KEY
2317 add r4, sp, #0x90 @ pass key schedule
2318#else
2319 add r4, $key, #248 @ pass key schedule
2320#endif
2321 veor @XMM[2], @XMM[2], @XMM[10]
2322 mov r5, $rounds @ pass rounds
2323 mov r0, sp
2324
2325 bl _bsaes_decrypt8
2326
2327 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
2328 vld1.64 {@XMM[10]}, [r0,:128]!
2329 veor @XMM[0], @XMM[0], @XMM[ 8]
2330 veor @XMM[1], @XMM[1], @XMM[ 9]
2331 veor @XMM[8], @XMM[6], @XMM[10]
2332 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2333 vst1.8 {@XMM[8]}, [$out]!
2334
2335 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2336 b .Lxts_dec_done
2337.align 4
2338.Lxts_dec_2:
2339 vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak
2340
2341 veor @XMM[0], @XMM[0], @XMM[8]
2342#ifndef BSAES_ASM_EXTENDED_KEY
2343 add r4, sp, #0x90 @ pass key schedule
2344#else
2345 add r4, $key, #248 @ pass key schedule
2346#endif
2347 veor @XMM[1], @XMM[1], @XMM[9]
2348 mov r5, $rounds @ pass rounds
2349 mov r0, sp
2350
2351 bl _bsaes_decrypt8
2352
2353 vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
2354 veor @XMM[0], @XMM[0], @XMM[ 8]
2355 veor @XMM[1], @XMM[1], @XMM[ 9]
2356 vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
2357
2358 vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
2359 b .Lxts_dec_done
2360.align 4
2361.Lxts_dec_1:
2362 mov r0, sp
2363 veor @XMM[0], @XMM[8]
2364 mov r1, sp
2365 vst1.8 {@XMM[0]}, [sp,:128]
2366 mov r2, $key
2367 mov r4, $fp @ preserve fp
2368 mov r5, $magic @ preserve magic
2369
2370 bl AES_decrypt
2371
2372 vld1.8 {@XMM[0]}, [sp,:128]
2373 veor @XMM[0], @XMM[0], @XMM[8]
2374 vst1.8 {@XMM[0]}, [$out]!
2375 mov $fp, r4
2376 mov $magic, r5
2377
2378 vmov @XMM[8], @XMM[9] @ next round tweak
2379
2380.Lxts_dec_done:
2381#ifndef XTS_CHAIN_TWEAK
2382 adds $len, #0x10
2383 beq .Lxts_dec_ret
2384
2385 @ calculate one round of extra tweak for the stolen ciphertext
2386 vldmia $magic, {$twmask}
2387 vshr.s64 @XMM[6], @XMM[8], #63
2388 vand @XMM[6], @XMM[6], $twmask
2389 vadd.u64 @XMM[9], @XMM[8], @XMM[8]
2390 vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")`
2391 veor @XMM[9], @XMM[9], @XMM[6]
2392
2393 @ perform the final decryption with the last tweak value
2394 vld1.8 {@XMM[0]}, [$inp]!
2395 mov r0, sp
2396 veor @XMM[0], @XMM[0], @XMM[9]
2397 mov r1, sp
2398 vst1.8 {@XMM[0]}, [sp,:128]
2399 mov r2, $key
2400 mov r4, $fp @ preserve fp
2401
2402 bl AES_decrypt
2403
2404 vld1.8 {@XMM[0]}, [sp,:128]
2405 veor @XMM[0], @XMM[0], @XMM[9]
2406 vst1.8 {@XMM[0]}, [$out]
2407
2408 mov r6, $out
2409.Lxts_dec_steal:
2410 ldrb r1, [$out]
2411 ldrb r0, [$inp], #1
2412 strb r1, [$out, #0x10]
2413 strb r0, [$out], #1
2414
2415 subs $len, #1
2416 bhi .Lxts_dec_steal
2417
2418 vld1.8 {@XMM[0]}, [r6]
2419 mov r0, sp
2420 veor @XMM[0], @XMM[8]
2421 mov r1, sp
2422 vst1.8 {@XMM[0]}, [sp,:128]
2423 mov r2, $key
2424
2425 bl AES_decrypt
2426
2427 vld1.8 {@XMM[0]}, [sp,:128]
2428 veor @XMM[0], @XMM[0], @XMM[8]
2429 vst1.8 {@XMM[0]}, [r6]
2430 mov $fp, r4
2431#endif
2432
2433.Lxts_dec_ret:
2434 bic r0, $fp, #0xf
2435 vmov.i32 q0, #0
2436 vmov.i32 q1, #0
2437#ifdef XTS_CHAIN_TWEAK
2438 ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak
2439#endif
2440.Lxts_dec_bzero: @ wipe key schedule [if any]
2441 vstmia sp!, {q0-q1}
2442 cmp sp, r0
2443 bne .Lxts_dec_bzero
2444
2445 mov sp, $fp
2446#ifdef XTS_CHAIN_TWEAK
2447 vst1.8 {@XMM[8]}, [r1]
2448#endif
2449 VFP_ABI_POP
2450 ldmia sp!, {r4-r10, pc} @ return
2451
2452.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
2453___
2454}
2455$code.=<<___;
2456#endif
2457___
2458
2459$code =~ s/\`([^\`]*)\`/eval($1)/gem;
2460
2461open SELF,$0;
2462while(<SELF>) {
2463 next if (/^#!/);
2464 last if (!s/^#/@/ and !/^$/);
2465 print;
2466}
2467close SELF;
2468
2469print $code;
2470
2471close STDOUT;
diff --git a/arch/arm/crypto/chacha20-neon-core.S b/arch/arm/crypto/chacha20-neon-core.S
new file mode 100644
index 000000000000..3fecb2124c35
--- /dev/null
+++ b/arch/arm/crypto/chacha20-neon-core.S
@@ -0,0 +1,523 @@
1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
3 *
4 * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Based on:
11 * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSE3 functions
12 *
13 * Copyright (C) 2015 Martin Willi
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 */
20
21#include <linux/linkage.h>
22
23 .text
24 .fpu neon
25 .align 5
26
27ENTRY(chacha20_block_xor_neon)
28 // r0: Input state matrix, s
29 // r1: 1 data block output, o
30 // r2: 1 data block input, i
31
32 //
33 // This function encrypts one ChaCha20 block by loading the state matrix
34 // in four NEON registers. It performs matrix operation on four words in
35 // parallel, but requireds shuffling to rearrange the words after each
36 // round.
37 //
38
39 // x0..3 = s0..3
40 add ip, r0, #0x20
41 vld1.32 {q0-q1}, [r0]
42 vld1.32 {q2-q3}, [ip]
43
44 vmov q8, q0
45 vmov q9, q1
46 vmov q10, q2
47 vmov q11, q3
48
49 mov r3, #10
50
51.Ldoubleround:
52 // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
53 vadd.i32 q0, q0, q1
54 veor q4, q3, q0
55 vshl.u32 q3, q4, #16
56 vsri.u32 q3, q4, #16
57
58 // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
59 vadd.i32 q2, q2, q3
60 veor q4, q1, q2
61 vshl.u32 q1, q4, #12
62 vsri.u32 q1, q4, #20
63
64 // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
65 vadd.i32 q0, q0, q1
66 veor q4, q3, q0
67 vshl.u32 q3, q4, #8
68 vsri.u32 q3, q4, #24
69
70 // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
71 vadd.i32 q2, q2, q3
72 veor q4, q1, q2
73 vshl.u32 q1, q4, #7
74 vsri.u32 q1, q4, #25
75
76 // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
77 vext.8 q1, q1, q1, #4
78 // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
79 vext.8 q2, q2, q2, #8
80 // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
81 vext.8 q3, q3, q3, #12
82
83 // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
84 vadd.i32 q0, q0, q1
85 veor q4, q3, q0
86 vshl.u32 q3, q4, #16
87 vsri.u32 q3, q4, #16
88
89 // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
90 vadd.i32 q2, q2, q3
91 veor q4, q1, q2
92 vshl.u32 q1, q4, #12
93 vsri.u32 q1, q4, #20
94
95 // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
96 vadd.i32 q0, q0, q1
97 veor q4, q3, q0
98 vshl.u32 q3, q4, #8
99 vsri.u32 q3, q4, #24
100
101 // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
102 vadd.i32 q2, q2, q3
103 veor q4, q1, q2
104 vshl.u32 q1, q4, #7
105 vsri.u32 q1, q4, #25
106
107 // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
108 vext.8 q1, q1, q1, #12
109 // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
110 vext.8 q2, q2, q2, #8
111 // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
112 vext.8 q3, q3, q3, #4
113
114 subs r3, r3, #1
115 bne .Ldoubleround
116
117 add ip, r2, #0x20
118 vld1.8 {q4-q5}, [r2]
119 vld1.8 {q6-q7}, [ip]
120
121 // o0 = i0 ^ (x0 + s0)
122 vadd.i32 q0, q0, q8
123 veor q0, q0, q4
124
125 // o1 = i1 ^ (x1 + s1)
126 vadd.i32 q1, q1, q9
127 veor q1, q1, q5
128
129 // o2 = i2 ^ (x2 + s2)
130 vadd.i32 q2, q2, q10
131 veor q2, q2, q6
132
133 // o3 = i3 ^ (x3 + s3)
134 vadd.i32 q3, q3, q11
135 veor q3, q3, q7
136
137 add ip, r1, #0x20
138 vst1.8 {q0-q1}, [r1]
139 vst1.8 {q2-q3}, [ip]
140
141 bx lr
142ENDPROC(chacha20_block_xor_neon)
143
144 .align 5
145ENTRY(chacha20_4block_xor_neon)
146 push {r4-r6, lr}
147 mov ip, sp // preserve the stack pointer
148 sub r3, sp, #0x20 // allocate a 32 byte buffer
149 bic r3, r3, #0x1f // aligned to 32 bytes
150 mov sp, r3
151
152 // r0: Input state matrix, s
153 // r1: 4 data blocks output, o
154 // r2: 4 data blocks input, i
155
156 //
157 // This function encrypts four consecutive ChaCha20 blocks by loading
158 // the state matrix in NEON registers four times. The algorithm performs
159 // each operation on the corresponding word of each state matrix, hence
160 // requires no word shuffling. For final XORing step we transpose the
161 // matrix by interleaving 32- and then 64-bit words, which allows us to
162 // do XOR in NEON registers.
163 //
164
165 // x0..15[0-3] = s0..3[0..3]
166 add r3, r0, #0x20
167 vld1.32 {q0-q1}, [r0]
168 vld1.32 {q2-q3}, [r3]
169
170 adr r3, CTRINC
171 vdup.32 q15, d7[1]
172 vdup.32 q14, d7[0]
173 vld1.32 {q11}, [r3, :128]
174 vdup.32 q13, d6[1]
175 vdup.32 q12, d6[0]
176 vadd.i32 q12, q12, q11 // x12 += counter values 0-3
177 vdup.32 q11, d5[1]
178 vdup.32 q10, d5[0]
179 vdup.32 q9, d4[1]
180 vdup.32 q8, d4[0]
181 vdup.32 q7, d3[1]
182 vdup.32 q6, d3[0]
183 vdup.32 q5, d2[1]
184 vdup.32 q4, d2[0]
185 vdup.32 q3, d1[1]
186 vdup.32 q2, d1[0]
187 vdup.32 q1, d0[1]
188 vdup.32 q0, d0[0]
189
190 mov r3, #10
191
192.Ldoubleround4:
193 // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
194 // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
195 // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
196 // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
197 vadd.i32 q0, q0, q4
198 vadd.i32 q1, q1, q5
199 vadd.i32 q2, q2, q6
200 vadd.i32 q3, q3, q7
201
202 veor q12, q12, q0
203 veor q13, q13, q1
204 veor q14, q14, q2
205 veor q15, q15, q3
206
207 vrev32.16 q12, q12
208 vrev32.16 q13, q13
209 vrev32.16 q14, q14
210 vrev32.16 q15, q15
211
212 // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
213 // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
214 // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
215 // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
216 vadd.i32 q8, q8, q12
217 vadd.i32 q9, q9, q13
218 vadd.i32 q10, q10, q14
219 vadd.i32 q11, q11, q15
220
221 vst1.32 {q8-q9}, [sp, :256]
222
223 veor q8, q4, q8
224 veor q9, q5, q9
225 vshl.u32 q4, q8, #12
226 vshl.u32 q5, q9, #12
227 vsri.u32 q4, q8, #20
228 vsri.u32 q5, q9, #20
229
230 veor q8, q6, q10
231 veor q9, q7, q11
232 vshl.u32 q6, q8, #12
233 vshl.u32 q7, q9, #12
234 vsri.u32 q6, q8, #20
235 vsri.u32 q7, q9, #20
236
237 // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
238 // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
239 // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
240 // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
241 vadd.i32 q0, q0, q4
242 vadd.i32 q1, q1, q5
243 vadd.i32 q2, q2, q6
244 vadd.i32 q3, q3, q7
245
246 veor q8, q12, q0
247 veor q9, q13, q1
248 vshl.u32 q12, q8, #8
249 vshl.u32 q13, q9, #8
250 vsri.u32 q12, q8, #24
251 vsri.u32 q13, q9, #24
252
253 veor q8, q14, q2
254 veor q9, q15, q3
255 vshl.u32 q14, q8, #8
256 vshl.u32 q15, q9, #8
257 vsri.u32 q14, q8, #24
258 vsri.u32 q15, q9, #24
259
260 vld1.32 {q8-q9}, [sp, :256]
261
262 // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
263 // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
264 // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
265 // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
266 vadd.i32 q8, q8, q12
267 vadd.i32 q9, q9, q13
268 vadd.i32 q10, q10, q14
269 vadd.i32 q11, q11, q15
270
271 vst1.32 {q8-q9}, [sp, :256]
272
273 veor q8, q4, q8
274 veor q9, q5, q9
275 vshl.u32 q4, q8, #7
276 vshl.u32 q5, q9, #7
277 vsri.u32 q4, q8, #25
278 vsri.u32 q5, q9, #25
279
280 veor q8, q6, q10
281 veor q9, q7, q11
282 vshl.u32 q6, q8, #7
283 vshl.u32 q7, q9, #7
284 vsri.u32 q6, q8, #25
285 vsri.u32 q7, q9, #25
286
287 vld1.32 {q8-q9}, [sp, :256]
288
289 // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
290 // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
291 // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
292 // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
293 vadd.i32 q0, q0, q5
294 vadd.i32 q1, q1, q6
295 vadd.i32 q2, q2, q7
296 vadd.i32 q3, q3, q4
297
298 veor q15, q15, q0
299 veor q12, q12, q1
300 veor q13, q13, q2
301 veor q14, q14, q3
302
303 vrev32.16 q15, q15
304 vrev32.16 q12, q12
305 vrev32.16 q13, q13
306 vrev32.16 q14, q14
307
308 // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
309 // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
310 // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
311 // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
312 vadd.i32 q10, q10, q15
313 vadd.i32 q11, q11, q12
314 vadd.i32 q8, q8, q13
315 vadd.i32 q9, q9, q14
316
317 vst1.32 {q8-q9}, [sp, :256]
318
319 veor q8, q7, q8
320 veor q9, q4, q9
321 vshl.u32 q7, q8, #12
322 vshl.u32 q4, q9, #12
323 vsri.u32 q7, q8, #20
324 vsri.u32 q4, q9, #20
325
326 veor q8, q5, q10
327 veor q9, q6, q11
328 vshl.u32 q5, q8, #12
329 vshl.u32 q6, q9, #12
330 vsri.u32 q5, q8, #20
331 vsri.u32 q6, q9, #20
332
333 // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
334 // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
335 // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
336 // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
337 vadd.i32 q0, q0, q5
338 vadd.i32 q1, q1, q6
339 vadd.i32 q2, q2, q7
340 vadd.i32 q3, q3, q4
341
342 veor q8, q15, q0
343 veor q9, q12, q1
344 vshl.u32 q15, q8, #8
345 vshl.u32 q12, q9, #8
346 vsri.u32 q15, q8, #24
347 vsri.u32 q12, q9, #24
348
349 veor q8, q13, q2
350 veor q9, q14, q3
351 vshl.u32 q13, q8, #8
352 vshl.u32 q14, q9, #8
353 vsri.u32 q13, q8, #24
354 vsri.u32 q14, q9, #24
355
356 vld1.32 {q8-q9}, [sp, :256]
357
358 // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
359 // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
360 // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
361 // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
362 vadd.i32 q10, q10, q15
363 vadd.i32 q11, q11, q12
364 vadd.i32 q8, q8, q13
365 vadd.i32 q9, q9, q14
366
367 vst1.32 {q8-q9}, [sp, :256]
368
369 veor q8, q7, q8
370 veor q9, q4, q9
371 vshl.u32 q7, q8, #7
372 vshl.u32 q4, q9, #7
373 vsri.u32 q7, q8, #25
374 vsri.u32 q4, q9, #25
375
376 veor q8, q5, q10
377 veor q9, q6, q11
378 vshl.u32 q5, q8, #7
379 vshl.u32 q6, q9, #7
380 vsri.u32 q5, q8, #25
381 vsri.u32 q6, q9, #25
382
383 subs r3, r3, #1
384 beq 0f
385
386 vld1.32 {q8-q9}, [sp, :256]
387 b .Ldoubleround4
388
389 // x0[0-3] += s0[0]
390 // x1[0-3] += s0[1]
391 // x2[0-3] += s0[2]
392 // x3[0-3] += s0[3]
3930: ldmia r0!, {r3-r6}
394 vdup.32 q8, r3
395 vdup.32 q9, r4
396 vadd.i32 q0, q0, q8
397 vadd.i32 q1, q1, q9
398 vdup.32 q8, r5
399 vdup.32 q9, r6
400 vadd.i32 q2, q2, q8
401 vadd.i32 q3, q3, q9
402
403 // x4[0-3] += s1[0]
404 // x5[0-3] += s1[1]
405 // x6[0-3] += s1[2]
406 // x7[0-3] += s1[3]
407 ldmia r0!, {r3-r6}
408 vdup.32 q8, r3
409 vdup.32 q9, r4
410 vadd.i32 q4, q4, q8
411 vadd.i32 q5, q5, q9
412 vdup.32 q8, r5
413 vdup.32 q9, r6
414 vadd.i32 q6, q6, q8
415 vadd.i32 q7, q7, q9
416
417 // interleave 32-bit words in state n, n+1
418 vzip.32 q0, q1
419 vzip.32 q2, q3
420 vzip.32 q4, q5
421 vzip.32 q6, q7
422
423 // interleave 64-bit words in state n, n+2
424 vswp d1, d4
425 vswp d3, d6
426 vswp d9, d12
427 vswp d11, d14
428
429 // xor with corresponding input, write to output
430 vld1.8 {q8-q9}, [r2]!
431 veor q8, q8, q0
432 veor q9, q9, q4
433 vst1.8 {q8-q9}, [r1]!
434
435 vld1.32 {q8-q9}, [sp, :256]
436
437 // x8[0-3] += s2[0]
438 // x9[0-3] += s2[1]
439 // x10[0-3] += s2[2]
440 // x11[0-3] += s2[3]
441 ldmia r0!, {r3-r6}
442 vdup.32 q0, r3
443 vdup.32 q4, r4
444 vadd.i32 q8, q8, q0
445 vadd.i32 q9, q9, q4
446 vdup.32 q0, r5
447 vdup.32 q4, r6
448 vadd.i32 q10, q10, q0
449 vadd.i32 q11, q11, q4
450
451 // x12[0-3] += s3[0]
452 // x13[0-3] += s3[1]
453 // x14[0-3] += s3[2]
454 // x15[0-3] += s3[3]
455 ldmia r0!, {r3-r6}
456 vdup.32 q0, r3
457 vdup.32 q4, r4
458 adr r3, CTRINC
459 vadd.i32 q12, q12, q0
460 vld1.32 {q0}, [r3, :128]
461 vadd.i32 q13, q13, q4
462 vadd.i32 q12, q12, q0 // x12 += counter values 0-3
463
464 vdup.32 q0, r5
465 vdup.32 q4, r6
466 vadd.i32 q14, q14, q0
467 vadd.i32 q15, q15, q4
468
469 // interleave 32-bit words in state n, n+1
470 vzip.32 q8, q9
471 vzip.32 q10, q11
472 vzip.32 q12, q13
473 vzip.32 q14, q15
474
475 // interleave 64-bit words in state n, n+2
476 vswp d17, d20
477 vswp d19, d22
478 vswp d25, d28
479 vswp d27, d30
480
481 vmov q4, q1
482
483 vld1.8 {q0-q1}, [r2]!
484 veor q0, q0, q8
485 veor q1, q1, q12
486 vst1.8 {q0-q1}, [r1]!
487
488 vld1.8 {q0-q1}, [r2]!
489 veor q0, q0, q2
490 veor q1, q1, q6
491 vst1.8 {q0-q1}, [r1]!
492
493 vld1.8 {q0-q1}, [r2]!
494 veor q0, q0, q10
495 veor q1, q1, q14
496 vst1.8 {q0-q1}, [r1]!
497
498 vld1.8 {q0-q1}, [r2]!
499 veor q0, q0, q4
500 veor q1, q1, q5
501 vst1.8 {q0-q1}, [r1]!
502
503 vld1.8 {q0-q1}, [r2]!
504 veor q0, q0, q9
505 veor q1, q1, q13
506 vst1.8 {q0-q1}, [r1]!
507
508 vld1.8 {q0-q1}, [r2]!
509 veor q0, q0, q3
510 veor q1, q1, q7
511 vst1.8 {q0-q1}, [r1]!
512
513 vld1.8 {q0-q1}, [r2]
514 veor q0, q0, q11
515 veor q1, q1, q15
516 vst1.8 {q0-q1}, [r1]
517
518 mov sp, ip
519 pop {r4-r6, pc}
520ENDPROC(chacha20_4block_xor_neon)
521
522 .align 4
523CTRINC: .word 0, 1, 2, 3
diff --git a/arch/arm/crypto/chacha20-neon-glue.c b/arch/arm/crypto/chacha20-neon-glue.c
new file mode 100644
index 000000000000..59a7be08e80c
--- /dev/null
+++ b/arch/arm/crypto/chacha20-neon-glue.c
@@ -0,0 +1,127 @@
1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
3 *
4 * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Based on:
11 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
12 *
13 * Copyright (C) 2015 Martin Willi
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 */
20
21#include <crypto/algapi.h>
22#include <crypto/chacha20.h>
23#include <crypto/internal/skcipher.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26
27#include <asm/hwcap.h>
28#include <asm/neon.h>
29#include <asm/simd.h>
30
31asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
32asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
33
34static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
35 unsigned int bytes)
36{
37 u8 buf[CHACHA20_BLOCK_SIZE];
38
39 while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
40 chacha20_4block_xor_neon(state, dst, src);
41 bytes -= CHACHA20_BLOCK_SIZE * 4;
42 src += CHACHA20_BLOCK_SIZE * 4;
43 dst += CHACHA20_BLOCK_SIZE * 4;
44 state[12] += 4;
45 }
46 while (bytes >= CHACHA20_BLOCK_SIZE) {
47 chacha20_block_xor_neon(state, dst, src);
48 bytes -= CHACHA20_BLOCK_SIZE;
49 src += CHACHA20_BLOCK_SIZE;
50 dst += CHACHA20_BLOCK_SIZE;
51 state[12]++;
52 }
53 if (bytes) {
54 memcpy(buf, src, bytes);
55 chacha20_block_xor_neon(state, buf, buf);
56 memcpy(dst, buf, bytes);
57 }
58}
59
60static int chacha20_neon(struct skcipher_request *req)
61{
62 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
63 struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
64 struct skcipher_walk walk;
65 u32 state[16];
66 int err;
67
68 if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
69 return crypto_chacha20_crypt(req);
70
71 err = skcipher_walk_virt(&walk, req, true);
72
73 crypto_chacha20_init(state, ctx, walk.iv);
74
75 kernel_neon_begin();
76 while (walk.nbytes > 0) {
77 unsigned int nbytes = walk.nbytes;
78
79 if (nbytes < walk.total)
80 nbytes = round_down(nbytes, walk.stride);
81
82 chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
83 nbytes);
84 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
85 }
86 kernel_neon_end();
87
88 return err;
89}
90
91static struct skcipher_alg alg = {
92 .base.cra_name = "chacha20",
93 .base.cra_driver_name = "chacha20-neon",
94 .base.cra_priority = 300,
95 .base.cra_blocksize = 1,
96 .base.cra_ctxsize = sizeof(struct chacha20_ctx),
97 .base.cra_module = THIS_MODULE,
98
99 .min_keysize = CHACHA20_KEY_SIZE,
100 .max_keysize = CHACHA20_KEY_SIZE,
101 .ivsize = CHACHA20_IV_SIZE,
102 .chunksize = CHACHA20_BLOCK_SIZE,
103 .walksize = 4 * CHACHA20_BLOCK_SIZE,
104 .setkey = crypto_chacha20_setkey,
105 .encrypt = chacha20_neon,
106 .decrypt = chacha20_neon,
107};
108
109static int __init chacha20_simd_mod_init(void)
110{
111 if (!(elf_hwcap & HWCAP_NEON))
112 return -ENODEV;
113
114 return crypto_register_skcipher(&alg);
115}
116
117static void __exit chacha20_simd_mod_fini(void)
118{
119 crypto_unregister_skcipher(&alg);
120}
121
122module_init(chacha20_simd_mod_init);
123module_exit(chacha20_simd_mod_fini);
124
125MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
126MODULE_LICENSE("GPL v2");
127MODULE_ALIAS_CRYPTO("chacha20");
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 33b744d54739..6fc6f5a2a6e5 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -516,4 +516,3 @@ CONFIG_CRYPTO_GHASH_ARM64_CE=y
516CONFIG_CRYPTO_AES_ARM64_CE_CCM=y 516CONFIG_CRYPTO_AES_ARM64_CE_CCM=y
517CONFIG_CRYPTO_AES_ARM64_CE_BLK=y 517CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
518# CONFIG_CRYPTO_AES_ARM64_NEON_BLK is not set 518# CONFIG_CRYPTO_AES_ARM64_NEON_BLK is not set
519CONFIG_CRYPTO_CRC32_ARM64=y
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 450a85df041a..d92293747d63 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -37,10 +37,14 @@ config CRYPTO_CRCT10DIF_ARM64_CE
37 select CRYPTO_HASH 37 select CRYPTO_HASH
38 38
39config CRYPTO_CRC32_ARM64_CE 39config CRYPTO_CRC32_ARM64_CE
40 tristate "CRC32 and CRC32C digest algorithms using PMULL instructions" 40 tristate "CRC32 and CRC32C digest algorithms using ARMv8 extensions"
41 depends on KERNEL_MODE_NEON && CRC32 41 depends on CRC32
42 select CRYPTO_HASH 42 select CRYPTO_HASH
43 43
44config CRYPTO_AES_ARM64
45 tristate "AES core cipher using scalar instructions"
46 select CRYPTO_AES
47
44config CRYPTO_AES_ARM64_CE 48config CRYPTO_AES_ARM64_CE
45 tristate "AES core cipher using ARMv8 Crypto Extensions" 49 tristate "AES core cipher using ARMv8 Crypto Extensions"
46 depends on ARM64 && KERNEL_MODE_NEON 50 depends on ARM64 && KERNEL_MODE_NEON
@@ -67,9 +71,17 @@ config CRYPTO_AES_ARM64_NEON_BLK
67 select CRYPTO_AES 71 select CRYPTO_AES
68 select CRYPTO_SIMD 72 select CRYPTO_SIMD
69 73
70config CRYPTO_CRC32_ARM64 74config CRYPTO_CHACHA20_NEON
71 tristate "CRC32 and CRC32C using optional ARMv8 instructions" 75 tristate "NEON accelerated ChaCha20 symmetric cipher"
72 depends on ARM64 76 depends on KERNEL_MODE_NEON
73 select CRYPTO_HASH 77 select CRYPTO_BLKCIPHER
78 select CRYPTO_CHACHA20
79
80config CRYPTO_AES_ARM64_BS
81 tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm"
82 depends on KERNEL_MODE_NEON
83 select CRYPTO_BLKCIPHER
84 select CRYPTO_AES_ARM64_NEON_BLK
85 select CRYPTO_SIMD
74 86
75endif 87endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index aa8888d7b744..b5edc5918c28 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -41,15 +41,20 @@ sha256-arm64-y := sha256-glue.o sha256-core.o
41obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o 41obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o
42sha512-arm64-y := sha512-glue.o sha512-core.o 42sha512-arm64-y := sha512-glue.o sha512-core.o
43 43
44obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
45chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
46
47obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
48aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
49
50obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o
51aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
52
44AFLAGS_aes-ce.o := -DINTERLEAVE=4 53AFLAGS_aes-ce.o := -DINTERLEAVE=4
45AFLAGS_aes-neon.o := -DINTERLEAVE=4 54AFLAGS_aes-neon.o := -DINTERLEAVE=4
46 55
47CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS 56CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS
48 57
49obj-$(CONFIG_CRYPTO_CRC32_ARM64) += crc32-arm64.o
50
51CFLAGS_crc32-arm64.o := -mcpu=generic+crc
52
53$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE 58$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE
54 $(call if_changed_rule,cc_o_c) 59 $(call if_changed_rule,cc_o_c)
55 60
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
index cc5515dac74a..6a7dbc7c83a6 100644
--- a/arch/arm64/crypto/aes-ce-ccm-glue.c
+++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
@@ -258,7 +258,6 @@ static struct aead_alg ccm_aes_alg = {
258 .cra_priority = 300, 258 .cra_priority = 300,
259 .cra_blocksize = 1, 259 .cra_blocksize = 1,
260 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 260 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
261 .cra_alignmask = 7,
262 .cra_module = THIS_MODULE, 261 .cra_module = THIS_MODULE,
263 }, 262 },
264 .ivsize = AES_BLOCK_SIZE, 263 .ivsize = AES_BLOCK_SIZE,
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..f2f9cc519309
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -0,0 +1,110 @@
1/*
2 * Scalar AES core transform
3 *
4 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/linkage.h>
12#include <asm/assembler.h>
13
14 .text
15
16 rk .req x0
17 out .req x1
18 in .req x2
19 rounds .req x3
20 tt .req x4
21 lt .req x2
22
23 .macro __pair, enc, reg0, reg1, in0, in1e, in1d, shift
24 ubfx \reg0, \in0, #\shift, #8
25 .if \enc
26 ubfx \reg1, \in1e, #\shift, #8
27 .else
28 ubfx \reg1, \in1d, #\shift, #8
29 .endif
30 ldr \reg0, [tt, \reg0, uxtw #2]
31 ldr \reg1, [tt, \reg1, uxtw #2]
32 .endm
33
34 .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc
35 ldp \out0, \out1, [rk], #8
36
37 __pair \enc, w13, w14, \in0, \in1, \in3, 0
38 __pair \enc, w15, w16, \in1, \in2, \in0, 8
39 __pair \enc, w17, w18, \in2, \in3, \in1, 16
40 __pair \enc, \t0, \t1, \in3, \in0, \in2, 24
41
42 eor \out0, \out0, w13
43 eor \out1, \out1, w14
44 eor \out0, \out0, w15, ror #24
45 eor \out1, \out1, w16, ror #24
46 eor \out0, \out0, w17, ror #16
47 eor \out1, \out1, w18, ror #16
48 eor \out0, \out0, \t0, ror #8
49 eor \out1, \out1, \t1, ror #8
50 .endm
51
52 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
53 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
54 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
55 .endm
56
57 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
58 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
59 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
60 .endm
61
62 .macro do_crypt, round, ttab, ltab
63 ldp w5, w6, [in]
64 ldp w7, w8, [in, #8]
65 ldp w9, w10, [rk], #16
66 ldp w11, w12, [rk, #-8]
67
68CPU_BE( rev w5, w5 )
69CPU_BE( rev w6, w6 )
70CPU_BE( rev w7, w7 )
71CPU_BE( rev w8, w8 )
72
73 eor w5, w5, w9
74 eor w6, w6, w10
75 eor w7, w7, w11
76 eor w8, w8, w12
77
78 adr_l tt, \ttab
79 adr_l lt, \ltab
80
81 tbnz rounds, #1, 1f
82
830: \round w9, w10, w11, w12, w5, w6, w7, w8
84 \round w5, w6, w7, w8, w9, w10, w11, w12
85
861: subs rounds, rounds, #4
87 \round w9, w10, w11, w12, w5, w6, w7, w8
88 csel tt, tt, lt, hi
89 \round w5, w6, w7, w8, w9, w10, w11, w12
90 b.hi 0b
91
92CPU_BE( rev w5, w5 )
93CPU_BE( rev w6, w6 )
94CPU_BE( rev w7, w7 )
95CPU_BE( rev w8, w8 )
96
97 stp w5, w6, [out]
98 stp w7, w8, [out, #8]
99 ret
100 .endm
101
102 .align 5
103ENTRY(__aes_arm64_encrypt)
104 do_crypt fround, crypto_ft_tab, crypto_fl_tab
105ENDPROC(__aes_arm64_encrypt)
106
107 .align 5
108ENTRY(__aes_arm64_decrypt)
109 do_crypt iround, crypto_it_tab, crypto_il_tab
110ENDPROC(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-cipher-glue.c b/arch/arm64/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..7288e7cbebff
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-glue.c
@@ -0,0 +1,69 @@
1/*
2 * Scalar AES core transform
3 *
4 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <crypto/aes.h>
12#include <linux/crypto.h>
13#include <linux/module.h>
14
15asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
16EXPORT_SYMBOL(__aes_arm64_encrypt);
17
18asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
19EXPORT_SYMBOL(__aes_arm64_decrypt);
20
21static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
22{
23 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
24 int rounds = 6 + ctx->key_length / 4;
25
26 __aes_arm64_encrypt(ctx->key_enc, out, in, rounds);
27}
28
29static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
30{
31 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
32 int rounds = 6 + ctx->key_length / 4;
33
34 __aes_arm64_decrypt(ctx->key_dec, out, in, rounds);
35}
36
37static struct crypto_alg aes_alg = {
38 .cra_name = "aes",
39 .cra_driver_name = "aes-arm64",
40 .cra_priority = 200,
41 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
42 .cra_blocksize = AES_BLOCK_SIZE,
43 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
44 .cra_module = THIS_MODULE,
45
46 .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
47 .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
48 .cra_cipher.cia_setkey = crypto_aes_set_key,
49 .cra_cipher.cia_encrypt = aes_encrypt,
50 .cra_cipher.cia_decrypt = aes_decrypt
51};
52
53static int __init aes_init(void)
54{
55 return crypto_register_alg(&aes_alg);
56}
57
58static void __exit aes_fini(void)
59{
60 crypto_unregister_alg(&aes_alg);
61}
62
63module_init(aes_init);
64module_exit(aes_fini);
65
66MODULE_DESCRIPTION("Scalar AES cipher for arm64");
67MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
68MODULE_LICENSE("GPL v2");
69MODULE_ALIAS_CRYPTO("aes");
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 4e3f8adb1793..bcf596b0197e 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES 2 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
3 * 3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -11,6 +11,7 @@
11#include <asm/neon.h> 11#include <asm/neon.h>
12#include <asm/hwcap.h> 12#include <asm/hwcap.h>
13#include <crypto/aes.h> 13#include <crypto/aes.h>
14#include <crypto/internal/hash.h>
14#include <crypto/internal/simd.h> 15#include <crypto/internal/simd.h>
15#include <crypto/internal/skcipher.h> 16#include <crypto/internal/skcipher.h>
16#include <linux/module.h> 17#include <linux/module.h>
@@ -31,6 +32,7 @@
31#define aes_ctr_encrypt ce_aes_ctr_encrypt 32#define aes_ctr_encrypt ce_aes_ctr_encrypt
32#define aes_xts_encrypt ce_aes_xts_encrypt 33#define aes_xts_encrypt ce_aes_xts_encrypt
33#define aes_xts_decrypt ce_aes_xts_decrypt 34#define aes_xts_decrypt ce_aes_xts_decrypt
35#define aes_mac_update ce_aes_mac_update
34MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 36MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
35#else 37#else
36#define MODE "neon" 38#define MODE "neon"
@@ -44,11 +46,15 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
44#define aes_ctr_encrypt neon_aes_ctr_encrypt 46#define aes_ctr_encrypt neon_aes_ctr_encrypt
45#define aes_xts_encrypt neon_aes_xts_encrypt 47#define aes_xts_encrypt neon_aes_xts_encrypt
46#define aes_xts_decrypt neon_aes_xts_decrypt 48#define aes_xts_decrypt neon_aes_xts_decrypt
49#define aes_mac_update neon_aes_mac_update
47MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); 50MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
48MODULE_ALIAS_CRYPTO("ecb(aes)"); 51MODULE_ALIAS_CRYPTO("ecb(aes)");
49MODULE_ALIAS_CRYPTO("cbc(aes)"); 52MODULE_ALIAS_CRYPTO("cbc(aes)");
50MODULE_ALIAS_CRYPTO("ctr(aes)"); 53MODULE_ALIAS_CRYPTO("ctr(aes)");
51MODULE_ALIAS_CRYPTO("xts(aes)"); 54MODULE_ALIAS_CRYPTO("xts(aes)");
55MODULE_ALIAS_CRYPTO("cmac(aes)");
56MODULE_ALIAS_CRYPTO("xcbc(aes)");
57MODULE_ALIAS_CRYPTO("cbcmac(aes)");
52#endif 58#endif
53 59
54MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 60MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
@@ -75,11 +81,25 @@ asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
75 int rounds, int blocks, u8 const rk2[], u8 iv[], 81 int rounds, int blocks, u8 const rk2[], u8 iv[],
76 int first); 82 int first);
77 83
84asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
85 int blocks, u8 dg[], int enc_before,
86 int enc_after);
87
78struct crypto_aes_xts_ctx { 88struct crypto_aes_xts_ctx {
79 struct crypto_aes_ctx key1; 89 struct crypto_aes_ctx key1;
80 struct crypto_aes_ctx __aligned(8) key2; 90 struct crypto_aes_ctx __aligned(8) key2;
81}; 91};
82 92
93struct mac_tfm_ctx {
94 struct crypto_aes_ctx key;
95 u8 __aligned(8) consts[];
96};
97
98struct mac_desc_ctx {
99 unsigned int len;
100 u8 dg[AES_BLOCK_SIZE];
101};
102
83static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 103static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
84 unsigned int key_len) 104 unsigned int key_len)
85{ 105{
@@ -215,14 +235,15 @@ static int ctr_encrypt(struct skcipher_request *req)
215 u8 *tsrc = walk.src.virt.addr; 235 u8 *tsrc = walk.src.virt.addr;
216 236
217 /* 237 /*
218 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need 238 * Tell aes_ctr_encrypt() to process a tail block.
219 * to tell aes_ctr_encrypt() to only read half a block.
220 */ 239 */
221 blocks = (nbytes <= 8) ? -1 : 1; 240 blocks = -1;
222 241
223 aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds, 242 aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
224 blocks, walk.iv, first); 243 blocks, walk.iv, first);
225 memcpy(tdst, tail, nbytes); 244 if (tdst != tsrc)
245 memcpy(tdst, tsrc, nbytes);
246 crypto_xor(tdst, tail, nbytes);
226 err = skcipher_walk_done(&walk, 0); 247 err = skcipher_walk_done(&walk, 0);
227 } 248 }
228 kernel_neon_end(); 249 kernel_neon_end();
@@ -282,7 +303,6 @@ static struct skcipher_alg aes_algs[] = { {
282 .cra_flags = CRYPTO_ALG_INTERNAL, 303 .cra_flags = CRYPTO_ALG_INTERNAL,
283 .cra_blocksize = AES_BLOCK_SIZE, 304 .cra_blocksize = AES_BLOCK_SIZE,
284 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 305 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
285 .cra_alignmask = 7,
286 .cra_module = THIS_MODULE, 306 .cra_module = THIS_MODULE,
287 }, 307 },
288 .min_keysize = AES_MIN_KEY_SIZE, 308 .min_keysize = AES_MIN_KEY_SIZE,
@@ -298,7 +318,6 @@ static struct skcipher_alg aes_algs[] = { {
298 .cra_flags = CRYPTO_ALG_INTERNAL, 318 .cra_flags = CRYPTO_ALG_INTERNAL,
299 .cra_blocksize = AES_BLOCK_SIZE, 319 .cra_blocksize = AES_BLOCK_SIZE,
300 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 320 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
301 .cra_alignmask = 7,
302 .cra_module = THIS_MODULE, 321 .cra_module = THIS_MODULE,
303 }, 322 },
304 .min_keysize = AES_MIN_KEY_SIZE, 323 .min_keysize = AES_MIN_KEY_SIZE,
@@ -315,7 +334,22 @@ static struct skcipher_alg aes_algs[] = { {
315 .cra_flags = CRYPTO_ALG_INTERNAL, 334 .cra_flags = CRYPTO_ALG_INTERNAL,
316 .cra_blocksize = 1, 335 .cra_blocksize = 1,
317 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 336 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
318 .cra_alignmask = 7, 337 .cra_module = THIS_MODULE,
338 },
339 .min_keysize = AES_MIN_KEY_SIZE,
340 .max_keysize = AES_MAX_KEY_SIZE,
341 .ivsize = AES_BLOCK_SIZE,
342 .chunksize = AES_BLOCK_SIZE,
343 .setkey = skcipher_aes_setkey,
344 .encrypt = ctr_encrypt,
345 .decrypt = ctr_encrypt,
346}, {
347 .base = {
348 .cra_name = "ctr(aes)",
349 .cra_driver_name = "ctr-aes-" MODE,
350 .cra_priority = PRIO - 1,
351 .cra_blocksize = 1,
352 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
319 .cra_module = THIS_MODULE, 353 .cra_module = THIS_MODULE,
320 }, 354 },
321 .min_keysize = AES_MIN_KEY_SIZE, 355 .min_keysize = AES_MIN_KEY_SIZE,
@@ -333,7 +367,6 @@ static struct skcipher_alg aes_algs[] = { {
333 .cra_flags = CRYPTO_ALG_INTERNAL, 367 .cra_flags = CRYPTO_ALG_INTERNAL,
334 .cra_blocksize = AES_BLOCK_SIZE, 368 .cra_blocksize = AES_BLOCK_SIZE,
335 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 369 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
336 .cra_alignmask = 7,
337 .cra_module = THIS_MODULE, 370 .cra_module = THIS_MODULE,
338 }, 371 },
339 .min_keysize = 2 * AES_MIN_KEY_SIZE, 372 .min_keysize = 2 * AES_MIN_KEY_SIZE,
@@ -344,15 +377,228 @@ static struct skcipher_alg aes_algs[] = { {
344 .decrypt = xts_decrypt, 377 .decrypt = xts_decrypt,
345} }; 378} };
346 379
380static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
381 unsigned int key_len)
382{
383 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
384 int err;
385
386 err = aes_expandkey(&ctx->key, in_key, key_len);
387 if (err)
388 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
389
390 return err;
391}
392
393static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
394{
395 u64 a = be64_to_cpu(x->a);
396 u64 b = be64_to_cpu(x->b);
397
398 y->a = cpu_to_be64((a << 1) | (b >> 63));
399 y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
400}
401
402static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
403 unsigned int key_len)
404{
405 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
406 be128 *consts = (be128 *)ctx->consts;
407 u8 *rk = (u8 *)ctx->key.key_enc;
408 int rounds = 6 + key_len / 4;
409 int err;
410
411 err = cbcmac_setkey(tfm, in_key, key_len);
412 if (err)
413 return err;
414
415 /* encrypt the zero vector */
416 kernel_neon_begin();
417 aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1, 1);
418 kernel_neon_end();
419
420 cmac_gf128_mul_by_x(consts, consts);
421 cmac_gf128_mul_by_x(consts + 1, consts);
422
423 return 0;
424}
425
426static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
427 unsigned int key_len)
428{
429 static u8 const ks[3][AES_BLOCK_SIZE] = {
430 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
431 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
432 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
433 };
434
435 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
436 u8 *rk = (u8 *)ctx->key.key_enc;
437 int rounds = 6 + key_len / 4;
438 u8 key[AES_BLOCK_SIZE];
439 int err;
440
441 err = cbcmac_setkey(tfm, in_key, key_len);
442 if (err)
443 return err;
444
445 kernel_neon_begin();
446 aes_ecb_encrypt(key, ks[0], rk, rounds, 1, 1);
447 aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2, 0);
448 kernel_neon_end();
449
450 return cbcmac_setkey(tfm, key, sizeof(key));
451}
452
453static int mac_init(struct shash_desc *desc)
454{
455 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
456
457 memset(ctx->dg, 0, AES_BLOCK_SIZE);
458 ctx->len = 0;
459
460 return 0;
461}
462
463static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
464{
465 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
466 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
467 int rounds = 6 + tctx->key.key_length / 4;
468
469 while (len > 0) {
470 unsigned int l;
471
472 if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
473 (ctx->len + len) > AES_BLOCK_SIZE) {
474
475 int blocks = len / AES_BLOCK_SIZE;
476
477 len %= AES_BLOCK_SIZE;
478
479 kernel_neon_begin();
480 aes_mac_update(p, tctx->key.key_enc, rounds, blocks,
481 ctx->dg, (ctx->len != 0), (len != 0));
482 kernel_neon_end();
483
484 p += blocks * AES_BLOCK_SIZE;
485
486 if (!len) {
487 ctx->len = AES_BLOCK_SIZE;
488 break;
489 }
490 ctx->len = 0;
491 }
492
493 l = min(len, AES_BLOCK_SIZE - ctx->len);
494
495 if (l <= AES_BLOCK_SIZE) {
496 crypto_xor(ctx->dg + ctx->len, p, l);
497 ctx->len += l;
498 len -= l;
499 p += l;
500 }
501 }
502
503 return 0;
504}
505
506static int cbcmac_final(struct shash_desc *desc, u8 *out)
507{
508 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
509 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
510 int rounds = 6 + tctx->key.key_length / 4;
511
512 kernel_neon_begin();
513 aes_mac_update(NULL, tctx->key.key_enc, rounds, 0, ctx->dg, 1, 0);
514 kernel_neon_end();
515
516 memcpy(out, ctx->dg, AES_BLOCK_SIZE);
517
518 return 0;
519}
520
521static int cmac_final(struct shash_desc *desc, u8 *out)
522{
523 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
524 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
525 int rounds = 6 + tctx->key.key_length / 4;
526 u8 *consts = tctx->consts;
527
528 if (ctx->len != AES_BLOCK_SIZE) {
529 ctx->dg[ctx->len] ^= 0x80;
530 consts += AES_BLOCK_SIZE;
531 }
532
533 kernel_neon_begin();
534 aes_mac_update(consts, tctx->key.key_enc, rounds, 1, ctx->dg, 0, 1);
535 kernel_neon_end();
536
537 memcpy(out, ctx->dg, AES_BLOCK_SIZE);
538
539 return 0;
540}
541
542static struct shash_alg mac_algs[] = { {
543 .base.cra_name = "cmac(aes)",
544 .base.cra_driver_name = "cmac-aes-" MODE,
545 .base.cra_priority = PRIO,
546 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
547 .base.cra_blocksize = AES_BLOCK_SIZE,
548 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) +
549 2 * AES_BLOCK_SIZE,
550 .base.cra_module = THIS_MODULE,
551
552 .digestsize = AES_BLOCK_SIZE,
553 .init = mac_init,
554 .update = mac_update,
555 .final = cmac_final,
556 .setkey = cmac_setkey,
557 .descsize = sizeof(struct mac_desc_ctx),
558}, {
559 .base.cra_name = "xcbc(aes)",
560 .base.cra_driver_name = "xcbc-aes-" MODE,
561 .base.cra_priority = PRIO,
562 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
563 .base.cra_blocksize = AES_BLOCK_SIZE,
564 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) +
565 2 * AES_BLOCK_SIZE,
566 .base.cra_module = THIS_MODULE,
567
568 .digestsize = AES_BLOCK_SIZE,
569 .init = mac_init,
570 .update = mac_update,
571 .final = cmac_final,
572 .setkey = xcbc_setkey,
573 .descsize = sizeof(struct mac_desc_ctx),
574}, {
575 .base.cra_name = "cbcmac(aes)",
576 .base.cra_driver_name = "cbcmac-aes-" MODE,
577 .base.cra_priority = PRIO,
578 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
579 .base.cra_blocksize = 1,
580 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx),
581 .base.cra_module = THIS_MODULE,
582
583 .digestsize = AES_BLOCK_SIZE,
584 .init = mac_init,
585 .update = mac_update,
586 .final = cbcmac_final,
587 .setkey = cbcmac_setkey,
588 .descsize = sizeof(struct mac_desc_ctx),
589} };
590
347static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 591static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
348 592
349static void aes_exit(void) 593static void aes_exit(void)
350{ 594{
351 int i; 595 int i;
352 596
353 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++) 597 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
354 simd_skcipher_free(aes_simd_algs[i]); 598 if (aes_simd_algs[i])
599 simd_skcipher_free(aes_simd_algs[i]);
355 600
601 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
356 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 602 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
357} 603}
358 604
@@ -369,7 +615,14 @@ static int __init aes_init(void)
369 if (err) 615 if (err)
370 return err; 616 return err;
371 617
618 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
619 if (err)
620 goto unregister_ciphers;
621
372 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 622 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
623 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
624 continue;
625
373 algname = aes_algs[i].base.cra_name + 2; 626 algname = aes_algs[i].base.cra_name + 2;
374 drvname = aes_algs[i].base.cra_driver_name + 2; 627 drvname = aes_algs[i].base.cra_driver_name + 2;
375 basename = aes_algs[i].base.cra_driver_name; 628 basename = aes_algs[i].base.cra_driver_name;
@@ -385,6 +638,8 @@ static int __init aes_init(void)
385 638
386unregister_simds: 639unregister_simds:
387 aes_exit(); 640 aes_exit();
641unregister_ciphers:
642 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
388 return err; 643 return err;
389} 644}
390 645
@@ -392,5 +647,7 @@ unregister_simds:
392module_cpu_feature_match(AES, aes_init); 647module_cpu_feature_match(AES, aes_init);
393#else 648#else
394module_init(aes_init); 649module_init(aes_init);
650EXPORT_SYMBOL(neon_aes_ecb_encrypt);
651EXPORT_SYMBOL(neon_aes_cbc_encrypt);
395#endif 652#endif
396module_exit(aes_exit); 653module_exit(aes_exit);
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index 838dad5c209f..2674d43d1384 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -1,7 +1,7 @@
1/* 1/*
2 * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES 2 * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
3 * 3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -337,7 +337,7 @@ AES_ENTRY(aes_ctr_encrypt)
337 337
338.Lctrcarrydone: 338.Lctrcarrydone:
339 subs w4, w4, #1 339 subs w4, w4, #1
340 bmi .Lctrhalfblock /* blocks < 0 means 1/2 block */ 340 bmi .Lctrtailblock /* blocks <0 means tail block */
341 ld1 {v3.16b}, [x1], #16 341 ld1 {v3.16b}, [x1], #16
342 eor v3.16b, v0.16b, v3.16b 342 eor v3.16b, v0.16b, v3.16b
343 st1 {v3.16b}, [x0], #16 343 st1 {v3.16b}, [x0], #16
@@ -348,10 +348,8 @@ AES_ENTRY(aes_ctr_encrypt)
348 FRAME_POP 348 FRAME_POP
349 ret 349 ret
350 350
351.Lctrhalfblock: 351.Lctrtailblock:
352 ld1 {v3.8b}, [x1] 352 st1 {v0.16b}, [x0]
353 eor v3.8b, v0.8b, v3.8b
354 st1 {v3.8b}, [x0]
355 FRAME_POP 353 FRAME_POP
356 ret 354 ret
357 355
@@ -527,3 +525,30 @@ AES_ENTRY(aes_xts_decrypt)
527 FRAME_POP 525 FRAME_POP
528 ret 526 ret
529AES_ENDPROC(aes_xts_decrypt) 527AES_ENDPROC(aes_xts_decrypt)
528
529 /*
530 * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
531 * int blocks, u8 dg[], int enc_before, int enc_after)
532 */
533AES_ENTRY(aes_mac_update)
534 ld1 {v0.16b}, [x4] /* get dg */
535 enc_prepare w2, x1, x7
536 cbnz w5, .Lmacenc
537
538.Lmacloop:
539 cbz w3, .Lmacout
540 ld1 {v1.16b}, [x0], #16 /* get next pt block */
541 eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
542
543 subs w3, w3, #1
544 csinv x5, x6, xzr, eq
545 cbz w5, .Lmacout
546
547.Lmacenc:
548 encrypt_block v0, w2, x1, x7, w8
549 b .Lmacloop
550
551.Lmacout:
552 st1 {v0.16b}, [x4] /* return dg */
553 ret
554AES_ENDPROC(aes_mac_update)
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
index 85f07ead7c5c..f1e3aa2732f9 100644
--- a/arch/arm64/crypto/aes-neon.S
+++ b/arch/arm64/crypto/aes-neon.S
@@ -1,7 +1,7 @@
1/* 1/*
2 * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON 2 * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON
3 * 3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> 4 * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -17,17 +17,25 @@
17 /* multiply by polynomial 'x' in GF(2^8) */ 17 /* multiply by polynomial 'x' in GF(2^8) */
18 .macro mul_by_x, out, in, temp, const 18 .macro mul_by_x, out, in, temp, const
19 sshr \temp, \in, #7 19 sshr \temp, \in, #7
20 add \out, \in, \in 20 shl \out, \in, #1
21 and \temp, \temp, \const 21 and \temp, \temp, \const
22 eor \out, \out, \temp 22 eor \out, \out, \temp
23 .endm 23 .endm
24 24
25 /* multiply by polynomial 'x^2' in GF(2^8) */
26 .macro mul_by_x2, out, in, temp, const
27 ushr \temp, \in, #6
28 shl \out, \in, #2
29 pmul \temp, \temp, \const
30 eor \out, \out, \temp
31 .endm
32
25 /* preload the entire Sbox */ 33 /* preload the entire Sbox */
26 .macro prepare, sbox, shiftrows, temp 34 .macro prepare, sbox, shiftrows, temp
27 adr \temp, \sbox 35 adr \temp, \sbox
28 movi v12.16b, #0x40 36 movi v12.16b, #0x1b
29 ldr q13, \shiftrows 37 ldr q13, \shiftrows
30 movi v14.16b, #0x1b 38 ldr q14, .Lror32by8
31 ld1 {v16.16b-v19.16b}, [\temp], #64 39 ld1 {v16.16b-v19.16b}, [\temp], #64
32 ld1 {v20.16b-v23.16b}, [\temp], #64 40 ld1 {v20.16b-v23.16b}, [\temp], #64
33 ld1 {v24.16b-v27.16b}, [\temp], #64 41 ld1 {v24.16b-v27.16b}, [\temp], #64
@@ -50,37 +58,31 @@
50 58
51 /* apply SubBytes transformation using the the preloaded Sbox */ 59 /* apply SubBytes transformation using the the preloaded Sbox */
52 .macro sub_bytes, in 60 .macro sub_bytes, in
53 sub v9.16b, \in\().16b, v12.16b 61 sub v9.16b, \in\().16b, v15.16b
54 tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b 62 tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b
55 sub v10.16b, v9.16b, v12.16b 63 sub v10.16b, v9.16b, v15.16b
56 tbx \in\().16b, {v20.16b-v23.16b}, v9.16b 64 tbx \in\().16b, {v20.16b-v23.16b}, v9.16b
57 sub v11.16b, v10.16b, v12.16b 65 sub v11.16b, v10.16b, v15.16b
58 tbx \in\().16b, {v24.16b-v27.16b}, v10.16b 66 tbx \in\().16b, {v24.16b-v27.16b}, v10.16b
59 tbx \in\().16b, {v28.16b-v31.16b}, v11.16b 67 tbx \in\().16b, {v28.16b-v31.16b}, v11.16b
60 .endm 68 .endm
61 69
62 /* apply MixColumns transformation */ 70 /* apply MixColumns transformation */
63 .macro mix_columns, in 71 .macro mix_columns, in, enc
64 mul_by_x v10.16b, \in\().16b, v9.16b, v14.16b 72 .if \enc == 0
65 rev32 v8.8h, \in\().8h
66 eor \in\().16b, v10.16b, \in\().16b
67 shl v9.4s, v8.4s, #24
68 shl v11.4s, \in\().4s, #24
69 sri v9.4s, v8.4s, #8
70 sri v11.4s, \in\().4s, #8
71 eor v9.16b, v9.16b, v8.16b
72 eor v10.16b, v10.16b, v9.16b
73 eor \in\().16b, v10.16b, v11.16b
74 .endm
75
76 /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */ 73 /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
77 .macro inv_mix_columns, in 74 mul_by_x2 v8.16b, \in\().16b, v9.16b, v12.16b
78 mul_by_x v11.16b, \in\().16b, v10.16b, v14.16b 75 eor \in\().16b, \in\().16b, v8.16b
79 mul_by_x v11.16b, v11.16b, v10.16b, v14.16b 76 rev32 v8.8h, v8.8h
80 eor \in\().16b, \in\().16b, v11.16b 77 eor \in\().16b, \in\().16b, v8.16b
81 rev32 v11.8h, v11.8h 78 .endif
82 eor \in\().16b, \in\().16b, v11.16b 79
83 mix_columns \in 80 mul_by_x v9.16b, \in\().16b, v8.16b, v12.16b
81 rev32 v8.8h, \in\().8h
82 eor v8.16b, v8.16b, v9.16b
83 eor \in\().16b, \in\().16b, v8.16b
84 tbl \in\().16b, {\in\().16b}, v14.16b
85 eor \in\().16b, \in\().16b, v8.16b
84 .endm 86 .endm
85 87
86 .macro do_block, enc, in, rounds, rk, rkp, i 88 .macro do_block, enc, in, rounds, rk, rkp, i
@@ -88,16 +90,13 @@
88 add \rkp, \rk, #16 90 add \rkp, \rk, #16
89 mov \i, \rounds 91 mov \i, \rounds
901111: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ 921111: eor \in\().16b, \in\().16b, v15.16b /* ^round key */
93 movi v15.16b, #0x40
91 tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */ 94 tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */
92 sub_bytes \in 95 sub_bytes \in
93 ld1 {v15.4s}, [\rkp], #16
94 subs \i, \i, #1 96 subs \i, \i, #1
97 ld1 {v15.4s}, [\rkp], #16
95 beq 2222f 98 beq 2222f
96 .if \enc == 1 99 mix_columns \in, \enc
97 mix_columns \in
98 .else
99 inv_mix_columns \in
100 .endif
101 b 1111b 100 b 1111b
1022222: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ 1012222: eor \in\().16b, \in\().16b, v15.16b /* ^round key */
103 .endm 102 .endm
@@ -116,139 +115,114 @@
116 */ 115 */
117 116
118 .macro sub_bytes_2x, in0, in1 117 .macro sub_bytes_2x, in0, in1
119 sub v8.16b, \in0\().16b, v12.16b 118 sub v8.16b, \in0\().16b, v15.16b
120 sub v9.16b, \in1\().16b, v12.16b
121 tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b 119 tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
120 sub v9.16b, \in1\().16b, v15.16b
122 tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b 121 tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
123 sub v10.16b, v8.16b, v12.16b 122 sub v10.16b, v8.16b, v15.16b
124 sub v11.16b, v9.16b, v12.16b
125 tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b 123 tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b
124 sub v11.16b, v9.16b, v15.16b
126 tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b 125 tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b
127 sub v8.16b, v10.16b, v12.16b 126 sub v8.16b, v10.16b, v15.16b
128 sub v9.16b, v11.16b, v12.16b
129 tbx \in0\().16b, {v24.16b-v27.16b}, v10.16b 127 tbx \in0\().16b, {v24.16b-v27.16b}, v10.16b
128 sub v9.16b, v11.16b, v15.16b
130 tbx \in1\().16b, {v24.16b-v27.16b}, v11.16b 129 tbx \in1\().16b, {v24.16b-v27.16b}, v11.16b
131 tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b 130 tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b
132 tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b 131 tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b
133 .endm 132 .endm
134 133
135 .macro sub_bytes_4x, in0, in1, in2, in3 134 .macro sub_bytes_4x, in0, in1, in2, in3
136 sub v8.16b, \in0\().16b, v12.16b 135 sub v8.16b, \in0\().16b, v15.16b
137 tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b 136 tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
138 sub v9.16b, \in1\().16b, v12.16b 137 sub v9.16b, \in1\().16b, v15.16b
139 tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b 138 tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
140 sub v10.16b, \in2\().16b, v12.16b 139 sub v10.16b, \in2\().16b, v15.16b
141 tbl \in2\().16b, {v16.16b-v19.16b}, \in2\().16b 140 tbl \in2\().16b, {v16.16b-v19.16b}, \in2\().16b
142 sub v11.16b, \in3\().16b, v12.16b 141 sub v11.16b, \in3\().16b, v15.16b
143 tbl \in3\().16b, {v16.16b-v19.16b}, \in3\().16b 142 tbl \in3\().16b, {v16.16b-v19.16b}, \in3\().16b
144 tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b 143 tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b
145 tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b 144 tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b
146 sub v8.16b, v8.16b, v12.16b 145 sub v8.16b, v8.16b, v15.16b
147 tbx \in2\().16b, {v20.16b-v23.16b}, v10.16b 146 tbx \in2\().16b, {v20.16b-v23.16b}, v10.16b
148 sub v9.16b, v9.16b, v12.16b 147 sub v9.16b, v9.16b, v15.16b
149 tbx \in3\().16b, {v20.16b-v23.16b}, v11.16b 148 tbx \in3\().16b, {v20.16b-v23.16b}, v11.16b
150 sub v10.16b, v10.16b, v12.16b 149 sub v10.16b, v10.16b, v15.16b
151 tbx \in0\().16b, {v24.16b-v27.16b}, v8.16b 150 tbx \in0\().16b, {v24.16b-v27.16b}, v8.16b
152 sub v11.16b, v11.16b, v12.16b 151 sub v11.16b, v11.16b, v15.16b
153 tbx \in1\().16b, {v24.16b-v27.16b}, v9.16b 152 tbx \in1\().16b, {v24.16b-v27.16b}, v9.16b
154 sub v8.16b, v8.16b, v12.16b 153 sub v8.16b, v8.16b, v15.16b
155 tbx \in2\().16b, {v24.16b-v27.16b}, v10.16b 154 tbx \in2\().16b, {v24.16b-v27.16b}, v10.16b
156 sub v9.16b, v9.16b, v12.16b 155 sub v9.16b, v9.16b, v15.16b
157 tbx \in3\().16b, {v24.16b-v27.16b}, v11.16b 156 tbx \in3\().16b, {v24.16b-v27.16b}, v11.16b
158 sub v10.16b, v10.16b, v12.16b 157 sub v10.16b, v10.16b, v15.16b
159 tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b 158 tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b
160 sub v11.16b, v11.16b, v12.16b 159 sub v11.16b, v11.16b, v15.16b
161 tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b 160 tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b
162 tbx \in2\().16b, {v28.16b-v31.16b}, v10.16b 161 tbx \in2\().16b, {v28.16b-v31.16b}, v10.16b
163 tbx \in3\().16b, {v28.16b-v31.16b}, v11.16b 162 tbx \in3\().16b, {v28.16b-v31.16b}, v11.16b
164 .endm 163 .endm
165 164
166 .macro mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const 165 .macro mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
167 sshr \tmp0\().16b, \in0\().16b, #7 166 sshr \tmp0\().16b, \in0\().16b, #7
168 add \out0\().16b, \in0\().16b, \in0\().16b 167 shl \out0\().16b, \in0\().16b, #1
169 sshr \tmp1\().16b, \in1\().16b, #7 168 sshr \tmp1\().16b, \in1\().16b, #7
170 and \tmp0\().16b, \tmp0\().16b, \const\().16b 169 and \tmp0\().16b, \tmp0\().16b, \const\().16b
171 add \out1\().16b, \in1\().16b, \in1\().16b 170 shl \out1\().16b, \in1\().16b, #1
172 and \tmp1\().16b, \tmp1\().16b, \const\().16b 171 and \tmp1\().16b, \tmp1\().16b, \const\().16b
173 eor \out0\().16b, \out0\().16b, \tmp0\().16b 172 eor \out0\().16b, \out0\().16b, \tmp0\().16b
174 eor \out1\().16b, \out1\().16b, \tmp1\().16b 173 eor \out1\().16b, \out1\().16b, \tmp1\().16b
175 .endm 174 .endm
176 175
177 .macro mix_columns_2x, in0, in1 176 .macro mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
178 mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v14 177 ushr \tmp0\().16b, \in0\().16b, #6
179 rev32 v10.8h, \in0\().8h 178 shl \out0\().16b, \in0\().16b, #2
180 rev32 v11.8h, \in1\().8h 179 ushr \tmp1\().16b, \in1\().16b, #6
181 eor \in0\().16b, v8.16b, \in0\().16b 180 pmul \tmp0\().16b, \tmp0\().16b, \const\().16b
182 eor \in1\().16b, v9.16b, \in1\().16b 181 shl \out1\().16b, \in1\().16b, #2
183 shl v12.4s, v10.4s, #24 182 pmul \tmp1\().16b, \tmp1\().16b, \const\().16b
184 shl v13.4s, v11.4s, #24 183 eor \out0\().16b, \out0\().16b, \tmp0\().16b
185 eor v8.16b, v8.16b, v10.16b 184 eor \out1\().16b, \out1\().16b, \tmp1\().16b
186 sri v12.4s, v10.4s, #8
187 shl v10.4s, \in0\().4s, #24
188 eor v9.16b, v9.16b, v11.16b
189 sri v13.4s, v11.4s, #8
190 shl v11.4s, \in1\().4s, #24
191 sri v10.4s, \in0\().4s, #8
192 eor \in0\().16b, v8.16b, v12.16b
193 sri v11.4s, \in1\().4s, #8
194 eor \in1\().16b, v9.16b, v13.16b
195 eor \in0\().16b, v10.16b, \in0\().16b
196 eor \in1\().16b, v11.16b, \in1\().16b
197 .endm 185 .endm
198 186
199 .macro inv_mix_cols_2x, in0, in1 187 .macro mix_columns_2x, in0, in1, enc
200 mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v14 188 .if \enc == 0
201 mul_by_x_2x v8, v9, v8, v9, v10, v11, v14 189 /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
190 mul_by_x2_2x v8, v9, \in0, \in1, v10, v11, v12
202 eor \in0\().16b, \in0\().16b, v8.16b 191 eor \in0\().16b, \in0\().16b, v8.16b
203 eor \in1\().16b, \in1\().16b, v9.16b
204 rev32 v8.8h, v8.8h 192 rev32 v8.8h, v8.8h
205 rev32 v9.8h, v9.8h
206 eor \in0\().16b, \in0\().16b, v8.16b
207 eor \in1\().16b, \in1\().16b, v9.16b
208 mix_columns_2x \in0, \in1
209 .endm
210
211 .macro inv_mix_cols_4x, in0, in1, in2, in3
212 mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v14
213 mul_by_x_2x v10, v11, \in2, \in3, v12, v13, v14
214 mul_by_x_2x v8, v9, v8, v9, v12, v13, v14
215 mul_by_x_2x v10, v11, v10, v11, v12, v13, v14
216 eor \in0\().16b, \in0\().16b, v8.16b
217 eor \in1\().16b, \in1\().16b, v9.16b 193 eor \in1\().16b, \in1\().16b, v9.16b
218 eor \in2\().16b, \in2\().16b, v10.16b
219 eor \in3\().16b, \in3\().16b, v11.16b
220 rev32 v8.8h, v8.8h
221 rev32 v9.8h, v9.8h 194 rev32 v9.8h, v9.8h
222 rev32 v10.8h, v10.8h
223 rev32 v11.8h, v11.8h
224 eor \in0\().16b, \in0\().16b, v8.16b 195 eor \in0\().16b, \in0\().16b, v8.16b
225 eor \in1\().16b, \in1\().16b, v9.16b 196 eor \in1\().16b, \in1\().16b, v9.16b
226 eor \in2\().16b, \in2\().16b, v10.16b 197 .endif
227 eor \in3\().16b, \in3\().16b, v11.16b 198
228 mix_columns_2x \in0, \in1 199 mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12
229 mix_columns_2x \in2, \in3 200 rev32 v10.8h, \in0\().8h
201 rev32 v11.8h, \in1\().8h
202 eor v10.16b, v10.16b, v8.16b
203 eor v11.16b, v11.16b, v9.16b
204 eor \in0\().16b, \in0\().16b, v10.16b
205 eor \in1\().16b, \in1\().16b, v11.16b
206 tbl \in0\().16b, {\in0\().16b}, v14.16b
207 tbl \in1\().16b, {\in1\().16b}, v14.16b
208 eor \in0\().16b, \in0\().16b, v10.16b
209 eor \in1\().16b, \in1\().16b, v11.16b
230 .endm 210 .endm
231 211
232 .macro do_block_2x, enc, in0, in1 rounds, rk, rkp, i 212 .macro do_block_2x, enc, in0, in1, rounds, rk, rkp, i
233 ld1 {v15.4s}, [\rk] 213 ld1 {v15.4s}, [\rk]
234 add \rkp, \rk, #16 214 add \rkp, \rk, #16
235 mov \i, \rounds 215 mov \i, \rounds
2361111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ 2161111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
237 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ 217 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
238 sub_bytes_2x \in0, \in1 218 movi v15.16b, #0x40
239 tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ 219 tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */
240 tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ 220 tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */
241 ld1 {v15.4s}, [\rkp], #16 221 sub_bytes_2x \in0, \in1
242 subs \i, \i, #1 222 subs \i, \i, #1
223 ld1 {v15.4s}, [\rkp], #16
243 beq 2222f 224 beq 2222f
244 .if \enc == 1 225 mix_columns_2x \in0, \in1, \enc
245 mix_columns_2x \in0, \in1
246 ldr q13, .LForward_ShiftRows
247 .else
248 inv_mix_cols_2x \in0, \in1
249 ldr q13, .LReverse_ShiftRows
250 .endif
251 movi v12.16b, #0x40
252 b 1111b 226 b 1111b
2532222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ 2272222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
254 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ 228 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
@@ -262,23 +236,17 @@
262 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ 236 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
263 eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */ 237 eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */
264 eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ 238 eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */
265 sub_bytes_4x \in0, \in1, \in2, \in3 239 movi v15.16b, #0x40
266 tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ 240 tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */
267 tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ 241 tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */
268 tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */ 242 tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */
269 tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */ 243 tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */
270 ld1 {v15.4s}, [\rkp], #16 244 sub_bytes_4x \in0, \in1, \in2, \in3
271 subs \i, \i, #1 245 subs \i, \i, #1
246 ld1 {v15.4s}, [\rkp], #16
272 beq 2222f 247 beq 2222f
273 .if \enc == 1 248 mix_columns_2x \in0, \in1, \enc
274 mix_columns_2x \in0, \in1 249 mix_columns_2x \in2, \in3, \enc
275 mix_columns_2x \in2, \in3
276 ldr q13, .LForward_ShiftRows
277 .else
278 inv_mix_cols_4x \in0, \in1, \in2, \in3
279 ldr q13, .LReverse_ShiftRows
280 .endif
281 movi v12.16b, #0x40
282 b 1111b 250 b 1111b
2832222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ 2512222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
284 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ 252 eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
@@ -305,19 +273,7 @@
305#include "aes-modes.S" 273#include "aes-modes.S"
306 274
307 .text 275 .text
308 .align 4 276 .align 6
309.LForward_ShiftRows:
310CPU_LE( .byte 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3 )
311CPU_LE( .byte 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb )
312CPU_BE( .byte 0xb, 0x6, 0x1, 0xc, 0x7, 0x2, 0xd, 0x8 )
313CPU_BE( .byte 0x3, 0xe, 0x9, 0x4, 0xf, 0xa, 0x5, 0x0 )
314
315.LReverse_ShiftRows:
316CPU_LE( .byte 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb )
317CPU_LE( .byte 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3 )
318CPU_BE( .byte 0x3, 0x6, 0x9, 0xc, 0xf, 0x2, 0x5, 0x8 )
319CPU_BE( .byte 0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0 )
320
321.LForward_Sbox: 277.LForward_Sbox:
322 .byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 278 .byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
323 .byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76 279 .byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
@@ -385,3 +341,12 @@ CPU_BE( .byte 0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0 )
385 .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 341 .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
386 .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 342 .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
387 .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d 343 .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
344
345.LForward_ShiftRows:
346 .octa 0x0b06010c07020d08030e09040f0a0500
347
348.LReverse_ShiftRows:
349 .octa 0x0306090c0f0205080b0e0104070a0d00
350
351.Lror32by8:
352 .octa 0x0c0f0e0d080b0a090407060500030201
diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S
new file mode 100644
index 000000000000..ca0472500433
--- /dev/null
+++ b/arch/arm64/crypto/aes-neonbs-core.S
@@ -0,0 +1,972 @@
1/*
2 * Bit sliced AES using NEON instructions
3 *
4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11/*
12 * The algorithm implemented here is described in detail by the paper
13 * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
14 * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
15 *
16 * This implementation is based primarily on the OpenSSL implementation
17 * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
18 */
19
20#include <linux/linkage.h>
21#include <asm/assembler.h>
22
23 .text
24
25 rounds .req x11
26 bskey .req x12
27
28 .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
29 eor \b2, \b2, \b1
30 eor \b5, \b5, \b6
31 eor \b3, \b3, \b0
32 eor \b6, \b6, \b2
33 eor \b5, \b5, \b0
34 eor \b6, \b6, \b3
35 eor \b3, \b3, \b7
36 eor \b7, \b7, \b5
37 eor \b3, \b3, \b4
38 eor \b4, \b4, \b5
39 eor \b2, \b2, \b7
40 eor \b3, \b3, \b1
41 eor \b1, \b1, \b5
42 .endm
43
44 .macro out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
45 eor \b0, \b0, \b6
46 eor \b1, \b1, \b4
47 eor \b4, \b4, \b6
48 eor \b2, \b2, \b0
49 eor \b6, \b6, \b1
50 eor \b1, \b1, \b5
51 eor \b5, \b5, \b3
52 eor \b3, \b3, \b7
53 eor \b7, \b7, \b5
54 eor \b2, \b2, \b5
55 eor \b4, \b4, \b7
56 .endm
57
58 .macro inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
59 eor \b1, \b1, \b7
60 eor \b4, \b4, \b7
61 eor \b7, \b7, \b5
62 eor \b1, \b1, \b3
63 eor \b2, \b2, \b5
64 eor \b3, \b3, \b7
65 eor \b6, \b6, \b1
66 eor \b2, \b2, \b0
67 eor \b5, \b5, \b3
68 eor \b4, \b4, \b6
69 eor \b0, \b0, \b6
70 eor \b1, \b1, \b4
71 .endm
72
73 .macro inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
74 eor \b1, \b1, \b5
75 eor \b2, \b2, \b7
76 eor \b3, \b3, \b1
77 eor \b4, \b4, \b5
78 eor \b7, \b7, \b5
79 eor \b3, \b3, \b4
80 eor \b5, \b5, \b0
81 eor \b3, \b3, \b7
82 eor \b6, \b6, \b2
83 eor \b2, \b2, \b1
84 eor \b6, \b6, \b3
85 eor \b3, \b3, \b0
86 eor \b5, \b5, \b6
87 .endm
88
89 .macro mul_gf4, x0, x1, y0, y1, t0, t1
90 eor \t0, \y0, \y1
91 and \t0, \t0, \x0
92 eor \x0, \x0, \x1
93 and \t1, \x1, \y0
94 and \x0, \x0, \y1
95 eor \x1, \t1, \t0
96 eor \x0, \x0, \t1
97 .endm
98
99 .macro mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
100 eor \t0, \y0, \y1
101 eor \t1, \y2, \y3
102 and \t0, \t0, \x0
103 and \t1, \t1, \x2
104 eor \x0, \x0, \x1
105 eor \x2, \x2, \x3
106 and \x1, \x1, \y0
107 and \x3, \x3, \y2
108 and \x0, \x0, \y1
109 and \x2, \x2, \y3
110 eor \x1, \x1, \x0
111 eor \x2, \x2, \x3
112 eor \x0, \x0, \t0
113 eor \x3, \x3, \t1
114 .endm
115
116 .macro mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
117 y0, y1, y2, y3, t0, t1, t2, t3
118 eor \t0, \x0, \x2
119 eor \t1, \x1, \x3
120 mul_gf4 \x0, \x1, \y0, \y1, \t2, \t3
121 eor \y0, \y0, \y2
122 eor \y1, \y1, \y3
123 mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
124 eor \x0, \x0, \t0
125 eor \x2, \x2, \t0
126 eor \x1, \x1, \t1
127 eor \x3, \x3, \t1
128 eor \t0, \x4, \x6
129 eor \t1, \x5, \x7
130 mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
131 eor \y0, \y0, \y2
132 eor \y1, \y1, \y3
133 mul_gf4 \x4, \x5, \y0, \y1, \t2, \t3
134 eor \x4, \x4, \t0
135 eor \x6, \x6, \t0
136 eor \x5, \x5, \t1
137 eor \x7, \x7, \t1
138 .endm
139
140 .macro inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
141 t0, t1, t2, t3, s0, s1, s2, s3
142 eor \t3, \x4, \x6
143 eor \t0, \x5, \x7
144 eor \t1, \x1, \x3
145 eor \s1, \x7, \x6
146 eor \s0, \x0, \x2
147 eor \s3, \t3, \t0
148 orr \t2, \t0, \t1
149 and \s2, \t3, \s0
150 orr \t3, \t3, \s0
151 eor \s0, \s0, \t1
152 and \t0, \t0, \t1
153 eor \t1, \x3, \x2
154 and \s3, \s3, \s0
155 and \s1, \s1, \t1
156 eor \t1, \x4, \x5
157 eor \s0, \x1, \x0
158 eor \t3, \t3, \s1
159 eor \t2, \t2, \s1
160 and \s1, \t1, \s0
161 orr \t1, \t1, \s0
162 eor \t3, \t3, \s3
163 eor \t0, \t0, \s1
164 eor \t2, \t2, \s2
165 eor \t1, \t1, \s3
166 eor \t0, \t0, \s2
167 and \s0, \x7, \x3
168 eor \t1, \t1, \s2
169 and \s1, \x6, \x2
170 and \s2, \x5, \x1
171 orr \s3, \x4, \x0
172 eor \t3, \t3, \s0
173 eor \t1, \t1, \s2
174 eor \s0, \t0, \s3
175 eor \t2, \t2, \s1
176 and \s2, \t3, \t1
177 eor \s1, \t2, \s2
178 eor \s3, \s0, \s2
179 bsl \s1, \t1, \s0
180 not \t0, \s0
181 bsl \s0, \s1, \s3
182 bsl \t0, \s1, \s3
183 bsl \s3, \t3, \t2
184 eor \t3, \t3, \t2
185 and \s2, \s0, \s3
186 eor \t1, \t1, \t0
187 eor \s2, \s2, \t3
188 mul_gf16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
189 \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
190 .endm
191
192 .macro sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
193 t0, t1, t2, t3, s0, s1, s2, s3
194 in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
195 \b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
196 inv_gf256 \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b, \
197 \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
198 \t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
199 \s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
200 out_bs_ch \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
201 \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b
202 .endm
203
204 .macro inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
205 t0, t1, t2, t3, s0, s1, s2, s3
206 inv_in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
207 \b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
208 inv_gf256 \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b, \
209 \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
210 \t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
211 \s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
212 inv_out_bs_ch \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
213 \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b
214 .endm
215
216 .macro enc_next_rk
217 ldp q16, q17, [bskey], #128
218 ldp q18, q19, [bskey, #-96]
219 ldp q20, q21, [bskey, #-64]
220 ldp q22, q23, [bskey, #-32]
221 .endm
222
223 .macro dec_next_rk
224 ldp q16, q17, [bskey, #-128]!
225 ldp q18, q19, [bskey, #32]
226 ldp q20, q21, [bskey, #64]
227 ldp q22, q23, [bskey, #96]
228 .endm
229
230 .macro add_round_key, x0, x1, x2, x3, x4, x5, x6, x7
231 eor \x0\().16b, \x0\().16b, v16.16b
232 eor \x1\().16b, \x1\().16b, v17.16b
233 eor \x2\().16b, \x2\().16b, v18.16b
234 eor \x3\().16b, \x3\().16b, v19.16b
235 eor \x4\().16b, \x4\().16b, v20.16b
236 eor \x5\().16b, \x5\().16b, v21.16b
237 eor \x6\().16b, \x6\().16b, v22.16b
238 eor \x7\().16b, \x7\().16b, v23.16b
239 .endm
240
241 .macro shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, mask
242 tbl \x0\().16b, {\x0\().16b}, \mask\().16b
243 tbl \x1\().16b, {\x1\().16b}, \mask\().16b
244 tbl \x2\().16b, {\x2\().16b}, \mask\().16b
245 tbl \x3\().16b, {\x3\().16b}, \mask\().16b
246 tbl \x4\().16b, {\x4\().16b}, \mask\().16b
247 tbl \x5\().16b, {\x5\().16b}, \mask\().16b
248 tbl \x6\().16b, {\x6\().16b}, \mask\().16b
249 tbl \x7\().16b, {\x7\().16b}, \mask\().16b
250 .endm
251
252 .macro mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
253 t0, t1, t2, t3, t4, t5, t6, t7, inv
254 ext \t0\().16b, \x0\().16b, \x0\().16b, #12
255 ext \t1\().16b, \x1\().16b, \x1\().16b, #12
256 eor \x0\().16b, \x0\().16b, \t0\().16b
257 ext \t2\().16b, \x2\().16b, \x2\().16b, #12
258 eor \x1\().16b, \x1\().16b, \t1\().16b
259 ext \t3\().16b, \x3\().16b, \x3\().16b, #12
260 eor \x2\().16b, \x2\().16b, \t2\().16b
261 ext \t4\().16b, \x4\().16b, \x4\().16b, #12
262 eor \x3\().16b, \x3\().16b, \t3\().16b
263 ext \t5\().16b, \x5\().16b, \x5\().16b, #12
264 eor \x4\().16b, \x4\().16b, \t4\().16b
265 ext \t6\().16b, \x6\().16b, \x6\().16b, #12
266 eor \x5\().16b, \x5\().16b, \t5\().16b
267 ext \t7\().16b, \x7\().16b, \x7\().16b, #12
268 eor \x6\().16b, \x6\().16b, \t6\().16b
269 eor \t1\().16b, \t1\().16b, \x0\().16b
270 eor \x7\().16b, \x7\().16b, \t7\().16b
271 ext \x0\().16b, \x0\().16b, \x0\().16b, #8
272 eor \t2\().16b, \t2\().16b, \x1\().16b
273 eor \t0\().16b, \t0\().16b, \x7\().16b
274 eor \t1\().16b, \t1\().16b, \x7\().16b
275 ext \x1\().16b, \x1\().16b, \x1\().16b, #8
276 eor \t5\().16b, \t5\().16b, \x4\().16b
277 eor \x0\().16b, \x0\().16b, \t0\().16b
278 eor \t6\().16b, \t6\().16b, \x5\().16b
279 eor \x1\().16b, \x1\().16b, \t1\().16b
280 ext \t0\().16b, \x4\().16b, \x4\().16b, #8
281 eor \t4\().16b, \t4\().16b, \x3\().16b
282 ext \t1\().16b, \x5\().16b, \x5\().16b, #8
283 eor \t7\().16b, \t7\().16b, \x6\().16b
284 ext \x4\().16b, \x3\().16b, \x3\().16b, #8
285 eor \t3\().16b, \t3\().16b, \x2\().16b
286 ext \x5\().16b, \x7\().16b, \x7\().16b, #8
287 eor \t4\().16b, \t4\().16b, \x7\().16b
288 ext \x3\().16b, \x6\().16b, \x6\().16b, #8
289 eor \t3\().16b, \t3\().16b, \x7\().16b
290 ext \x6\().16b, \x2\().16b, \x2\().16b, #8
291 eor \x7\().16b, \t1\().16b, \t5\().16b
292 .ifb \inv
293 eor \x2\().16b, \t0\().16b, \t4\().16b
294 eor \x4\().16b, \x4\().16b, \t3\().16b
295 eor \x5\().16b, \x5\().16b, \t7\().16b
296 eor \x3\().16b, \x3\().16b, \t6\().16b
297 eor \x6\().16b, \x6\().16b, \t2\().16b
298 .else
299 eor \t3\().16b, \t3\().16b, \x4\().16b
300 eor \x5\().16b, \x5\().16b, \t7\().16b
301 eor \x2\().16b, \x3\().16b, \t6\().16b
302 eor \x3\().16b, \t0\().16b, \t4\().16b
303 eor \x4\().16b, \x6\().16b, \t2\().16b
304 mov \x6\().16b, \t3\().16b
305 .endif
306 .endm
307
308 .macro inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
309 t0, t1, t2, t3, t4, t5, t6, t7
310 ext \t0\().16b, \x0\().16b, \x0\().16b, #8
311 ext \t6\().16b, \x6\().16b, \x6\().16b, #8
312 ext \t7\().16b, \x7\().16b, \x7\().16b, #8
313 eor \t0\().16b, \t0\().16b, \x0\().16b
314 ext \t1\().16b, \x1\().16b, \x1\().16b, #8
315 eor \t6\().16b, \t6\().16b, \x6\().16b
316 ext \t2\().16b, \x2\().16b, \x2\().16b, #8
317 eor \t7\().16b, \t7\().16b, \x7\().16b
318 ext \t3\().16b, \x3\().16b, \x3\().16b, #8
319 eor \t1\().16b, \t1\().16b, \x1\().16b
320 ext \t4\().16b, \x4\().16b, \x4\().16b, #8
321 eor \t2\().16b, \t2\().16b, \x2\().16b
322 ext \t5\().16b, \x5\().16b, \x5\().16b, #8
323 eor \t3\().16b, \t3\().16b, \x3\().16b
324 eor \t4\().16b, \t4\().16b, \x4\().16b
325 eor \t5\().16b, \t5\().16b, \x5\().16b
326 eor \x0\().16b, \x0\().16b, \t6\().16b
327 eor \x1\().16b, \x1\().16b, \t6\().16b
328 eor \x2\().16b, \x2\().16b, \t0\().16b
329 eor \x4\().16b, \x4\().16b, \t2\().16b
330 eor \x3\().16b, \x3\().16b, \t1\().16b
331 eor \x1\().16b, \x1\().16b, \t7\().16b
332 eor \x2\().16b, \x2\().16b, \t7\().16b
333 eor \x4\().16b, \x4\().16b, \t6\().16b
334 eor \x5\().16b, \x5\().16b, \t3\().16b
335 eor \x3\().16b, \x3\().16b, \t6\().16b
336 eor \x6\().16b, \x6\().16b, \t4\().16b
337 eor \x4\().16b, \x4\().16b, \t7\().16b
338 eor \x5\().16b, \x5\().16b, \t7\().16b
339 eor \x7\().16b, \x7\().16b, \t5\().16b
340 mix_cols \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
341 \t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
342 .endm
343
344 .macro swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
345 ushr \t0\().2d, \b0\().2d, #\n
346 ushr \t1\().2d, \b1\().2d, #\n
347 eor \t0\().16b, \t0\().16b, \a0\().16b
348 eor \t1\().16b, \t1\().16b, \a1\().16b
349 and \t0\().16b, \t0\().16b, \mask\().16b
350 and \t1\().16b, \t1\().16b, \mask\().16b
351 eor \a0\().16b, \a0\().16b, \t0\().16b
352 shl \t0\().2d, \t0\().2d, #\n
353 eor \a1\().16b, \a1\().16b, \t1\().16b
354 shl \t1\().2d, \t1\().2d, #\n
355 eor \b0\().16b, \b0\().16b, \t0\().16b
356 eor \b1\().16b, \b1\().16b, \t1\().16b
357 .endm
358
359 .macro bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
360 movi \t0\().16b, #0x55
361 movi \t1\().16b, #0x33
362 swapmove_2x \x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
363 swapmove_2x \x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
364 movi \t0\().16b, #0x0f
365 swapmove_2x \x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
366 swapmove_2x \x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
367 swapmove_2x \x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
368 swapmove_2x \x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
369 .endm
370
371
372 .align 6
373M0: .octa 0x0004080c0105090d02060a0e03070b0f
374
375M0SR: .octa 0x0004080c05090d010a0e02060f03070b
376SR: .octa 0x0f0e0d0c0a09080b0504070600030201
377SRM0: .octa 0x01060b0c0207080d0304090e00050a0f
378
379M0ISR: .octa 0x0004080c0d0105090a0e0206070b0f03
380ISR: .octa 0x0f0e0d0c080b0a090504070602010003
381ISRM0: .octa 0x0306090c00070a0d01040b0e0205080f
382
383 /*
384 * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
385 */
386ENTRY(aesbs_convert_key)
387 ld1 {v7.4s}, [x1], #16 // load round 0 key
388 ld1 {v17.4s}, [x1], #16 // load round 1 key
389
390 movi v8.16b, #0x01 // bit masks
391 movi v9.16b, #0x02
392 movi v10.16b, #0x04
393 movi v11.16b, #0x08
394 movi v12.16b, #0x10
395 movi v13.16b, #0x20
396 movi v14.16b, #0x40
397 movi v15.16b, #0x80
398 ldr q16, M0
399
400 sub x2, x2, #1
401 str q7, [x0], #16 // save round 0 key
402
403.Lkey_loop:
404 tbl v7.16b ,{v17.16b}, v16.16b
405 ld1 {v17.4s}, [x1], #16 // load next round key
406
407 cmtst v0.16b, v7.16b, v8.16b
408 cmtst v1.16b, v7.16b, v9.16b
409 cmtst v2.16b, v7.16b, v10.16b
410 cmtst v3.16b, v7.16b, v11.16b
411 cmtst v4.16b, v7.16b, v12.16b
412 cmtst v5.16b, v7.16b, v13.16b
413 cmtst v6.16b, v7.16b, v14.16b
414 cmtst v7.16b, v7.16b, v15.16b
415 not v0.16b, v0.16b
416 not v1.16b, v1.16b
417 not v5.16b, v5.16b
418 not v6.16b, v6.16b
419
420 subs x2, x2, #1
421 stp q0, q1, [x0], #128
422 stp q2, q3, [x0, #-96]
423 stp q4, q5, [x0, #-64]
424 stp q6, q7, [x0, #-32]
425 b.ne .Lkey_loop
426
427 movi v7.16b, #0x63 // compose .L63
428 eor v17.16b, v17.16b, v7.16b
429 str q17, [x0]
430 ret
431ENDPROC(aesbs_convert_key)
432
433 .align 4
434aesbs_encrypt8:
435 ldr q9, [bskey], #16 // round 0 key
436 ldr q8, M0SR
437 ldr q24, SR
438
439 eor v10.16b, v0.16b, v9.16b // xor with round0 key
440 eor v11.16b, v1.16b, v9.16b
441 tbl v0.16b, {v10.16b}, v8.16b
442 eor v12.16b, v2.16b, v9.16b
443 tbl v1.16b, {v11.16b}, v8.16b
444 eor v13.16b, v3.16b, v9.16b
445 tbl v2.16b, {v12.16b}, v8.16b
446 eor v14.16b, v4.16b, v9.16b
447 tbl v3.16b, {v13.16b}, v8.16b
448 eor v15.16b, v5.16b, v9.16b
449 tbl v4.16b, {v14.16b}, v8.16b
450 eor v10.16b, v6.16b, v9.16b
451 tbl v5.16b, {v15.16b}, v8.16b
452 eor v11.16b, v7.16b, v9.16b
453 tbl v6.16b, {v10.16b}, v8.16b
454 tbl v7.16b, {v11.16b}, v8.16b
455
456 bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
457
458 sub rounds, rounds, #1
459 b .Lenc_sbox
460
461.Lenc_loop:
462 shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
463.Lenc_sbox:
464 sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
465 v13, v14, v15
466 subs rounds, rounds, #1
467 b.cc .Lenc_done
468
469 enc_next_rk
470
471 mix_cols v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11, v12, \
472 v13, v14, v15
473
474 add_round_key v0, v1, v2, v3, v4, v5, v6, v7
475
476 b.ne .Lenc_loop
477 ldr q24, SRM0
478 b .Lenc_loop
479
480.Lenc_done:
481 ldr q12, [bskey] // last round key
482
483 bitslice v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11
484
485 eor v0.16b, v0.16b, v12.16b
486 eor v1.16b, v1.16b, v12.16b
487 eor v4.16b, v4.16b, v12.16b
488 eor v6.16b, v6.16b, v12.16b
489 eor v3.16b, v3.16b, v12.16b
490 eor v7.16b, v7.16b, v12.16b
491 eor v2.16b, v2.16b, v12.16b
492 eor v5.16b, v5.16b, v12.16b
493 ret
494ENDPROC(aesbs_encrypt8)
495
496 .align 4
497aesbs_decrypt8:
498 lsl x9, rounds, #7
499 add bskey, bskey, x9
500
501 ldr q9, [bskey, #-112]! // round 0 key
502 ldr q8, M0ISR
503 ldr q24, ISR
504
505 eor v10.16b, v0.16b, v9.16b // xor with round0 key
506 eor v11.16b, v1.16b, v9.16b
507 tbl v0.16b, {v10.16b}, v8.16b
508 eor v12.16b, v2.16b, v9.16b
509 tbl v1.16b, {v11.16b}, v8.16b
510 eor v13.16b, v3.16b, v9.16b
511 tbl v2.16b, {v12.16b}, v8.16b
512 eor v14.16b, v4.16b, v9.16b
513 tbl v3.16b, {v13.16b}, v8.16b
514 eor v15.16b, v5.16b, v9.16b
515 tbl v4.16b, {v14.16b}, v8.16b
516 eor v10.16b, v6.16b, v9.16b
517 tbl v5.16b, {v15.16b}, v8.16b
518 eor v11.16b, v7.16b, v9.16b
519 tbl v6.16b, {v10.16b}, v8.16b
520 tbl v7.16b, {v11.16b}, v8.16b
521
522 bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
523
524 sub rounds, rounds, #1
525 b .Ldec_sbox
526
527.Ldec_loop:
528 shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
529.Ldec_sbox:
530 inv_sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
531 v13, v14, v15
532 subs rounds, rounds, #1
533 b.cc .Ldec_done
534
535 dec_next_rk
536
537 add_round_key v0, v1, v6, v4, v2, v7, v3, v5
538
539 inv_mix_cols v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11, v12, \
540 v13, v14, v15
541
542 b.ne .Ldec_loop
543 ldr q24, ISRM0
544 b .Ldec_loop
545.Ldec_done:
546 ldr q12, [bskey, #-16] // last round key
547
548 bitslice v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11
549
550 eor v0.16b, v0.16b, v12.16b
551 eor v1.16b, v1.16b, v12.16b
552 eor v6.16b, v6.16b, v12.16b
553 eor v4.16b, v4.16b, v12.16b
554 eor v2.16b, v2.16b, v12.16b
555 eor v7.16b, v7.16b, v12.16b
556 eor v3.16b, v3.16b, v12.16b
557 eor v5.16b, v5.16b, v12.16b
558 ret
559ENDPROC(aesbs_decrypt8)
560
561 /*
562 * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
563 * int blocks)
564 * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
565 * int blocks)
566 */
567 .macro __ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
568 stp x29, x30, [sp, #-16]!
569 mov x29, sp
570
57199: mov x5, #1
572 lsl x5, x5, x4
573 subs w4, w4, #8
574 csel x4, x4, xzr, pl
575 csel x5, x5, xzr, mi
576
577 ld1 {v0.16b}, [x1], #16
578 tbnz x5, #1, 0f
579 ld1 {v1.16b}, [x1], #16
580 tbnz x5, #2, 0f
581 ld1 {v2.16b}, [x1], #16
582 tbnz x5, #3, 0f
583 ld1 {v3.16b}, [x1], #16
584 tbnz x5, #4, 0f
585 ld1 {v4.16b}, [x1], #16
586 tbnz x5, #5, 0f
587 ld1 {v5.16b}, [x1], #16
588 tbnz x5, #6, 0f
589 ld1 {v6.16b}, [x1], #16
590 tbnz x5, #7, 0f
591 ld1 {v7.16b}, [x1], #16
592
5930: mov bskey, x2
594 mov rounds, x3
595 bl \do8
596
597 st1 {\o0\().16b}, [x0], #16
598 tbnz x5, #1, 1f
599 st1 {\o1\().16b}, [x0], #16
600 tbnz x5, #2, 1f
601 st1 {\o2\().16b}, [x0], #16
602 tbnz x5, #3, 1f
603 st1 {\o3\().16b}, [x0], #16
604 tbnz x5, #4, 1f
605 st1 {\o4\().16b}, [x0], #16
606 tbnz x5, #5, 1f
607 st1 {\o5\().16b}, [x0], #16
608 tbnz x5, #6, 1f
609 st1 {\o6\().16b}, [x0], #16
610 tbnz x5, #7, 1f
611 st1 {\o7\().16b}, [x0], #16
612
613 cbnz x4, 99b
614
6151: ldp x29, x30, [sp], #16
616 ret
617 .endm
618
619 .align 4
620ENTRY(aesbs_ecb_encrypt)
621 __ecb_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
622ENDPROC(aesbs_ecb_encrypt)
623
624 .align 4
625ENTRY(aesbs_ecb_decrypt)
626 __ecb_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
627ENDPROC(aesbs_ecb_decrypt)
628
629 /*
630 * aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
631 * int blocks, u8 iv[])
632 */
633 .align 4
634ENTRY(aesbs_cbc_decrypt)
635 stp x29, x30, [sp, #-16]!
636 mov x29, sp
637
63899: mov x6, #1
639 lsl x6, x6, x4
640 subs w4, w4, #8
641 csel x4, x4, xzr, pl
642 csel x6, x6, xzr, mi
643
644 ld1 {v0.16b}, [x1], #16
645 mov v25.16b, v0.16b
646 tbnz x6, #1, 0f
647 ld1 {v1.16b}, [x1], #16
648 mov v26.16b, v1.16b
649 tbnz x6, #2, 0f
650 ld1 {v2.16b}, [x1], #16
651 mov v27.16b, v2.16b
652 tbnz x6, #3, 0f
653 ld1 {v3.16b}, [x1], #16
654 mov v28.16b, v3.16b
655 tbnz x6, #4, 0f
656 ld1 {v4.16b}, [x1], #16
657 mov v29.16b, v4.16b
658 tbnz x6, #5, 0f
659 ld1 {v5.16b}, [x1], #16
660 mov v30.16b, v5.16b
661 tbnz x6, #6, 0f
662 ld1 {v6.16b}, [x1], #16
663 mov v31.16b, v6.16b
664 tbnz x6, #7, 0f
665 ld1 {v7.16b}, [x1]
666
6670: mov bskey, x2
668 mov rounds, x3
669 bl aesbs_decrypt8
670
671 ld1 {v24.16b}, [x5] // load IV
672
673 eor v1.16b, v1.16b, v25.16b
674 eor v6.16b, v6.16b, v26.16b
675 eor v4.16b, v4.16b, v27.16b
676 eor v2.16b, v2.16b, v28.16b
677 eor v7.16b, v7.16b, v29.16b
678 eor v0.16b, v0.16b, v24.16b
679 eor v3.16b, v3.16b, v30.16b
680 eor v5.16b, v5.16b, v31.16b
681
682 st1 {v0.16b}, [x0], #16
683 mov v24.16b, v25.16b
684 tbnz x6, #1, 1f
685 st1 {v1.16b}, [x0], #16
686 mov v24.16b, v26.16b
687 tbnz x6, #2, 1f
688 st1 {v6.16b}, [x0], #16
689 mov v24.16b, v27.16b
690 tbnz x6, #3, 1f
691 st1 {v4.16b}, [x0], #16
692 mov v24.16b, v28.16b
693 tbnz x6, #4, 1f
694 st1 {v2.16b}, [x0], #16
695 mov v24.16b, v29.16b
696 tbnz x6, #5, 1f
697 st1 {v7.16b}, [x0], #16
698 mov v24.16b, v30.16b
699 tbnz x6, #6, 1f
700 st1 {v3.16b}, [x0], #16
701 mov v24.16b, v31.16b
702 tbnz x6, #7, 1f
703 ld1 {v24.16b}, [x1], #16
704 st1 {v5.16b}, [x0], #16
7051: st1 {v24.16b}, [x5] // store IV
706
707 cbnz x4, 99b
708
709 ldp x29, x30, [sp], #16
710 ret
711ENDPROC(aesbs_cbc_decrypt)
712
713 .macro next_tweak, out, in, const, tmp
714 sshr \tmp\().2d, \in\().2d, #63
715 and \tmp\().16b, \tmp\().16b, \const\().16b
716 add \out\().2d, \in\().2d, \in\().2d
717 ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
718 eor \out\().16b, \out\().16b, \tmp\().16b
719 .endm
720
721 .align 4
722.Lxts_mul_x:
723CPU_LE( .quad 1, 0x87 )
724CPU_BE( .quad 0x87, 1 )
725
726 /*
727 * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
728 * int blocks, u8 iv[])
729 * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
730 * int blocks, u8 iv[])
731 */
732__xts_crypt8:
733 mov x6, #1
734 lsl x6, x6, x4
735 subs w4, w4, #8
736 csel x4, x4, xzr, pl
737 csel x6, x6, xzr, mi
738
739 ld1 {v0.16b}, [x1], #16
740 next_tweak v26, v25, v30, v31
741 eor v0.16b, v0.16b, v25.16b
742 tbnz x6, #1, 0f
743
744 ld1 {v1.16b}, [x1], #16
745 next_tweak v27, v26, v30, v31
746 eor v1.16b, v1.16b, v26.16b
747 tbnz x6, #2, 0f
748
749 ld1 {v2.16b}, [x1], #16
750 next_tweak v28, v27, v30, v31
751 eor v2.16b, v2.16b, v27.16b
752 tbnz x6, #3, 0f
753
754 ld1 {v3.16b}, [x1], #16
755 next_tweak v29, v28, v30, v31
756 eor v3.16b, v3.16b, v28.16b
757 tbnz x6, #4, 0f
758
759 ld1 {v4.16b}, [x1], #16
760 str q29, [sp, #16]
761 eor v4.16b, v4.16b, v29.16b
762 next_tweak v29, v29, v30, v31
763 tbnz x6, #5, 0f
764
765 ld1 {v5.16b}, [x1], #16
766 str q29, [sp, #32]
767 eor v5.16b, v5.16b, v29.16b
768 next_tweak v29, v29, v30, v31
769 tbnz x6, #6, 0f
770
771 ld1 {v6.16b}, [x1], #16
772 str q29, [sp, #48]
773 eor v6.16b, v6.16b, v29.16b
774 next_tweak v29, v29, v30, v31
775 tbnz x6, #7, 0f
776
777 ld1 {v7.16b}, [x1], #16
778 str q29, [sp, #64]
779 eor v7.16b, v7.16b, v29.16b
780 next_tweak v29, v29, v30, v31
781
7820: mov bskey, x2
783 mov rounds, x3
784 br x7
785ENDPROC(__xts_crypt8)
786
787 .macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
788 stp x29, x30, [sp, #-80]!
789 mov x29, sp
790
791 ldr q30, .Lxts_mul_x
792 ld1 {v25.16b}, [x5]
793
79499: adr x7, \do8
795 bl __xts_crypt8
796
797 ldp q16, q17, [sp, #16]
798 ldp q18, q19, [sp, #48]
799
800 eor \o0\().16b, \o0\().16b, v25.16b
801 eor \o1\().16b, \o1\().16b, v26.16b
802 eor \o2\().16b, \o2\().16b, v27.16b
803 eor \o3\().16b, \o3\().16b, v28.16b
804
805 st1 {\o0\().16b}, [x0], #16
806 mov v25.16b, v26.16b
807 tbnz x6, #1, 1f
808 st1 {\o1\().16b}, [x0], #16
809 mov v25.16b, v27.16b
810 tbnz x6, #2, 1f
811 st1 {\o2\().16b}, [x0], #16
812 mov v25.16b, v28.16b
813 tbnz x6, #3, 1f
814 st1 {\o3\().16b}, [x0], #16
815 mov v25.16b, v29.16b
816 tbnz x6, #4, 1f
817
818 eor \o4\().16b, \o4\().16b, v16.16b
819 eor \o5\().16b, \o5\().16b, v17.16b
820 eor \o6\().16b, \o6\().16b, v18.16b
821 eor \o7\().16b, \o7\().16b, v19.16b
822
823 st1 {\o4\().16b}, [x0], #16
824 tbnz x6, #5, 1f
825 st1 {\o5\().16b}, [x0], #16
826 tbnz x6, #6, 1f
827 st1 {\o6\().16b}, [x0], #16
828 tbnz x6, #7, 1f
829 st1 {\o7\().16b}, [x0], #16
830
831 cbnz x4, 99b
832
8331: st1 {v25.16b}, [x5]
834 ldp x29, x30, [sp], #80
835 ret
836 .endm
837
838ENTRY(aesbs_xts_encrypt)
839 __xts_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
840ENDPROC(aesbs_xts_encrypt)
841
842ENTRY(aesbs_xts_decrypt)
843 __xts_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
844ENDPROC(aesbs_xts_decrypt)
845
846 .macro next_ctr, v
847 mov \v\().d[1], x8
848 adds x8, x8, #1
849 mov \v\().d[0], x7
850 adc x7, x7, xzr
851 rev64 \v\().16b, \v\().16b
852 .endm
853
854 /*
855 * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
856 * int rounds, int blocks, u8 iv[], u8 final[])
857 */
858ENTRY(aesbs_ctr_encrypt)
859 stp x29, x30, [sp, #-16]!
860 mov x29, sp
861
862 cmp x6, #0
863 cset x10, ne
864 add x4, x4, x10 // do one extra block if final
865
866 ldp x7, x8, [x5]
867 ld1 {v0.16b}, [x5]
868CPU_LE( rev x7, x7 )
869CPU_LE( rev x8, x8 )
870 adds x8, x8, #1
871 adc x7, x7, xzr
872
87399: mov x9, #1
874 lsl x9, x9, x4
875 subs w4, w4, #8
876 csel x4, x4, xzr, pl
877 csel x9, x9, xzr, le
878
879 tbnz x9, #1, 0f
880 next_ctr v1
881 tbnz x9, #2, 0f
882 next_ctr v2
883 tbnz x9, #3, 0f
884 next_ctr v3
885 tbnz x9, #4, 0f
886 next_ctr v4
887 tbnz x9, #5, 0f
888 next_ctr v5
889 tbnz x9, #6, 0f
890 next_ctr v6
891 tbnz x9, #7, 0f
892 next_ctr v7
893
8940: mov bskey, x2
895 mov rounds, x3
896 bl aesbs_encrypt8
897
898 lsr x9, x9, x10 // disregard the extra block
899 tbnz x9, #0, 0f
900
901 ld1 {v8.16b}, [x1], #16
902 eor v0.16b, v0.16b, v8.16b
903 st1 {v0.16b}, [x0], #16
904 tbnz x9, #1, 1f
905
906 ld1 {v9.16b}, [x1], #16
907 eor v1.16b, v1.16b, v9.16b
908 st1 {v1.16b}, [x0], #16
909 tbnz x9, #2, 2f
910
911 ld1 {v10.16b}, [x1], #16
912 eor v4.16b, v4.16b, v10.16b
913 st1 {v4.16b}, [x0], #16
914 tbnz x9, #3, 3f
915
916 ld1 {v11.16b}, [x1], #16
917 eor v6.16b, v6.16b, v11.16b
918 st1 {v6.16b}, [x0], #16
919 tbnz x9, #4, 4f
920
921 ld1 {v12.16b}, [x1], #16
922 eor v3.16b, v3.16b, v12.16b
923 st1 {v3.16b}, [x0], #16
924 tbnz x9, #5, 5f
925
926 ld1 {v13.16b}, [x1], #16
927 eor v7.16b, v7.16b, v13.16b
928 st1 {v7.16b}, [x0], #16
929 tbnz x9, #6, 6f
930
931 ld1 {v14.16b}, [x1], #16
932 eor v2.16b, v2.16b, v14.16b
933 st1 {v2.16b}, [x0], #16
934 tbnz x9, #7, 7f
935
936 ld1 {v15.16b}, [x1], #16
937 eor v5.16b, v5.16b, v15.16b
938 st1 {v5.16b}, [x0], #16
939
9408: next_ctr v0
941 cbnz x4, 99b
942
9430: st1 {v0.16b}, [x5]
944 ldp x29, x30, [sp], #16
945 ret
946
947 /*
948 * If we are handling the tail of the input (x6 != NULL), return the
949 * final keystream block back to the caller.
950 */
9511: cbz x6, 8b
952 st1 {v1.16b}, [x6]
953 b 8b
9542: cbz x6, 8b
955 st1 {v4.16b}, [x6]
956 b 8b
9573: cbz x6, 8b
958 st1 {v6.16b}, [x6]
959 b 8b
9604: cbz x6, 8b
961 st1 {v3.16b}, [x6]
962 b 8b
9635: cbz x6, 8b
964 st1 {v7.16b}, [x6]
965 b 8b
9666: cbz x6, 8b
967 st1 {v2.16b}, [x6]
968 b 8b
9697: cbz x6, 8b
970 st1 {v5.16b}, [x6]
971 b 8b
972ENDPROC(aesbs_ctr_encrypt)
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
new file mode 100644
index 000000000000..db2501d93550
--- /dev/null
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -0,0 +1,439 @@
1/*
2 * Bit sliced AES using NEON instructions
3 *
4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <crypto/aes.h>
13#include <crypto/internal/simd.h>
14#include <crypto/internal/skcipher.h>
15#include <crypto/xts.h>
16#include <linux/module.h>
17
18MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
19MODULE_LICENSE("GPL v2");
20
21MODULE_ALIAS_CRYPTO("ecb(aes)");
22MODULE_ALIAS_CRYPTO("cbc(aes)");
23MODULE_ALIAS_CRYPTO("ctr(aes)");
24MODULE_ALIAS_CRYPTO("xts(aes)");
25
26asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
27
28asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
29 int rounds, int blocks);
30asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
31 int rounds, int blocks);
32
33asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
34 int rounds, int blocks, u8 iv[]);
35
36asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
37 int rounds, int blocks, u8 iv[], u8 final[]);
38
39asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
40 int rounds, int blocks, u8 iv[]);
41asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
42 int rounds, int blocks, u8 iv[]);
43
44/* borrowed from aes-neon-blk.ko */
45asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
46 int rounds, int blocks, int first);
47asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
48 int rounds, int blocks, u8 iv[],
49 int first);
50
51struct aesbs_ctx {
52 u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32];
53 int rounds;
54} __aligned(AES_BLOCK_SIZE);
55
56struct aesbs_cbc_ctx {
57 struct aesbs_ctx key;
58 u32 enc[AES_MAX_KEYLENGTH_U32];
59};
60
61struct aesbs_xts_ctx {
62 struct aesbs_ctx key;
63 u32 twkey[AES_MAX_KEYLENGTH_U32];
64};
65
66static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
67 unsigned int key_len)
68{
69 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
70 struct crypto_aes_ctx rk;
71 int err;
72
73 err = crypto_aes_expand_key(&rk, in_key, key_len);
74 if (err)
75 return err;
76
77 ctx->rounds = 6 + key_len / 4;
78
79 kernel_neon_begin();
80 aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
81 kernel_neon_end();
82
83 return 0;
84}
85
86static int __ecb_crypt(struct skcipher_request *req,
87 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
88 int rounds, int blocks))
89{
90 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
91 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
92 struct skcipher_walk walk;
93 int err;
94
95 err = skcipher_walk_virt(&walk, req, true);
96
97 kernel_neon_begin();
98 while (walk.nbytes >= AES_BLOCK_SIZE) {
99 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
100
101 if (walk.nbytes < walk.total)
102 blocks = round_down(blocks,
103 walk.stride / AES_BLOCK_SIZE);
104
105 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
106 ctx->rounds, blocks);
107 err = skcipher_walk_done(&walk,
108 walk.nbytes - blocks * AES_BLOCK_SIZE);
109 }
110 kernel_neon_end();
111
112 return err;
113}
114
115static int ecb_encrypt(struct skcipher_request *req)
116{
117 return __ecb_crypt(req, aesbs_ecb_encrypt);
118}
119
120static int ecb_decrypt(struct skcipher_request *req)
121{
122 return __ecb_crypt(req, aesbs_ecb_decrypt);
123}
124
125static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
126 unsigned int key_len)
127{
128 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
129 struct crypto_aes_ctx rk;
130 int err;
131
132 err = crypto_aes_expand_key(&rk, in_key, key_len);
133 if (err)
134 return err;
135
136 ctx->key.rounds = 6 + key_len / 4;
137
138 memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc));
139
140 kernel_neon_begin();
141 aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
142 kernel_neon_end();
143
144 return 0;
145}
146
147static int cbc_encrypt(struct skcipher_request *req)
148{
149 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
150 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
151 struct skcipher_walk walk;
152 int err, first = 1;
153
154 err = skcipher_walk_virt(&walk, req, true);
155
156 kernel_neon_begin();
157 while (walk.nbytes >= AES_BLOCK_SIZE) {
158 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
159
160 /* fall back to the non-bitsliced NEON implementation */
161 neon_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
162 ctx->enc, ctx->key.rounds, blocks, walk.iv,
163 first);
164 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
165 first = 0;
166 }
167 kernel_neon_end();
168 return err;
169}
170
171static int cbc_decrypt(struct skcipher_request *req)
172{
173 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
174 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
175 struct skcipher_walk walk;
176 int err;
177
178 err = skcipher_walk_virt(&walk, req, true);
179
180 kernel_neon_begin();
181 while (walk.nbytes >= AES_BLOCK_SIZE) {
182 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
183
184 if (walk.nbytes < walk.total)
185 blocks = round_down(blocks,
186 walk.stride / AES_BLOCK_SIZE);
187
188 aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
189 ctx->key.rk, ctx->key.rounds, blocks,
190 walk.iv);
191 err = skcipher_walk_done(&walk,
192 walk.nbytes - blocks * AES_BLOCK_SIZE);
193 }
194 kernel_neon_end();
195
196 return err;
197}
198
199static int ctr_encrypt(struct skcipher_request *req)
200{
201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
202 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
203 struct skcipher_walk walk;
204 u8 buf[AES_BLOCK_SIZE];
205 int err;
206
207 err = skcipher_walk_virt(&walk, req, true);
208
209 kernel_neon_begin();
210 while (walk.nbytes > 0) {
211 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
212 u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL;
213
214 if (walk.nbytes < walk.total) {
215 blocks = round_down(blocks,
216 walk.stride / AES_BLOCK_SIZE);
217 final = NULL;
218 }
219
220 aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
221 ctx->rk, ctx->rounds, blocks, walk.iv, final);
222
223 if (final) {
224 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
225 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
226
227 if (dst != src)
228 memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
229 crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
230
231 err = skcipher_walk_done(&walk, 0);
232 break;
233 }
234 err = skcipher_walk_done(&walk,
235 walk.nbytes - blocks * AES_BLOCK_SIZE);
236 }
237 kernel_neon_end();
238
239 return err;
240}
241
242static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
243 unsigned int key_len)
244{
245 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
246 struct crypto_aes_ctx rk;
247 int err;
248
249 err = xts_verify_key(tfm, in_key, key_len);
250 if (err)
251 return err;
252
253 key_len /= 2;
254 err = crypto_aes_expand_key(&rk, in_key + key_len, key_len);
255 if (err)
256 return err;
257
258 memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey));
259
260 return aesbs_setkey(tfm, in_key, key_len);
261}
262
263static int __xts_crypt(struct skcipher_request *req,
264 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
265 int rounds, int blocks, u8 iv[]))
266{
267 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
268 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
269 struct skcipher_walk walk;
270 int err;
271
272 err = skcipher_walk_virt(&walk, req, true);
273
274 kernel_neon_begin();
275
276 neon_aes_ecb_encrypt(walk.iv, walk.iv, ctx->twkey,
277 ctx->key.rounds, 1, 1);
278
279 while (walk.nbytes >= AES_BLOCK_SIZE) {
280 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
281
282 if (walk.nbytes < walk.total)
283 blocks = round_down(blocks,
284 walk.stride / AES_BLOCK_SIZE);
285
286 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
287 ctx->key.rounds, blocks, walk.iv);
288 err = skcipher_walk_done(&walk,
289 walk.nbytes - blocks * AES_BLOCK_SIZE);
290 }
291 kernel_neon_end();
292
293 return err;
294}
295
296static int xts_encrypt(struct skcipher_request *req)
297{
298 return __xts_crypt(req, aesbs_xts_encrypt);
299}
300
301static int xts_decrypt(struct skcipher_request *req)
302{
303 return __xts_crypt(req, aesbs_xts_decrypt);
304}
305
306static struct skcipher_alg aes_algs[] = { {
307 .base.cra_name = "__ecb(aes)",
308 .base.cra_driver_name = "__ecb-aes-neonbs",
309 .base.cra_priority = 250,
310 .base.cra_blocksize = AES_BLOCK_SIZE,
311 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
312 .base.cra_module = THIS_MODULE,
313 .base.cra_flags = CRYPTO_ALG_INTERNAL,
314
315 .min_keysize = AES_MIN_KEY_SIZE,
316 .max_keysize = AES_MAX_KEY_SIZE,
317 .walksize = 8 * AES_BLOCK_SIZE,
318 .setkey = aesbs_setkey,
319 .encrypt = ecb_encrypt,
320 .decrypt = ecb_decrypt,
321}, {
322 .base.cra_name = "__cbc(aes)",
323 .base.cra_driver_name = "__cbc-aes-neonbs",
324 .base.cra_priority = 250,
325 .base.cra_blocksize = AES_BLOCK_SIZE,
326 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
327 .base.cra_module = THIS_MODULE,
328 .base.cra_flags = CRYPTO_ALG_INTERNAL,
329
330 .min_keysize = AES_MIN_KEY_SIZE,
331 .max_keysize = AES_MAX_KEY_SIZE,
332 .walksize = 8 * AES_BLOCK_SIZE,
333 .ivsize = AES_BLOCK_SIZE,
334 .setkey = aesbs_cbc_setkey,
335 .encrypt = cbc_encrypt,
336 .decrypt = cbc_decrypt,
337}, {
338 .base.cra_name = "__ctr(aes)",
339 .base.cra_driver_name = "__ctr-aes-neonbs",
340 .base.cra_priority = 250,
341 .base.cra_blocksize = 1,
342 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
343 .base.cra_module = THIS_MODULE,
344 .base.cra_flags = CRYPTO_ALG_INTERNAL,
345
346 .min_keysize = AES_MIN_KEY_SIZE,
347 .max_keysize = AES_MAX_KEY_SIZE,
348 .chunksize = AES_BLOCK_SIZE,
349 .walksize = 8 * AES_BLOCK_SIZE,
350 .ivsize = AES_BLOCK_SIZE,
351 .setkey = aesbs_setkey,
352 .encrypt = ctr_encrypt,
353 .decrypt = ctr_encrypt,
354}, {
355 .base.cra_name = "ctr(aes)",
356 .base.cra_driver_name = "ctr-aes-neonbs",
357 .base.cra_priority = 250 - 1,
358 .base.cra_blocksize = 1,
359 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
360 .base.cra_module = THIS_MODULE,
361
362 .min_keysize = AES_MIN_KEY_SIZE,
363 .max_keysize = AES_MAX_KEY_SIZE,
364 .chunksize = AES_BLOCK_SIZE,
365 .walksize = 8 * AES_BLOCK_SIZE,
366 .ivsize = AES_BLOCK_SIZE,
367 .setkey = aesbs_setkey,
368 .encrypt = ctr_encrypt,
369 .decrypt = ctr_encrypt,
370}, {
371 .base.cra_name = "__xts(aes)",
372 .base.cra_driver_name = "__xts-aes-neonbs",
373 .base.cra_priority = 250,
374 .base.cra_blocksize = AES_BLOCK_SIZE,
375 .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
376 .base.cra_module = THIS_MODULE,
377 .base.cra_flags = CRYPTO_ALG_INTERNAL,
378
379 .min_keysize = 2 * AES_MIN_KEY_SIZE,
380 .max_keysize = 2 * AES_MAX_KEY_SIZE,
381 .walksize = 8 * AES_BLOCK_SIZE,
382 .ivsize = AES_BLOCK_SIZE,
383 .setkey = aesbs_xts_setkey,
384 .encrypt = xts_encrypt,
385 .decrypt = xts_decrypt,
386} };
387
388static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
389
390static void aes_exit(void)
391{
392 int i;
393
394 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
395 if (aes_simd_algs[i])
396 simd_skcipher_free(aes_simd_algs[i]);
397
398 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
399}
400
401static int __init aes_init(void)
402{
403 struct simd_skcipher_alg *simd;
404 const char *basename;
405 const char *algname;
406 const char *drvname;
407 int err;
408 int i;
409
410 if (!(elf_hwcap & HWCAP_ASIMD))
411 return -ENODEV;
412
413 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
414 if (err)
415 return err;
416
417 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
418 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
419 continue;
420
421 algname = aes_algs[i].base.cra_name + 2;
422 drvname = aes_algs[i].base.cra_driver_name + 2;
423 basename = aes_algs[i].base.cra_driver_name;
424 simd = simd_skcipher_create_compat(algname, drvname, basename);
425 err = PTR_ERR(simd);
426 if (IS_ERR(simd))
427 goto unregister_simds;
428
429 aes_simd_algs[i] = simd;
430 }
431 return 0;
432
433unregister_simds:
434 aes_exit();
435 return err;
436}
437
438module_init(aes_init);
439module_exit(aes_exit);
diff --git a/arch/arm64/crypto/chacha20-neon-core.S b/arch/arm64/crypto/chacha20-neon-core.S
new file mode 100644
index 000000000000..13c85e272c2a
--- /dev/null
+++ b/arch/arm64/crypto/chacha20-neon-core.S
@@ -0,0 +1,450 @@
1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
3 *
4 * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Based on:
11 * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions
12 *
13 * Copyright (C) 2015 Martin Willi
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 */
20
21#include <linux/linkage.h>
22
23 .text
24 .align 6
25
26ENTRY(chacha20_block_xor_neon)
27 // x0: Input state matrix, s
28 // x1: 1 data block output, o
29 // x2: 1 data block input, i
30
31 //
32 // This function encrypts one ChaCha20 block by loading the state matrix
33 // in four NEON registers. It performs matrix operation on four words in
34 // parallel, but requires shuffling to rearrange the words after each
35 // round.
36 //
37
38 // x0..3 = s0..3
39 adr x3, ROT8
40 ld1 {v0.4s-v3.4s}, [x0]
41 ld1 {v8.4s-v11.4s}, [x0]
42 ld1 {v12.4s}, [x3]
43
44 mov x3, #10
45
46.Ldoubleround:
47 // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
48 add v0.4s, v0.4s, v1.4s
49 eor v3.16b, v3.16b, v0.16b
50 rev32 v3.8h, v3.8h
51
52 // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
53 add v2.4s, v2.4s, v3.4s
54 eor v4.16b, v1.16b, v2.16b
55 shl v1.4s, v4.4s, #12
56 sri v1.4s, v4.4s, #20
57
58 // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
59 add v0.4s, v0.4s, v1.4s
60 eor v3.16b, v3.16b, v0.16b
61 tbl v3.16b, {v3.16b}, v12.16b
62
63 // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
64 add v2.4s, v2.4s, v3.4s
65 eor v4.16b, v1.16b, v2.16b
66 shl v1.4s, v4.4s, #7
67 sri v1.4s, v4.4s, #25
68
69 // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
70 ext v1.16b, v1.16b, v1.16b, #4
71 // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
72 ext v2.16b, v2.16b, v2.16b, #8
73 // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
74 ext v3.16b, v3.16b, v3.16b, #12
75
76 // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
77 add v0.4s, v0.4s, v1.4s
78 eor v3.16b, v3.16b, v0.16b
79 rev32 v3.8h, v3.8h
80
81 // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
82 add v2.4s, v2.4s, v3.4s
83 eor v4.16b, v1.16b, v2.16b
84 shl v1.4s, v4.4s, #12
85 sri v1.4s, v4.4s, #20
86
87 // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
88 add v0.4s, v0.4s, v1.4s
89 eor v3.16b, v3.16b, v0.16b
90 tbl v3.16b, {v3.16b}, v12.16b
91
92 // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
93 add v2.4s, v2.4s, v3.4s
94 eor v4.16b, v1.16b, v2.16b
95 shl v1.4s, v4.4s, #7
96 sri v1.4s, v4.4s, #25
97
98 // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
99 ext v1.16b, v1.16b, v1.16b, #12
100 // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
101 ext v2.16b, v2.16b, v2.16b, #8
102 // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
103 ext v3.16b, v3.16b, v3.16b, #4
104
105 subs x3, x3, #1
106 b.ne .Ldoubleround
107
108 ld1 {v4.16b-v7.16b}, [x2]
109
110 // o0 = i0 ^ (x0 + s0)
111 add v0.4s, v0.4s, v8.4s
112 eor v0.16b, v0.16b, v4.16b
113
114 // o1 = i1 ^ (x1 + s1)
115 add v1.4s, v1.4s, v9.4s
116 eor v1.16b, v1.16b, v5.16b
117
118 // o2 = i2 ^ (x2 + s2)
119 add v2.4s, v2.4s, v10.4s
120 eor v2.16b, v2.16b, v6.16b
121
122 // o3 = i3 ^ (x3 + s3)
123 add v3.4s, v3.4s, v11.4s
124 eor v3.16b, v3.16b, v7.16b
125
126 st1 {v0.16b-v3.16b}, [x1]
127
128 ret
129ENDPROC(chacha20_block_xor_neon)
130
131 .align 6
132ENTRY(chacha20_4block_xor_neon)
133 // x0: Input state matrix, s
134 // x1: 4 data blocks output, o
135 // x2: 4 data blocks input, i
136
137 //
138 // This function encrypts four consecutive ChaCha20 blocks by loading
139 // the state matrix in NEON registers four times. The algorithm performs
140 // each operation on the corresponding word of each state matrix, hence
141 // requires no word shuffling. For final XORing step we transpose the
142 // matrix by interleaving 32- and then 64-bit words, which allows us to
143 // do XOR in NEON registers.
144 //
145 adr x3, CTRINC // ... and ROT8
146 ld1 {v30.4s-v31.4s}, [x3]
147
148 // x0..15[0-3] = s0..3[0..3]
149 mov x4, x0
150 ld4r { v0.4s- v3.4s}, [x4], #16
151 ld4r { v4.4s- v7.4s}, [x4], #16
152 ld4r { v8.4s-v11.4s}, [x4], #16
153 ld4r {v12.4s-v15.4s}, [x4]
154
155 // x12 += counter values 0-3
156 add v12.4s, v12.4s, v30.4s
157
158 mov x3, #10
159
160.Ldoubleround4:
161 // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
162 // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
163 // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
164 // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
165 add v0.4s, v0.4s, v4.4s
166 add v1.4s, v1.4s, v5.4s
167 add v2.4s, v2.4s, v6.4s
168 add v3.4s, v3.4s, v7.4s
169
170 eor v12.16b, v12.16b, v0.16b
171 eor v13.16b, v13.16b, v1.16b
172 eor v14.16b, v14.16b, v2.16b
173 eor v15.16b, v15.16b, v3.16b
174
175 rev32 v12.8h, v12.8h
176 rev32 v13.8h, v13.8h
177 rev32 v14.8h, v14.8h
178 rev32 v15.8h, v15.8h
179
180 // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
181 // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
182 // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
183 // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
184 add v8.4s, v8.4s, v12.4s
185 add v9.4s, v9.4s, v13.4s
186 add v10.4s, v10.4s, v14.4s
187 add v11.4s, v11.4s, v15.4s
188
189 eor v16.16b, v4.16b, v8.16b
190 eor v17.16b, v5.16b, v9.16b
191 eor v18.16b, v6.16b, v10.16b
192 eor v19.16b, v7.16b, v11.16b
193
194 shl v4.4s, v16.4s, #12
195 shl v5.4s, v17.4s, #12
196 shl v6.4s, v18.4s, #12
197 shl v7.4s, v19.4s, #12
198
199 sri v4.4s, v16.4s, #20
200 sri v5.4s, v17.4s, #20
201 sri v6.4s, v18.4s, #20
202 sri v7.4s, v19.4s, #20
203
204 // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
205 // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
206 // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
207 // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
208 add v0.4s, v0.4s, v4.4s
209 add v1.4s, v1.4s, v5.4s
210 add v2.4s, v2.4s, v6.4s
211 add v3.4s, v3.4s, v7.4s
212
213 eor v12.16b, v12.16b, v0.16b
214 eor v13.16b, v13.16b, v1.16b
215 eor v14.16b, v14.16b, v2.16b
216 eor v15.16b, v15.16b, v3.16b
217
218 tbl v12.16b, {v12.16b}, v31.16b
219 tbl v13.16b, {v13.16b}, v31.16b
220 tbl v14.16b, {v14.16b}, v31.16b
221 tbl v15.16b, {v15.16b}, v31.16b
222
223 // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
224 // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
225 // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
226 // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
227 add v8.4s, v8.4s, v12.4s
228 add v9.4s, v9.4s, v13.4s
229 add v10.4s, v10.4s, v14.4s
230 add v11.4s, v11.4s, v15.4s
231
232 eor v16.16b, v4.16b, v8.16b
233 eor v17.16b, v5.16b, v9.16b
234 eor v18.16b, v6.16b, v10.16b
235 eor v19.16b, v7.16b, v11.16b
236
237 shl v4.4s, v16.4s, #7
238 shl v5.4s, v17.4s, #7
239 shl v6.4s, v18.4s, #7
240 shl v7.4s, v19.4s, #7
241
242 sri v4.4s, v16.4s, #25
243 sri v5.4s, v17.4s, #25
244 sri v6.4s, v18.4s, #25
245 sri v7.4s, v19.4s, #25
246
247 // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
248 // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
249 // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
250 // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
251 add v0.4s, v0.4s, v5.4s
252 add v1.4s, v1.4s, v6.4s
253 add v2.4s, v2.4s, v7.4s
254 add v3.4s, v3.4s, v4.4s
255
256 eor v15.16b, v15.16b, v0.16b
257 eor v12.16b, v12.16b, v1.16b
258 eor v13.16b, v13.16b, v2.16b
259 eor v14.16b, v14.16b, v3.16b
260
261 rev32 v15.8h, v15.8h
262 rev32 v12.8h, v12.8h
263 rev32 v13.8h, v13.8h
264 rev32 v14.8h, v14.8h
265
266 // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
267 // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
268 // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
269 // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
270 add v10.4s, v10.4s, v15.4s
271 add v11.4s, v11.4s, v12.4s
272 add v8.4s, v8.4s, v13.4s
273 add v9.4s, v9.4s, v14.4s
274
275 eor v16.16b, v5.16b, v10.16b
276 eor v17.16b, v6.16b, v11.16b
277 eor v18.16b, v7.16b, v8.16b
278 eor v19.16b, v4.16b, v9.16b
279
280 shl v5.4s, v16.4s, #12
281 shl v6.4s, v17.4s, #12
282 shl v7.4s, v18.4s, #12
283 shl v4.4s, v19.4s, #12
284
285 sri v5.4s, v16.4s, #20
286 sri v6.4s, v17.4s, #20
287 sri v7.4s, v18.4s, #20
288 sri v4.4s, v19.4s, #20
289
290 // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
291 // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
292 // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
293 // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
294 add v0.4s, v0.4s, v5.4s
295 add v1.4s, v1.4s, v6.4s
296 add v2.4s, v2.4s, v7.4s
297 add v3.4s, v3.4s, v4.4s
298
299 eor v15.16b, v15.16b, v0.16b
300 eor v12.16b, v12.16b, v1.16b
301 eor v13.16b, v13.16b, v2.16b
302 eor v14.16b, v14.16b, v3.16b
303
304 tbl v15.16b, {v15.16b}, v31.16b
305 tbl v12.16b, {v12.16b}, v31.16b
306 tbl v13.16b, {v13.16b}, v31.16b
307 tbl v14.16b, {v14.16b}, v31.16b
308
309 // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
310 // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
311 // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
312 // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
313 add v10.4s, v10.4s, v15.4s
314 add v11.4s, v11.4s, v12.4s
315 add v8.4s, v8.4s, v13.4s
316 add v9.4s, v9.4s, v14.4s
317
318 eor v16.16b, v5.16b, v10.16b
319 eor v17.16b, v6.16b, v11.16b
320 eor v18.16b, v7.16b, v8.16b
321 eor v19.16b, v4.16b, v9.16b
322
323 shl v5.4s, v16.4s, #7
324 shl v6.4s, v17.4s, #7
325 shl v7.4s, v18.4s, #7
326 shl v4.4s, v19.4s, #7
327
328 sri v5.4s, v16.4s, #25
329 sri v6.4s, v17.4s, #25
330 sri v7.4s, v18.4s, #25
331 sri v4.4s, v19.4s, #25
332
333 subs x3, x3, #1
334 b.ne .Ldoubleround4
335
336 ld4r {v16.4s-v19.4s}, [x0], #16
337 ld4r {v20.4s-v23.4s}, [x0], #16
338
339 // x12 += counter values 0-3
340 add v12.4s, v12.4s, v30.4s
341
342 // x0[0-3] += s0[0]
343 // x1[0-3] += s0[1]
344 // x2[0-3] += s0[2]
345 // x3[0-3] += s0[3]
346 add v0.4s, v0.4s, v16.4s
347 add v1.4s, v1.4s, v17.4s
348 add v2.4s, v2.4s, v18.4s
349 add v3.4s, v3.4s, v19.4s
350
351 ld4r {v24.4s-v27.4s}, [x0], #16
352 ld4r {v28.4s-v31.4s}, [x0]
353
354 // x4[0-3] += s1[0]
355 // x5[0-3] += s1[1]
356 // x6[0-3] += s1[2]
357 // x7[0-3] += s1[3]
358 add v4.4s, v4.4s, v20.4s
359 add v5.4s, v5.4s, v21.4s
360 add v6.4s, v6.4s, v22.4s
361 add v7.4s, v7.4s, v23.4s
362
363 // x8[0-3] += s2[0]
364 // x9[0-3] += s2[1]
365 // x10[0-3] += s2[2]
366 // x11[0-3] += s2[3]
367 add v8.4s, v8.4s, v24.4s
368 add v9.4s, v9.4s, v25.4s
369 add v10.4s, v10.4s, v26.4s
370 add v11.4s, v11.4s, v27.4s
371
372 // x12[0-3] += s3[0]
373 // x13[0-3] += s3[1]
374 // x14[0-3] += s3[2]
375 // x15[0-3] += s3[3]
376 add v12.4s, v12.4s, v28.4s
377 add v13.4s, v13.4s, v29.4s
378 add v14.4s, v14.4s, v30.4s
379 add v15.4s, v15.4s, v31.4s
380
381 // interleave 32-bit words in state n, n+1
382 zip1 v16.4s, v0.4s, v1.4s
383 zip2 v17.4s, v0.4s, v1.4s
384 zip1 v18.4s, v2.4s, v3.4s
385 zip2 v19.4s, v2.4s, v3.4s
386 zip1 v20.4s, v4.4s, v5.4s
387 zip2 v21.4s, v4.4s, v5.4s
388 zip1 v22.4s, v6.4s, v7.4s
389 zip2 v23.4s, v6.4s, v7.4s
390 zip1 v24.4s, v8.4s, v9.4s
391 zip2 v25.4s, v8.4s, v9.4s
392 zip1 v26.4s, v10.4s, v11.4s
393 zip2 v27.4s, v10.4s, v11.4s
394 zip1 v28.4s, v12.4s, v13.4s
395 zip2 v29.4s, v12.4s, v13.4s
396 zip1 v30.4s, v14.4s, v15.4s
397 zip2 v31.4s, v14.4s, v15.4s
398
399 // interleave 64-bit words in state n, n+2
400 zip1 v0.2d, v16.2d, v18.2d
401 zip2 v4.2d, v16.2d, v18.2d
402 zip1 v8.2d, v17.2d, v19.2d
403 zip2 v12.2d, v17.2d, v19.2d
404 ld1 {v16.16b-v19.16b}, [x2], #64
405
406 zip1 v1.2d, v20.2d, v22.2d
407 zip2 v5.2d, v20.2d, v22.2d
408 zip1 v9.2d, v21.2d, v23.2d
409 zip2 v13.2d, v21.2d, v23.2d
410 ld1 {v20.16b-v23.16b}, [x2], #64
411
412 zip1 v2.2d, v24.2d, v26.2d
413 zip2 v6.2d, v24.2d, v26.2d
414 zip1 v10.2d, v25.2d, v27.2d
415 zip2 v14.2d, v25.2d, v27.2d
416 ld1 {v24.16b-v27.16b}, [x2], #64
417
418 zip1 v3.2d, v28.2d, v30.2d
419 zip2 v7.2d, v28.2d, v30.2d
420 zip1 v11.2d, v29.2d, v31.2d
421 zip2 v15.2d, v29.2d, v31.2d
422 ld1 {v28.16b-v31.16b}, [x2]
423
424 // xor with corresponding input, write to output
425 eor v16.16b, v16.16b, v0.16b
426 eor v17.16b, v17.16b, v1.16b
427 eor v18.16b, v18.16b, v2.16b
428 eor v19.16b, v19.16b, v3.16b
429 eor v20.16b, v20.16b, v4.16b
430 eor v21.16b, v21.16b, v5.16b
431 st1 {v16.16b-v19.16b}, [x1], #64
432 eor v22.16b, v22.16b, v6.16b
433 eor v23.16b, v23.16b, v7.16b
434 eor v24.16b, v24.16b, v8.16b
435 eor v25.16b, v25.16b, v9.16b
436 st1 {v20.16b-v23.16b}, [x1], #64
437 eor v26.16b, v26.16b, v10.16b
438 eor v27.16b, v27.16b, v11.16b
439 eor v28.16b, v28.16b, v12.16b
440 st1 {v24.16b-v27.16b}, [x1], #64
441 eor v29.16b, v29.16b, v13.16b
442 eor v30.16b, v30.16b, v14.16b
443 eor v31.16b, v31.16b, v15.16b
444 st1 {v28.16b-v31.16b}, [x1]
445
446 ret
447ENDPROC(chacha20_4block_xor_neon)
448
449CTRINC: .word 0, 1, 2, 3
450ROT8: .word 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f
diff --git a/arch/arm64/crypto/chacha20-neon-glue.c b/arch/arm64/crypto/chacha20-neon-glue.c
new file mode 100644
index 000000000000..a7cd575ea223
--- /dev/null
+++ b/arch/arm64/crypto/chacha20-neon-glue.c
@@ -0,0 +1,126 @@
1/*
2 * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
3 *
4 * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Based on:
11 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
12 *
13 * Copyright (C) 2015 Martin Willi
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 */
20
21#include <crypto/algapi.h>
22#include <crypto/chacha20.h>
23#include <crypto/internal/skcipher.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26
27#include <asm/hwcap.h>
28#include <asm/neon.h>
29
30asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
31asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
32
33static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
34 unsigned int bytes)
35{
36 u8 buf[CHACHA20_BLOCK_SIZE];
37
38 while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
39 chacha20_4block_xor_neon(state, dst, src);
40 bytes -= CHACHA20_BLOCK_SIZE * 4;
41 src += CHACHA20_BLOCK_SIZE * 4;
42 dst += CHACHA20_BLOCK_SIZE * 4;
43 state[12] += 4;
44 }
45 while (bytes >= CHACHA20_BLOCK_SIZE) {
46 chacha20_block_xor_neon(state, dst, src);
47 bytes -= CHACHA20_BLOCK_SIZE;
48 src += CHACHA20_BLOCK_SIZE;
49 dst += CHACHA20_BLOCK_SIZE;
50 state[12]++;
51 }
52 if (bytes) {
53 memcpy(buf, src, bytes);
54 chacha20_block_xor_neon(state, buf, buf);
55 memcpy(dst, buf, bytes);
56 }
57}
58
59static int chacha20_neon(struct skcipher_request *req)
60{
61 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
62 struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
63 struct skcipher_walk walk;
64 u32 state[16];
65 int err;
66
67 if (req->cryptlen <= CHACHA20_BLOCK_SIZE)
68 return crypto_chacha20_crypt(req);
69
70 err = skcipher_walk_virt(&walk, req, true);
71
72 crypto_chacha20_init(state, ctx, walk.iv);
73
74 kernel_neon_begin();
75 while (walk.nbytes > 0) {
76 unsigned int nbytes = walk.nbytes;
77
78 if (nbytes < walk.total)
79 nbytes = round_down(nbytes, walk.stride);
80
81 chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
82 nbytes);
83 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
84 }
85 kernel_neon_end();
86
87 return err;
88}
89
90static struct skcipher_alg alg = {
91 .base.cra_name = "chacha20",
92 .base.cra_driver_name = "chacha20-neon",
93 .base.cra_priority = 300,
94 .base.cra_blocksize = 1,
95 .base.cra_ctxsize = sizeof(struct chacha20_ctx),
96 .base.cra_module = THIS_MODULE,
97
98 .min_keysize = CHACHA20_KEY_SIZE,
99 .max_keysize = CHACHA20_KEY_SIZE,
100 .ivsize = CHACHA20_IV_SIZE,
101 .chunksize = CHACHA20_BLOCK_SIZE,
102 .walksize = 4 * CHACHA20_BLOCK_SIZE,
103 .setkey = crypto_chacha20_setkey,
104 .encrypt = chacha20_neon,
105 .decrypt = chacha20_neon,
106};
107
108static int __init chacha20_simd_mod_init(void)
109{
110 if (!(elf_hwcap & HWCAP_ASIMD))
111 return -ENODEV;
112
113 return crypto_register_skcipher(&alg);
114}
115
116static void __exit chacha20_simd_mod_fini(void)
117{
118 crypto_unregister_skcipher(&alg);
119}
120
121module_init(chacha20_simd_mod_init);
122module_exit(chacha20_simd_mod_fini);
123
124MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
125MODULE_LICENSE("GPL v2");
126MODULE_ALIAS_CRYPTO("chacha20");
diff --git a/arch/arm64/crypto/crc32-arm64.c b/arch/arm64/crypto/crc32-arm64.c
deleted file mode 100644
index 6a37c3c6b11d..000000000000
--- a/arch/arm64/crypto/crc32-arm64.c
+++ /dev/null
@@ -1,290 +0,0 @@
1/*
2 * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions
3 *
4 * Module based on crypto/crc32c_generic.c
5 *
6 * CRC32 loop taken from Ed Nevill's Hadoop CRC patch
7 * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E
8 *
9 * Using inline assembly instead of intrinsics in order to be backwards
10 * compatible with older compilers.
11 *
12 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License version 2 as
16 * published by the Free Software Foundation.
17 */
18
19#include <linux/unaligned/access_ok.h>
20#include <linux/cpufeature.h>
21#include <linux/init.h>
22#include <linux/kernel.h>
23#include <linux/module.h>
24#include <linux/string.h>
25
26#include <crypto/internal/hash.h>
27
28MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");
29MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");
30MODULE_LICENSE("GPL v2");
31
32#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
33#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
34#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
35#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
36#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
37#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
38#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
39#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
40
41static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
42{
43 s64 length = len;
44
45 while ((length -= sizeof(u64)) >= 0) {
46 CRC32X(crc, get_unaligned_le64(p));
47 p += sizeof(u64);
48 }
49
50 /* The following is more efficient than the straight loop */
51 if (length & sizeof(u32)) {
52 CRC32W(crc, get_unaligned_le32(p));
53 p += sizeof(u32);
54 }
55 if (length & sizeof(u16)) {
56 CRC32H(crc, get_unaligned_le16(p));
57 p += sizeof(u16);
58 }
59 if (length & sizeof(u8))
60 CRC32B(crc, *p);
61
62 return crc;
63}
64
65static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
66{
67 s64 length = len;
68
69 while ((length -= sizeof(u64)) >= 0) {
70 CRC32CX(crc, get_unaligned_le64(p));
71 p += sizeof(u64);
72 }
73
74 /* The following is more efficient than the straight loop */
75 if (length & sizeof(u32)) {
76 CRC32CW(crc, get_unaligned_le32(p));
77 p += sizeof(u32);
78 }
79 if (length & sizeof(u16)) {
80 CRC32CH(crc, get_unaligned_le16(p));
81 p += sizeof(u16);
82 }
83 if (length & sizeof(u8))
84 CRC32CB(crc, *p);
85
86 return crc;
87}
88
89#define CHKSUM_BLOCK_SIZE 1
90#define CHKSUM_DIGEST_SIZE 4
91
92struct chksum_ctx {
93 u32 key;
94};
95
96struct chksum_desc_ctx {
97 u32 crc;
98};
99
100static int chksum_init(struct shash_desc *desc)
101{
102 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
103 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
104
105 ctx->crc = mctx->key;
106
107 return 0;
108}
109
110/*
111 * Setting the seed allows arbitrary accumulators and flexible XOR policy
112 * If your algorithm starts with ~0, then XOR with ~0 before you set
113 * the seed.
114 */
115static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
116 unsigned int keylen)
117{
118 struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
119
120 if (keylen != sizeof(mctx->key)) {
121 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
122 return -EINVAL;
123 }
124 mctx->key = get_unaligned_le32(key);
125 return 0;
126}
127
128static int chksum_update(struct shash_desc *desc, const u8 *data,
129 unsigned int length)
130{
131 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
132
133 ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);
134 return 0;
135}
136
137static int chksumc_update(struct shash_desc *desc, const u8 *data,
138 unsigned int length)
139{
140 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
141
142 ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);
143 return 0;
144}
145
146static int chksum_final(struct shash_desc *desc, u8 *out)
147{
148 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
149
150 put_unaligned_le32(ctx->crc, out);
151 return 0;
152}
153
154static int chksumc_final(struct shash_desc *desc, u8 *out)
155{
156 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
157
158 put_unaligned_le32(~ctx->crc, out);
159 return 0;
160}
161
162static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
163{
164 put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);
165 return 0;
166}
167
168static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
169{
170 put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);
171 return 0;
172}
173
174static int chksum_finup(struct shash_desc *desc, const u8 *data,
175 unsigned int len, u8 *out)
176{
177 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
178
179 return __chksum_finup(ctx->crc, data, len, out);
180}
181
182static int chksumc_finup(struct shash_desc *desc, const u8 *data,
183 unsigned int len, u8 *out)
184{
185 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
186
187 return __chksumc_finup(ctx->crc, data, len, out);
188}
189
190static int chksum_digest(struct shash_desc *desc, const u8 *data,
191 unsigned int length, u8 *out)
192{
193 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
194
195 return __chksum_finup(mctx->key, data, length, out);
196}
197
198static int chksumc_digest(struct shash_desc *desc, const u8 *data,
199 unsigned int length, u8 *out)
200{
201 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
202
203 return __chksumc_finup(mctx->key, data, length, out);
204}
205
206static int crc32_cra_init(struct crypto_tfm *tfm)
207{
208 struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
209
210 mctx->key = 0;
211 return 0;
212}
213
214static int crc32c_cra_init(struct crypto_tfm *tfm)
215{
216 struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
217
218 mctx->key = ~0;
219 return 0;
220}
221
222static struct shash_alg crc32_alg = {
223 .digestsize = CHKSUM_DIGEST_SIZE,
224 .setkey = chksum_setkey,
225 .init = chksum_init,
226 .update = chksum_update,
227 .final = chksum_final,
228 .finup = chksum_finup,
229 .digest = chksum_digest,
230 .descsize = sizeof(struct chksum_desc_ctx),
231 .base = {
232 .cra_name = "crc32",
233 .cra_driver_name = "crc32-arm64-hw",
234 .cra_priority = 300,
235 .cra_blocksize = CHKSUM_BLOCK_SIZE,
236 .cra_alignmask = 0,
237 .cra_ctxsize = sizeof(struct chksum_ctx),
238 .cra_module = THIS_MODULE,
239 .cra_init = crc32_cra_init,
240 }
241};
242
243static struct shash_alg crc32c_alg = {
244 .digestsize = CHKSUM_DIGEST_SIZE,
245 .setkey = chksum_setkey,
246 .init = chksum_init,
247 .update = chksumc_update,
248 .final = chksumc_final,
249 .finup = chksumc_finup,
250 .digest = chksumc_digest,
251 .descsize = sizeof(struct chksum_desc_ctx),
252 .base = {
253 .cra_name = "crc32c",
254 .cra_driver_name = "crc32c-arm64-hw",
255 .cra_priority = 300,
256 .cra_blocksize = CHKSUM_BLOCK_SIZE,
257 .cra_alignmask = 0,
258 .cra_ctxsize = sizeof(struct chksum_ctx),
259 .cra_module = THIS_MODULE,
260 .cra_init = crc32c_cra_init,
261 }
262};
263
264static int __init crc32_mod_init(void)
265{
266 int err;
267
268 err = crypto_register_shash(&crc32_alg);
269
270 if (err)
271 return err;
272
273 err = crypto_register_shash(&crc32c_alg);
274
275 if (err) {
276 crypto_unregister_shash(&crc32_alg);
277 return err;
278 }
279
280 return 0;
281}
282
283static void __exit crc32_mod_exit(void)
284{
285 crypto_unregister_shash(&crc32_alg);
286 crypto_unregister_shash(&crc32c_alg);
287}
288
289module_cpu_feature_match(CRC32, crc32_mod_init);
290module_exit(crc32_mod_exit);
diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
index 8594127d5e01..eccb1ae90064 100644
--- a/arch/arm64/crypto/crc32-ce-glue.c
+++ b/arch/arm64/crypto/crc32-ce-glue.c
@@ -72,6 +72,24 @@ static int crc32_pmull_init(struct shash_desc *desc)
72 return 0; 72 return 0;
73} 73}
74 74
75static int crc32_update(struct shash_desc *desc, const u8 *data,
76 unsigned int length)
77{
78 u32 *crc = shash_desc_ctx(desc);
79
80 *crc = crc32_armv8_le(*crc, data, length);
81 return 0;
82}
83
84static int crc32c_update(struct shash_desc *desc, const u8 *data,
85 unsigned int length)
86{
87 u32 *crc = shash_desc_ctx(desc);
88
89 *crc = crc32c_armv8_le(*crc, data, length);
90 return 0;
91}
92
75static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, 93static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
76 unsigned int length) 94 unsigned int length)
77{ 95{
@@ -156,7 +174,7 @@ static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
156static struct shash_alg crc32_pmull_algs[] = { { 174static struct shash_alg crc32_pmull_algs[] = { {
157 .setkey = crc32_pmull_setkey, 175 .setkey = crc32_pmull_setkey,
158 .init = crc32_pmull_init, 176 .init = crc32_pmull_init,
159 .update = crc32_pmull_update, 177 .update = crc32_update,
160 .final = crc32_pmull_final, 178 .final = crc32_pmull_final,
161 .descsize = sizeof(u32), 179 .descsize = sizeof(u32),
162 .digestsize = sizeof(u32), 180 .digestsize = sizeof(u32),
@@ -171,7 +189,7 @@ static struct shash_alg crc32_pmull_algs[] = { {
171}, { 189}, {
172 .setkey = crc32_pmull_setkey, 190 .setkey = crc32_pmull_setkey,
173 .init = crc32_pmull_init, 191 .init = crc32_pmull_init,
174 .update = crc32c_pmull_update, 192 .update = crc32c_update,
175 .final = crc32c_pmull_final, 193 .final = crc32c_pmull_final,
176 .descsize = sizeof(u32), 194 .descsize = sizeof(u32),
177 .digestsize = sizeof(u32), 195 .digestsize = sizeof(u32),
@@ -187,14 +205,20 @@ static struct shash_alg crc32_pmull_algs[] = { {
187 205
188static int __init crc32_pmull_mod_init(void) 206static int __init crc32_pmull_mod_init(void)
189{ 207{
190 if (elf_hwcap & HWCAP_CRC32) { 208 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
191 fallback_crc32 = crc32_armv8_le; 209 crc32_pmull_algs[0].update = crc32_pmull_update;
192 fallback_crc32c = crc32c_armv8_le; 210 crc32_pmull_algs[1].update = crc32c_pmull_update;
193 } else { 211
194 fallback_crc32 = crc32_le; 212 if (elf_hwcap & HWCAP_CRC32) {
195 fallback_crc32c = __crc32c_le; 213 fallback_crc32 = crc32_armv8_le;
214 fallback_crc32c = crc32c_armv8_le;
215 } else {
216 fallback_crc32 = crc32_le;
217 fallback_crc32c = __crc32c_le;
218 }
219 } else if (!(elf_hwcap & HWCAP_CRC32)) {
220 return -ENODEV;
196 } 221 }
197
198 return crypto_register_shashes(crc32_pmull_algs, 222 return crypto_register_shashes(crc32_pmull_algs,
199 ARRAY_SIZE(crc32_pmull_algs)); 223 ARRAY_SIZE(crc32_pmull_algs));
200} 224}
@@ -205,7 +229,12 @@ static void __exit crc32_pmull_mod_exit(void)
205 ARRAY_SIZE(crc32_pmull_algs)); 229 ARRAY_SIZE(crc32_pmull_algs));
206} 230}
207 231
208module_cpu_feature_match(PMULL, crc32_pmull_mod_init); 232static const struct cpu_feature crc32_cpu_feature[] = {
233 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
234};
235MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
236
237module_init(crc32_pmull_mod_init);
209module_exit(crc32_pmull_mod_exit); 238module_exit(crc32_pmull_mod_exit);
210 239
211MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 240MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 383a6f84a060..3c465184ff8a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -46,28 +46,49 @@
46 46
47#ifdef __x86_64__ 47#ifdef __x86_64__
48 48
49.data 49# constants in mergeable sections, linker can reorder and merge
50.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16
50.align 16 51.align 16
51.Lgf128mul_x_ble_mask: 52.Lgf128mul_x_ble_mask:
52 .octa 0x00000000000000010000000000000087 53 .octa 0x00000000000000010000000000000087
54.section .rodata.cst16.POLY, "aM", @progbits, 16
55.align 16
53POLY: .octa 0xC2000000000000000000000000000001 56POLY: .octa 0xC2000000000000000000000000000001
57.section .rodata.cst16.TWOONE, "aM", @progbits, 16
58.align 16
54TWOONE: .octa 0x00000001000000000000000000000001 59TWOONE: .octa 0x00000001000000000000000000000001
55 60
56# order of these constants should not change. 61.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16
57# more specifically, ALL_F should follow SHIFT_MASK, 62.align 16
58# and ZERO should follow ALL_F
59
60SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F 63SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F
64.section .rodata.cst16.MASK1, "aM", @progbits, 16
65.align 16
61MASK1: .octa 0x0000000000000000ffffffffffffffff 66MASK1: .octa 0x0000000000000000ffffffffffffffff
67.section .rodata.cst16.MASK2, "aM", @progbits, 16
68.align 16
62MASK2: .octa 0xffffffffffffffff0000000000000000 69MASK2: .octa 0xffffffffffffffff0000000000000000
63SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 70.section .rodata.cst16.ONE, "aM", @progbits, 16
64ALL_F: .octa 0xffffffffffffffffffffffffffffffff 71.align 16
65ZERO: .octa 0x00000000000000000000000000000000
66ONE: .octa 0x00000000000000000000000000000001 72ONE: .octa 0x00000000000000000000000000000001
73.section .rodata.cst16.F_MIN_MASK, "aM", @progbits, 16
74.align 16
67F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0 75F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0
76.section .rodata.cst16.dec, "aM", @progbits, 16
77.align 16
68dec: .octa 0x1 78dec: .octa 0x1
79.section .rodata.cst16.enc, "aM", @progbits, 16
80.align 16
69enc: .octa 0x2 81enc: .octa 0x2
70 82
83# order of these constants should not change.
84# more specifically, ALL_F should follow SHIFT_MASK,
85# and zero should follow ALL_F
86.section .rodata, "a", @progbits
87.align 16
88SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
89ALL_F: .octa 0xffffffffffffffffffffffffffffffff
90 .octa 0x00000000000000000000000000000000
91
71 92
72.text 93.text
73 94
diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S
index 522ab68d1c88..d664382c6e56 100644
--- a/arch/x86/crypto/aesni-intel_avx-x86_64.S
+++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S
@@ -122,23 +122,39 @@
122#include <linux/linkage.h> 122#include <linux/linkage.h>
123#include <asm/inst.h> 123#include <asm/inst.h>
124 124
125.data 125# constants in mergeable sections, linker can reorder and merge
126.section .rodata.cst16.POLY, "aM", @progbits, 16
126.align 16 127.align 16
127
128POLY: .octa 0xC2000000000000000000000000000001 128POLY: .octa 0xC2000000000000000000000000000001
129
130.section .rodata.cst16.POLY2, "aM", @progbits, 16
131.align 16
129POLY2: .octa 0xC20000000000000000000001C2000000 132POLY2: .octa 0xC20000000000000000000001C2000000
130TWOONE: .octa 0x00000001000000000000000000000001
131 133
132# order of these constants should not change. 134.section .rodata.cst16.TWOONE, "aM", @progbits, 16
133# more specifically, ALL_F should follow SHIFT_MASK, and ZERO should follow ALL_F 135.align 16
136TWOONE: .octa 0x00000001000000000000000000000001
134 137
138.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16
139.align 16
135SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F 140SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F
136SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 141
137ALL_F: .octa 0xffffffffffffffffffffffffffffffff 142.section .rodata.cst16.ONE, "aM", @progbits, 16
138ZERO: .octa 0x00000000000000000000000000000000 143.align 16
139ONE: .octa 0x00000000000000000000000000000001 144ONE: .octa 0x00000000000000000000000000000001
145
146.section .rodata.cst16.ONEf, "aM", @progbits, 16
147.align 16
140ONEf: .octa 0x01000000000000000000000000000000 148ONEf: .octa 0x01000000000000000000000000000000
141 149
150# order of these constants should not change.
151# more specifically, ALL_F should follow SHIFT_MASK, and zero should follow ALL_F
152.section .rodata, "a", @progbits
153.align 16
154SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
155ALL_F: .octa 0xffffffffffffffffffffffffffffffff
156 .octa 0x00000000000000000000000000000000
157
142.text 158.text
143 159
144 160
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 7ff1b0c86a8e..93de8ea51548 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -740,9 +740,11 @@ static int helper_rfc4106_encrypt(struct aead_request *req)
740 *((__be32 *)(iv+12)) = counter; 740 *((__be32 *)(iv+12)) = counter;
741 741
742 if (sg_is_last(req->src) && 742 if (sg_is_last(req->src) &&
743 req->src->offset + req->src->length <= PAGE_SIZE && 743 (!PageHighMem(sg_page(req->src)) ||
744 req->src->offset + req->src->length <= PAGE_SIZE) &&
744 sg_is_last(req->dst) && 745 sg_is_last(req->dst) &&
745 req->dst->offset + req->dst->length <= PAGE_SIZE) { 746 (!PageHighMem(sg_page(req->dst)) ||
747 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
746 one_entry_in_sg = 1; 748 one_entry_in_sg = 1;
747 scatterwalk_start(&src_sg_walk, req->src); 749 scatterwalk_start(&src_sg_walk, req->src);
748 assoc = scatterwalk_map(&src_sg_walk); 750 assoc = scatterwalk_map(&src_sg_walk);
@@ -822,9 +824,11 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
822 *((__be32 *)(iv+12)) = counter; 824 *((__be32 *)(iv+12)) = counter;
823 825
824 if (sg_is_last(req->src) && 826 if (sg_is_last(req->src) &&
825 req->src->offset + req->src->length <= PAGE_SIZE && 827 (!PageHighMem(sg_page(req->src)) ||
828 req->src->offset + req->src->length <= PAGE_SIZE) &&
826 sg_is_last(req->dst) && 829 sg_is_last(req->dst) &&
827 req->dst->offset + req->dst->length <= PAGE_SIZE) { 830 (!PageHighMem(sg_page(req->dst)) ||
831 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
828 one_entry_in_sg = 1; 832 one_entry_in_sg = 1;
829 scatterwalk_start(&src_sg_walk, req->src); 833 scatterwalk_start(&src_sg_walk, req->src);
830 assoc = scatterwalk_map(&src_sg_walk); 834 assoc = scatterwalk_map(&src_sg_walk);
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index aa9e8bd163f6..f7c495e2863c 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -571,7 +571,9 @@ ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
571 vmovdqu y6, 14 * 16(rio); \ 571 vmovdqu y6, 14 * 16(rio); \
572 vmovdqu y7, 15 * 16(rio); 572 vmovdqu y7, 15 * 16(rio);
573 573
574.data 574
575/* NB: section is mergeable, all elements must be aligned 16-byte blocks */
576.section .rodata.cst16, "aM", @progbits, 16
575.align 16 577.align 16
576 578
577#define SHUFB_BYTES(idx) \ 579#define SHUFB_BYTES(idx) \
@@ -711,6 +713,7 @@ ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
711 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 713 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
712 714
713/* 4-bit mask */ 715/* 4-bit mask */
716.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
714.align 4 717.align 4
715.L0f0f0f0f: 718.L0f0f0f0f:
716 .long 0x0f0f0f0f 719 .long 0x0f0f0f0f
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index 16186c18656d..eee5b3982cfd 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -610,20 +610,25 @@ ENDPROC(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
610 vmovdqu y6, 14 * 32(rio); \ 610 vmovdqu y6, 14 * 32(rio); \
611 vmovdqu y7, 15 * 32(rio); 611 vmovdqu y7, 15 * 32(rio);
612 612
613.data
614.align 32
615 613
614.section .rodata.cst32.shufb_16x16b, "aM", @progbits, 32
615.align 32
616#define SHUFB_BYTES(idx) \ 616#define SHUFB_BYTES(idx) \
617 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) 617 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx)
618
619.Lshufb_16x16b: 618.Lshufb_16x16b:
620 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) 619 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
621 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) 620 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
622 621
622.section .rodata.cst32.pack_bswap, "aM", @progbits, 32
623.align 32
623.Lpack_bswap: 624.Lpack_bswap:
624 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 625 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080
625 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 626 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080
626 627
628/* NB: section is mergeable, all elements must be aligned 16-byte blocks */
629.section .rodata.cst16, "aM", @progbits, 16
630.align 16
631
627/* For CTR-mode IV byteswap */ 632/* For CTR-mode IV byteswap */
628.Lbswap128_mask: 633.Lbswap128_mask:
629 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 634 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
@@ -750,6 +755,7 @@ ENDPROC(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
750 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b 755 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
751 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 756 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
752 757
758.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
753.align 4 759.align 4
754/* 4-bit mask */ 760/* 4-bit mask */
755.L0f0f0f0f: 761.L0f0f0f0f:
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index 14fa1966bf01..b4a8806234ea 100644
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -195,19 +195,29 @@
195 vpshufb rmask, x0, x0; \ 195 vpshufb rmask, x0, x0; \
196 vpshufb rmask, x1, x1; 196 vpshufb rmask, x1, x1;
197 197
198.data 198.section .rodata.cst16.bswap_mask, "aM", @progbits, 16
199
200.align 16 199.align 16
201.Lbswap_mask: 200.Lbswap_mask:
202 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 201 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
202.section .rodata.cst16.bswap128_mask, "aM", @progbits, 16
203.align 16
203.Lbswap128_mask: 204.Lbswap128_mask:
204 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 205 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
206.section .rodata.cst16.bswap_iv_mask, "aM", @progbits, 16
207.align 16
205.Lbswap_iv_mask: 208.Lbswap_iv_mask:
206 .byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0 209 .byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0
210
211.section .rodata.cst4.16_mask, "aM", @progbits, 4
212.align 4
207.L16_mask: 213.L16_mask:
208 .byte 16, 16, 16, 16 214 .byte 16, 16, 16, 16
215.section .rodata.cst4.32_mask, "aM", @progbits, 4
216.align 4
209.L32_mask: 217.L32_mask:
210 .byte 32, 0, 0, 0 218 .byte 32, 0, 0, 0
219.section .rodata.cst4.first_mask, "aM", @progbits, 4
220.align 4
211.Lfirst_mask: 221.Lfirst_mask:
212 .byte 0x1f, 0, 0, 0 222 .byte 0x1f, 0, 0, 0
213 223
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
index c419389889cd..952d3156a933 100644
--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -225,8 +225,7 @@
225 vpshufb rmask, x2, x2; \ 225 vpshufb rmask, x2, x2; \
226 vpshufb rmask, x3, x3; 226 vpshufb rmask, x3, x3;
227 227
228.data 228.section .rodata.cst16, "aM", @progbits, 16
229
230.align 16 229.align 16
231.Lxts_gf128mul_and_shl1_mask: 230.Lxts_gf128mul_and_shl1_mask:
232 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 231 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
@@ -244,10 +243,19 @@
244 .byte 12, 13, 14, 15, 8, 9, 10, 11, 7, 6, 5, 4, 3, 2, 1, 0 243 .byte 12, 13, 14, 15, 8, 9, 10, 11, 7, 6, 5, 4, 3, 2, 1, 0
245.Lrkr_dec_QBAR_QBAR_QBAR_QBAR: 244.Lrkr_dec_QBAR_QBAR_QBAR_QBAR:
246 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 245 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
246
247.section .rodata.cst4.L16_mask, "aM", @progbits, 4
248.align 4
247.L16_mask: 249.L16_mask:
248 .byte 16, 16, 16, 16 250 .byte 16, 16, 16, 16
251
252.section .rodata.cst4.L32_mask, "aM", @progbits, 4
253.align 4
249.L32_mask: 254.L32_mask:
250 .byte 32, 0, 0, 0 255 .byte 32, 0, 0, 0
256
257.section .rodata.cst4.first_mask, "aM", @progbits, 4
258.align 4
251.Lfirst_mask: 259.Lfirst_mask:
252 .byte 0x1f, 0, 0, 0 260 .byte 0x1f, 0, 0, 0
253 261
diff --git a/arch/x86/crypto/chacha20-avx2-x86_64.S b/arch/x86/crypto/chacha20-avx2-x86_64.S
index 16694e625f77..3a2dc3dc6cac 100644
--- a/arch/x86/crypto/chacha20-avx2-x86_64.S
+++ b/arch/x86/crypto/chacha20-avx2-x86_64.S
@@ -11,13 +11,18 @@
11 11
12#include <linux/linkage.h> 12#include <linux/linkage.h>
13 13
14.data 14.section .rodata.cst32.ROT8, "aM", @progbits, 32
15.align 32 15.align 32
16
17ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003 16ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003
18 .octa 0x0e0d0c0f0a09080b0605040702010003 17 .octa 0x0e0d0c0f0a09080b0605040702010003
18
19.section .rodata.cst32.ROT16, "aM", @progbits, 32
20.align 32
19ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302 21ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302
20 .octa 0x0d0c0f0e09080b0a0504070601000302 22 .octa 0x0d0c0f0e09080b0a0504070601000302
23
24.section .rodata.cst32.CTRINC, "aM", @progbits, 32
25.align 32
21CTRINC: .octa 0x00000003000000020000000100000000 26CTRINC: .octa 0x00000003000000020000000100000000
22 .octa 0x00000007000000060000000500000004 27 .octa 0x00000007000000060000000500000004
23 28
diff --git a/arch/x86/crypto/chacha20-ssse3-x86_64.S b/arch/x86/crypto/chacha20-ssse3-x86_64.S
index 3a33124e9112..3f511a7d73b8 100644
--- a/arch/x86/crypto/chacha20-ssse3-x86_64.S
+++ b/arch/x86/crypto/chacha20-ssse3-x86_64.S
@@ -11,11 +11,14 @@
11 11
12#include <linux/linkage.h> 12#include <linux/linkage.h>
13 13
14.data 14.section .rodata.cst16.ROT8, "aM", @progbits, 16
15.align 16 15.align 16
16
17ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003 16ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003
17.section .rodata.cst16.ROT16, "aM", @progbits, 16
18.align 16
18ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302 19ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302
20.section .rodata.cst16.CTRINC, "aM", @progbits, 16
21.align 16
19CTRINC: .octa 0x00000003000000020000000100000000 22CTRINC: .octa 0x00000003000000020000000100000000
20 23
21.text 24.text
diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c
index f910d1d449f0..1e6af1b35f7b 100644
--- a/arch/x86/crypto/chacha20_glue.c
+++ b/arch/x86/crypto/chacha20_glue.c
@@ -11,7 +11,7 @@
11 11
12#include <crypto/algapi.h> 12#include <crypto/algapi.h>
13#include <crypto/chacha20.h> 13#include <crypto/chacha20.h>
14#include <linux/crypto.h> 14#include <crypto/internal/skcipher.h>
15#include <linux/kernel.h> 15#include <linux/kernel.h>
16#include <linux/module.h> 16#include <linux/module.h>
17#include <asm/fpu/api.h> 17#include <asm/fpu/api.h>
@@ -63,36 +63,37 @@ static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src,
63 } 63 }
64} 64}
65 65
66static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst, 66static int chacha20_simd(struct skcipher_request *req)
67 struct scatterlist *src, unsigned int nbytes)
68{ 67{
69 u32 *state, state_buf[16 + (CHACHA20_STATE_ALIGN / sizeof(u32)) - 1]; 68 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
70 struct blkcipher_walk walk; 69 struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
70 u32 *state, state_buf[16 + 2] __aligned(8);
71 struct skcipher_walk walk;
71 int err; 72 int err;
72 73
73 if (nbytes <= CHACHA20_BLOCK_SIZE || !may_use_simd()) 74 BUILD_BUG_ON(CHACHA20_STATE_ALIGN != 16);
74 return crypto_chacha20_crypt(desc, dst, src, nbytes); 75 state = PTR_ALIGN(state_buf + 0, CHACHA20_STATE_ALIGN);
75 76
76 state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN); 77 if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
78 return crypto_chacha20_crypt(req);
77 79
78 blkcipher_walk_init(&walk, dst, src, nbytes); 80 err = skcipher_walk_virt(&walk, req, true);
79 err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE);
80 81
81 crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv); 82 crypto_chacha20_init(state, ctx, walk.iv);
82 83
83 kernel_fpu_begin(); 84 kernel_fpu_begin();
84 85
85 while (walk.nbytes >= CHACHA20_BLOCK_SIZE) { 86 while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
86 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, 87 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
87 rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE)); 88 rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
88 err = blkcipher_walk_done(desc, &walk, 89 err = skcipher_walk_done(&walk,
89 walk.nbytes % CHACHA20_BLOCK_SIZE); 90 walk.nbytes % CHACHA20_BLOCK_SIZE);
90 } 91 }
91 92
92 if (walk.nbytes) { 93 if (walk.nbytes) {
93 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, 94 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
94 walk.nbytes); 95 walk.nbytes);
95 err = blkcipher_walk_done(desc, &walk, 0); 96 err = skcipher_walk_done(&walk, 0);
96 } 97 }
97 98
98 kernel_fpu_end(); 99 kernel_fpu_end();
@@ -100,27 +101,22 @@ static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst,
100 return err; 101 return err;
101} 102}
102 103
103static struct crypto_alg alg = { 104static struct skcipher_alg alg = {
104 .cra_name = "chacha20", 105 .base.cra_name = "chacha20",
105 .cra_driver_name = "chacha20-simd", 106 .base.cra_driver_name = "chacha20-simd",
106 .cra_priority = 300, 107 .base.cra_priority = 300,
107 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 108 .base.cra_blocksize = 1,
108 .cra_blocksize = 1, 109 .base.cra_ctxsize = sizeof(struct chacha20_ctx),
109 .cra_type = &crypto_blkcipher_type, 110 .base.cra_alignmask = sizeof(u32) - 1,
110 .cra_ctxsize = sizeof(struct chacha20_ctx), 111 .base.cra_module = THIS_MODULE,
111 .cra_alignmask = sizeof(u32) - 1, 112
112 .cra_module = THIS_MODULE, 113 .min_keysize = CHACHA20_KEY_SIZE,
113 .cra_u = { 114 .max_keysize = CHACHA20_KEY_SIZE,
114 .blkcipher = { 115 .ivsize = CHACHA20_IV_SIZE,
115 .min_keysize = CHACHA20_KEY_SIZE, 116 .chunksize = CHACHA20_BLOCK_SIZE,
116 .max_keysize = CHACHA20_KEY_SIZE, 117 .setkey = crypto_chacha20_setkey,
117 .ivsize = CHACHA20_IV_SIZE, 118 .encrypt = chacha20_simd,
118 .geniv = "seqiv", 119 .decrypt = chacha20_simd,
119 .setkey = crypto_chacha20_setkey,
120 .encrypt = chacha20_simd,
121 .decrypt = chacha20_simd,
122 },
123 },
124}; 120};
125 121
126static int __init chacha20_simd_mod_init(void) 122static int __init chacha20_simd_mod_init(void)
@@ -133,12 +129,12 @@ static int __init chacha20_simd_mod_init(void)
133 boot_cpu_has(X86_FEATURE_AVX2) && 129 boot_cpu_has(X86_FEATURE_AVX2) &&
134 cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); 130 cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
135#endif 131#endif
136 return crypto_register_alg(&alg); 132 return crypto_register_skcipher(&alg);
137} 133}
138 134
139static void __exit chacha20_simd_mod_fini(void) 135static void __exit chacha20_simd_mod_fini(void)
140{ 136{
141 crypto_unregister_alg(&alg); 137 crypto_unregister_skcipher(&alg);
142} 138}
143 139
144module_init(chacha20_simd_mod_init); 140module_init(chacha20_simd_mod_init);
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index dc05f010ca9b..7a7de27c6f41 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -312,7 +312,7 @@ do_return:
312 ret 312 ret
313ENDPROC(crc_pcl) 313ENDPROC(crc_pcl)
314 314
315.section .rodata, "a", %progbits 315.section .rodata, "a", @progbits
316 ################################################################ 316 ################################################################
317 ## jump table Table is 129 entries x 2 bytes each 317 ## jump table Table is 129 entries x 2 bytes each
318 ################################################################ 318 ################################################################
diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S
index 35e97569d05f..de04d3e98d8d 100644
--- a/arch/x86/crypto/crct10dif-pcl-asm_64.S
+++ b/arch/x86/crypto/crct10dif-pcl-asm_64.S
@@ -554,12 +554,11 @@ _only_less_than_2:
554 554
555ENDPROC(crc_t10dif_pcl) 555ENDPROC(crc_t10dif_pcl)
556 556
557.data 557.section .rodata, "a", @progbits
558 558.align 16
559# precomputed constants 559# precomputed constants
560# these constants are precomputed from the poly: 560# these constants are precomputed from the poly:
561# 0x8bb70000 (0x8bb7 scaled to 32 bits) 561# 0x8bb70000 (0x8bb7 scaled to 32 bits)
562.align 16
563# Q = 0x18BB70000 562# Q = 0x18BB70000
564# rk1 = 2^(32*3) mod Q << 32 563# rk1 = 2^(32*3) mod Q << 32
565# rk2 = 2^(32*5) mod Q << 32 564# rk2 = 2^(32*5) mod Q << 32
@@ -613,14 +612,23 @@ rk20:
613 612
614 613
615 614
615.section .rodata.cst16.mask1, "aM", @progbits, 16
616.align 16
616mask1: 617mask1:
617.octa 0x80808080808080808080808080808080 618.octa 0x80808080808080808080808080808080
619
620.section .rodata.cst16.mask2, "aM", @progbits, 16
621.align 16
618mask2: 622mask2:
619.octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF 623.octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF
620 624
625.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16
626.align 16
621SHUF_MASK: 627SHUF_MASK:
622.octa 0x000102030405060708090A0B0C0D0E0F 628.octa 0x000102030405060708090A0B0C0D0E0F
623 629
630.section .rodata.cst32.pshufb_shf_table, "aM", @progbits, 32
631.align 32
624pshufb_shf_table: 632pshufb_shf_table:
625# use these values for shift constants for the pshufb instruction 633# use these values for shift constants for the pshufb instruction
626# different alignments result in values as shown: 634# different alignments result in values as shown:
diff --git a/arch/x86/crypto/des3_ede-asm_64.S b/arch/x86/crypto/des3_ede-asm_64.S
index 038f6ae87c5e..f3e91647ca27 100644
--- a/arch/x86/crypto/des3_ede-asm_64.S
+++ b/arch/x86/crypto/des3_ede-asm_64.S
@@ -537,7 +537,7 @@ ENTRY(des3_ede_x86_64_crypt_blk_3way)
537 ret; 537 ret;
538ENDPROC(des3_ede_x86_64_crypt_blk_3way) 538ENDPROC(des3_ede_x86_64_crypt_blk_3way)
539 539
540.data 540.section .rodata, "a", @progbits
541.align 16 541.align 16
542.L_s1: 542.L_s1:
543 .quad 0x0010100001010400, 0x0000000000000000 543 .quad 0x0010100001010400, 0x0000000000000000
diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
index eed55c8cca4f..f94375a8dcd1 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -20,8 +20,7 @@
20#include <asm/inst.h> 20#include <asm/inst.h>
21#include <asm/frame.h> 21#include <asm/frame.h>
22 22
23.data 23.section .rodata.cst16.bswap_mask, "aM", @progbits, 16
24
25.align 16 24.align 16
26.Lbswap_mask: 25.Lbswap_mask:
27 .octa 0x000102030405060708090a0b0c0d0e0f 26 .octa 0x000102030405060708090a0b0c0d0e0f
diff --git a/arch/x86/crypto/poly1305-avx2-x86_64.S b/arch/x86/crypto/poly1305-avx2-x86_64.S
index eff2f414e22b..3b6e70d085da 100644
--- a/arch/x86/crypto/poly1305-avx2-x86_64.S
+++ b/arch/x86/crypto/poly1305-avx2-x86_64.S
@@ -11,11 +11,13 @@
11 11
12#include <linux/linkage.h> 12#include <linux/linkage.h>
13 13
14.data 14.section .rodata.cst32.ANMASK, "aM", @progbits, 32
15.align 32 15.align 32
16
17ANMASK: .octa 0x0000000003ffffff0000000003ffffff 16ANMASK: .octa 0x0000000003ffffff0000000003ffffff
18 .octa 0x0000000003ffffff0000000003ffffff 17 .octa 0x0000000003ffffff0000000003ffffff
18
19.section .rodata.cst32.ORMASK, "aM", @progbits, 32
20.align 32
19ORMASK: .octa 0x00000000010000000000000001000000 21ORMASK: .octa 0x00000000010000000000000001000000
20 .octa 0x00000000010000000000000001000000 22 .octa 0x00000000010000000000000001000000
21 23
diff --git a/arch/x86/crypto/poly1305-sse2-x86_64.S b/arch/x86/crypto/poly1305-sse2-x86_64.S
index 338c748054ed..c88c670cb5fc 100644
--- a/arch/x86/crypto/poly1305-sse2-x86_64.S
+++ b/arch/x86/crypto/poly1305-sse2-x86_64.S
@@ -11,10 +11,12 @@
11 11
12#include <linux/linkage.h> 12#include <linux/linkage.h>
13 13
14.data 14.section .rodata.cst16.ANMASK, "aM", @progbits, 16
15.align 16 15.align 16
16
17ANMASK: .octa 0x0000000003ffffff0000000003ffffff 16ANMASK: .octa 0x0000000003ffffff0000000003ffffff
17
18.section .rodata.cst16.ORMASK, "aM", @progbits, 16
19.align 16
18ORMASK: .octa 0x00000000010000000000000001000000 20ORMASK: .octa 0x00000000010000000000000001000000
19 21
20.text 22.text
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 8be571808342..2925077f8c6a 100644
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -29,11 +29,12 @@
29 29
30.file "serpent-avx-x86_64-asm_64.S" 30.file "serpent-avx-x86_64-asm_64.S"
31 31
32.data 32.section .rodata.cst16.bswap128_mask, "aM", @progbits, 16
33.align 16 33.align 16
34
35.Lbswap128_mask: 34.Lbswap128_mask:
36 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 35 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
36.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16
37.align 16
37.Lxts_gf128mul_and_shl1_mask: 38.Lxts_gf128mul_and_shl1_mask:
38 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 39 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
39 40
diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S
index 97c48add33ed..d67888f2a52a 100644
--- a/arch/x86/crypto/serpent-avx2-asm_64.S
+++ b/arch/x86/crypto/serpent-avx2-asm_64.S
@@ -20,13 +20,18 @@
20 20
21.file "serpent-avx2-asm_64.S" 21.file "serpent-avx2-asm_64.S"
22 22
23.data 23.section .rodata.cst16.bswap128_mask, "aM", @progbits, 16
24.align 16 24.align 16
25
26.Lbswap128_mask: 25.Lbswap128_mask:
27 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 26 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
27
28.section .rodata.cst16.xts_gf128mul_and_shl1_mask_0, "aM", @progbits, 16
29.align 16
28.Lxts_gf128mul_and_shl1_mask_0: 30.Lxts_gf128mul_and_shl1_mask_0:
29 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 31 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
32
33.section .rodata.cst16.xts_gf128mul_and_shl1_mask_1, "aM", @progbits, 16
34.align 16
30.Lxts_gf128mul_and_shl1_mask_1: 35.Lxts_gf128mul_and_shl1_mask_1:
31 .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 36 .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0
32 37
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S
index 96df6a39d7e2..93b945597ecf 100644
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S
+++ b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S
@@ -281,11 +281,13 @@ ENTRY(sha1_mb_mgr_get_comp_job_avx2)
281 ret 281 ret
282ENDPROC(sha1_mb_mgr_get_comp_job_avx2) 282ENDPROC(sha1_mb_mgr_get_comp_job_avx2)
283 283
284.data 284.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
285
286.align 16 285.align 16
287clear_low_nibble: 286clear_low_nibble:
288.octa 0x000000000000000000000000FFFFFFF0 287.octa 0x000000000000000000000000FFFFFFF0
288
289.section .rodata.cst8, "aM", @progbits, 8
290.align 8
289one: 291one:
290.quad 1 292.quad 1
291two: 293two:
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S
index 63a0d9c8e31f..7a93b1c0d69a 100644
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S
+++ b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S
@@ -203,8 +203,7 @@ return_null:
203 203
204ENDPROC(sha1_mb_mgr_submit_avx2) 204ENDPROC(sha1_mb_mgr_submit_avx2)
205 205
206.data 206.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
207
208.align 16 207.align 16
209clear_low_nibble: 208clear_low_nibble:
210 .octa 0x000000000000000000000000FFFFFFF0 209 .octa 0x000000000000000000000000FFFFFFF0
diff --git a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S
index c9dae1cd2919..20f77aa633de 100644
--- a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S
+++ b/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S
@@ -461,21 +461,32 @@ lloop:
461ENDPROC(sha1_x8_avx2) 461ENDPROC(sha1_x8_avx2)
462 462
463 463
464.data 464.section .rodata.cst32.K00_19, "aM", @progbits, 32
465
466.align 32 465.align 32
467K00_19: 466K00_19:
468.octa 0x5A8279995A8279995A8279995A827999 467.octa 0x5A8279995A8279995A8279995A827999
469.octa 0x5A8279995A8279995A8279995A827999 468.octa 0x5A8279995A8279995A8279995A827999
469
470.section .rodata.cst32.K20_39, "aM", @progbits, 32
471.align 32
470K20_39: 472K20_39:
471.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 473.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1
472.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 474.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1
475
476.section .rodata.cst32.K40_59, "aM", @progbits, 32
477.align 32
473K40_59: 478K40_59:
474.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC 479.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC
475.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC 480.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC
481
482.section .rodata.cst32.K60_79, "aM", @progbits, 32
483.align 32
476K60_79: 484K60_79:
477.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 485.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6
478.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 486.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6
487
488.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
489.align 32
479PSHUFFLE_BYTE_FLIP_MASK: 490PSHUFFLE_BYTE_FLIP_MASK:
480.octa 0x0c0d0e0f08090a0b0405060700010203 491.octa 0x0c0d0e0f08090a0b0405060700010203
481.octa 0x0c0d0e0f08090a0b0405060700010203 492.octa 0x0c0d0e0f08090a0b0405060700010203
diff --git a/arch/x86/crypto/sha1_ni_asm.S b/arch/x86/crypto/sha1_ni_asm.S
index 874a651b9e7d..ebbdba72ae07 100644
--- a/arch/x86/crypto/sha1_ni_asm.S
+++ b/arch/x86/crypto/sha1_ni_asm.S
@@ -293,10 +293,12 @@ ENTRY(sha1_ni_transform)
293 ret 293 ret
294ENDPROC(sha1_ni_transform) 294ENDPROC(sha1_ni_transform)
295 295
296.data 296.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
297 297.align 16
298.align 64
299PSHUFFLE_BYTE_FLIP_MASK: 298PSHUFFLE_BYTE_FLIP_MASK:
300 .octa 0x000102030405060708090a0b0c0d0e0f 299 .octa 0x000102030405060708090a0b0c0d0e0f
300
301.section .rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16
302.align 16
301UPPER_WORD_MASK: 303UPPER_WORD_MASK:
302 .octa 0xFFFFFFFF000000000000000000000000 304 .octa 0xFFFFFFFF000000000000000000000000
diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/crypto/sha256-avx-asm.S
index 92b3b5d75ba9..e08888a1a5f2 100644
--- a/arch/x86/crypto/sha256-avx-asm.S
+++ b/arch/x86/crypto/sha256-avx-asm.S
@@ -463,7 +463,7 @@ done_hash:
463 ret 463 ret
464ENDPROC(sha256_transform_avx) 464ENDPROC(sha256_transform_avx)
465 465
466.data 466.section .rodata.cst256.K256, "aM", @progbits, 256
467.align 64 467.align 64
468K256: 468K256:
469 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 469 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
@@ -483,14 +483,21 @@ K256:
483 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 483 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
484 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 484 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
485 485
486.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
487.align 16
486PSHUFFLE_BYTE_FLIP_MASK: 488PSHUFFLE_BYTE_FLIP_MASK:
487 .octa 0x0c0d0e0f08090a0b0405060700010203 489 .octa 0x0c0d0e0f08090a0b0405060700010203
488 490
491.section .rodata.cst16._SHUF_00BA, "aM", @progbits, 16
492.align 16
489# shuffle xBxA -> 00BA 493# shuffle xBxA -> 00BA
490_SHUF_00BA: 494_SHUF_00BA:
491 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100 495 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
492 496
497.section .rodata.cst16._SHUF_DC00, "aM", @progbits, 16
498.align 16
493# shuffle xDxC -> DC00 499# shuffle xDxC -> DC00
494_SHUF_DC00: 500_SHUF_DC00:
495 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF 501 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
502
496#endif 503#endif
diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/crypto/sha256-avx2-asm.S
index 570ec5ec62d7..89c8f09787d2 100644
--- a/arch/x86/crypto/sha256-avx2-asm.S
+++ b/arch/x86/crypto/sha256-avx2-asm.S
@@ -723,7 +723,7 @@ done_hash:
723 ret 723 ret
724ENDPROC(sha256_transform_rorx) 724ENDPROC(sha256_transform_rorx)
725 725
726.data 726.section .rodata.cst512.K256, "aM", @progbits, 512
727.align 64 727.align 64
728K256: 728K256:
729 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 729 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
@@ -759,14 +759,21 @@ K256:
759 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 759 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
760 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 760 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
761 761
762.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
763.align 32
762PSHUFFLE_BYTE_FLIP_MASK: 764PSHUFFLE_BYTE_FLIP_MASK:
763 .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203 765 .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203
764 766
765# shuffle xBxA -> 00BA 767# shuffle xBxA -> 00BA
768.section .rodata.cst32._SHUF_00BA, "aM", @progbits, 32
769.align 32
766_SHUF_00BA: 770_SHUF_00BA:
767 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100 771 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100
768 772
769# shuffle xDxC -> DC00 773# shuffle xDxC -> DC00
774.section .rodata.cst32._SHUF_DC00, "aM", @progbits, 32
775.align 32
770_SHUF_DC00: 776_SHUF_DC00:
771 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF 777 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF
778
772#endif 779#endif
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S
index a78a0694ddef..8fe6338bcc84 100644
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S
+++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S
@@ -284,11 +284,13 @@ ENTRY(sha256_mb_mgr_get_comp_job_avx2)
284 ret 284 ret
285ENDPROC(sha256_mb_mgr_get_comp_job_avx2) 285ENDPROC(sha256_mb_mgr_get_comp_job_avx2)
286 286
287.data 287.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
288
289.align 16 288.align 16
290clear_low_nibble: 289clear_low_nibble:
291.octa 0x000000000000000000000000FFFFFFF0 290.octa 0x000000000000000000000000FFFFFFF0
291
292.section .rodata.cst8, "aM", @progbits, 8
293.align 8
292one: 294one:
293.quad 1 295.quad 1
294two: 296two:
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S
index 7ea670e25acc..b36ae7454084 100644
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S
+++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S
@@ -208,8 +208,7 @@ return_null:
208 208
209ENDPROC(sha256_mb_mgr_submit_avx2) 209ENDPROC(sha256_mb_mgr_submit_avx2)
210 210
211.data 211.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
212
213.align 16 212.align 16
214clear_low_nibble: 213clear_low_nibble:
215 .octa 0x000000000000000000000000FFFFFFF0 214 .octa 0x000000000000000000000000FFFFFFF0
diff --git a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S
index aa21aea4c722..1687c80c5995 100644
--- a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S
+++ b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S
@@ -437,7 +437,8 @@ Lrounds_16_xx:
437 437
438 ret 438 ret
439ENDPROC(sha256_x8_avx2) 439ENDPROC(sha256_x8_avx2)
440.data 440
441.section .rodata.K256_8, "a", @progbits
441.align 64 442.align 64
442K256_8: 443K256_8:
443 .octa 0x428a2f98428a2f98428a2f98428a2f98 444 .octa 0x428a2f98428a2f98428a2f98428a2f98
@@ -568,10 +569,14 @@ K256_8:
568 .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 569 .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7
569 .octa 0xc67178f2c67178f2c67178f2c67178f2 570 .octa 0xc67178f2c67178f2c67178f2c67178f2
570 .octa 0xc67178f2c67178f2c67178f2c67178f2 571 .octa 0xc67178f2c67178f2c67178f2c67178f2
572
573.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
574.align 32
571PSHUFFLE_BYTE_FLIP_MASK: 575PSHUFFLE_BYTE_FLIP_MASK:
572.octa 0x0c0d0e0f08090a0b0405060700010203 576.octa 0x0c0d0e0f08090a0b0405060700010203
573.octa 0x0c0d0e0f08090a0b0405060700010203 577.octa 0x0c0d0e0f08090a0b0405060700010203
574 578
579.section .rodata.cst256.K256, "aM", @progbits, 256
575.align 64 580.align 64
576.global K256 581.global K256
577K256: 582K256:
diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/crypto/sha256-ssse3-asm.S
index 2cedc44e8121..39b83c93e7fd 100644
--- a/arch/x86/crypto/sha256-ssse3-asm.S
+++ b/arch/x86/crypto/sha256-ssse3-asm.S
@@ -474,7 +474,7 @@ done_hash:
474 ret 474 ret
475ENDPROC(sha256_transform_ssse3) 475ENDPROC(sha256_transform_ssse3)
476 476
477.data 477.section .rodata.cst256.K256, "aM", @progbits, 256
478.align 64 478.align 64
479K256: 479K256:
480 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 480 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
@@ -494,13 +494,19 @@ K256:
494 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 494 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
495 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 495 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
496 496
497.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
498.align 16
497PSHUFFLE_BYTE_FLIP_MASK: 499PSHUFFLE_BYTE_FLIP_MASK:
498 .octa 0x0c0d0e0f08090a0b0405060700010203 500 .octa 0x0c0d0e0f08090a0b0405060700010203
499 501
502.section .rodata.cst16._SHUF_00BA, "aM", @progbits, 16
503.align 16
500# shuffle xBxA -> 00BA 504# shuffle xBxA -> 00BA
501_SHUF_00BA: 505_SHUF_00BA:
502 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100 506 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
503 507
508.section .rodata.cst16._SHUF_DC00, "aM", @progbits, 16
509.align 16
504# shuffle xDxC -> DC00 510# shuffle xDxC -> DC00
505_SHUF_DC00: 511_SHUF_DC00:
506 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF 512 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/crypto/sha256_ni_asm.S
index 748cdf21a938..fb58f58ecfbc 100644
--- a/arch/x86/crypto/sha256_ni_asm.S
+++ b/arch/x86/crypto/sha256_ni_asm.S
@@ -329,7 +329,7 @@ ENTRY(sha256_ni_transform)
329 ret 329 ret
330ENDPROC(sha256_ni_transform) 330ENDPROC(sha256_ni_transform)
331 331
332.data 332.section .rodata.cst256.K256, "aM", @progbits, 256
333.align 64 333.align 64
334K256: 334K256:
335 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 335 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
@@ -349,5 +349,7 @@ K256:
349 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 349 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
350 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 350 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
351 351
352.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
353.align 16
352PSHUFFLE_BYTE_FLIP_MASK: 354PSHUFFLE_BYTE_FLIP_MASK:
353 .octa 0x0c0d0e0f08090a0b0405060700010203 355 .octa 0x0c0d0e0f08090a0b0405060700010203
diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S
index 565274d6a641..39235fefe6f7 100644
--- a/arch/x86/crypto/sha512-avx-asm.S
+++ b/arch/x86/crypto/sha512-avx-asm.S
@@ -370,14 +370,17 @@ ENDPROC(sha512_transform_avx)
370######################################################################## 370########################################################################
371### Binary Data 371### Binary Data
372 372
373.data 373.section .rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
374
375.align 16 374.align 16
376
377# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. 375# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
378XMM_QWORD_BSWAP: 376XMM_QWORD_BSWAP:
379 .octa 0x08090a0b0c0d0e0f0001020304050607 377 .octa 0x08090a0b0c0d0e0f0001020304050607
380 378
379# Mergeable 640-byte rodata section. This allows linker to merge the table
380# with other, exactly the same 640-byte fragment of another rodata section
381# (if such section exists).
382.section .rodata.cst640.K512, "aM", @progbits, 640
383.align 64
381# K[t] used in SHA512 hashing 384# K[t] used in SHA512 hashing
382K512: 385K512:
383 .quad 0x428a2f98d728ae22,0x7137449123ef65cd 386 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/arch/x86/crypto/sha512-avx2-asm.S
index 1f20b35d8573..7f5f6c6ec72e 100644
--- a/arch/x86/crypto/sha512-avx2-asm.S
+++ b/arch/x86/crypto/sha512-avx2-asm.S
@@ -684,8 +684,11 @@ ENDPROC(sha512_transform_rorx)
684######################################################################## 684########################################################################
685### Binary Data 685### Binary Data
686 686
687.data
688 687
688# Mergeable 640-byte rodata section. This allows linker to merge the table
689# with other, exactly the same 640-byte fragment of another rodata section
690# (if such section exists).
691.section .rodata.cst640.K512, "aM", @progbits, 640
689.align 64 692.align 64
690# K[t] used in SHA512 hashing 693# K[t] used in SHA512 hashing
691K512: 694K512:
@@ -730,14 +733,17 @@ K512:
730 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a 733 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
731 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 734 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
732 735
736.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
733.align 32 737.align 32
734
735# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. 738# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
736PSHUFFLE_BYTE_FLIP_MASK: 739PSHUFFLE_BYTE_FLIP_MASK:
737 .octa 0x08090a0b0c0d0e0f0001020304050607 740 .octa 0x08090a0b0c0d0e0f0001020304050607
738 .octa 0x18191a1b1c1d1e1f1011121314151617 741 .octa 0x18191a1b1c1d1e1f1011121314151617
739 742
743.section .rodata.cst32.MASK_YMM_LO, "aM", @progbits, 32
744.align 32
740MASK_YMM_LO: 745MASK_YMM_LO:
741 .octa 0x00000000000000000000000000000000 746 .octa 0x00000000000000000000000000000000
742 .octa 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 747 .octa 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
748
743#endif 749#endif
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c
index 9c1bb6d58141..2dd3674b5a1e 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb.c
+++ b/arch/x86/crypto/sha512-mb/sha512_mb.c
@@ -221,7 +221,7 @@ static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit
221} 221}
222 222
223static struct sha512_hash_ctx 223static struct sha512_hash_ctx
224 *sha512_ctx_mgr_get_comp_ctx(struct sha512_ctx_mgr *mgr) 224 *sha512_ctx_mgr_get_comp_ctx(struct mcryptd_alg_cstate *cstate)
225{ 225{
226 /* 226 /*
227 * If get_comp_job returns NULL, there are no jobs complete. 227 * If get_comp_job returns NULL, there are no jobs complete.
@@ -233,11 +233,17 @@ static struct sha512_hash_ctx
233 * Otherwise, all jobs currently being managed by the hash_ctx_mgr 233 * Otherwise, all jobs currently being managed by the hash_ctx_mgr
234 * still need processing. 234 * still need processing.
235 */ 235 */
236 struct sha512_ctx_mgr *mgr;
236 struct sha512_hash_ctx *ctx; 237 struct sha512_hash_ctx *ctx;
238 unsigned long flags;
237 239
240 mgr = cstate->mgr;
241 spin_lock_irqsave(&cstate->work_lock, flags);
238 ctx = (struct sha512_hash_ctx *) 242 ctx = (struct sha512_hash_ctx *)
239 sha512_job_mgr_get_comp_job(&mgr->mgr); 243 sha512_job_mgr_get_comp_job(&mgr->mgr);
240 return sha512_ctx_mgr_resubmit(mgr, ctx); 244 ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
245 spin_unlock_irqrestore(&cstate->work_lock, flags);
246 return ctx;
241} 247}
242 248
243static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr) 249static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
@@ -246,12 +252,17 @@ static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
246} 252}
247 253
248static struct sha512_hash_ctx 254static struct sha512_hash_ctx
249 *sha512_ctx_mgr_submit(struct sha512_ctx_mgr *mgr, 255 *sha512_ctx_mgr_submit(struct mcryptd_alg_cstate *cstate,
250 struct sha512_hash_ctx *ctx, 256 struct sha512_hash_ctx *ctx,
251 const void *buffer, 257 const void *buffer,
252 uint32_t len, 258 uint32_t len,
253 int flags) 259 int flags)
254{ 260{
261 struct sha512_ctx_mgr *mgr;
262 unsigned long irqflags;
263
264 mgr = cstate->mgr;
265 spin_lock_irqsave(&cstate->work_lock, irqflags);
255 if (flags & (~HASH_ENTIRE)) { 266 if (flags & (~HASH_ENTIRE)) {
256 /* 267 /*
257 * User should not pass anything other than FIRST, UPDATE, or 268 * User should not pass anything other than FIRST, UPDATE, or
@@ -351,20 +362,26 @@ static struct sha512_hash_ctx
351 } 362 }
352 } 363 }
353 364
354 return sha512_ctx_mgr_resubmit(mgr, ctx); 365 ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
366 spin_unlock_irqrestore(&cstate->work_lock, irqflags);
367 return ctx;
355} 368}
356 369
357static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct sha512_ctx_mgr *mgr) 370static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct mcryptd_alg_cstate *cstate)
358{ 371{
372 struct sha512_ctx_mgr *mgr;
359 struct sha512_hash_ctx *ctx; 373 struct sha512_hash_ctx *ctx;
374 unsigned long flags;
360 375
376 mgr = cstate->mgr;
377 spin_lock_irqsave(&cstate->work_lock, flags);
361 while (1) { 378 while (1) {
362 ctx = (struct sha512_hash_ctx *) 379 ctx = (struct sha512_hash_ctx *)
363 sha512_job_mgr_flush(&mgr->mgr); 380 sha512_job_mgr_flush(&mgr->mgr);
364 381
365 /* If flush returned 0, there are no more jobs in flight. */ 382 /* If flush returned 0, there are no more jobs in flight. */
366 if (!ctx) 383 if (!ctx)
367 return NULL; 384 break;
368 385
369 /* 386 /*
370 * If flush returned a job, resubmit the job to finish 387 * If flush returned a job, resubmit the job to finish
@@ -378,8 +395,10 @@ static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct sha512_ctx_mgr *mgr)
378 * the sha512_ctx_mgr still need processing. Loop. 395 * the sha512_ctx_mgr still need processing. Loop.
379 */ 396 */
380 if (ctx) 397 if (ctx)
381 return ctx; 398 break;
382 } 399 }
400 spin_unlock_irqrestore(&cstate->work_lock, flags);
401 return ctx;
383} 402}
384 403
385static int sha512_mb_init(struct ahash_request *areq) 404static int sha512_mb_init(struct ahash_request *areq)
@@ -439,11 +458,11 @@ static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
439 sha_ctx = (struct sha512_hash_ctx *) 458 sha_ctx = (struct sha512_hash_ctx *)
440 ahash_request_ctx(&rctx->areq); 459 ahash_request_ctx(&rctx->areq);
441 kernel_fpu_begin(); 460 kernel_fpu_begin();
442 sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, 461 sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx,
443 rctx->walk.data, nbytes, flag); 462 rctx->walk.data, nbytes, flag);
444 if (!sha_ctx) { 463 if (!sha_ctx) {
445 if (flush) 464 if (flush)
446 sha_ctx = sha512_ctx_mgr_flush(cstate->mgr); 465 sha_ctx = sha512_ctx_mgr_flush(cstate);
447 } 466 }
448 kernel_fpu_end(); 467 kernel_fpu_end();
449 if (sha_ctx) 468 if (sha_ctx)
@@ -471,11 +490,12 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
471 struct sha512_hash_ctx *sha_ctx; 490 struct sha512_hash_ctx *sha_ctx;
472 struct mcryptd_hash_request_ctx *req_ctx; 491 struct mcryptd_hash_request_ctx *req_ctx;
473 int ret; 492 int ret;
493 unsigned long flags;
474 494
475 /* remove from work list */ 495 /* remove from work list */
476 spin_lock(&cstate->work_lock); 496 spin_lock_irqsave(&cstate->work_lock, flags);
477 list_del(&rctx->waiter); 497 list_del(&rctx->waiter);
478 spin_unlock(&cstate->work_lock); 498 spin_unlock_irqrestore(&cstate->work_lock, flags);
479 499
480 if (irqs_disabled()) 500 if (irqs_disabled())
481 rctx->complete(&req->base, err); 501 rctx->complete(&req->base, err);
@@ -486,14 +506,14 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
486 } 506 }
487 507
488 /* check to see if there are other jobs that are done */ 508 /* check to see if there are other jobs that are done */
489 sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr); 509 sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
490 while (sha_ctx) { 510 while (sha_ctx) {
491 req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); 511 req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
492 ret = sha_finish_walk(&req_ctx, cstate, false); 512 ret = sha_finish_walk(&req_ctx, cstate, false);
493 if (req_ctx) { 513 if (req_ctx) {
494 spin_lock(&cstate->work_lock); 514 spin_lock_irqsave(&cstate->work_lock, flags);
495 list_del(&req_ctx->waiter); 515 list_del(&req_ctx->waiter);
496 spin_unlock(&cstate->work_lock); 516 spin_unlock_irqrestore(&cstate->work_lock, flags);
497 517
498 req = cast_mcryptd_ctx_to_req(req_ctx); 518 req = cast_mcryptd_ctx_to_req(req_ctx);
499 if (irqs_disabled()) 519 if (irqs_disabled())
@@ -504,7 +524,7 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
504 local_bh_enable(); 524 local_bh_enable();
505 } 525 }
506 } 526 }
507 sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr); 527 sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
508 } 528 }
509 529
510 return 0; 530 return 0;
@@ -515,6 +535,7 @@ static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
515{ 535{
516 unsigned long next_flush; 536 unsigned long next_flush;
517 unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); 537 unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
538 unsigned long flags;
518 539
519 /* initialize tag */ 540 /* initialize tag */
520 rctx->tag.arrival = jiffies; /* tag the arrival time */ 541 rctx->tag.arrival = jiffies; /* tag the arrival time */
@@ -522,9 +543,9 @@ static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
522 next_flush = rctx->tag.arrival + delay; 543 next_flush = rctx->tag.arrival + delay;
523 rctx->tag.expire = next_flush; 544 rctx->tag.expire = next_flush;
524 545
525 spin_lock(&cstate->work_lock); 546 spin_lock_irqsave(&cstate->work_lock, flags);
526 list_add_tail(&rctx->waiter, &cstate->work_list); 547 list_add_tail(&rctx->waiter, &cstate->work_list);
527 spin_unlock(&cstate->work_lock); 548 spin_unlock_irqrestore(&cstate->work_lock, flags);
528 549
529 mcryptd_arm_flusher(cstate, delay); 550 mcryptd_arm_flusher(cstate, delay);
530} 551}
@@ -565,7 +586,7 @@ static int sha512_mb_update(struct ahash_request *areq)
565 sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); 586 sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
566 sha512_mb_add_list(rctx, cstate); 587 sha512_mb_add_list(rctx, cstate);
567 kernel_fpu_begin(); 588 kernel_fpu_begin();
568 sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, 589 sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
569 nbytes, HASH_UPDATE); 590 nbytes, HASH_UPDATE);
570 kernel_fpu_end(); 591 kernel_fpu_end();
571 592
@@ -628,7 +649,7 @@ static int sha512_mb_finup(struct ahash_request *areq)
628 sha512_mb_add_list(rctx, cstate); 649 sha512_mb_add_list(rctx, cstate);
629 650
630 kernel_fpu_begin(); 651 kernel_fpu_begin();
631 sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, 652 sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
632 nbytes, flag); 653 nbytes, flag);
633 kernel_fpu_end(); 654 kernel_fpu_end();
634 655
@@ -677,8 +698,7 @@ static int sha512_mb_final(struct ahash_request *areq)
677 /* flag HASH_FINAL and 0 data size */ 698 /* flag HASH_FINAL and 0 data size */
678 sha512_mb_add_list(rctx, cstate); 699 sha512_mb_add_list(rctx, cstate);
679 kernel_fpu_begin(); 700 kernel_fpu_begin();
680 sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, 701 sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, &data, 0, HASH_LAST);
681 HASH_LAST);
682 kernel_fpu_end(); 702 kernel_fpu_end();
683 703
684 /* check if anything is returned */ 704 /* check if anything is returned */
@@ -940,7 +960,7 @@ static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate)
940 break; 960 break;
941 kernel_fpu_begin(); 961 kernel_fpu_begin();
942 sha_ctx = (struct sha512_hash_ctx *) 962 sha_ctx = (struct sha512_hash_ctx *)
943 sha512_ctx_mgr_flush(cstate->mgr); 963 sha512_ctx_mgr_flush(cstate);
944 kernel_fpu_end(); 964 kernel_fpu_end();
945 if (!sha_ctx) { 965 if (!sha_ctx) {
946 pr_err("sha512_mb error: nothing got flushed for" 966 pr_err("sha512_mb error: nothing got flushed for"
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S
index 3ddba19a0db6..7c629caebc05 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S
+++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S
@@ -280,12 +280,18 @@ ENTRY(sha512_mb_mgr_get_comp_job_avx2)
280 pop %rbx 280 pop %rbx
281 ret 281 ret
282ENDPROC(sha512_mb_mgr_get_comp_job_avx2) 282ENDPROC(sha512_mb_mgr_get_comp_job_avx2)
283.data
284 283
285.align 16 284.section .rodata.cst8.one, "aM", @progbits, 8
285.align 8
286one: 286one:
287.quad 1 287.quad 1
288
289.section .rodata.cst8.two, "aM", @progbits, 8
290.align 8
288two: 291two:
289.quad 2 292.quad 2
293
294.section .rodata.cst8.three, "aM", @progbits, 8
295.align 8
290three: 296three:
291.quad 3 297.quad 3
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S
index 815f07bdd1f8..4ba709ba78e5 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S
+++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S
@@ -209,8 +209,9 @@ return_null:
209 xor job_rax, job_rax 209 xor job_rax, job_rax
210 jmp return 210 jmp return
211ENDPROC(sha512_mb_mgr_submit_avx2) 211ENDPROC(sha512_mb_mgr_submit_avx2)
212.data
213 212
213/* UNUSED?
214.section .rodata.cst16, "aM", @progbits, 16
214.align 16 215.align 16
215H0: .int 0x6a09e667 216H0: .int 0x6a09e667
216H1: .int 0xbb67ae85 217H1: .int 0xbb67ae85
@@ -220,3 +221,4 @@ H4: .int 0x510e527f
220H5: .int 0x9b05688c 221H5: .int 0x9b05688c
221H6: .int 0x1f83d9ab 222H6: .int 0x1f83d9ab
222H7: .int 0x5be0cd19 223H7: .int 0x5be0cd19
224*/
diff --git a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S
index 31ab1eff6413..e22e907643a6 100644
--- a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S
+++ b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S
@@ -361,7 +361,7 @@ Lrounds_16_xx:
361 ret 361 ret
362ENDPROC(sha512_x4_avx2) 362ENDPROC(sha512_x4_avx2)
363 363
364.data 364.section .rodata.K512_4, "a", @progbits
365.align 64 365.align 64
366K512_4: 366K512_4:
367 .octa 0x428a2f98d728ae22428a2f98d728ae22,\ 367 .octa 0x428a2f98d728ae22428a2f98d728ae22,\
@@ -525,5 +525,7 @@ K512_4:
525 .octa 0x6c44198c4a4758176c44198c4a475817,\ 525 .octa 0x6c44198c4a4758176c44198c4a475817,\
526 0x6c44198c4a4758176c44198c4a475817 526 0x6c44198c4a4758176c44198c4a475817
527 527
528.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
529.align 32
528PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607 530PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607
529 .octa 0x18191a1b1c1d1e1f1011121314151617 531 .octa 0x18191a1b1c1d1e1f1011121314151617
diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S
index e610e29cbc81..66bbd9058a90 100644
--- a/arch/x86/crypto/sha512-ssse3-asm.S
+++ b/arch/x86/crypto/sha512-ssse3-asm.S
@@ -369,14 +369,17 @@ ENDPROC(sha512_transform_ssse3)
369######################################################################## 369########################################################################
370### Binary Data 370### Binary Data
371 371
372.data 372.section .rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
373
374.align 16 373.align 16
375
376# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. 374# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
377XMM_QWORD_BSWAP: 375XMM_QWORD_BSWAP:
378 .octa 0x08090a0b0c0d0e0f0001020304050607 376 .octa 0x08090a0b0c0d0e0f0001020304050607
379 377
378# Mergeable 640-byte rodata section. This allows linker to merge the table
379# with other, exactly the same 640-byte fragment of another rodata section
380# (if such section exists).
381.section .rodata.cst640.K512, "aM", @progbits, 640
382.align 64
380# K[t] used in SHA512 hashing 383# K[t] used in SHA512 hashing
381K512: 384K512:
382 .quad 0x428a2f98d728ae22,0x7137449123ef65cd 385 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index dc66273e610d..b3f49d286348 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -29,11 +29,13 @@
29 29
30.file "twofish-avx-x86_64-asm_64.S" 30.file "twofish-avx-x86_64-asm_64.S"
31 31
32.data 32.section .rodata.cst16.bswap128_mask, "aM", @progbits, 16
33.align 16 33.align 16
34
35.Lbswap128_mask: 34.Lbswap128_mask:
36 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 35 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
36
37.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16
38.align 16
37.Lxts_gf128mul_and_shl1_mask: 39.Lxts_gf128mul_and_shl1_mask:
38 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 40 .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
39 41
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 160f08e721cc..f37e9cca50e1 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -263,6 +263,7 @@ comment "Authenticated Encryption with Associated Data"
263config CRYPTO_CCM 263config CRYPTO_CCM
264 tristate "CCM support" 264 tristate "CCM support"
265 select CRYPTO_CTR 265 select CRYPTO_CTR
266 select CRYPTO_HASH
266 select CRYPTO_AEAD 267 select CRYPTO_AEAD
267 help 268 help
268 Support for Counter with CBC MAC. Required for IPsec. 269 Support for Counter with CBC MAC. Required for IPsec.
@@ -374,6 +375,7 @@ config CRYPTO_XTS
374 select CRYPTO_BLKCIPHER 375 select CRYPTO_BLKCIPHER
375 select CRYPTO_MANAGER 376 select CRYPTO_MANAGER
376 select CRYPTO_GF128MUL 377 select CRYPTO_GF128MUL
378 select CRYPTO_ECB
377 help 379 help
378 XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain, 380 XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain,
379 key size 256, 384 or 512 bits. This implementation currently 381 key size 256, 384 or 512 bits. This implementation currently
@@ -895,6 +897,23 @@ config CRYPTO_AES
895 897
896 See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information. 898 See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information.
897 899
900config CRYPTO_AES_TI
901 tristate "Fixed time AES cipher"
902 select CRYPTO_ALGAPI
903 help
904 This is a generic implementation of AES that attempts to eliminate
905 data dependent latencies as much as possible without affecting
906 performance too much. It is intended for use by the generic CCM
907 and GCM drivers, and other CTR or CMAC/XCBC based modes that rely
908 solely on encryption (although decryption is supported as well, but
909 with a more dramatic performance hit)
910
911 Instead of using 16 lookup tables of 1 KB each, (8 for encryption and
912 8 for decryption), this implementation only uses just two S-boxes of
913 256 bytes each, and attempts to eliminate data dependent latencies by
914 prefetching the entire table into the cache at the start of each
915 block.
916
898config CRYPTO_AES_586 917config CRYPTO_AES_586
899 tristate "AES cipher algorithms (i586)" 918 tristate "AES cipher algorithms (i586)"
900 depends on (X86 || UML_X86) && !64BIT 919 depends on (X86 || UML_X86) && !64BIT
diff --git a/crypto/Makefile b/crypto/Makefile
index b8f0e3eb0791..8a44057240d5 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -75,6 +75,7 @@ obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o
75obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o 75obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o
76obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o 76obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o
77obj-$(CONFIG_CRYPTO_WP512) += wp512.o 77obj-$(CONFIG_CRYPTO_WP512) += wp512.o
78CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149
78obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o 79obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o
79obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o 80obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o
80obj-$(CONFIG_CRYPTO_ECB) += ecb.o 81obj-$(CONFIG_CRYPTO_ECB) += ecb.o
@@ -98,7 +99,9 @@ obj-$(CONFIG_CRYPTO_BLOWFISH_COMMON) += blowfish_common.o
98obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o 99obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o
99obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o 100obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
100obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o 101obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
102CFLAGS_serpent_generic.o := $(call cc-option,-fsched-pressure) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149
101obj-$(CONFIG_CRYPTO_AES) += aes_generic.o 103obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
104obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o
102obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o 105obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
103obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o 106obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o
104obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o 107obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o
diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index d676fc59521a..d880a4897159 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -19,6 +19,7 @@
19#include <linux/slab.h> 19#include <linux/slab.h>
20#include <linux/seq_file.h> 20#include <linux/seq_file.h>
21#include <linux/cryptouser.h> 21#include <linux/cryptouser.h>
22#include <linux/compiler.h>
22#include <net/netlink.h> 23#include <net/netlink.h>
23 24
24#include <crypto/scatterwalk.h> 25#include <crypto/scatterwalk.h>
@@ -394,7 +395,7 @@ static int crypto_ablkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
394#endif 395#endif
395 396
396static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg) 397static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg)
397 __attribute__ ((unused)); 398 __maybe_unused;
398static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg) 399static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg)
399{ 400{
400 struct ablkcipher_alg *ablkcipher = &alg->cra_ablkcipher; 401 struct ablkcipher_alg *ablkcipher = &alg->cra_ablkcipher;
@@ -468,7 +469,7 @@ static int crypto_givcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
468#endif 469#endif
469 470
470static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg) 471static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg)
471 __attribute__ ((unused)); 472 __maybe_unused;
472static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg) 473static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg)
473{ 474{
474 struct ablkcipher_alg *ablkcipher = &alg->cra_ablkcipher; 475 struct ablkcipher_alg *ablkcipher = &alg->cra_ablkcipher;
diff --git a/crypto/acompress.c b/crypto/acompress.c
index 887783d8e9a9..47d11627cd20 100644
--- a/crypto/acompress.c
+++ b/crypto/acompress.c
@@ -20,6 +20,7 @@
20#include <linux/crypto.h> 20#include <linux/crypto.h>
21#include <crypto/algapi.h> 21#include <crypto/algapi.h>
22#include <linux/cryptouser.h> 22#include <linux/cryptouser.h>
23#include <linux/compiler.h>
23#include <net/netlink.h> 24#include <net/netlink.h>
24#include <crypto/internal/acompress.h> 25#include <crypto/internal/acompress.h>
25#include <crypto/internal/scompress.h> 26#include <crypto/internal/scompress.h>
@@ -50,7 +51,7 @@ static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg)
50#endif 51#endif
51 52
52static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) 53static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
53 __attribute__ ((unused)); 54 __maybe_unused;
54 55
55static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) 56static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
56{ 57{
diff --git a/crypto/aead.c b/crypto/aead.c
index 3f5c5ff004ab..f794b30a9407 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -24,6 +24,7 @@
24#include <linux/slab.h> 24#include <linux/slab.h>
25#include <linux/seq_file.h> 25#include <linux/seq_file.h>
26#include <linux/cryptouser.h> 26#include <linux/cryptouser.h>
27#include <linux/compiler.h>
27#include <net/netlink.h> 28#include <net/netlink.h>
28 29
29#include "internal.h" 30#include "internal.h"
@@ -132,7 +133,7 @@ static int crypto_aead_report(struct sk_buff *skb, struct crypto_alg *alg)
132#endif 133#endif
133 134
134static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg) 135static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg)
135 __attribute__ ((unused)); 136 __maybe_unused;
136static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg) 137static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg)
137{ 138{
138 struct aead_alg *aead = container_of(alg, struct aead_alg, base); 139 struct aead_alg *aead = container_of(alg, struct aead_alg, base);
diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c
index 3dd101144a58..ca554d57d01e 100644
--- a/crypto/aes_generic.c
+++ b/crypto/aes_generic.c
@@ -54,6 +54,7 @@
54#include <linux/errno.h> 54#include <linux/errno.h>
55#include <linux/crypto.h> 55#include <linux/crypto.h>
56#include <asm/byteorder.h> 56#include <asm/byteorder.h>
57#include <asm/unaligned.h>
57 58
58static inline u8 byte(const u32 x, const unsigned n) 59static inline u8 byte(const u32 x, const unsigned n)
59{ 60{
@@ -1216,7 +1217,6 @@ EXPORT_SYMBOL_GPL(crypto_il_tab);
1216int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 1217int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
1217 unsigned int key_len) 1218 unsigned int key_len)
1218{ 1219{
1219 const __le32 *key = (const __le32 *)in_key;
1220 u32 i, t, u, v, w, j; 1220 u32 i, t, u, v, w, j;
1221 1221
1222 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 1222 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
@@ -1225,10 +1225,15 @@ int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
1225 1225
1226 ctx->key_length = key_len; 1226 ctx->key_length = key_len;
1227 1227
1228 ctx->key_dec[key_len + 24] = ctx->key_enc[0] = le32_to_cpu(key[0]); 1228 ctx->key_enc[0] = get_unaligned_le32(in_key);
1229 ctx->key_dec[key_len + 25] = ctx->key_enc[1] = le32_to_cpu(key[1]); 1229 ctx->key_enc[1] = get_unaligned_le32(in_key + 4);
1230 ctx->key_dec[key_len + 26] = ctx->key_enc[2] = le32_to_cpu(key[2]); 1230 ctx->key_enc[2] = get_unaligned_le32(in_key + 8);
1231 ctx->key_dec[key_len + 27] = ctx->key_enc[3] = le32_to_cpu(key[3]); 1231 ctx->key_enc[3] = get_unaligned_le32(in_key + 12);
1232
1233 ctx->key_dec[key_len + 24] = ctx->key_enc[0];
1234 ctx->key_dec[key_len + 25] = ctx->key_enc[1];
1235 ctx->key_dec[key_len + 26] = ctx->key_enc[2];
1236 ctx->key_dec[key_len + 27] = ctx->key_enc[3];
1232 1237
1233 switch (key_len) { 1238 switch (key_len) {
1234 case AES_KEYSIZE_128: 1239 case AES_KEYSIZE_128:
@@ -1238,17 +1243,17 @@ int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
1238 break; 1243 break;
1239 1244
1240 case AES_KEYSIZE_192: 1245 case AES_KEYSIZE_192:
1241 ctx->key_enc[4] = le32_to_cpu(key[4]); 1246 ctx->key_enc[4] = get_unaligned_le32(in_key + 16);
1242 t = ctx->key_enc[5] = le32_to_cpu(key[5]); 1247 t = ctx->key_enc[5] = get_unaligned_le32(in_key + 20);
1243 for (i = 0; i < 8; ++i) 1248 for (i = 0; i < 8; ++i)
1244 loop6(i); 1249 loop6(i);
1245 break; 1250 break;
1246 1251
1247 case AES_KEYSIZE_256: 1252 case AES_KEYSIZE_256:
1248 ctx->key_enc[4] = le32_to_cpu(key[4]); 1253 ctx->key_enc[4] = get_unaligned_le32(in_key + 16);
1249 ctx->key_enc[5] = le32_to_cpu(key[5]); 1254 ctx->key_enc[5] = get_unaligned_le32(in_key + 20);
1250 ctx->key_enc[6] = le32_to_cpu(key[6]); 1255 ctx->key_enc[6] = get_unaligned_le32(in_key + 24);
1251 t = ctx->key_enc[7] = le32_to_cpu(key[7]); 1256 t = ctx->key_enc[7] = get_unaligned_le32(in_key + 28);
1252 for (i = 0; i < 6; ++i) 1257 for (i = 0; i < 6; ++i)
1253 loop8(i); 1258 loop8(i);
1254 loop8tophalf(i); 1259 loop8tophalf(i);
@@ -1329,16 +1334,14 @@ EXPORT_SYMBOL_GPL(crypto_aes_set_key);
1329static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 1334static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1330{ 1335{
1331 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); 1336 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
1332 const __le32 *src = (const __le32 *)in;
1333 __le32 *dst = (__le32 *)out;
1334 u32 b0[4], b1[4]; 1337 u32 b0[4], b1[4];
1335 const u32 *kp = ctx->key_enc + 4; 1338 const u32 *kp = ctx->key_enc + 4;
1336 const int key_len = ctx->key_length; 1339 const int key_len = ctx->key_length;
1337 1340
1338 b0[0] = le32_to_cpu(src[0]) ^ ctx->key_enc[0]; 1341 b0[0] = ctx->key_enc[0] ^ get_unaligned_le32(in);
1339 b0[1] = le32_to_cpu(src[1]) ^ ctx->key_enc[1]; 1342 b0[1] = ctx->key_enc[1] ^ get_unaligned_le32(in + 4);
1340 b0[2] = le32_to_cpu(src[2]) ^ ctx->key_enc[2]; 1343 b0[2] = ctx->key_enc[2] ^ get_unaligned_le32(in + 8);
1341 b0[3] = le32_to_cpu(src[3]) ^ ctx->key_enc[3]; 1344 b0[3] = ctx->key_enc[3] ^ get_unaligned_le32(in + 12);
1342 1345
1343 if (key_len > 24) { 1346 if (key_len > 24) {
1344 f_nround(b1, b0, kp); 1347 f_nround(b1, b0, kp);
@@ -1361,10 +1364,10 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1361 f_nround(b1, b0, kp); 1364 f_nround(b1, b0, kp);
1362 f_lround(b0, b1, kp); 1365 f_lround(b0, b1, kp);
1363 1366
1364 dst[0] = cpu_to_le32(b0[0]); 1367 put_unaligned_le32(b0[0], out);
1365 dst[1] = cpu_to_le32(b0[1]); 1368 put_unaligned_le32(b0[1], out + 4);
1366 dst[2] = cpu_to_le32(b0[2]); 1369 put_unaligned_le32(b0[2], out + 8);
1367 dst[3] = cpu_to_le32(b0[3]); 1370 put_unaligned_le32(b0[3], out + 12);
1368} 1371}
1369 1372
1370/* decrypt a block of text */ 1373/* decrypt a block of text */
@@ -1401,16 +1404,14 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1401static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 1404static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1402{ 1405{
1403 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); 1406 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
1404 const __le32 *src = (const __le32 *)in;
1405 __le32 *dst = (__le32 *)out;
1406 u32 b0[4], b1[4]; 1407 u32 b0[4], b1[4];
1407 const int key_len = ctx->key_length; 1408 const int key_len = ctx->key_length;
1408 const u32 *kp = ctx->key_dec + 4; 1409 const u32 *kp = ctx->key_dec + 4;
1409 1410
1410 b0[0] = le32_to_cpu(src[0]) ^ ctx->key_dec[0]; 1411 b0[0] = ctx->key_dec[0] ^ get_unaligned_le32(in);
1411 b0[1] = le32_to_cpu(src[1]) ^ ctx->key_dec[1]; 1412 b0[1] = ctx->key_dec[1] ^ get_unaligned_le32(in + 4);
1412 b0[2] = le32_to_cpu(src[2]) ^ ctx->key_dec[2]; 1413 b0[2] = ctx->key_dec[2] ^ get_unaligned_le32(in + 8);
1413 b0[3] = le32_to_cpu(src[3]) ^ ctx->key_dec[3]; 1414 b0[3] = ctx->key_dec[3] ^ get_unaligned_le32(in + 12);
1414 1415
1415 if (key_len > 24) { 1416 if (key_len > 24) {
1416 i_nround(b1, b0, kp); 1417 i_nround(b1, b0, kp);
@@ -1433,10 +1434,10 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
1433 i_nround(b1, b0, kp); 1434 i_nround(b1, b0, kp);
1434 i_lround(b0, b1, kp); 1435 i_lround(b0, b1, kp);
1435 1436
1436 dst[0] = cpu_to_le32(b0[0]); 1437 put_unaligned_le32(b0[0], out);
1437 dst[1] = cpu_to_le32(b0[1]); 1438 put_unaligned_le32(b0[1], out + 4);
1438 dst[2] = cpu_to_le32(b0[2]); 1439 put_unaligned_le32(b0[2], out + 8);
1439 dst[3] = cpu_to_le32(b0[3]); 1440 put_unaligned_le32(b0[3], out + 12);
1440} 1441}
1441 1442
1442static struct crypto_alg aes_alg = { 1443static struct crypto_alg aes_alg = {
@@ -1446,7 +1447,6 @@ static struct crypto_alg aes_alg = {
1446 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1447 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1447 .cra_blocksize = AES_BLOCK_SIZE, 1448 .cra_blocksize = AES_BLOCK_SIZE,
1448 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 1449 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
1449 .cra_alignmask = 3,
1450 .cra_module = THIS_MODULE, 1450 .cra_module = THIS_MODULE,
1451 .cra_u = { 1451 .cra_u = {
1452 .cipher = { 1452 .cipher = {
diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c
new file mode 100644
index 000000000000..92644fd1ac19
--- /dev/null
+++ b/crypto/aes_ti.c
@@ -0,0 +1,375 @@
1/*
2 * Scalar fixed time AES core transform
3 *
4 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <crypto/aes.h>
12#include <linux/crypto.h>
13#include <linux/module.h>
14#include <asm/unaligned.h>
15
16/*
17 * Emit the sbox as volatile const to prevent the compiler from doing
18 * constant folding on sbox references involving fixed indexes.
19 */
20static volatile const u8 __cacheline_aligned __aesti_sbox[] = {
21 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
22 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
23 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
24 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
25 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
26 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
27 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
28 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
29 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
30 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
31 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
32 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
33 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
34 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
35 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
36 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
37 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
38 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
39 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
40 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
41 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
42 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
43 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
44 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
45 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
46 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
47 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
48 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
49 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
50 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
51 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
52 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
53};
54
55static volatile const u8 __cacheline_aligned __aesti_inv_sbox[] = {
56 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38,
57 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
58 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
59 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
60 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d,
61 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
62 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2,
63 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
64 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16,
65 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
66 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda,
67 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
68 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a,
69 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
70 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02,
71 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
72 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea,
73 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
74 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85,
75 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
76 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89,
77 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
78 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20,
79 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
80 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31,
81 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
82 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d,
83 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
84 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0,
85 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
86 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26,
87 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d,
88};
89
90static u32 mul_by_x(u32 w)
91{
92 u32 x = w & 0x7f7f7f7f;
93 u32 y = w & 0x80808080;
94
95 /* multiply by polynomial 'x' (0b10) in GF(2^8) */
96 return (x << 1) ^ (y >> 7) * 0x1b;
97}
98
99static u32 mul_by_x2(u32 w)
100{
101 u32 x = w & 0x3f3f3f3f;
102 u32 y = w & 0x80808080;
103 u32 z = w & 0x40404040;
104
105 /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */
106 return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b;
107}
108
109static u32 mix_columns(u32 x)
110{
111 /*
112 * Perform the following matrix multiplication in GF(2^8)
113 *
114 * | 0x2 0x3 0x1 0x1 | | x[0] |
115 * | 0x1 0x2 0x3 0x1 | | x[1] |
116 * | 0x1 0x1 0x2 0x3 | x | x[2] |
117 * | 0x3 0x1 0x1 0x3 | | x[3] |
118 */
119 u32 y = mul_by_x(x) ^ ror32(x, 16);
120
121 return y ^ ror32(x ^ y, 8);
122}
123
124static u32 inv_mix_columns(u32 x)
125{
126 /*
127 * Perform the following matrix multiplication in GF(2^8)
128 *
129 * | 0xe 0xb 0xd 0x9 | | x[0] |
130 * | 0x9 0xe 0xb 0xd | | x[1] |
131 * | 0xd 0x9 0xe 0xb | x | x[2] |
132 * | 0xb 0xd 0x9 0xe | | x[3] |
133 *
134 * which can conveniently be reduced to
135 *
136 * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] |
137 * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] |
138 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] |
139 * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] |
140 */
141 u32 y = mul_by_x2(x);
142
143 return mix_columns(x ^ y ^ ror32(y, 16));
144}
145
146static __always_inline u32 subshift(u32 in[], int pos)
147{
148 return (__aesti_sbox[in[pos] & 0xff]) ^
149 (__aesti_sbox[(in[(pos + 1) % 4] >> 8) & 0xff] << 8) ^
150 (__aesti_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
151 (__aesti_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
152}
153
154static __always_inline u32 inv_subshift(u32 in[], int pos)
155{
156 return (__aesti_inv_sbox[in[pos] & 0xff]) ^
157 (__aesti_inv_sbox[(in[(pos + 3) % 4] >> 8) & 0xff] << 8) ^
158 (__aesti_inv_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
159 (__aesti_inv_sbox[(in[(pos + 1) % 4] >> 24) & 0xff] << 24);
160}
161
162static u32 subw(u32 in)
163{
164 return (__aesti_sbox[in & 0xff]) ^
165 (__aesti_sbox[(in >> 8) & 0xff] << 8) ^
166 (__aesti_sbox[(in >> 16) & 0xff] << 16) ^
167 (__aesti_sbox[(in >> 24) & 0xff] << 24);
168}
169
170static int aesti_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
171 unsigned int key_len)
172{
173 u32 kwords = key_len / sizeof(u32);
174 u32 rc, i, j;
175
176 if (key_len != AES_KEYSIZE_128 &&
177 key_len != AES_KEYSIZE_192 &&
178 key_len != AES_KEYSIZE_256)
179 return -EINVAL;
180
181 ctx->key_length = key_len;
182
183 for (i = 0; i < kwords; i++)
184 ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
185
186 for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) {
187 u32 *rki = ctx->key_enc + (i * kwords);
188 u32 *rko = rki + kwords;
189
190 rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0];
191 rko[1] = rko[0] ^ rki[1];
192 rko[2] = rko[1] ^ rki[2];
193 rko[3] = rko[2] ^ rki[3];
194
195 if (key_len == 24) {
196 if (i >= 7)
197 break;
198 rko[4] = rko[3] ^ rki[4];
199 rko[5] = rko[4] ^ rki[5];
200 } else if (key_len == 32) {
201 if (i >= 6)
202 break;
203 rko[4] = subw(rko[3]) ^ rki[4];
204 rko[5] = rko[4] ^ rki[5];
205 rko[6] = rko[5] ^ rki[6];
206 rko[7] = rko[6] ^ rki[7];
207 }
208 }
209
210 /*
211 * Generate the decryption keys for the Equivalent Inverse Cipher.
212 * This involves reversing the order of the round keys, and applying
213 * the Inverse Mix Columns transformation to all but the first and
214 * the last one.
215 */
216 ctx->key_dec[0] = ctx->key_enc[key_len + 24];
217 ctx->key_dec[1] = ctx->key_enc[key_len + 25];
218 ctx->key_dec[2] = ctx->key_enc[key_len + 26];
219 ctx->key_dec[3] = ctx->key_enc[key_len + 27];
220
221 for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
222 ctx->key_dec[i] = inv_mix_columns(ctx->key_enc[j]);
223 ctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]);
224 ctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]);
225 ctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]);
226 }
227
228 ctx->key_dec[i] = ctx->key_enc[0];
229 ctx->key_dec[i + 1] = ctx->key_enc[1];
230 ctx->key_dec[i + 2] = ctx->key_enc[2];
231 ctx->key_dec[i + 3] = ctx->key_enc[3];
232
233 return 0;
234}
235
236static int aesti_set_key(struct crypto_tfm *tfm, const u8 *in_key,
237 unsigned int key_len)
238{
239 struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
240 int err;
241
242 err = aesti_expand_key(ctx, in_key, key_len);
243 if (err)
244 return err;
245
246 /*
247 * In order to force the compiler to emit data independent Sbox lookups
248 * at the start of each block, xor the first round key with values at
249 * fixed indexes in the Sbox. This will need to be repeated each time
250 * the key is used, which will pull the entire Sbox into the D-cache
251 * before any data dependent Sbox lookups are performed.
252 */
253 ctx->key_enc[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
254 ctx->key_enc[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
255 ctx->key_enc[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
256 ctx->key_enc[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
257
258 ctx->key_dec[0] ^= __aesti_inv_sbox[ 0] ^ __aesti_inv_sbox[128];
259 ctx->key_dec[1] ^= __aesti_inv_sbox[32] ^ __aesti_inv_sbox[160];
260 ctx->key_dec[2] ^= __aesti_inv_sbox[64] ^ __aesti_inv_sbox[192];
261 ctx->key_dec[3] ^= __aesti_inv_sbox[96] ^ __aesti_inv_sbox[224];
262
263 return 0;
264}
265
266static void aesti_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
267{
268 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
269 const u32 *rkp = ctx->key_enc + 4;
270 int rounds = 6 + ctx->key_length / 4;
271 u32 st0[4], st1[4];
272 int round;
273
274 st0[0] = ctx->key_enc[0] ^ get_unaligned_le32(in);
275 st0[1] = ctx->key_enc[1] ^ get_unaligned_le32(in + 4);
276 st0[2] = ctx->key_enc[2] ^ get_unaligned_le32(in + 8);
277 st0[3] = ctx->key_enc[3] ^ get_unaligned_le32(in + 12);
278
279 st0[0] ^= __aesti_sbox[ 0] ^ __aesti_sbox[128];
280 st0[1] ^= __aesti_sbox[32] ^ __aesti_sbox[160];
281 st0[2] ^= __aesti_sbox[64] ^ __aesti_sbox[192];
282 st0[3] ^= __aesti_sbox[96] ^ __aesti_sbox[224];
283
284 for (round = 0;; round += 2, rkp += 8) {
285 st1[0] = mix_columns(subshift(st0, 0)) ^ rkp[0];
286 st1[1] = mix_columns(subshift(st0, 1)) ^ rkp[1];
287 st1[2] = mix_columns(subshift(st0, 2)) ^ rkp[2];
288 st1[3] = mix_columns(subshift(st0, 3)) ^ rkp[3];
289
290 if (round == rounds - 2)
291 break;
292
293 st0[0] = mix_columns(subshift(st1, 0)) ^ rkp[4];
294 st0[1] = mix_columns(subshift(st1, 1)) ^ rkp[5];
295 st0[2] = mix_columns(subshift(st1, 2)) ^ rkp[6];
296 st0[3] = mix_columns(subshift(st1, 3)) ^ rkp[7];
297 }
298
299 put_unaligned_le32(subshift(st1, 0) ^ rkp[4], out);
300 put_unaligned_le32(subshift(st1, 1) ^ rkp[5], out + 4);
301 put_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8);
302 put_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12);
303}
304
305static void aesti_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
306{
307 const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
308 const u32 *rkp = ctx->key_dec + 4;
309 int rounds = 6 + ctx->key_length / 4;
310 u32 st0[4], st1[4];
311 int round;
312
313 st0[0] = ctx->key_dec[0] ^ get_unaligned_le32(in);
314 st0[1] = ctx->key_dec[1] ^ get_unaligned_le32(in + 4);
315 st0[2] = ctx->key_dec[2] ^ get_unaligned_le32(in + 8);
316 st0[3] = ctx->key_dec[3] ^ get_unaligned_le32(in + 12);
317
318 st0[0] ^= __aesti_inv_sbox[ 0] ^ __aesti_inv_sbox[128];
319 st0[1] ^= __aesti_inv_sbox[32] ^ __aesti_inv_sbox[160];
320 st0[2] ^= __aesti_inv_sbox[64] ^ __aesti_inv_sbox[192];
321 st0[3] ^= __aesti_inv_sbox[96] ^ __aesti_inv_sbox[224];
322
323 for (round = 0;; round += 2, rkp += 8) {
324 st1[0] = inv_mix_columns(inv_subshift(st0, 0)) ^ rkp[0];
325 st1[1] = inv_mix_columns(inv_subshift(st0, 1)) ^ rkp[1];
326 st1[2] = inv_mix_columns(inv_subshift(st0, 2)) ^ rkp[2];
327 st1[3] = inv_mix_columns(inv_subshift(st0, 3)) ^ rkp[3];
328
329 if (round == rounds - 2)
330 break;
331
332 st0[0] = inv_mix_columns(inv_subshift(st1, 0)) ^ rkp[4];
333 st0[1] = inv_mix_columns(inv_subshift(st1, 1)) ^ rkp[5];
334 st0[2] = inv_mix_columns(inv_subshift(st1, 2)) ^ rkp[6];
335 st0[3] = inv_mix_columns(inv_subshift(st1, 3)) ^ rkp[7];
336 }
337
338 put_unaligned_le32(inv_subshift(st1, 0) ^ rkp[4], out);
339 put_unaligned_le32(inv_subshift(st1, 1) ^ rkp[5], out + 4);
340 put_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8);
341 put_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12);
342}
343
344static struct crypto_alg aes_alg = {
345 .cra_name = "aes",
346 .cra_driver_name = "aes-fixed-time",
347 .cra_priority = 100 + 1,
348 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
349 .cra_blocksize = AES_BLOCK_SIZE,
350 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
351 .cra_module = THIS_MODULE,
352
353 .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
354 .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
355 .cra_cipher.cia_setkey = aesti_set_key,
356 .cra_cipher.cia_encrypt = aesti_encrypt,
357 .cra_cipher.cia_decrypt = aesti_decrypt
358};
359
360static int __init aes_init(void)
361{
362 return crypto_register_alg(&aes_alg);
363}
364
365static void __exit aes_fini(void)
366{
367 crypto_unregister_alg(&aes_alg);
368}
369
370module_init(aes_init);
371module_exit(aes_fini);
372
373MODULE_DESCRIPTION("Generic fixed time AES");
374MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
375MODULE_LICENSE("GPL v2");
diff --git a/crypto/ahash.c b/crypto/ahash.c
index 2ce8bcb9049c..e58c4970c22b 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -23,6 +23,7 @@
23#include <linux/slab.h> 23#include <linux/slab.h>
24#include <linux/seq_file.h> 24#include <linux/seq_file.h>
25#include <linux/cryptouser.h> 25#include <linux/cryptouser.h>
26#include <linux/compiler.h>
26#include <net/netlink.h> 27#include <net/netlink.h>
27 28
28#include "internal.h" 29#include "internal.h"
@@ -493,7 +494,7 @@ static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
493#endif 494#endif
494 495
495static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 496static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
496 __attribute__ ((unused)); 497 __maybe_unused;
497static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 498static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
498{ 499{
499 seq_printf(m, "type : ahash\n"); 500 seq_printf(m, "type : ahash\n");
diff --git a/crypto/akcipher.c b/crypto/akcipher.c
index def301ed1288..cfbdb06d8ca8 100644
--- a/crypto/akcipher.c
+++ b/crypto/akcipher.c
@@ -17,6 +17,7 @@
17#include <linux/slab.h> 17#include <linux/slab.h>
18#include <linux/string.h> 18#include <linux/string.h>
19#include <linux/crypto.h> 19#include <linux/crypto.h>
20#include <linux/compiler.h>
20#include <crypto/algapi.h> 21#include <crypto/algapi.h>
21#include <linux/cryptouser.h> 22#include <linux/cryptouser.h>
22#include <net/netlink.h> 23#include <net/netlink.h>
@@ -47,7 +48,7 @@ static int crypto_akcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
47#endif 48#endif
48 49
49static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg) 50static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg)
50 __attribute__ ((unused)); 51 __maybe_unused;
51 52
52static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg) 53static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg)
53{ 54{
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 1fad2a6b3bbb..6b52e8f0b95f 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -962,34 +962,66 @@ void crypto_inc(u8 *a, unsigned int size)
962 __be32 *b = (__be32 *)(a + size); 962 __be32 *b = (__be32 *)(a + size);
963 u32 c; 963 u32 c;
964 964
965 for (; size >= 4; size -= 4) { 965 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
966 c = be32_to_cpu(*--b) + 1; 966 !((unsigned long)b & (__alignof__(*b) - 1)))
967 *b = cpu_to_be32(c); 967 for (; size >= 4; size -= 4) {
968 if (c) 968 c = be32_to_cpu(*--b) + 1;
969 return; 969 *b = cpu_to_be32(c);
970 } 970 if (c)
971 return;
972 }
971 973
972 crypto_inc_byte(a, size); 974 crypto_inc_byte(a, size);
973} 975}
974EXPORT_SYMBOL_GPL(crypto_inc); 976EXPORT_SYMBOL_GPL(crypto_inc);
975 977
976static inline void crypto_xor_byte(u8 *a, const u8 *b, unsigned int size) 978void __crypto_xor(u8 *dst, const u8 *src, unsigned int len)
977{ 979{
978 for (; size; size--) 980 int relalign = 0;
979 *a++ ^= *b++; 981
980} 982 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
983 int size = sizeof(unsigned long);
984 int d = ((unsigned long)dst ^ (unsigned long)src) & (size - 1);
985
986 relalign = d ? 1 << __ffs(d) : size;
987
988 /*
989 * If we care about alignment, process as many bytes as
990 * needed to advance dst and src to values whose alignments
991 * equal their relative alignment. This will allow us to
992 * process the remainder of the input using optimal strides.
993 */
994 while (((unsigned long)dst & (relalign - 1)) && len > 0) {
995 *dst++ ^= *src++;
996 len--;
997 }
998 }
981 999
982void crypto_xor(u8 *dst, const u8 *src, unsigned int size) 1000 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) {
983{ 1001 *(u64 *)dst ^= *(u64 *)src;
984 u32 *a = (u32 *)dst; 1002 dst += 8;
985 u32 *b = (u32 *)src; 1003 src += 8;
1004 len -= 8;
1005 }
986 1006
987 for (; size >= 4; size -= 4) 1007 while (len >= 4 && !(relalign & 3)) {
988 *a++ ^= *b++; 1008 *(u32 *)dst ^= *(u32 *)src;
1009 dst += 4;
1010 src += 4;
1011 len -= 4;
1012 }
1013
1014 while (len >= 2 && !(relalign & 1)) {
1015 *(u16 *)dst ^= *(u16 *)src;
1016 dst += 2;
1017 src += 2;
1018 len -= 2;
1019 }
989 1020
990 crypto_xor_byte((u8 *)a, (u8 *)b, size); 1021 while (len--)
1022 *dst++ ^= *src++;
991} 1023}
992EXPORT_SYMBOL_GPL(crypto_xor); 1024EXPORT_SYMBOL_GPL(__crypto_xor);
993 1025
994unsigned int crypto_alg_extsize(struct crypto_alg *alg) 1026unsigned int crypto_alg_extsize(struct crypto_alg *alg)
995{ 1027{
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index d19b09cdf284..54fc90e8339c 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -245,7 +245,7 @@ static int hash_accept(struct socket *sock, struct socket *newsock, int flags)
245 struct alg_sock *ask = alg_sk(sk); 245 struct alg_sock *ask = alg_sk(sk);
246 struct hash_ctx *ctx = ask->private; 246 struct hash_ctx *ctx = ask->private;
247 struct ahash_request *req = &ctx->req; 247 struct ahash_request *req = &ctx->req;
248 char state[crypto_ahash_statesize(crypto_ahash_reqtfm(req))]; 248 char state[crypto_ahash_statesize(crypto_ahash_reqtfm(req)) ? : 1];
249 struct sock *sk2; 249 struct sock *sk2;
250 struct alg_sock *ask2; 250 struct alg_sock *ask2;
251 struct hash_ctx *ctx2; 251 struct hash_ctx *ctx2;
diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c
index a832426820e8..6c43a0a17a55 100644
--- a/crypto/blkcipher.c
+++ b/crypto/blkcipher.c
@@ -1,6 +1,6 @@
1/* 1/*
2 * Block chaining cipher operations. 2 * Block chaining cipher operations.
3 * 3 *
4 * Generic encrypt/decrypt wrapper for ciphers, handles operations across 4 * Generic encrypt/decrypt wrapper for ciphers, handles operations across
5 * multiple page boundaries by using temporary blocks. In user context, 5 * multiple page boundaries by using temporary blocks. In user context,
6 * the kernel is given a chance to schedule us once per page. 6 * the kernel is given a chance to schedule us once per page.
@@ -9,7 +9,7 @@
9 * 9 *
10 * This program is free software; you can redistribute it and/or modify it 10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 11 * under the terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 2 of the License, or (at your option) 12 * Software Foundation; either version 2 of the License, or (at your option)
13 * any later version. 13 * any later version.
14 * 14 *
15 */ 15 */
@@ -25,6 +25,7 @@
25#include <linux/slab.h> 25#include <linux/slab.h>
26#include <linux/string.h> 26#include <linux/string.h>
27#include <linux/cryptouser.h> 27#include <linux/cryptouser.h>
28#include <linux/compiler.h>
28#include <net/netlink.h> 29#include <net/netlink.h>
29 30
30#include "internal.h" 31#include "internal.h"
@@ -534,7 +535,7 @@ static int crypto_blkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
534#endif 535#endif
535 536
536static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg) 537static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
537 __attribute__ ((unused)); 538 __maybe_unused;
538static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg) 539static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
539{ 540{
540 seq_printf(m, "type : blkcipher\n"); 541 seq_printf(m, "type : blkcipher\n");
diff --git a/crypto/cbc.c b/crypto/cbc.c
index 68f751a41a84..bc160a3186dc 100644
--- a/crypto/cbc.c
+++ b/crypto/cbc.c
@@ -145,9 +145,6 @@ static int crypto_cbc_create(struct crypto_template *tmpl, struct rtattr **tb)
145 inst->alg.base.cra_blocksize = alg->cra_blocksize; 145 inst->alg.base.cra_blocksize = alg->cra_blocksize;
146 inst->alg.base.cra_alignmask = alg->cra_alignmask; 146 inst->alg.base.cra_alignmask = alg->cra_alignmask;
147 147
148 /* We access the data as u32s when xoring. */
149 inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
150
151 inst->alg.ivsize = alg->cra_blocksize; 148 inst->alg.ivsize = alg->cra_blocksize;
152 inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize; 149 inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
153 inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize; 150 inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 26b924d1e582..442848807a52 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -11,6 +11,7 @@
11 */ 11 */
12 12
13#include <crypto/internal/aead.h> 13#include <crypto/internal/aead.h>
14#include <crypto/internal/hash.h>
14#include <crypto/internal/skcipher.h> 15#include <crypto/internal/skcipher.h>
15#include <crypto/scatterwalk.h> 16#include <crypto/scatterwalk.h>
16#include <linux/err.h> 17#include <linux/err.h>
@@ -23,11 +24,11 @@
23 24
24struct ccm_instance_ctx { 25struct ccm_instance_ctx {
25 struct crypto_skcipher_spawn ctr; 26 struct crypto_skcipher_spawn ctr;
26 struct crypto_spawn cipher; 27 struct crypto_ahash_spawn mac;
27}; 28};
28 29
29struct crypto_ccm_ctx { 30struct crypto_ccm_ctx {
30 struct crypto_cipher *cipher; 31 struct crypto_ahash *mac;
31 struct crypto_skcipher *ctr; 32 struct crypto_skcipher *ctr;
32}; 33};
33 34
@@ -44,15 +45,21 @@ struct crypto_rfc4309_req_ctx {
44 45
45struct crypto_ccm_req_priv_ctx { 46struct crypto_ccm_req_priv_ctx {
46 u8 odata[16]; 47 u8 odata[16];
47 u8 idata[16];
48 u8 auth_tag[16]; 48 u8 auth_tag[16];
49 u32 ilen;
50 u32 flags; 49 u32 flags;
51 struct scatterlist src[3]; 50 struct scatterlist src[3];
52 struct scatterlist dst[3]; 51 struct scatterlist dst[3];
53 struct skcipher_request skreq; 52 struct skcipher_request skreq;
54}; 53};
55 54
55struct cbcmac_tfm_ctx {
56 struct crypto_cipher *child;
57};
58
59struct cbcmac_desc_ctx {
60 unsigned int len;
61};
62
56static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx( 63static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
57 struct aead_request *req) 64 struct aead_request *req)
58{ 65{
@@ -84,7 +91,7 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
84{ 91{
85 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); 92 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
86 struct crypto_skcipher *ctr = ctx->ctr; 93 struct crypto_skcipher *ctr = ctx->ctr;
87 struct crypto_cipher *tfm = ctx->cipher; 94 struct crypto_ahash *mac = ctx->mac;
88 int err = 0; 95 int err = 0;
89 96
90 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 97 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
@@ -96,11 +103,11 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
96 if (err) 103 if (err)
97 goto out; 104 goto out;
98 105
99 crypto_cipher_clear_flags(tfm, CRYPTO_TFM_REQ_MASK); 106 crypto_ahash_clear_flags(mac, CRYPTO_TFM_REQ_MASK);
100 crypto_cipher_set_flags(tfm, crypto_aead_get_flags(aead) & 107 crypto_ahash_set_flags(mac, crypto_aead_get_flags(aead) &
101 CRYPTO_TFM_REQ_MASK); 108 CRYPTO_TFM_REQ_MASK);
102 err = crypto_cipher_setkey(tfm, key, keylen); 109 err = crypto_ahash_setkey(mac, key, keylen);
103 crypto_aead_set_flags(aead, crypto_cipher_get_flags(tfm) & 110 crypto_aead_set_flags(aead, crypto_ahash_get_flags(mac) &
104 CRYPTO_TFM_RES_MASK); 111 CRYPTO_TFM_RES_MASK);
105 112
106out: 113out:
@@ -167,119 +174,61 @@ static int format_adata(u8 *adata, unsigned int a)
167 return len; 174 return len;
168} 175}
169 176
170static void compute_mac(struct crypto_cipher *tfm, u8 *data, int n,
171 struct crypto_ccm_req_priv_ctx *pctx)
172{
173 unsigned int bs = 16;
174 u8 *odata = pctx->odata;
175 u8 *idata = pctx->idata;
176 int datalen, getlen;
177
178 datalen = n;
179
180 /* first time in here, block may be partially filled. */
181 getlen = bs - pctx->ilen;
182 if (datalen >= getlen) {
183 memcpy(idata + pctx->ilen, data, getlen);
184 crypto_xor(odata, idata, bs);
185 crypto_cipher_encrypt_one(tfm, odata, odata);
186 datalen -= getlen;
187 data += getlen;
188 pctx->ilen = 0;
189 }
190
191 /* now encrypt rest of data */
192 while (datalen >= bs) {
193 crypto_xor(odata, data, bs);
194 crypto_cipher_encrypt_one(tfm, odata, odata);
195
196 datalen -= bs;
197 data += bs;
198 }
199
200 /* check and see if there's leftover data that wasn't
201 * enough to fill a block.
202 */
203 if (datalen) {
204 memcpy(idata + pctx->ilen, data, datalen);
205 pctx->ilen += datalen;
206 }
207}
208
209static void get_data_to_compute(struct crypto_cipher *tfm,
210 struct crypto_ccm_req_priv_ctx *pctx,
211 struct scatterlist *sg, unsigned int len)
212{
213 struct scatter_walk walk;
214 u8 *data_src;
215 int n;
216
217 scatterwalk_start(&walk, sg);
218
219 while (len) {
220 n = scatterwalk_clamp(&walk, len);
221 if (!n) {
222 scatterwalk_start(&walk, sg_next(walk.sg));
223 n = scatterwalk_clamp(&walk, len);
224 }
225 data_src = scatterwalk_map(&walk);
226
227 compute_mac(tfm, data_src, n, pctx);
228 len -= n;
229
230 scatterwalk_unmap(data_src);
231 scatterwalk_advance(&walk, n);
232 scatterwalk_done(&walk, 0, len);
233 if (len)
234 crypto_yield(pctx->flags);
235 }
236
237 /* any leftover needs padding and then encrypted */
238 if (pctx->ilen) {
239 int padlen;
240 u8 *odata = pctx->odata;
241 u8 *idata = pctx->idata;
242
243 padlen = 16 - pctx->ilen;
244 memset(idata + pctx->ilen, 0, padlen);
245 crypto_xor(odata, idata, 16);
246 crypto_cipher_encrypt_one(tfm, odata, odata);
247 pctx->ilen = 0;
248 }
249}
250
251static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain, 177static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
252 unsigned int cryptlen) 178 unsigned int cryptlen)
253{ 179{
180 struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
254 struct crypto_aead *aead = crypto_aead_reqtfm(req); 181 struct crypto_aead *aead = crypto_aead_reqtfm(req);
255 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); 182 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
256 struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); 183 AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
257 struct crypto_cipher *cipher = ctx->cipher;
258 unsigned int assoclen = req->assoclen; 184 unsigned int assoclen = req->assoclen;
259 u8 *odata = pctx->odata; 185 struct scatterlist sg[3];
260 u8 *idata = pctx->idata; 186 u8 odata[16];
261 int err; 187 u8 idata[16];
188 int ilen, err;
262 189
263 /* format control data for input */ 190 /* format control data for input */
264 err = format_input(odata, req, cryptlen); 191 err = format_input(odata, req, cryptlen);
265 if (err) 192 if (err)
266 goto out; 193 goto out;
267 194
268 /* encrypt first block to use as start in computing mac */ 195 sg_init_table(sg, 3);
269 crypto_cipher_encrypt_one(cipher, odata, odata); 196 sg_set_buf(&sg[0], odata, 16);
270 197
271 /* format associated data and compute into mac */ 198 /* format associated data and compute into mac */
272 if (assoclen) { 199 if (assoclen) {
273 pctx->ilen = format_adata(idata, assoclen); 200 ilen = format_adata(idata, assoclen);
274 get_data_to_compute(cipher, pctx, req->src, req->assoclen); 201 sg_set_buf(&sg[1], idata, ilen);
202 sg_chain(sg, 3, req->src);
275 } else { 203 } else {
276 pctx->ilen = 0; 204 ilen = 0;
205 sg_chain(sg, 2, req->src);
277 } 206 }
278 207
279 /* compute plaintext into mac */ 208 ahash_request_set_tfm(ahreq, ctx->mac);
280 if (cryptlen) 209 ahash_request_set_callback(ahreq, pctx->flags, NULL, NULL);
281 get_data_to_compute(cipher, pctx, plain, cryptlen); 210 ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16);
211 err = crypto_ahash_init(ahreq);
212 if (err)
213 goto out;
214 err = crypto_ahash_update(ahreq);
215 if (err)
216 goto out;
282 217
218 /* we need to pad the MAC input to a round multiple of the block size */
219 ilen = 16 - (assoclen + ilen) % 16;
220 if (ilen < 16) {
221 memset(idata, 0, ilen);
222 sg_init_table(sg, 2);
223 sg_set_buf(&sg[0], idata, ilen);
224 if (plain)
225 sg_chain(sg, 2, plain);
226 plain = sg;
227 cryptlen += ilen;
228 }
229
230 ahash_request_set_crypt(ahreq, plain, pctx->odata, cryptlen);
231 err = crypto_ahash_finup(ahreq);
283out: 232out:
284 return err; 233 return err;
285} 234}
@@ -453,21 +402,21 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
453 struct aead_instance *inst = aead_alg_instance(tfm); 402 struct aead_instance *inst = aead_alg_instance(tfm);
454 struct ccm_instance_ctx *ictx = aead_instance_ctx(inst); 403 struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
455 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); 404 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
456 struct crypto_cipher *cipher; 405 struct crypto_ahash *mac;
457 struct crypto_skcipher *ctr; 406 struct crypto_skcipher *ctr;
458 unsigned long align; 407 unsigned long align;
459 int err; 408 int err;
460 409
461 cipher = crypto_spawn_cipher(&ictx->cipher); 410 mac = crypto_spawn_ahash(&ictx->mac);
462 if (IS_ERR(cipher)) 411 if (IS_ERR(mac))
463 return PTR_ERR(cipher); 412 return PTR_ERR(mac);
464 413
465 ctr = crypto_spawn_skcipher(&ictx->ctr); 414 ctr = crypto_spawn_skcipher(&ictx->ctr);
466 err = PTR_ERR(ctr); 415 err = PTR_ERR(ctr);
467 if (IS_ERR(ctr)) 416 if (IS_ERR(ctr))
468 goto err_free_cipher; 417 goto err_free_mac;
469 418
470 ctx->cipher = cipher; 419 ctx->mac = mac;
471 ctx->ctr = ctr; 420 ctx->ctr = ctr;
472 421
473 align = crypto_aead_alignmask(tfm); 422 align = crypto_aead_alignmask(tfm);
@@ -479,8 +428,8 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
479 428
480 return 0; 429 return 0;
481 430
482err_free_cipher: 431err_free_mac:
483 crypto_free_cipher(cipher); 432 crypto_free_ahash(mac);
484 return err; 433 return err;
485} 434}
486 435
@@ -488,7 +437,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm)
488{ 437{
489 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); 438 struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
490 439
491 crypto_free_cipher(ctx->cipher); 440 crypto_free_ahash(ctx->mac);
492 crypto_free_skcipher(ctx->ctr); 441 crypto_free_skcipher(ctx->ctr);
493} 442}
494 443
@@ -496,7 +445,7 @@ static void crypto_ccm_free(struct aead_instance *inst)
496{ 445{
497 struct ccm_instance_ctx *ctx = aead_instance_ctx(inst); 446 struct ccm_instance_ctx *ctx = aead_instance_ctx(inst);
498 447
499 crypto_drop_spawn(&ctx->cipher); 448 crypto_drop_ahash(&ctx->mac);
500 crypto_drop_skcipher(&ctx->ctr); 449 crypto_drop_skcipher(&ctx->ctr);
501 kfree(inst); 450 kfree(inst);
502} 451}
@@ -505,12 +454,13 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
505 struct rtattr **tb, 454 struct rtattr **tb,
506 const char *full_name, 455 const char *full_name,
507 const char *ctr_name, 456 const char *ctr_name,
508 const char *cipher_name) 457 const char *mac_name)
509{ 458{
510 struct crypto_attr_type *algt; 459 struct crypto_attr_type *algt;
511 struct aead_instance *inst; 460 struct aead_instance *inst;
512 struct skcipher_alg *ctr; 461 struct skcipher_alg *ctr;
513 struct crypto_alg *cipher; 462 struct crypto_alg *mac_alg;
463 struct hash_alg_common *mac;
514 struct ccm_instance_ctx *ictx; 464 struct ccm_instance_ctx *ictx;
515 int err; 465 int err;
516 466
@@ -521,25 +471,26 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
521 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 471 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
522 return -EINVAL; 472 return -EINVAL;
523 473
524 cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER, 474 mac_alg = crypto_find_alg(mac_name, &crypto_ahash_type,
525 CRYPTO_ALG_TYPE_MASK); 475 CRYPTO_ALG_TYPE_HASH,
526 if (IS_ERR(cipher)) 476 CRYPTO_ALG_TYPE_AHASH_MASK |
527 return PTR_ERR(cipher); 477 CRYPTO_ALG_ASYNC);
478 if (IS_ERR(mac_alg))
479 return PTR_ERR(mac_alg);
528 480
481 mac = __crypto_hash_alg_common(mac_alg);
529 err = -EINVAL; 482 err = -EINVAL;
530 if (cipher->cra_blocksize != 16) 483 if (mac->digestsize != 16)
531 goto out_put_cipher; 484 goto out_put_mac;
532 485
533 inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); 486 inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
534 err = -ENOMEM; 487 err = -ENOMEM;
535 if (!inst) 488 if (!inst)
536 goto out_put_cipher; 489 goto out_put_mac;
537 490
538 ictx = aead_instance_ctx(inst); 491 ictx = aead_instance_ctx(inst);
539 492 err = crypto_init_ahash_spawn(&ictx->mac, mac,
540 err = crypto_init_spawn(&ictx->cipher, cipher, 493 aead_crypto_instance(inst));
541 aead_crypto_instance(inst),
542 CRYPTO_ALG_TYPE_MASK);
543 if (err) 494 if (err)
544 goto err_free_inst; 495 goto err_free_inst;
545 496
@@ -548,7 +499,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
548 crypto_requires_sync(algt->type, 499 crypto_requires_sync(algt->type,
549 algt->mask)); 500 algt->mask));
550 if (err) 501 if (err)
551 goto err_drop_cipher; 502 goto err_drop_mac;
552 503
553 ctr = crypto_spawn_skcipher_alg(&ictx->ctr); 504 ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
554 505
@@ -564,18 +515,17 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
564 err = -ENAMETOOLONG; 515 err = -ENAMETOOLONG;
565 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 516 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
566 "ccm_base(%s,%s)", ctr->base.cra_driver_name, 517 "ccm_base(%s,%s)", ctr->base.cra_driver_name,
567 cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 518 mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
568 goto err_drop_ctr; 519 goto err_drop_ctr;
569 520
570 memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); 521 memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
571 522
572 inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC; 523 inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
573 inst->alg.base.cra_priority = (cipher->cra_priority + 524 inst->alg.base.cra_priority = (mac->base.cra_priority +
574 ctr->base.cra_priority) / 2; 525 ctr->base.cra_priority) / 2;
575 inst->alg.base.cra_blocksize = 1; 526 inst->alg.base.cra_blocksize = 1;
576 inst->alg.base.cra_alignmask = cipher->cra_alignmask | 527 inst->alg.base.cra_alignmask = mac->base.cra_alignmask |
577 ctr->base.cra_alignmask | 528 ctr->base.cra_alignmask;
578 (__alignof__(u32) - 1);
579 inst->alg.ivsize = 16; 529 inst->alg.ivsize = 16;
580 inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr); 530 inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
581 inst->alg.maxauthsize = 16; 531 inst->alg.maxauthsize = 16;
@@ -593,23 +543,24 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
593 if (err) 543 if (err)
594 goto err_drop_ctr; 544 goto err_drop_ctr;
595 545
596out_put_cipher: 546out_put_mac:
597 crypto_mod_put(cipher); 547 crypto_mod_put(mac_alg);
598 return err; 548 return err;
599 549
600err_drop_ctr: 550err_drop_ctr:
601 crypto_drop_skcipher(&ictx->ctr); 551 crypto_drop_skcipher(&ictx->ctr);
602err_drop_cipher: 552err_drop_mac:
603 crypto_drop_spawn(&ictx->cipher); 553 crypto_drop_ahash(&ictx->mac);
604err_free_inst: 554err_free_inst:
605 kfree(inst); 555 kfree(inst);
606 goto out_put_cipher; 556 goto out_put_mac;
607} 557}
608 558
609static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) 559static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
610{ 560{
611 const char *cipher_name; 561 const char *cipher_name;
612 char ctr_name[CRYPTO_MAX_ALG_NAME]; 562 char ctr_name[CRYPTO_MAX_ALG_NAME];
563 char mac_name[CRYPTO_MAX_ALG_NAME];
613 char full_name[CRYPTO_MAX_ALG_NAME]; 564 char full_name[CRYPTO_MAX_ALG_NAME];
614 565
615 cipher_name = crypto_attr_alg_name(tb[1]); 566 cipher_name = crypto_attr_alg_name(tb[1]);
@@ -620,12 +571,16 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
620 cipher_name) >= CRYPTO_MAX_ALG_NAME) 571 cipher_name) >= CRYPTO_MAX_ALG_NAME)
621 return -ENAMETOOLONG; 572 return -ENAMETOOLONG;
622 573
574 if (snprintf(mac_name, CRYPTO_MAX_ALG_NAME, "cbcmac(%s)",
575 cipher_name) >= CRYPTO_MAX_ALG_NAME)
576 return -ENAMETOOLONG;
577
623 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >= 578 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >=
624 CRYPTO_MAX_ALG_NAME) 579 CRYPTO_MAX_ALG_NAME)
625 return -ENAMETOOLONG; 580 return -ENAMETOOLONG;
626 581
627 return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, 582 return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name,
628 cipher_name); 583 mac_name);
629} 584}
630 585
631static struct crypto_template crypto_ccm_tmpl = { 586static struct crypto_template crypto_ccm_tmpl = {
@@ -899,14 +854,164 @@ static struct crypto_template crypto_rfc4309_tmpl = {
899 .module = THIS_MODULE, 854 .module = THIS_MODULE,
900}; 855};
901 856
857static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent,
858 const u8 *inkey, unsigned int keylen)
859{
860 struct cbcmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
861
862 return crypto_cipher_setkey(ctx->child, inkey, keylen);
863}
864
865static int crypto_cbcmac_digest_init(struct shash_desc *pdesc)
866{
867 struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
868 int bs = crypto_shash_digestsize(pdesc->tfm);
869 u8 *dg = (u8 *)ctx + crypto_shash_descsize(pdesc->tfm) - bs;
870
871 ctx->len = 0;
872 memset(dg, 0, bs);
873
874 return 0;
875}
876
877static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p,
878 unsigned int len)
879{
880 struct crypto_shash *parent = pdesc->tfm;
881 struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
882 struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
883 struct crypto_cipher *tfm = tctx->child;
884 int bs = crypto_shash_digestsize(parent);
885 u8 *dg = (u8 *)ctx + crypto_shash_descsize(parent) - bs;
886
887 while (len > 0) {
888 unsigned int l = min(len, bs - ctx->len);
889
890 crypto_xor(dg + ctx->len, p, l);
891 ctx->len +=l;
892 len -= l;
893 p += l;
894
895 if (ctx->len == bs) {
896 crypto_cipher_encrypt_one(tfm, dg, dg);
897 ctx->len = 0;
898 }
899 }
900
901 return 0;
902}
903
904static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out)
905{
906 struct crypto_shash *parent = pdesc->tfm;
907 struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
908 struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
909 struct crypto_cipher *tfm = tctx->child;
910 int bs = crypto_shash_digestsize(parent);
911 u8 *dg = (u8 *)ctx + crypto_shash_descsize(parent) - bs;
912
913 if (ctx->len)
914 crypto_cipher_encrypt_one(tfm, dg, dg);
915
916 memcpy(out, dg, bs);
917 return 0;
918}
919
920static int cbcmac_init_tfm(struct crypto_tfm *tfm)
921{
922 struct crypto_cipher *cipher;
923 struct crypto_instance *inst = (void *)tfm->__crt_alg;
924 struct crypto_spawn *spawn = crypto_instance_ctx(inst);
925 struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
926
927 cipher = crypto_spawn_cipher(spawn);
928 if (IS_ERR(cipher))
929 return PTR_ERR(cipher);
930
931 ctx->child = cipher;
932
933 return 0;
934};
935
936static void cbcmac_exit_tfm(struct crypto_tfm *tfm)
937{
938 struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
939 crypto_free_cipher(ctx->child);
940}
941
942static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb)
943{
944 struct shash_instance *inst;
945 struct crypto_alg *alg;
946 int err;
947
948 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
949 if (err)
950 return err;
951
952 alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
953 CRYPTO_ALG_TYPE_MASK);
954 if (IS_ERR(alg))
955 return PTR_ERR(alg);
956
957 inst = shash_alloc_instance("cbcmac", alg);
958 err = PTR_ERR(inst);
959 if (IS_ERR(inst))
960 goto out_put_alg;
961
962 err = crypto_init_spawn(shash_instance_ctx(inst), alg,
963 shash_crypto_instance(inst),
964 CRYPTO_ALG_TYPE_MASK);
965 if (err)
966 goto out_free_inst;
967
968 inst->alg.base.cra_priority = alg->cra_priority;
969 inst->alg.base.cra_blocksize = 1;
970
971 inst->alg.digestsize = alg->cra_blocksize;
972 inst->alg.descsize = ALIGN(sizeof(struct cbcmac_desc_ctx),
973 alg->cra_alignmask + 1) +
974 alg->cra_blocksize;
975
976 inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx);
977 inst->alg.base.cra_init = cbcmac_init_tfm;
978 inst->alg.base.cra_exit = cbcmac_exit_tfm;
979
980 inst->alg.init = crypto_cbcmac_digest_init;
981 inst->alg.update = crypto_cbcmac_digest_update;
982 inst->alg.final = crypto_cbcmac_digest_final;
983 inst->alg.setkey = crypto_cbcmac_digest_setkey;
984
985 err = shash_register_instance(tmpl, inst);
986
987out_free_inst:
988 if (err)
989 shash_free_instance(shash_crypto_instance(inst));
990
991out_put_alg:
992 crypto_mod_put(alg);
993 return err;
994}
995
996static struct crypto_template crypto_cbcmac_tmpl = {
997 .name = "cbcmac",
998 .create = cbcmac_create,
999 .free = shash_free_instance,
1000 .module = THIS_MODULE,
1001};
1002
902static int __init crypto_ccm_module_init(void) 1003static int __init crypto_ccm_module_init(void)
903{ 1004{
904 int err; 1005 int err;
905 1006
906 err = crypto_register_template(&crypto_ccm_base_tmpl); 1007 err = crypto_register_template(&crypto_cbcmac_tmpl);
907 if (err) 1008 if (err)
908 goto out; 1009 goto out;
909 1010
1011 err = crypto_register_template(&crypto_ccm_base_tmpl);
1012 if (err)
1013 goto out_undo_cbcmac;
1014
910 err = crypto_register_template(&crypto_ccm_tmpl); 1015 err = crypto_register_template(&crypto_ccm_tmpl);
911 if (err) 1016 if (err)
912 goto out_undo_base; 1017 goto out_undo_base;
@@ -922,6 +1027,8 @@ out_undo_ccm:
922 crypto_unregister_template(&crypto_ccm_tmpl); 1027 crypto_unregister_template(&crypto_ccm_tmpl);
923out_undo_base: 1028out_undo_base:
924 crypto_unregister_template(&crypto_ccm_base_tmpl); 1029 crypto_unregister_template(&crypto_ccm_base_tmpl);
1030out_undo_cbcmac:
1031 crypto_register_template(&crypto_cbcmac_tmpl);
925 goto out; 1032 goto out;
926} 1033}
927 1034
@@ -930,6 +1037,7 @@ static void __exit crypto_ccm_module_exit(void)
930 crypto_unregister_template(&crypto_rfc4309_tmpl); 1037 crypto_unregister_template(&crypto_rfc4309_tmpl);
931 crypto_unregister_template(&crypto_ccm_tmpl); 1038 crypto_unregister_template(&crypto_ccm_tmpl);
932 crypto_unregister_template(&crypto_ccm_base_tmpl); 1039 crypto_unregister_template(&crypto_ccm_base_tmpl);
1040 crypto_unregister_template(&crypto_cbcmac_tmpl);
933} 1041}
934 1042
935module_init(crypto_ccm_module_init); 1043module_init(crypto_ccm_module_init);
diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c
index 1cab83146e33..8b3c04d625c3 100644
--- a/crypto/chacha20_generic.c
+++ b/crypto/chacha20_generic.c
@@ -10,10 +10,9 @@
10 */ 10 */
11 11
12#include <crypto/algapi.h> 12#include <crypto/algapi.h>
13#include <linux/crypto.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <crypto/chacha20.h> 13#include <crypto/chacha20.h>
14#include <crypto/internal/skcipher.h>
15#include <linux/module.h>
17 16
18static inline u32 le32_to_cpuvp(const void *p) 17static inline u32 le32_to_cpuvp(const void *p)
19{ 18{
@@ -63,10 +62,10 @@ void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv)
63} 62}
64EXPORT_SYMBOL_GPL(crypto_chacha20_init); 63EXPORT_SYMBOL_GPL(crypto_chacha20_init);
65 64
66int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key, 65int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
67 unsigned int keysize) 66 unsigned int keysize)
68{ 67{
69 struct chacha20_ctx *ctx = crypto_tfm_ctx(tfm); 68 struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
70 int i; 69 int i;
71 70
72 if (keysize != CHACHA20_KEY_SIZE) 71 if (keysize != CHACHA20_KEY_SIZE)
@@ -79,66 +78,54 @@ int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key,
79} 78}
80EXPORT_SYMBOL_GPL(crypto_chacha20_setkey); 79EXPORT_SYMBOL_GPL(crypto_chacha20_setkey);
81 80
82int crypto_chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 81int crypto_chacha20_crypt(struct skcipher_request *req)
83 struct scatterlist *src, unsigned int nbytes)
84{ 82{
85 struct blkcipher_walk walk; 83 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
84 struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
85 struct skcipher_walk walk;
86 u32 state[16]; 86 u32 state[16];
87 int err; 87 int err;
88 88
89 blkcipher_walk_init(&walk, dst, src, nbytes); 89 err = skcipher_walk_virt(&walk, req, true);
90 err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE);
91
92 crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv);
93 90
94 while (walk.nbytes >= CHACHA20_BLOCK_SIZE) { 91 crypto_chacha20_init(state, ctx, walk.iv);
95 chacha20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr,
96 rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
97 err = blkcipher_walk_done(desc, &walk,
98 walk.nbytes % CHACHA20_BLOCK_SIZE);
99 }
100 92
101 if (walk.nbytes) { 93 while (walk.nbytes > 0) {
102 chacha20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, 94 chacha20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr,
103 walk.nbytes); 95 walk.nbytes);
104 err = blkcipher_walk_done(desc, &walk, 0); 96 err = skcipher_walk_done(&walk, 0);
105 } 97 }
106 98
107 return err; 99 return err;
108} 100}
109EXPORT_SYMBOL_GPL(crypto_chacha20_crypt); 101EXPORT_SYMBOL_GPL(crypto_chacha20_crypt);
110 102
111static struct crypto_alg alg = { 103static struct skcipher_alg alg = {
112 .cra_name = "chacha20", 104 .base.cra_name = "chacha20",
113 .cra_driver_name = "chacha20-generic", 105 .base.cra_driver_name = "chacha20-generic",
114 .cra_priority = 100, 106 .base.cra_priority = 100,
115 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 107 .base.cra_blocksize = 1,
116 .cra_blocksize = 1, 108 .base.cra_ctxsize = sizeof(struct chacha20_ctx),
117 .cra_type = &crypto_blkcipher_type, 109 .base.cra_alignmask = sizeof(u32) - 1,
118 .cra_ctxsize = sizeof(struct chacha20_ctx), 110 .base.cra_module = THIS_MODULE,
119 .cra_alignmask = sizeof(u32) - 1, 111
120 .cra_module = THIS_MODULE, 112 .min_keysize = CHACHA20_KEY_SIZE,
121 .cra_u = { 113 .max_keysize = CHACHA20_KEY_SIZE,
122 .blkcipher = { 114 .ivsize = CHACHA20_IV_SIZE,
123 .min_keysize = CHACHA20_KEY_SIZE, 115 .chunksize = CHACHA20_BLOCK_SIZE,
124 .max_keysize = CHACHA20_KEY_SIZE, 116 .setkey = crypto_chacha20_setkey,
125 .ivsize = CHACHA20_IV_SIZE, 117 .encrypt = crypto_chacha20_crypt,
126 .geniv = "seqiv", 118 .decrypt = crypto_chacha20_crypt,
127 .setkey = crypto_chacha20_setkey,
128 .encrypt = crypto_chacha20_crypt,
129 .decrypt = crypto_chacha20_crypt,
130 },
131 },
132}; 119};
133 120
134static int __init chacha20_generic_mod_init(void) 121static int __init chacha20_generic_mod_init(void)
135{ 122{
136 return crypto_register_alg(&alg); 123 return crypto_register_skcipher(&alg);
137} 124}
138 125
139static void __exit chacha20_generic_mod_fini(void) 126static void __exit chacha20_generic_mod_fini(void)
140{ 127{
141 crypto_unregister_alg(&alg); 128 crypto_unregister_skcipher(&alg);
142} 129}
143 130
144module_init(chacha20_generic_mod_init); 131module_init(chacha20_generic_mod_init);
diff --git a/crypto/cmac.c b/crypto/cmac.c
index 04080dca8f0c..16301f52858c 100644
--- a/crypto/cmac.c
+++ b/crypto/cmac.c
@@ -260,8 +260,7 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb)
260 if (err) 260 if (err)
261 goto out_free_inst; 261 goto out_free_inst;
262 262
263 /* We access the data as u32s when xoring. */ 263 alignmask = alg->cra_alignmask;
264 alignmask = alg->cra_alignmask | (__alignof__(u32) - 1);
265 inst->alg.base.cra_alignmask = alignmask; 264 inst->alg.base.cra_alignmask = alignmask;
266 inst->alg.base.cra_priority = alg->cra_priority; 265 inst->alg.base.cra_priority = alg->cra_priority;
267 inst->alg.base.cra_blocksize = alg->cra_blocksize; 266 inst->alg.base.cra_blocksize = alg->cra_blocksize;
diff --git a/crypto/ctr.c b/crypto/ctr.c
index a9a7a44f2783..a4f4a8983169 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -209,7 +209,7 @@ static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb)
209 inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; 209 inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
210 inst->alg.cra_priority = alg->cra_priority; 210 inst->alg.cra_priority = alg->cra_priority;
211 inst->alg.cra_blocksize = 1; 211 inst->alg.cra_blocksize = 1;
212 inst->alg.cra_alignmask = alg->cra_alignmask | (__alignof__(u32) - 1); 212 inst->alg.cra_alignmask = alg->cra_alignmask;
213 inst->alg.cra_type = &crypto_blkcipher_type; 213 inst->alg.cra_type = &crypto_blkcipher_type;
214 214
215 inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; 215 inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
diff --git a/crypto/cts.c b/crypto/cts.c
index 00254d76b21b..243f591dc409 100644
--- a/crypto/cts.c
+++ b/crypto/cts.c
@@ -49,6 +49,7 @@
49#include <linux/scatterlist.h> 49#include <linux/scatterlist.h>
50#include <crypto/scatterwalk.h> 50#include <crypto/scatterwalk.h>
51#include <linux/slab.h> 51#include <linux/slab.h>
52#include <linux/compiler.h>
52 53
53struct crypto_cts_ctx { 54struct crypto_cts_ctx {
54 struct crypto_skcipher *child; 55 struct crypto_skcipher *child;
@@ -103,7 +104,7 @@ static int cts_cbc_encrypt(struct skcipher_request *req)
103 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 104 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
104 struct skcipher_request *subreq = &rctx->subreq; 105 struct skcipher_request *subreq = &rctx->subreq;
105 int bsize = crypto_skcipher_blocksize(tfm); 106 int bsize = crypto_skcipher_blocksize(tfm);
106 u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32)))); 107 u8 d[bsize * 2] __aligned(__alignof__(u32));
107 struct scatterlist *sg; 108 struct scatterlist *sg;
108 unsigned int offset; 109 unsigned int offset;
109 int lastn; 110 int lastn;
@@ -183,7 +184,7 @@ static int cts_cbc_decrypt(struct skcipher_request *req)
183 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 184 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
184 struct skcipher_request *subreq = &rctx->subreq; 185 struct skcipher_request *subreq = &rctx->subreq;
185 int bsize = crypto_skcipher_blocksize(tfm); 186 int bsize = crypto_skcipher_blocksize(tfm);
186 u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32)))); 187 u8 d[bsize * 2] __aligned(__alignof__(u32));
187 struct scatterlist *sg; 188 struct scatterlist *sg;
188 unsigned int offset; 189 unsigned int offset;
189 u8 *space; 190 u8 *space;
@@ -373,9 +374,6 @@ static int crypto_cts_create(struct crypto_template *tmpl, struct rtattr **tb)
373 inst->alg.base.cra_blocksize = alg->base.cra_blocksize; 374 inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
374 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 375 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
375 376
376 /* We access the data as u32s when xoring. */
377 inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
378
379 inst->alg.ivsize = alg->base.cra_blocksize; 377 inst->alg.ivsize = alg->base.cra_blocksize;
380 inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); 378 inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
381 inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg); 379 inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
diff --git a/crypto/kpp.c b/crypto/kpp.c
index d36ce05eee43..a90edc27af77 100644
--- a/crypto/kpp.c
+++ b/crypto/kpp.c
@@ -19,6 +19,7 @@
19#include <linux/crypto.h> 19#include <linux/crypto.h>
20#include <crypto/algapi.h> 20#include <crypto/algapi.h>
21#include <linux/cryptouser.h> 21#include <linux/cryptouser.h>
22#include <linux/compiler.h>
22#include <net/netlink.h> 23#include <net/netlink.h>
23#include <crypto/kpp.h> 24#include <crypto/kpp.h>
24#include <crypto/internal/kpp.h> 25#include <crypto/internal/kpp.h>
@@ -47,7 +48,7 @@ static int crypto_kpp_report(struct sk_buff *skb, struct crypto_alg *alg)
47#endif 48#endif
48 49
49static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg) 50static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg)
50 __attribute__ ((unused)); 51 __maybe_unused;
51 52
52static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg) 53static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg)
53{ 54{
diff --git a/crypto/pcbc.c b/crypto/pcbc.c
index e4538e07f7ca..29dd2b4a3b85 100644
--- a/crypto/pcbc.c
+++ b/crypto/pcbc.c
@@ -20,6 +20,7 @@
20#include <linux/kernel.h> 20#include <linux/kernel.h>
21#include <linux/module.h> 21#include <linux/module.h>
22#include <linux/slab.h> 22#include <linux/slab.h>
23#include <linux/compiler.h>
23 24
24struct crypto_pcbc_ctx { 25struct crypto_pcbc_ctx {
25 struct crypto_cipher *child; 26 struct crypto_cipher *child;
@@ -146,7 +147,7 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req,
146 unsigned int nbytes = walk->nbytes; 147 unsigned int nbytes = walk->nbytes;
147 u8 *src = walk->src.virt.addr; 148 u8 *src = walk->src.virt.addr;
148 u8 *iv = walk->iv; 149 u8 *iv = walk->iv;
149 u8 tmpbuf[bsize] __attribute__ ((aligned(__alignof__(u32)))); 150 u8 tmpbuf[bsize] __aligned(__alignof__(u32));
150 151
151 do { 152 do {
152 memcpy(tmpbuf, src, bsize); 153 memcpy(tmpbuf, src, bsize);
@@ -259,9 +260,6 @@ static int crypto_pcbc_create(struct crypto_template *tmpl, struct rtattr **tb)
259 inst->alg.base.cra_blocksize = alg->cra_blocksize; 260 inst->alg.base.cra_blocksize = alg->cra_blocksize;
260 inst->alg.base.cra_alignmask = alg->cra_alignmask; 261 inst->alg.base.cra_alignmask = alg->cra_alignmask;
261 262
262 /* We access the data as u32s when xoring. */
263 inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
264
265 inst->alg.ivsize = alg->cra_blocksize; 263 inst->alg.ivsize = alg->cra_blocksize;
266 inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize; 264 inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
267 inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize; 265 inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
diff --git a/crypto/rng.c b/crypto/rng.c
index b81cffb13bab..f46dac5288b9 100644
--- a/crypto/rng.c
+++ b/crypto/rng.c
@@ -23,6 +23,7 @@
23#include <linux/slab.h> 23#include <linux/slab.h>
24#include <linux/string.h> 24#include <linux/string.h>
25#include <linux/cryptouser.h> 25#include <linux/cryptouser.h>
26#include <linux/compiler.h>
26#include <net/netlink.h> 27#include <net/netlink.h>
27 28
28#include "internal.h" 29#include "internal.h"
@@ -95,7 +96,7 @@ static int crypto_rng_report(struct sk_buff *skb, struct crypto_alg *alg)
95#endif 96#endif
96 97
97static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg) 98static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg)
98 __attribute__ ((unused)); 99 __maybe_unused;
99static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg) 100static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg)
100{ 101{
101 seq_printf(m, "type : rng\n"); 102 seq_printf(m, "type : rng\n");
diff --git a/crypto/scompress.c b/crypto/scompress.c
index 35e396d154b7..6b048b36312d 100644
--- a/crypto/scompress.c
+++ b/crypto/scompress.c
@@ -18,6 +18,7 @@
18#include <linux/slab.h> 18#include <linux/slab.h>
19#include <linux/string.h> 19#include <linux/string.h>
20#include <linux/crypto.h> 20#include <linux/crypto.h>
21#include <linux/compiler.h>
21#include <linux/vmalloc.h> 22#include <linux/vmalloc.h>
22#include <crypto/algapi.h> 23#include <crypto/algapi.h>
23#include <linux/cryptouser.h> 24#include <linux/cryptouser.h>
@@ -57,7 +58,7 @@ static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg)
57#endif 58#endif
58 59
59static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) 60static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
60 __attribute__ ((unused)); 61 __maybe_unused;
61 62
62static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) 63static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
63{ 64{
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index c7049231861f..570b7d1aa0ca 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -153,8 +153,6 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb)
153 if (IS_ERR(inst)) 153 if (IS_ERR(inst))
154 return PTR_ERR(inst); 154 return PTR_ERR(inst);
155 155
156 inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
157
158 spawn = aead_instance_ctx(inst); 156 spawn = aead_instance_ctx(inst);
159 alg = crypto_spawn_aead_alg(spawn); 157 alg = crypto_spawn_aead_alg(spawn);
160 158
diff --git a/crypto/shash.c b/crypto/shash.c
index a051541a4a17..5e31c8d776df 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -19,6 +19,7 @@
19#include <linux/seq_file.h> 19#include <linux/seq_file.h>
20#include <linux/cryptouser.h> 20#include <linux/cryptouser.h>
21#include <net/netlink.h> 21#include <net/netlink.h>
22#include <linux/compiler.h>
22 23
23#include "internal.h" 24#include "internal.h"
24 25
@@ -67,7 +68,7 @@ EXPORT_SYMBOL_GPL(crypto_shash_setkey);
67static inline unsigned int shash_align_buffer_size(unsigned len, 68static inline unsigned int shash_align_buffer_size(unsigned len,
68 unsigned long mask) 69 unsigned long mask)
69{ 70{
70 typedef u8 __attribute__ ((aligned)) u8_aligned; 71 typedef u8 __aligned_largest u8_aligned;
71 return len + (mask & ~(__alignof__(u8_aligned) - 1)); 72 return len + (mask & ~(__alignof__(u8_aligned) - 1));
72} 73}
73 74
@@ -80,7 +81,7 @@ static int shash_update_unaligned(struct shash_desc *desc, const u8 *data,
80 unsigned int unaligned_len = alignmask + 1 - 81 unsigned int unaligned_len = alignmask + 1 -
81 ((unsigned long)data & alignmask); 82 ((unsigned long)data & alignmask);
82 u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)] 83 u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)]
83 __attribute__ ((aligned)); 84 __aligned_largest;
84 u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1); 85 u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
85 int err; 86 int err;
86 87
@@ -116,7 +117,7 @@ static int shash_final_unaligned(struct shash_desc *desc, u8 *out)
116 struct shash_alg *shash = crypto_shash_alg(tfm); 117 struct shash_alg *shash = crypto_shash_alg(tfm);
117 unsigned int ds = crypto_shash_digestsize(tfm); 118 unsigned int ds = crypto_shash_digestsize(tfm);
118 u8 ubuf[shash_align_buffer_size(ds, alignmask)] 119 u8 ubuf[shash_align_buffer_size(ds, alignmask)]
119 __attribute__ ((aligned)); 120 __aligned_largest;
120 u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1); 121 u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
121 int err; 122 int err;
122 123
@@ -403,7 +404,7 @@ static int crypto_shash_report(struct sk_buff *skb, struct crypto_alg *alg)
403#endif 404#endif
404 405
405static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg) 406static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg)
406 __attribute__ ((unused)); 407 __maybe_unused;
407static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg) 408static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg)
408{ 409{
409 struct shash_alg *salg = __crypto_shash_alg(alg); 410 struct shash_alg *salg = __crypto_shash_alg(alg);
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index 0e1e6c35188e..014af741fc6a 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -19,6 +19,7 @@
19#include <crypto/scatterwalk.h> 19#include <crypto/scatterwalk.h>
20#include <linux/bug.h> 20#include <linux/bug.h>
21#include <linux/cryptouser.h> 21#include <linux/cryptouser.h>
22#include <linux/compiler.h>
22#include <linux/list.h> 23#include <linux/list.h>
23#include <linux/module.h> 24#include <linux/module.h>
24#include <linux/rtnetlink.h> 25#include <linux/rtnetlink.h>
@@ -185,12 +186,12 @@ void skcipher_walk_complete(struct skcipher_walk *walk, int err)
185 data = p->data; 186 data = p->data;
186 if (!data) { 187 if (!data) {
187 data = PTR_ALIGN(&p->buffer[0], walk->alignmask + 1); 188 data = PTR_ALIGN(&p->buffer[0], walk->alignmask + 1);
188 data = skcipher_get_spot(data, walk->chunksize); 189 data = skcipher_get_spot(data, walk->stride);
189 } 190 }
190 191
191 scatterwalk_copychunks(data, &p->dst, p->len, 1); 192 scatterwalk_copychunks(data, &p->dst, p->len, 1);
192 193
193 if (offset_in_page(p->data) + p->len + walk->chunksize > 194 if (offset_in_page(p->data) + p->len + walk->stride >
194 PAGE_SIZE) 195 PAGE_SIZE)
195 free_page((unsigned long)p->data); 196 free_page((unsigned long)p->data);
196 197
@@ -299,7 +300,7 @@ static int skcipher_next_copy(struct skcipher_walk *walk)
299 p->len = walk->nbytes; 300 p->len = walk->nbytes;
300 skcipher_queue_write(walk, p); 301 skcipher_queue_write(walk, p);
301 302
302 if (offset_in_page(walk->page) + walk->nbytes + walk->chunksize > 303 if (offset_in_page(walk->page) + walk->nbytes + walk->stride >
303 PAGE_SIZE) 304 PAGE_SIZE)
304 walk->page = NULL; 305 walk->page = NULL;
305 else 306 else
@@ -344,7 +345,7 @@ static int skcipher_walk_next(struct skcipher_walk *walk)
344 SKCIPHER_WALK_DIFF); 345 SKCIPHER_WALK_DIFF);
345 346
346 n = walk->total; 347 n = walk->total;
347 bsize = min(walk->chunksize, max(n, walk->blocksize)); 348 bsize = min(walk->stride, max(n, walk->blocksize));
348 n = scatterwalk_clamp(&walk->in, n); 349 n = scatterwalk_clamp(&walk->in, n);
349 n = scatterwalk_clamp(&walk->out, n); 350 n = scatterwalk_clamp(&walk->out, n);
350 351
@@ -393,7 +394,7 @@ static int skcipher_copy_iv(struct skcipher_walk *walk)
393 unsigned a = crypto_tfm_ctx_alignment() - 1; 394 unsigned a = crypto_tfm_ctx_alignment() - 1;
394 unsigned alignmask = walk->alignmask; 395 unsigned alignmask = walk->alignmask;
395 unsigned ivsize = walk->ivsize; 396 unsigned ivsize = walk->ivsize;
396 unsigned bs = walk->chunksize; 397 unsigned bs = walk->stride;
397 unsigned aligned_bs; 398 unsigned aligned_bs;
398 unsigned size; 399 unsigned size;
399 u8 *iv; 400 u8 *iv;
@@ -463,7 +464,7 @@ static int skcipher_walk_skcipher(struct skcipher_walk *walk,
463 SKCIPHER_WALK_SLEEP : 0; 464 SKCIPHER_WALK_SLEEP : 0;
464 465
465 walk->blocksize = crypto_skcipher_blocksize(tfm); 466 walk->blocksize = crypto_skcipher_blocksize(tfm);
466 walk->chunksize = crypto_skcipher_chunksize(tfm); 467 walk->stride = crypto_skcipher_walksize(tfm);
467 walk->ivsize = crypto_skcipher_ivsize(tfm); 468 walk->ivsize = crypto_skcipher_ivsize(tfm);
468 walk->alignmask = crypto_skcipher_alignmask(tfm); 469 walk->alignmask = crypto_skcipher_alignmask(tfm);
469 470
@@ -525,7 +526,7 @@ static int skcipher_walk_aead_common(struct skcipher_walk *walk,
525 walk->flags &= ~SKCIPHER_WALK_SLEEP; 526 walk->flags &= ~SKCIPHER_WALK_SLEEP;
526 527
527 walk->blocksize = crypto_aead_blocksize(tfm); 528 walk->blocksize = crypto_aead_blocksize(tfm);
528 walk->chunksize = crypto_aead_chunksize(tfm); 529 walk->stride = crypto_aead_chunksize(tfm);
529 walk->ivsize = crypto_aead_ivsize(tfm); 530 walk->ivsize = crypto_aead_ivsize(tfm);
530 walk->alignmask = crypto_aead_alignmask(tfm); 531 walk->alignmask = crypto_aead_alignmask(tfm);
531 532
@@ -807,7 +808,7 @@ static void crypto_skcipher_free_instance(struct crypto_instance *inst)
807} 808}
808 809
809static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) 810static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
810 __attribute__ ((unused)); 811 __maybe_unused;
811static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) 812static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
812{ 813{
813 struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg, 814 struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg,
@@ -821,6 +822,7 @@ static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
821 seq_printf(m, "max keysize : %u\n", skcipher->max_keysize); 822 seq_printf(m, "max keysize : %u\n", skcipher->max_keysize);
822 seq_printf(m, "ivsize : %u\n", skcipher->ivsize); 823 seq_printf(m, "ivsize : %u\n", skcipher->ivsize);
823 seq_printf(m, "chunksize : %u\n", skcipher->chunksize); 824 seq_printf(m, "chunksize : %u\n", skcipher->chunksize);
825 seq_printf(m, "walksize : %u\n", skcipher->walksize);
824} 826}
825 827
826#ifdef CONFIG_NET 828#ifdef CONFIG_NET
@@ -893,11 +895,14 @@ static int skcipher_prepare_alg(struct skcipher_alg *alg)
893{ 895{
894 struct crypto_alg *base = &alg->base; 896 struct crypto_alg *base = &alg->base;
895 897
896 if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8) 898 if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8 ||
899 alg->walksize > PAGE_SIZE / 8)
897 return -EINVAL; 900 return -EINVAL;
898 901
899 if (!alg->chunksize) 902 if (!alg->chunksize)
900 alg->chunksize = base->cra_blocksize; 903 alg->chunksize = base->cra_blocksize;
904 if (!alg->walksize)
905 alg->walksize = alg->chunksize;
901 906
902 base->cra_type = &crypto_skcipher_type2; 907 base->cra_type = &crypto_skcipher_type2;
903 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; 908 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index ae22f05d5936..9a11f3c2bf98 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -22,6 +22,8 @@
22 * 22 *
23 */ 23 */
24 24
25#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
26
25#include <crypto/aead.h> 27#include <crypto/aead.h>
26#include <crypto/hash.h> 28#include <crypto/hash.h>
27#include <crypto/skcipher.h> 29#include <crypto/skcipher.h>
@@ -1010,6 +1012,8 @@ static inline int tcrypt_test(const char *alg)
1010{ 1012{
1011 int ret; 1013 int ret;
1012 1014
1015 pr_debug("testing %s\n", alg);
1016
1013 ret = alg_test(alg, alg, 0, 0); 1017 ret = alg_test(alg, alg, 0, 0);
1014 /* non-fips algs return -EINVAL in fips mode */ 1018 /* non-fips algs return -EINVAL in fips mode */
1015 if (fips_enabled && ret == -EINVAL) 1019 if (fips_enabled && ret == -EINVAL)
@@ -2059,6 +2063,8 @@ static int __init tcrypt_mod_init(void)
2059 if (err) { 2063 if (err) {
2060 printk(KERN_ERR "tcrypt: one or more tests failed!\n"); 2064 printk(KERN_ERR "tcrypt: one or more tests failed!\n");
2061 goto err_free_tv; 2065 goto err_free_tv;
2066 } else {
2067 pr_debug("all tests passed\n");
2062 } 2068 }
2063 2069
2064 /* We intentionaly return -EAGAIN to prevent keeping the module, 2070 /* We intentionaly return -EAGAIN to prevent keeping the module,
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 44e888b0b041..f9c378af3907 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -265,6 +265,7 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
265 const int align_offset) 265 const int align_offset)
266{ 266{
267 const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); 267 const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm));
268 size_t digest_size = crypto_ahash_digestsize(tfm);
268 unsigned int i, j, k, temp; 269 unsigned int i, j, k, temp;
269 struct scatterlist sg[8]; 270 struct scatterlist sg[8];
270 char *result; 271 char *result;
@@ -275,7 +276,7 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
275 char *xbuf[XBUFSIZE]; 276 char *xbuf[XBUFSIZE];
276 int ret = -ENOMEM; 277 int ret = -ENOMEM;
277 278
278 result = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL); 279 result = kmalloc(digest_size, GFP_KERNEL);
279 if (!result) 280 if (!result)
280 return ret; 281 return ret;
281 key = kmalloc(MAX_KEYLEN, GFP_KERNEL); 282 key = kmalloc(MAX_KEYLEN, GFP_KERNEL);
@@ -305,7 +306,7 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
305 goto out; 306 goto out;
306 307
307 j++; 308 j++;
308 memset(result, 0, MAX_DIGEST_SIZE); 309 memset(result, 0, digest_size);
309 310
310 hash_buff = xbuf[0]; 311 hash_buff = xbuf[0];
311 hash_buff += align_offset; 312 hash_buff += align_offset;
@@ -380,7 +381,7 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
380 continue; 381 continue;
381 382
382 j++; 383 j++;
383 memset(result, 0, MAX_DIGEST_SIZE); 384 memset(result, 0, digest_size);
384 385
385 temp = 0; 386 temp = 0;
386 sg_init_table(sg, template[i].np); 387 sg_init_table(sg, template[i].np);
@@ -458,7 +459,7 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
458 continue; 459 continue;
459 460
460 j++; 461 j++;
461 memset(result, 0, MAX_DIGEST_SIZE); 462 memset(result, 0, digest_size);
462 463
463 ret = -EINVAL; 464 ret = -EINVAL;
464 hash_buff = xbuf[0]; 465 hash_buff = xbuf[0];
@@ -1463,13 +1464,12 @@ static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate,
1463 int ilen = ctemplate[i].inlen; 1464 int ilen = ctemplate[i].inlen;
1464 void *input_vec; 1465 void *input_vec;
1465 1466
1466 input_vec = kmalloc(ilen, GFP_KERNEL); 1467 input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
1467 if (!input_vec) { 1468 if (!input_vec) {
1468 ret = -ENOMEM; 1469 ret = -ENOMEM;
1469 goto out; 1470 goto out;
1470 } 1471 }
1471 1472
1472 memcpy(input_vec, ctemplate[i].input, ilen);
1473 memset(output, 0, dlen); 1473 memset(output, 0, dlen);
1474 init_completion(&result.completion); 1474 init_completion(&result.completion);
1475 sg_init_one(&src, input_vec, ilen); 1475 sg_init_one(&src, input_vec, ilen);
@@ -1525,13 +1525,12 @@ static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate,
1525 int ilen = dtemplate[i].inlen; 1525 int ilen = dtemplate[i].inlen;
1526 void *input_vec; 1526 void *input_vec;
1527 1527
1528 input_vec = kmalloc(ilen, GFP_KERNEL); 1528 input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
1529 if (!input_vec) { 1529 if (!input_vec) {
1530 ret = -ENOMEM; 1530 ret = -ENOMEM;
1531 goto out; 1531 goto out;
1532 } 1532 }
1533 1533
1534 memcpy(input_vec, dtemplate[i].input, ilen);
1535 memset(output, 0, dlen); 1534 memset(output, 0, dlen);
1536 init_completion(&result.completion); 1535 init_completion(&result.completion);
1537 sg_init_one(&src, input_vec, ilen); 1536 sg_init_one(&src, input_vec, ilen);
@@ -2251,30 +2250,23 @@ static int alg_test_null(const struct alg_test_desc *desc,
2251 return 0; 2250 return 0;
2252} 2251}
2253 2252
2253#define __VECS(tv) { .vecs = tv, .count = ARRAY_SIZE(tv) }
2254
2254/* Please keep this list sorted by algorithm name. */ 2255/* Please keep this list sorted by algorithm name. */
2255static const struct alg_test_desc alg_test_descs[] = { 2256static const struct alg_test_desc alg_test_descs[] = {
2256 { 2257 {
2257 .alg = "ansi_cprng", 2258 .alg = "ansi_cprng",
2258 .test = alg_test_cprng, 2259 .test = alg_test_cprng,
2259 .suite = { 2260 .suite = {
2260 .cprng = { 2261 .cprng = __VECS(ansi_cprng_aes_tv_template)
2261 .vecs = ansi_cprng_aes_tv_template,
2262 .count = ANSI_CPRNG_AES_TEST_VECTORS
2263 }
2264 } 2262 }
2265 }, { 2263 }, {
2266 .alg = "authenc(hmac(md5),ecb(cipher_null))", 2264 .alg = "authenc(hmac(md5),ecb(cipher_null))",
2267 .test = alg_test_aead, 2265 .test = alg_test_aead,
2268 .suite = { 2266 .suite = {
2269 .aead = { 2267 .aead = {
2270 .enc = { 2268 .enc = __VECS(hmac_md5_ecb_cipher_null_enc_tv_template),
2271 .vecs = hmac_md5_ecb_cipher_null_enc_tv_template, 2269 .dec = __VECS(hmac_md5_ecb_cipher_null_dec_tv_template)
2272 .count = HMAC_MD5_ECB_CIPHER_NULL_ENC_TEST_VECTORS
2273 },
2274 .dec = {
2275 .vecs = hmac_md5_ecb_cipher_null_dec_tv_template,
2276 .count = HMAC_MD5_ECB_CIPHER_NULL_DEC_TEST_VECTORS
2277 }
2278 } 2270 }
2279 } 2271 }
2280 }, { 2272 }, {
@@ -2282,12 +2274,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2282 .test = alg_test_aead, 2274 .test = alg_test_aead,
2283 .suite = { 2275 .suite = {
2284 .aead = { 2276 .aead = {
2285 .enc = { 2277 .enc = __VECS(hmac_sha1_aes_cbc_enc_tv_temp)
2286 .vecs =
2287 hmac_sha1_aes_cbc_enc_tv_temp,
2288 .count =
2289 HMAC_SHA1_AES_CBC_ENC_TEST_VEC
2290 }
2291 } 2278 }
2292 } 2279 }
2293 }, { 2280 }, {
@@ -2295,12 +2282,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2295 .test = alg_test_aead, 2282 .test = alg_test_aead,
2296 .suite = { 2283 .suite = {
2297 .aead = { 2284 .aead = {
2298 .enc = { 2285 .enc = __VECS(hmac_sha1_des_cbc_enc_tv_temp)
2299 .vecs =
2300 hmac_sha1_des_cbc_enc_tv_temp,
2301 .count =
2302 HMAC_SHA1_DES_CBC_ENC_TEST_VEC
2303 }
2304 } 2286 }
2305 } 2287 }
2306 }, { 2288 }, {
@@ -2309,12 +2291,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2309 .fips_allowed = 1, 2291 .fips_allowed = 1,
2310 .suite = { 2292 .suite = {
2311 .aead = { 2293 .aead = {
2312 .enc = { 2294 .enc = __VECS(hmac_sha1_des3_ede_cbc_enc_tv_temp)
2313 .vecs =
2314 hmac_sha1_des3_ede_cbc_enc_tv_temp,
2315 .count =
2316 HMAC_SHA1_DES3_EDE_CBC_ENC_TEST_VEC
2317 }
2318 } 2295 }
2319 } 2296 }
2320 }, { 2297 }, {
@@ -2326,18 +2303,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2326 .test = alg_test_aead, 2303 .test = alg_test_aead,
2327 .suite = { 2304 .suite = {
2328 .aead = { 2305 .aead = {
2329 .enc = { 2306 .enc = __VECS(hmac_sha1_ecb_cipher_null_enc_tv_temp),
2330 .vecs = 2307 .dec = __VECS(hmac_sha1_ecb_cipher_null_dec_tv_temp)
2331 hmac_sha1_ecb_cipher_null_enc_tv_temp,
2332 .count =
2333 HMAC_SHA1_ECB_CIPHER_NULL_ENC_TEST_VEC
2334 },
2335 .dec = {
2336 .vecs =
2337 hmac_sha1_ecb_cipher_null_dec_tv_temp,
2338 .count =
2339 HMAC_SHA1_ECB_CIPHER_NULL_DEC_TEST_VEC
2340 }
2341 } 2308 }
2342 } 2309 }
2343 }, { 2310 }, {
@@ -2349,12 +2316,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2349 .test = alg_test_aead, 2316 .test = alg_test_aead,
2350 .suite = { 2317 .suite = {
2351 .aead = { 2318 .aead = {
2352 .enc = { 2319 .enc = __VECS(hmac_sha224_des_cbc_enc_tv_temp)
2353 .vecs =
2354 hmac_sha224_des_cbc_enc_tv_temp,
2355 .count =
2356 HMAC_SHA224_DES_CBC_ENC_TEST_VEC
2357 }
2358 } 2320 }
2359 } 2321 }
2360 }, { 2322 }, {
@@ -2363,12 +2325,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2363 .fips_allowed = 1, 2325 .fips_allowed = 1,
2364 .suite = { 2326 .suite = {
2365 .aead = { 2327 .aead = {
2366 .enc = { 2328 .enc = __VECS(hmac_sha224_des3_ede_cbc_enc_tv_temp)
2367 .vecs =
2368 hmac_sha224_des3_ede_cbc_enc_tv_temp,
2369 .count =
2370 HMAC_SHA224_DES3_EDE_CBC_ENC_TEST_VEC
2371 }
2372 } 2329 }
2373 } 2330 }
2374 }, { 2331 }, {
@@ -2377,12 +2334,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2377 .fips_allowed = 1, 2334 .fips_allowed = 1,
2378 .suite = { 2335 .suite = {
2379 .aead = { 2336 .aead = {
2380 .enc = { 2337 .enc = __VECS(hmac_sha256_aes_cbc_enc_tv_temp)
2381 .vecs =
2382 hmac_sha256_aes_cbc_enc_tv_temp,
2383 .count =
2384 HMAC_SHA256_AES_CBC_ENC_TEST_VEC
2385 }
2386 } 2338 }
2387 } 2339 }
2388 }, { 2340 }, {
@@ -2390,12 +2342,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2390 .test = alg_test_aead, 2342 .test = alg_test_aead,
2391 .suite = { 2343 .suite = {
2392 .aead = { 2344 .aead = {
2393 .enc = { 2345 .enc = __VECS(hmac_sha256_des_cbc_enc_tv_temp)
2394 .vecs =
2395 hmac_sha256_des_cbc_enc_tv_temp,
2396 .count =
2397 HMAC_SHA256_DES_CBC_ENC_TEST_VEC
2398 }
2399 } 2346 }
2400 } 2347 }
2401 }, { 2348 }, {
@@ -2404,12 +2351,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2404 .fips_allowed = 1, 2351 .fips_allowed = 1,
2405 .suite = { 2352 .suite = {
2406 .aead = { 2353 .aead = {
2407 .enc = { 2354 .enc = __VECS(hmac_sha256_des3_ede_cbc_enc_tv_temp)
2408 .vecs =
2409 hmac_sha256_des3_ede_cbc_enc_tv_temp,
2410 .count =
2411 HMAC_SHA256_DES3_EDE_CBC_ENC_TEST_VEC
2412 }
2413 } 2355 }
2414 } 2356 }
2415 }, { 2357 }, {
@@ -2425,12 +2367,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2425 .test = alg_test_aead, 2367 .test = alg_test_aead,
2426 .suite = { 2368 .suite = {
2427 .aead = { 2369 .aead = {
2428 .enc = { 2370 .enc = __VECS(hmac_sha384_des_cbc_enc_tv_temp)
2429 .vecs =
2430 hmac_sha384_des_cbc_enc_tv_temp,
2431 .count =
2432 HMAC_SHA384_DES_CBC_ENC_TEST_VEC
2433 }
2434 } 2371 }
2435 } 2372 }
2436 }, { 2373 }, {
@@ -2439,12 +2376,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2439 .fips_allowed = 1, 2376 .fips_allowed = 1,
2440 .suite = { 2377 .suite = {
2441 .aead = { 2378 .aead = {
2442 .enc = { 2379 .enc = __VECS(hmac_sha384_des3_ede_cbc_enc_tv_temp)
2443 .vecs =
2444 hmac_sha384_des3_ede_cbc_enc_tv_temp,
2445 .count =
2446 HMAC_SHA384_DES3_EDE_CBC_ENC_TEST_VEC
2447 }
2448 } 2380 }
2449 } 2381 }
2450 }, { 2382 }, {
@@ -2461,12 +2393,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2461 .test = alg_test_aead, 2393 .test = alg_test_aead,
2462 .suite = { 2394 .suite = {
2463 .aead = { 2395 .aead = {
2464 .enc = { 2396 .enc = __VECS(hmac_sha512_aes_cbc_enc_tv_temp)
2465 .vecs =
2466 hmac_sha512_aes_cbc_enc_tv_temp,
2467 .count =
2468 HMAC_SHA512_AES_CBC_ENC_TEST_VEC
2469 }
2470 } 2397 }
2471 } 2398 }
2472 }, { 2399 }, {
@@ -2474,12 +2401,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2474 .test = alg_test_aead, 2401 .test = alg_test_aead,
2475 .suite = { 2402 .suite = {
2476 .aead = { 2403 .aead = {
2477 .enc = { 2404 .enc = __VECS(hmac_sha512_des_cbc_enc_tv_temp)
2478 .vecs =
2479 hmac_sha512_des_cbc_enc_tv_temp,
2480 .count =
2481 HMAC_SHA512_DES_CBC_ENC_TEST_VEC
2482 }
2483 } 2405 }
2484 } 2406 }
2485 }, { 2407 }, {
@@ -2488,12 +2410,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2488 .fips_allowed = 1, 2410 .fips_allowed = 1,
2489 .suite = { 2411 .suite = {
2490 .aead = { 2412 .aead = {
2491 .enc = { 2413 .enc = __VECS(hmac_sha512_des3_ede_cbc_enc_tv_temp)
2492 .vecs =
2493 hmac_sha512_des3_ede_cbc_enc_tv_temp,
2494 .count =
2495 HMAC_SHA512_DES3_EDE_CBC_ENC_TEST_VEC
2496 }
2497 } 2414 }
2498 } 2415 }
2499 }, { 2416 }, {
@@ -2510,14 +2427,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2510 .fips_allowed = 1, 2427 .fips_allowed = 1,
2511 .suite = { 2428 .suite = {
2512 .cipher = { 2429 .cipher = {
2513 .enc = { 2430 .enc = __VECS(aes_cbc_enc_tv_template),
2514 .vecs = aes_cbc_enc_tv_template, 2431 .dec = __VECS(aes_cbc_dec_tv_template)
2515 .count = AES_CBC_ENC_TEST_VECTORS
2516 },
2517 .dec = {
2518 .vecs = aes_cbc_dec_tv_template,
2519 .count = AES_CBC_DEC_TEST_VECTORS
2520 }
2521 } 2432 }
2522 } 2433 }
2523 }, { 2434 }, {
@@ -2525,14 +2436,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2525 .test = alg_test_skcipher, 2436 .test = alg_test_skcipher,
2526 .suite = { 2437 .suite = {
2527 .cipher = { 2438 .cipher = {
2528 .enc = { 2439 .enc = __VECS(anubis_cbc_enc_tv_template),
2529 .vecs = anubis_cbc_enc_tv_template, 2440 .dec = __VECS(anubis_cbc_dec_tv_template)
2530 .count = ANUBIS_CBC_ENC_TEST_VECTORS
2531 },
2532 .dec = {
2533 .vecs = anubis_cbc_dec_tv_template,
2534 .count = ANUBIS_CBC_DEC_TEST_VECTORS
2535 }
2536 } 2441 }
2537 } 2442 }
2538 }, { 2443 }, {
@@ -2540,14 +2445,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2540 .test = alg_test_skcipher, 2445 .test = alg_test_skcipher,
2541 .suite = { 2446 .suite = {
2542 .cipher = { 2447 .cipher = {
2543 .enc = { 2448 .enc = __VECS(bf_cbc_enc_tv_template),
2544 .vecs = bf_cbc_enc_tv_template, 2449 .dec = __VECS(bf_cbc_dec_tv_template)
2545 .count = BF_CBC_ENC_TEST_VECTORS
2546 },
2547 .dec = {
2548 .vecs = bf_cbc_dec_tv_template,
2549 .count = BF_CBC_DEC_TEST_VECTORS
2550 }
2551 } 2450 }
2552 } 2451 }
2553 }, { 2452 }, {
@@ -2555,14 +2454,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2555 .test = alg_test_skcipher, 2454 .test = alg_test_skcipher,
2556 .suite = { 2455 .suite = {
2557 .cipher = { 2456 .cipher = {
2558 .enc = { 2457 .enc = __VECS(camellia_cbc_enc_tv_template),
2559 .vecs = camellia_cbc_enc_tv_template, 2458 .dec = __VECS(camellia_cbc_dec_tv_template)
2560 .count = CAMELLIA_CBC_ENC_TEST_VECTORS
2561 },
2562 .dec = {
2563 .vecs = camellia_cbc_dec_tv_template,
2564 .count = CAMELLIA_CBC_DEC_TEST_VECTORS
2565 }
2566 } 2459 }
2567 } 2460 }
2568 }, { 2461 }, {
@@ -2570,14 +2463,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2570 .test = alg_test_skcipher, 2463 .test = alg_test_skcipher,
2571 .suite = { 2464 .suite = {
2572 .cipher = { 2465 .cipher = {
2573 .enc = { 2466 .enc = __VECS(cast5_cbc_enc_tv_template),
2574 .vecs = cast5_cbc_enc_tv_template, 2467 .dec = __VECS(cast5_cbc_dec_tv_template)
2575 .count = CAST5_CBC_ENC_TEST_VECTORS
2576 },
2577 .dec = {
2578 .vecs = cast5_cbc_dec_tv_template,
2579 .count = CAST5_CBC_DEC_TEST_VECTORS
2580 }
2581 } 2468 }
2582 } 2469 }
2583 }, { 2470 }, {
@@ -2585,14 +2472,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2585 .test = alg_test_skcipher, 2472 .test = alg_test_skcipher,
2586 .suite = { 2473 .suite = {
2587 .cipher = { 2474 .cipher = {
2588 .enc = { 2475 .enc = __VECS(cast6_cbc_enc_tv_template),
2589 .vecs = cast6_cbc_enc_tv_template, 2476 .dec = __VECS(cast6_cbc_dec_tv_template)
2590 .count = CAST6_CBC_ENC_TEST_VECTORS
2591 },
2592 .dec = {
2593 .vecs = cast6_cbc_dec_tv_template,
2594 .count = CAST6_CBC_DEC_TEST_VECTORS
2595 }
2596 } 2477 }
2597 } 2478 }
2598 }, { 2479 }, {
@@ -2600,14 +2481,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2600 .test = alg_test_skcipher, 2481 .test = alg_test_skcipher,
2601 .suite = { 2482 .suite = {
2602 .cipher = { 2483 .cipher = {
2603 .enc = { 2484 .enc = __VECS(des_cbc_enc_tv_template),
2604 .vecs = des_cbc_enc_tv_template, 2485 .dec = __VECS(des_cbc_dec_tv_template)
2605 .count = DES_CBC_ENC_TEST_VECTORS
2606 },
2607 .dec = {
2608 .vecs = des_cbc_dec_tv_template,
2609 .count = DES_CBC_DEC_TEST_VECTORS
2610 }
2611 } 2486 }
2612 } 2487 }
2613 }, { 2488 }, {
@@ -2616,14 +2491,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2616 .fips_allowed = 1, 2491 .fips_allowed = 1,
2617 .suite = { 2492 .suite = {
2618 .cipher = { 2493 .cipher = {
2619 .enc = { 2494 .enc = __VECS(des3_ede_cbc_enc_tv_template),
2620 .vecs = des3_ede_cbc_enc_tv_template, 2495 .dec = __VECS(des3_ede_cbc_dec_tv_template)
2621 .count = DES3_EDE_CBC_ENC_TEST_VECTORS
2622 },
2623 .dec = {
2624 .vecs = des3_ede_cbc_dec_tv_template,
2625 .count = DES3_EDE_CBC_DEC_TEST_VECTORS
2626 }
2627 } 2496 }
2628 } 2497 }
2629 }, { 2498 }, {
@@ -2631,14 +2500,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2631 .test = alg_test_skcipher, 2500 .test = alg_test_skcipher,
2632 .suite = { 2501 .suite = {
2633 .cipher = { 2502 .cipher = {
2634 .enc = { 2503 .enc = __VECS(serpent_cbc_enc_tv_template),
2635 .vecs = serpent_cbc_enc_tv_template, 2504 .dec = __VECS(serpent_cbc_dec_tv_template)
2636 .count = SERPENT_CBC_ENC_TEST_VECTORS
2637 },
2638 .dec = {
2639 .vecs = serpent_cbc_dec_tv_template,
2640 .count = SERPENT_CBC_DEC_TEST_VECTORS
2641 }
2642 } 2505 }
2643 } 2506 }
2644 }, { 2507 }, {
@@ -2646,30 +2509,25 @@ static const struct alg_test_desc alg_test_descs[] = {
2646 .test = alg_test_skcipher, 2509 .test = alg_test_skcipher,
2647 .suite = { 2510 .suite = {
2648 .cipher = { 2511 .cipher = {
2649 .enc = { 2512 .enc = __VECS(tf_cbc_enc_tv_template),
2650 .vecs = tf_cbc_enc_tv_template, 2513 .dec = __VECS(tf_cbc_dec_tv_template)
2651 .count = TF_CBC_ENC_TEST_VECTORS
2652 },
2653 .dec = {
2654 .vecs = tf_cbc_dec_tv_template,
2655 .count = TF_CBC_DEC_TEST_VECTORS
2656 }
2657 } 2514 }
2658 } 2515 }
2659 }, { 2516 }, {
2517 .alg = "cbcmac(aes)",
2518 .fips_allowed = 1,
2519 .test = alg_test_hash,
2520 .suite = {
2521 .hash = __VECS(aes_cbcmac_tv_template)
2522 }
2523 }, {
2660 .alg = "ccm(aes)", 2524 .alg = "ccm(aes)",
2661 .test = alg_test_aead, 2525 .test = alg_test_aead,
2662 .fips_allowed = 1, 2526 .fips_allowed = 1,
2663 .suite = { 2527 .suite = {
2664 .aead = { 2528 .aead = {
2665 .enc = { 2529 .enc = __VECS(aes_ccm_enc_tv_template),
2666 .vecs = aes_ccm_enc_tv_template, 2530 .dec = __VECS(aes_ccm_dec_tv_template)
2667 .count = AES_CCM_ENC_TEST_VECTORS
2668 },
2669 .dec = {
2670 .vecs = aes_ccm_dec_tv_template,
2671 .count = AES_CCM_DEC_TEST_VECTORS
2672 }
2673 } 2531 }
2674 } 2532 }
2675 }, { 2533 }, {
@@ -2677,14 +2535,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2677 .test = alg_test_skcipher, 2535 .test = alg_test_skcipher,
2678 .suite = { 2536 .suite = {
2679 .cipher = { 2537 .cipher = {
2680 .enc = { 2538 .enc = __VECS(chacha20_enc_tv_template),
2681 .vecs = chacha20_enc_tv_template, 2539 .dec = __VECS(chacha20_enc_tv_template),
2682 .count = CHACHA20_ENC_TEST_VECTORS
2683 },
2684 .dec = {
2685 .vecs = chacha20_enc_tv_template,
2686 .count = CHACHA20_ENC_TEST_VECTORS
2687 },
2688 } 2540 }
2689 } 2541 }
2690 }, { 2542 }, {
@@ -2692,20 +2544,14 @@ static const struct alg_test_desc alg_test_descs[] = {
2692 .fips_allowed = 1, 2544 .fips_allowed = 1,
2693 .test = alg_test_hash, 2545 .test = alg_test_hash,
2694 .suite = { 2546 .suite = {
2695 .hash = { 2547 .hash = __VECS(aes_cmac128_tv_template)
2696 .vecs = aes_cmac128_tv_template,
2697 .count = CMAC_AES_TEST_VECTORS
2698 }
2699 } 2548 }
2700 }, { 2549 }, {
2701 .alg = "cmac(des3_ede)", 2550 .alg = "cmac(des3_ede)",
2702 .fips_allowed = 1, 2551 .fips_allowed = 1,
2703 .test = alg_test_hash, 2552 .test = alg_test_hash,
2704 .suite = { 2553 .suite = {
2705 .hash = { 2554 .hash = __VECS(des3_ede_cmac64_tv_template)
2706 .vecs = des3_ede_cmac64_tv_template,
2707 .count = CMAC_DES3_EDE_TEST_VECTORS
2708 }
2709 } 2555 }
2710 }, { 2556 }, {
2711 .alg = "compress_null", 2557 .alg = "compress_null",
@@ -2714,30 +2560,21 @@ static const struct alg_test_desc alg_test_descs[] = {
2714 .alg = "crc32", 2560 .alg = "crc32",
2715 .test = alg_test_hash, 2561 .test = alg_test_hash,
2716 .suite = { 2562 .suite = {
2717 .hash = { 2563 .hash = __VECS(crc32_tv_template)
2718 .vecs = crc32_tv_template,
2719 .count = CRC32_TEST_VECTORS
2720 }
2721 } 2564 }
2722 }, { 2565 }, {
2723 .alg = "crc32c", 2566 .alg = "crc32c",
2724 .test = alg_test_crc32c, 2567 .test = alg_test_crc32c,
2725 .fips_allowed = 1, 2568 .fips_allowed = 1,
2726 .suite = { 2569 .suite = {
2727 .hash = { 2570 .hash = __VECS(crc32c_tv_template)
2728 .vecs = crc32c_tv_template,
2729 .count = CRC32C_TEST_VECTORS
2730 }
2731 } 2571 }
2732 }, { 2572 }, {
2733 .alg = "crct10dif", 2573 .alg = "crct10dif",
2734 .test = alg_test_hash, 2574 .test = alg_test_hash,
2735 .fips_allowed = 1, 2575 .fips_allowed = 1,
2736 .suite = { 2576 .suite = {
2737 .hash = { 2577 .hash = __VECS(crct10dif_tv_template)
2738 .vecs = crct10dif_tv_template,
2739 .count = CRCT10DIF_TEST_VECTORS
2740 }
2741 } 2578 }
2742 }, { 2579 }, {
2743 .alg = "ctr(aes)", 2580 .alg = "ctr(aes)",
@@ -2745,14 +2582,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2745 .fips_allowed = 1, 2582 .fips_allowed = 1,
2746 .suite = { 2583 .suite = {
2747 .cipher = { 2584 .cipher = {
2748 .enc = { 2585 .enc = __VECS(aes_ctr_enc_tv_template),
2749 .vecs = aes_ctr_enc_tv_template, 2586 .dec = __VECS(aes_ctr_dec_tv_template)
2750 .count = AES_CTR_ENC_TEST_VECTORS
2751 },
2752 .dec = {
2753 .vecs = aes_ctr_dec_tv_template,
2754 .count = AES_CTR_DEC_TEST_VECTORS
2755 }
2756 } 2587 }
2757 } 2588 }
2758 }, { 2589 }, {
@@ -2760,14 +2591,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2760 .test = alg_test_skcipher, 2591 .test = alg_test_skcipher,
2761 .suite = { 2592 .suite = {
2762 .cipher = { 2593 .cipher = {
2763 .enc = { 2594 .enc = __VECS(bf_ctr_enc_tv_template),
2764 .vecs = bf_ctr_enc_tv_template, 2595 .dec = __VECS(bf_ctr_dec_tv_template)
2765 .count = BF_CTR_ENC_TEST_VECTORS
2766 },
2767 .dec = {
2768 .vecs = bf_ctr_dec_tv_template,
2769 .count = BF_CTR_DEC_TEST_VECTORS
2770 }
2771 } 2596 }
2772 } 2597 }
2773 }, { 2598 }, {
@@ -2775,14 +2600,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2775 .test = alg_test_skcipher, 2600 .test = alg_test_skcipher,
2776 .suite = { 2601 .suite = {
2777 .cipher = { 2602 .cipher = {
2778 .enc = { 2603 .enc = __VECS(camellia_ctr_enc_tv_template),
2779 .vecs = camellia_ctr_enc_tv_template, 2604 .dec = __VECS(camellia_ctr_dec_tv_template)
2780 .count = CAMELLIA_CTR_ENC_TEST_VECTORS
2781 },
2782 .dec = {
2783 .vecs = camellia_ctr_dec_tv_template,
2784 .count = CAMELLIA_CTR_DEC_TEST_VECTORS
2785 }
2786 } 2605 }
2787 } 2606 }
2788 }, { 2607 }, {
@@ -2790,14 +2609,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2790 .test = alg_test_skcipher, 2609 .test = alg_test_skcipher,
2791 .suite = { 2610 .suite = {
2792 .cipher = { 2611 .cipher = {
2793 .enc = { 2612 .enc = __VECS(cast5_ctr_enc_tv_template),
2794 .vecs = cast5_ctr_enc_tv_template, 2613 .dec = __VECS(cast5_ctr_dec_tv_template)
2795 .count = CAST5_CTR_ENC_TEST_VECTORS
2796 },
2797 .dec = {
2798 .vecs = cast5_ctr_dec_tv_template,
2799 .count = CAST5_CTR_DEC_TEST_VECTORS
2800 }
2801 } 2614 }
2802 } 2615 }
2803 }, { 2616 }, {
@@ -2805,14 +2618,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2805 .test = alg_test_skcipher, 2618 .test = alg_test_skcipher,
2806 .suite = { 2619 .suite = {
2807 .cipher = { 2620 .cipher = {
2808 .enc = { 2621 .enc = __VECS(cast6_ctr_enc_tv_template),
2809 .vecs = cast6_ctr_enc_tv_template, 2622 .dec = __VECS(cast6_ctr_dec_tv_template)
2810 .count = CAST6_CTR_ENC_TEST_VECTORS
2811 },
2812 .dec = {
2813 .vecs = cast6_ctr_dec_tv_template,
2814 .count = CAST6_CTR_DEC_TEST_VECTORS
2815 }
2816 } 2623 }
2817 } 2624 }
2818 }, { 2625 }, {
@@ -2820,14 +2627,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2820 .test = alg_test_skcipher, 2627 .test = alg_test_skcipher,
2821 .suite = { 2628 .suite = {
2822 .cipher = { 2629 .cipher = {
2823 .enc = { 2630 .enc = __VECS(des_ctr_enc_tv_template),
2824 .vecs = des_ctr_enc_tv_template, 2631 .dec = __VECS(des_ctr_dec_tv_template)
2825 .count = DES_CTR_ENC_TEST_VECTORS
2826 },
2827 .dec = {
2828 .vecs = des_ctr_dec_tv_template,
2829 .count = DES_CTR_DEC_TEST_VECTORS
2830 }
2831 } 2632 }
2832 } 2633 }
2833 }, { 2634 }, {
@@ -2835,14 +2636,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2835 .test = alg_test_skcipher, 2636 .test = alg_test_skcipher,
2836 .suite = { 2637 .suite = {
2837 .cipher = { 2638 .cipher = {
2838 .enc = { 2639 .enc = __VECS(des3_ede_ctr_enc_tv_template),
2839 .vecs = des3_ede_ctr_enc_tv_template, 2640 .dec = __VECS(des3_ede_ctr_dec_tv_template)
2840 .count = DES3_EDE_CTR_ENC_TEST_VECTORS
2841 },
2842 .dec = {
2843 .vecs = des3_ede_ctr_dec_tv_template,
2844 .count = DES3_EDE_CTR_DEC_TEST_VECTORS
2845 }
2846 } 2641 }
2847 } 2642 }
2848 }, { 2643 }, {
@@ -2850,14 +2645,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2850 .test = alg_test_skcipher, 2645 .test = alg_test_skcipher,
2851 .suite = { 2646 .suite = {
2852 .cipher = { 2647 .cipher = {
2853 .enc = { 2648 .enc = __VECS(serpent_ctr_enc_tv_template),
2854 .vecs = serpent_ctr_enc_tv_template, 2649 .dec = __VECS(serpent_ctr_dec_tv_template)
2855 .count = SERPENT_CTR_ENC_TEST_VECTORS
2856 },
2857 .dec = {
2858 .vecs = serpent_ctr_dec_tv_template,
2859 .count = SERPENT_CTR_DEC_TEST_VECTORS
2860 }
2861 } 2650 }
2862 } 2651 }
2863 }, { 2652 }, {
@@ -2865,14 +2654,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2865 .test = alg_test_skcipher, 2654 .test = alg_test_skcipher,
2866 .suite = { 2655 .suite = {
2867 .cipher = { 2656 .cipher = {
2868 .enc = { 2657 .enc = __VECS(tf_ctr_enc_tv_template),
2869 .vecs = tf_ctr_enc_tv_template, 2658 .dec = __VECS(tf_ctr_dec_tv_template)
2870 .count = TF_CTR_ENC_TEST_VECTORS
2871 },
2872 .dec = {
2873 .vecs = tf_ctr_dec_tv_template,
2874 .count = TF_CTR_DEC_TEST_VECTORS
2875 }
2876 } 2659 }
2877 } 2660 }
2878 }, { 2661 }, {
@@ -2880,14 +2663,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2880 .test = alg_test_skcipher, 2663 .test = alg_test_skcipher,
2881 .suite = { 2664 .suite = {
2882 .cipher = { 2665 .cipher = {
2883 .enc = { 2666 .enc = __VECS(cts_mode_enc_tv_template),
2884 .vecs = cts_mode_enc_tv_template, 2667 .dec = __VECS(cts_mode_dec_tv_template)
2885 .count = CTS_MODE_ENC_TEST_VECTORS
2886 },
2887 .dec = {
2888 .vecs = cts_mode_dec_tv_template,
2889 .count = CTS_MODE_DEC_TEST_VECTORS
2890 }
2891 } 2668 }
2892 } 2669 }
2893 }, { 2670 }, {
@@ -2896,14 +2673,8 @@ static const struct alg_test_desc alg_test_descs[] = {
2896 .fips_allowed = 1, 2673 .fips_allowed = 1,
2897 .suite = { 2674 .suite = {
2898 .comp = { 2675 .comp = {
2899 .comp = { 2676 .comp = __VECS(deflate_comp_tv_template),
2900 .vecs = deflate_comp_tv_template, 2677 .decomp = __VECS(deflate_decomp_tv_template)
2901 .count = DEFLATE_COMP_TEST_VECTORS
2902 },
2903 .decomp = {
2904 .vecs = deflate_decomp_tv_template,
2905 .count = DEFLATE_DECOMP_TEST_VECTORS
2906 }
2907 } 2678 }
2908 } 2679 }
2909 }, { 2680 }, {
@@ -2911,10 +2682,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2911 .test = alg_test_kpp, 2682 .test = alg_test_kpp,
2912 .fips_allowed = 1, 2683 .fips_allowed = 1,
2913 .suite = { 2684 .suite = {
2914 .kpp = { 2685 .kpp = __VECS(dh_tv_template)
2915 .vecs = dh_tv_template,
2916 .count = DH_TEST_VECTORS
2917 }
2918 } 2686 }
2919 }, { 2687 }, {
2920 .alg = "digest_null", 2688 .alg = "digest_null",
@@ -2924,30 +2692,21 @@ static const struct alg_test_desc alg_test_descs[] = {
2924 .test = alg_test_drbg, 2692 .test = alg_test_drbg,
2925 .fips_allowed = 1, 2693 .fips_allowed = 1,
2926 .suite = { 2694 .suite = {
2927 .drbg = { 2695 .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
2928 .vecs = drbg_nopr_ctr_aes128_tv_template,
2929 .count = ARRAY_SIZE(drbg_nopr_ctr_aes128_tv_template)
2930 }
2931 } 2696 }
2932 }, { 2697 }, {
2933 .alg = "drbg_nopr_ctr_aes192", 2698 .alg = "drbg_nopr_ctr_aes192",
2934 .test = alg_test_drbg, 2699 .test = alg_test_drbg,
2935 .fips_allowed = 1, 2700 .fips_allowed = 1,
2936 .suite = { 2701 .suite = {
2937 .drbg = { 2702 .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
2938 .vecs = drbg_nopr_ctr_aes192_tv_template,
2939 .count = ARRAY_SIZE(drbg_nopr_ctr_aes192_tv_template)
2940 }
2941 } 2703 }
2942 }, { 2704 }, {
2943 .alg = "drbg_nopr_ctr_aes256", 2705 .alg = "drbg_nopr_ctr_aes256",
2944 .test = alg_test_drbg, 2706 .test = alg_test_drbg,
2945 .fips_allowed = 1, 2707 .fips_allowed = 1,
2946 .suite = { 2708 .suite = {
2947 .drbg = { 2709 .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
2948 .vecs = drbg_nopr_ctr_aes256_tv_template,
2949 .count = ARRAY_SIZE(drbg_nopr_ctr_aes256_tv_template)
2950 }
2951 } 2710 }
2952 }, { 2711 }, {
2953 /* 2712 /*
@@ -2962,11 +2721,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2962 .test = alg_test_drbg, 2721 .test = alg_test_drbg,
2963 .fips_allowed = 1, 2722 .fips_allowed = 1,
2964 .suite = { 2723 .suite = {
2965 .drbg = { 2724 .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
2966 .vecs = drbg_nopr_hmac_sha256_tv_template,
2967 .count =
2968 ARRAY_SIZE(drbg_nopr_hmac_sha256_tv_template)
2969 }
2970 } 2725 }
2971 }, { 2726 }, {
2972 /* covered by drbg_nopr_hmac_sha256 test */ 2727 /* covered by drbg_nopr_hmac_sha256 test */
@@ -2986,10 +2741,7 @@ static const struct alg_test_desc alg_test_descs[] = {
2986 .test = alg_test_drbg, 2741 .test = alg_test_drbg,
2987 .fips_allowed = 1, 2742 .fips_allowed = 1,
2988 .suite = { 2743 .suite = {
2989 .drbg = { 2744 .drbg = __VECS(drbg_nopr_sha256_tv_template)
2990 .vecs = drbg_nopr_sha256_tv_template,
2991 .count = ARRAY_SIZE(drbg_nopr_sha256_tv_template)
2992 }
2993 } 2745 }
2994 }, { 2746 }, {
2995 /* covered by drbg_nopr_sha256 test */ 2747 /* covered by drbg_nopr_sha256 test */
@@ -3005,10 +2757,7 @@ static const struct alg_test_desc alg_test_descs[] = {
3005 .test = alg_test_drbg, 2757 .test = alg_test_drbg,
3006 .fips_allowed = 1, 2758 .fips_allowed = 1,
3007 .suite = { 2759 .suite = {
3008 .drbg = { 2760 .drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
3009 .vecs = drbg_pr_ctr_aes128_tv_template,
3010 .count = ARRAY_SIZE(drbg_pr_ctr_aes128_tv_template)
3011 }
3012 } 2761 }
3013 }, { 2762 }, {
3014 /* covered by drbg_pr_ctr_aes128 test */ 2763 /* covered by drbg_pr_ctr_aes128 test */
@@ -3028,10 +2777,7 @@ static const struct alg_test_desc alg_test_descs[] = {
3028 .test = alg_test_drbg, 2777 .test = alg_test_drbg,
3029 .fips_allowed = 1, 2778 .fips_allowed = 1,
3030 .suite = { 2779 .suite = {
3031 .drbg = { 2780 .drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
3032 .vecs = drbg_pr_hmac_sha256_tv_template,
3033 .count = ARRAY_SIZE(drbg_pr_hmac_sha256_tv_template)
3034 }
3035 } 2781 }
3036 }, { 2782 }, {
3037 /* covered by drbg_pr_hmac_sha256 test */ 2783 /* covered by drbg_pr_hmac_sha256 test */
@@ -3051,10 +2797,7 @@ static const struct alg_test_desc alg_test_descs[] = {
3051 .test = alg_test_drbg, 2797 .test = alg_test_drbg,
3052 .fips_allowed = 1, 2798 .fips_allowed = 1,
3053 .suite = { 2799 .suite = {
3054 .drbg = { 2800 .drbg = __VECS(drbg_pr_sha256_tv_template)
3055 .vecs = drbg_pr_sha256_tv_template,
3056 .count = ARRAY_SIZE(drbg_pr_sha256_tv_template)
3057 }
3058 } 2801 }
3059 }, { 2802 }, {
3060 /* covered by drbg_pr_sha256 test */ 2803 /* covered by drbg_pr_sha256 test */
@@ -3071,14 +2814,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3071 .fips_allowed = 1, 2814 .fips_allowed = 1,
3072 .suite = { 2815 .suite = {
3073 .cipher = { 2816 .cipher = {
3074 .enc = { 2817 .enc = __VECS(aes_enc_tv_template),
3075 .vecs = aes_enc_tv_template, 2818 .dec = __VECS(aes_dec_tv_template)
3076 .count = AES_ENC_TEST_VECTORS
3077 },
3078 .dec = {
3079 .vecs = aes_dec_tv_template,
3080 .count = AES_DEC_TEST_VECTORS
3081 }
3082 } 2819 }
3083 } 2820 }
3084 }, { 2821 }, {
@@ -3086,14 +2823,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3086 .test = alg_test_skcipher, 2823 .test = alg_test_skcipher,
3087 .suite = { 2824 .suite = {
3088 .cipher = { 2825 .cipher = {
3089 .enc = { 2826 .enc = __VECS(anubis_enc_tv_template),
3090 .vecs = anubis_enc_tv_template, 2827 .dec = __VECS(anubis_dec_tv_template)
3091 .count = ANUBIS_ENC_TEST_VECTORS
3092 },
3093 .dec = {
3094 .vecs = anubis_dec_tv_template,
3095 .count = ANUBIS_DEC_TEST_VECTORS
3096 }
3097 } 2828 }
3098 } 2829 }
3099 }, { 2830 }, {
@@ -3101,14 +2832,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3101 .test = alg_test_skcipher, 2832 .test = alg_test_skcipher,
3102 .suite = { 2833 .suite = {
3103 .cipher = { 2834 .cipher = {
3104 .enc = { 2835 .enc = __VECS(arc4_enc_tv_template),
3105 .vecs = arc4_enc_tv_template, 2836 .dec = __VECS(arc4_dec_tv_template)
3106 .count = ARC4_ENC_TEST_VECTORS
3107 },
3108 .dec = {
3109 .vecs = arc4_dec_tv_template,
3110 .count = ARC4_DEC_TEST_VECTORS
3111 }
3112 } 2837 }
3113 } 2838 }
3114 }, { 2839 }, {
@@ -3116,14 +2841,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3116 .test = alg_test_skcipher, 2841 .test = alg_test_skcipher,
3117 .suite = { 2842 .suite = {
3118 .cipher = { 2843 .cipher = {
3119 .enc = { 2844 .enc = __VECS(bf_enc_tv_template),
3120 .vecs = bf_enc_tv_template, 2845 .dec = __VECS(bf_dec_tv_template)
3121 .count = BF_ENC_TEST_VECTORS
3122 },
3123 .dec = {
3124 .vecs = bf_dec_tv_template,
3125 .count = BF_DEC_TEST_VECTORS
3126 }
3127 } 2846 }
3128 } 2847 }
3129 }, { 2848 }, {
@@ -3131,14 +2850,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3131 .test = alg_test_skcipher, 2850 .test = alg_test_skcipher,
3132 .suite = { 2851 .suite = {
3133 .cipher = { 2852 .cipher = {
3134 .enc = { 2853 .enc = __VECS(camellia_enc_tv_template),
3135 .vecs = camellia_enc_tv_template, 2854 .dec = __VECS(camellia_dec_tv_template)
3136 .count = CAMELLIA_ENC_TEST_VECTORS
3137 },
3138 .dec = {
3139 .vecs = camellia_dec_tv_template,
3140 .count = CAMELLIA_DEC_TEST_VECTORS
3141 }
3142 } 2855 }
3143 } 2856 }
3144 }, { 2857 }, {
@@ -3146,14 +2859,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3146 .test = alg_test_skcipher, 2859 .test = alg_test_skcipher,
3147 .suite = { 2860 .suite = {
3148 .cipher = { 2861 .cipher = {
3149 .enc = { 2862 .enc = __VECS(cast5_enc_tv_template),
3150 .vecs = cast5_enc_tv_template, 2863 .dec = __VECS(cast5_dec_tv_template)
3151 .count = CAST5_ENC_TEST_VECTORS
3152 },
3153 .dec = {
3154 .vecs = cast5_dec_tv_template,
3155 .count = CAST5_DEC_TEST_VECTORS
3156 }
3157 } 2864 }
3158 } 2865 }
3159 }, { 2866 }, {
@@ -3161,14 +2868,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3161 .test = alg_test_skcipher, 2868 .test = alg_test_skcipher,
3162 .suite = { 2869 .suite = {
3163 .cipher = { 2870 .cipher = {
3164 .enc = { 2871 .enc = __VECS(cast6_enc_tv_template),
3165 .vecs = cast6_enc_tv_template, 2872 .dec = __VECS(cast6_dec_tv_template)
3166 .count = CAST6_ENC_TEST_VECTORS
3167 },
3168 .dec = {
3169 .vecs = cast6_dec_tv_template,
3170 .count = CAST6_DEC_TEST_VECTORS
3171 }
3172 } 2873 }
3173 } 2874 }
3174 }, { 2875 }, {
@@ -3179,14 +2880,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3179 .test = alg_test_skcipher, 2880 .test = alg_test_skcipher,
3180 .suite = { 2881 .suite = {
3181 .cipher = { 2882 .cipher = {
3182 .enc = { 2883 .enc = __VECS(des_enc_tv_template),
3183 .vecs = des_enc_tv_template, 2884 .dec = __VECS(des_dec_tv_template)
3184 .count = DES_ENC_TEST_VECTORS
3185 },
3186 .dec = {
3187 .vecs = des_dec_tv_template,
3188 .count = DES_DEC_TEST_VECTORS
3189 }
3190 } 2885 }
3191 } 2886 }
3192 }, { 2887 }, {
@@ -3195,14 +2890,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3195 .fips_allowed = 1, 2890 .fips_allowed = 1,
3196 .suite = { 2891 .suite = {
3197 .cipher = { 2892 .cipher = {
3198 .enc = { 2893 .enc = __VECS(des3_ede_enc_tv_template),
3199 .vecs = des3_ede_enc_tv_template, 2894 .dec = __VECS(des3_ede_dec_tv_template)
3200 .count = DES3_EDE_ENC_TEST_VECTORS
3201 },
3202 .dec = {
3203 .vecs = des3_ede_dec_tv_template,
3204 .count = DES3_EDE_DEC_TEST_VECTORS
3205 }
3206 } 2895 }
3207 } 2896 }
3208 }, { 2897 }, {
@@ -3225,14 +2914,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3225 .test = alg_test_skcipher, 2914 .test = alg_test_skcipher,
3226 .suite = { 2915 .suite = {
3227 .cipher = { 2916 .cipher = {
3228 .enc = { 2917 .enc = __VECS(khazad_enc_tv_template),
3229 .vecs = khazad_enc_tv_template, 2918 .dec = __VECS(khazad_dec_tv_template)
3230 .count = KHAZAD_ENC_TEST_VECTORS
3231 },
3232 .dec = {
3233 .vecs = khazad_dec_tv_template,
3234 .count = KHAZAD_DEC_TEST_VECTORS
3235 }
3236 } 2919 }
3237 } 2920 }
3238 }, { 2921 }, {
@@ -3240,14 +2923,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3240 .test = alg_test_skcipher, 2923 .test = alg_test_skcipher,
3241 .suite = { 2924 .suite = {
3242 .cipher = { 2925 .cipher = {
3243 .enc = { 2926 .enc = __VECS(seed_enc_tv_template),
3244 .vecs = seed_enc_tv_template, 2927 .dec = __VECS(seed_dec_tv_template)
3245 .count = SEED_ENC_TEST_VECTORS
3246 },
3247 .dec = {
3248 .vecs = seed_dec_tv_template,
3249 .count = SEED_DEC_TEST_VECTORS
3250 }
3251 } 2928 }
3252 } 2929 }
3253 }, { 2930 }, {
@@ -3255,14 +2932,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3255 .test = alg_test_skcipher, 2932 .test = alg_test_skcipher,
3256 .suite = { 2933 .suite = {
3257 .cipher = { 2934 .cipher = {
3258 .enc = { 2935 .enc = __VECS(serpent_enc_tv_template),
3259 .vecs = serpent_enc_tv_template, 2936 .dec = __VECS(serpent_dec_tv_template)
3260 .count = SERPENT_ENC_TEST_VECTORS
3261 },
3262 .dec = {
3263 .vecs = serpent_dec_tv_template,
3264 .count = SERPENT_DEC_TEST_VECTORS
3265 }
3266 } 2937 }
3267 } 2938 }
3268 }, { 2939 }, {
@@ -3270,14 +2941,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3270 .test = alg_test_skcipher, 2941 .test = alg_test_skcipher,
3271 .suite = { 2942 .suite = {
3272 .cipher = { 2943 .cipher = {
3273 .enc = { 2944 .enc = __VECS(tea_enc_tv_template),
3274 .vecs = tea_enc_tv_template, 2945 .dec = __VECS(tea_dec_tv_template)
3275 .count = TEA_ENC_TEST_VECTORS
3276 },
3277 .dec = {
3278 .vecs = tea_dec_tv_template,
3279 .count = TEA_DEC_TEST_VECTORS
3280 }
3281 } 2946 }
3282 } 2947 }
3283 }, { 2948 }, {
@@ -3285,14 +2950,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3285 .test = alg_test_skcipher, 2950 .test = alg_test_skcipher,
3286 .suite = { 2951 .suite = {
3287 .cipher = { 2952 .cipher = {
3288 .enc = { 2953 .enc = __VECS(tnepres_enc_tv_template),
3289 .vecs = tnepres_enc_tv_template, 2954 .dec = __VECS(tnepres_dec_tv_template)
3290 .count = TNEPRES_ENC_TEST_VECTORS
3291 },
3292 .dec = {
3293 .vecs = tnepres_dec_tv_template,
3294 .count = TNEPRES_DEC_TEST_VECTORS
3295 }
3296 } 2955 }
3297 } 2956 }
3298 }, { 2957 }, {
@@ -3300,14 +2959,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3300 .test = alg_test_skcipher, 2959 .test = alg_test_skcipher,
3301 .suite = { 2960 .suite = {
3302 .cipher = { 2961 .cipher = {
3303 .enc = { 2962 .enc = __VECS(tf_enc_tv_template),
3304 .vecs = tf_enc_tv_template, 2963 .dec = __VECS(tf_dec_tv_template)
3305 .count = TF_ENC_TEST_VECTORS
3306 },
3307 .dec = {
3308 .vecs = tf_dec_tv_template,
3309 .count = TF_DEC_TEST_VECTORS
3310 }
3311 } 2964 }
3312 } 2965 }
3313 }, { 2966 }, {
@@ -3315,14 +2968,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3315 .test = alg_test_skcipher, 2968 .test = alg_test_skcipher,
3316 .suite = { 2969 .suite = {
3317 .cipher = { 2970 .cipher = {
3318 .enc = { 2971 .enc = __VECS(xeta_enc_tv_template),
3319 .vecs = xeta_enc_tv_template, 2972 .dec = __VECS(xeta_dec_tv_template)
3320 .count = XETA_ENC_TEST_VECTORS
3321 },
3322 .dec = {
3323 .vecs = xeta_dec_tv_template,
3324 .count = XETA_DEC_TEST_VECTORS
3325 }
3326 } 2973 }
3327 } 2974 }
3328 }, { 2975 }, {
@@ -3330,14 +2977,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3330 .test = alg_test_skcipher, 2977 .test = alg_test_skcipher,
3331 .suite = { 2978 .suite = {
3332 .cipher = { 2979 .cipher = {
3333 .enc = { 2980 .enc = __VECS(xtea_enc_tv_template),
3334 .vecs = xtea_enc_tv_template, 2981 .dec = __VECS(xtea_dec_tv_template)
3335 .count = XTEA_ENC_TEST_VECTORS
3336 },
3337 .dec = {
3338 .vecs = xtea_dec_tv_template,
3339 .count = XTEA_DEC_TEST_VECTORS
3340 }
3341 } 2982 }
3342 } 2983 }
3343 }, { 2984 }, {
@@ -3345,10 +2986,7 @@ static const struct alg_test_desc alg_test_descs[] = {
3345 .test = alg_test_kpp, 2986 .test = alg_test_kpp,
3346 .fips_allowed = 1, 2987 .fips_allowed = 1,
3347 .suite = { 2988 .suite = {
3348 .kpp = { 2989 .kpp = __VECS(ecdh_tv_template)
3349 .vecs = ecdh_tv_template,
3350 .count = ECDH_TEST_VECTORS
3351 }
3352 } 2990 }
3353 }, { 2991 }, {
3354 .alg = "gcm(aes)", 2992 .alg = "gcm(aes)",
@@ -3356,14 +2994,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3356 .fips_allowed = 1, 2994 .fips_allowed = 1,
3357 .suite = { 2995 .suite = {
3358 .aead = { 2996 .aead = {
3359 .enc = { 2997 .enc = __VECS(aes_gcm_enc_tv_template),
3360 .vecs = aes_gcm_enc_tv_template, 2998 .dec = __VECS(aes_gcm_dec_tv_template)
3361 .count = AES_GCM_ENC_TEST_VECTORS
3362 },
3363 .dec = {
3364 .vecs = aes_gcm_dec_tv_template,
3365 .count = AES_GCM_DEC_TEST_VECTORS
3366 }
3367 } 2999 }
3368 } 3000 }
3369 }, { 3001 }, {
@@ -3371,136 +3003,94 @@ static const struct alg_test_desc alg_test_descs[] = {
3371 .test = alg_test_hash, 3003 .test = alg_test_hash,
3372 .fips_allowed = 1, 3004 .fips_allowed = 1,
3373 .suite = { 3005 .suite = {
3374 .hash = { 3006 .hash = __VECS(ghash_tv_template)
3375 .vecs = ghash_tv_template,
3376 .count = GHASH_TEST_VECTORS
3377 }
3378 } 3007 }
3379 }, { 3008 }, {
3380 .alg = "hmac(crc32)", 3009 .alg = "hmac(crc32)",
3381 .test = alg_test_hash, 3010 .test = alg_test_hash,
3382 .suite = { 3011 .suite = {
3383 .hash = { 3012 .hash = __VECS(bfin_crc_tv_template)
3384 .vecs = bfin_crc_tv_template,
3385 .count = BFIN_CRC_TEST_VECTORS
3386 }
3387 } 3013 }
3388 }, { 3014 }, {
3389 .alg = "hmac(md5)", 3015 .alg = "hmac(md5)",
3390 .test = alg_test_hash, 3016 .test = alg_test_hash,
3391 .suite = { 3017 .suite = {
3392 .hash = { 3018 .hash = __VECS(hmac_md5_tv_template)
3393 .vecs = hmac_md5_tv_template,
3394 .count = HMAC_MD5_TEST_VECTORS
3395 }
3396 } 3019 }
3397 }, { 3020 }, {
3398 .alg = "hmac(rmd128)", 3021 .alg = "hmac(rmd128)",
3399 .test = alg_test_hash, 3022 .test = alg_test_hash,
3400 .suite = { 3023 .suite = {
3401 .hash = { 3024 .hash = __VECS(hmac_rmd128_tv_template)
3402 .vecs = hmac_rmd128_tv_template,
3403 .count = HMAC_RMD128_TEST_VECTORS
3404 }
3405 } 3025 }
3406 }, { 3026 }, {
3407 .alg = "hmac(rmd160)", 3027 .alg = "hmac(rmd160)",
3408 .test = alg_test_hash, 3028 .test = alg_test_hash,
3409 .suite = { 3029 .suite = {
3410 .hash = { 3030 .hash = __VECS(hmac_rmd160_tv_template)
3411 .vecs = hmac_rmd160_tv_template,
3412 .count = HMAC_RMD160_TEST_VECTORS
3413 }
3414 } 3031 }
3415 }, { 3032 }, {
3416 .alg = "hmac(sha1)", 3033 .alg = "hmac(sha1)",
3417 .test = alg_test_hash, 3034 .test = alg_test_hash,
3418 .fips_allowed = 1, 3035 .fips_allowed = 1,
3419 .suite = { 3036 .suite = {
3420 .hash = { 3037 .hash = __VECS(hmac_sha1_tv_template)
3421 .vecs = hmac_sha1_tv_template,
3422 .count = HMAC_SHA1_TEST_VECTORS
3423 }
3424 } 3038 }
3425 }, { 3039 }, {
3426 .alg = "hmac(sha224)", 3040 .alg = "hmac(sha224)",
3427 .test = alg_test_hash, 3041 .test = alg_test_hash,
3428 .fips_allowed = 1, 3042 .fips_allowed = 1,
3429 .suite = { 3043 .suite = {
3430 .hash = { 3044 .hash = __VECS(hmac_sha224_tv_template)
3431 .vecs = hmac_sha224_tv_template,
3432 .count = HMAC_SHA224_TEST_VECTORS
3433 }
3434 } 3045 }
3435 }, { 3046 }, {
3436 .alg = "hmac(sha256)", 3047 .alg = "hmac(sha256)",
3437 .test = alg_test_hash, 3048 .test = alg_test_hash,
3438 .fips_allowed = 1, 3049 .fips_allowed = 1,
3439 .suite = { 3050 .suite = {
3440 .hash = { 3051 .hash = __VECS(hmac_sha256_tv_template)
3441 .vecs = hmac_sha256_tv_template,
3442 .count = HMAC_SHA256_TEST_VECTORS
3443 }
3444 } 3052 }
3445 }, { 3053 }, {
3446 .alg = "hmac(sha3-224)", 3054 .alg = "hmac(sha3-224)",
3447 .test = alg_test_hash, 3055 .test = alg_test_hash,
3448 .fips_allowed = 1, 3056 .fips_allowed = 1,
3449 .suite = { 3057 .suite = {
3450 .hash = { 3058 .hash = __VECS(hmac_sha3_224_tv_template)
3451 .vecs = hmac_sha3_224_tv_template,
3452 .count = HMAC_SHA3_224_TEST_VECTORS
3453 }
3454 } 3059 }
3455 }, { 3060 }, {
3456 .alg = "hmac(sha3-256)", 3061 .alg = "hmac(sha3-256)",
3457 .test = alg_test_hash, 3062 .test = alg_test_hash,
3458 .fips_allowed = 1, 3063 .fips_allowed = 1,
3459 .suite = { 3064 .suite = {
3460 .hash = { 3065 .hash = __VECS(hmac_sha3_256_tv_template)
3461 .vecs = hmac_sha3_256_tv_template,
3462 .count = HMAC_SHA3_256_TEST_VECTORS
3463 }
3464 } 3066 }
3465 }, { 3067 }, {
3466 .alg = "hmac(sha3-384)", 3068 .alg = "hmac(sha3-384)",
3467 .test = alg_test_hash, 3069 .test = alg_test_hash,
3468 .fips_allowed = 1, 3070 .fips_allowed = 1,
3469 .suite = { 3071 .suite = {
3470 .hash = { 3072 .hash = __VECS(hmac_sha3_384_tv_template)
3471 .vecs = hmac_sha3_384_tv_template,
3472 .count = HMAC_SHA3_384_TEST_VECTORS
3473 }
3474 } 3073 }
3475 }, { 3074 }, {
3476 .alg = "hmac(sha3-512)", 3075 .alg = "hmac(sha3-512)",
3477 .test = alg_test_hash, 3076 .test = alg_test_hash,
3478 .fips_allowed = 1, 3077 .fips_allowed = 1,
3479 .suite = { 3078 .suite = {
3480 .hash = { 3079 .hash = __VECS(hmac_sha3_512_tv_template)
3481 .vecs = hmac_sha3_512_tv_template,
3482 .count = HMAC_SHA3_512_TEST_VECTORS
3483 }
3484 } 3080 }
3485 }, { 3081 }, {
3486 .alg = "hmac(sha384)", 3082 .alg = "hmac(sha384)",
3487 .test = alg_test_hash, 3083 .test = alg_test_hash,
3488 .fips_allowed = 1, 3084 .fips_allowed = 1,
3489 .suite = { 3085 .suite = {
3490 .hash = { 3086 .hash = __VECS(hmac_sha384_tv_template)
3491 .vecs = hmac_sha384_tv_template,
3492 .count = HMAC_SHA384_TEST_VECTORS
3493 }
3494 } 3087 }
3495 }, { 3088 }, {
3496 .alg = "hmac(sha512)", 3089 .alg = "hmac(sha512)",
3497 .test = alg_test_hash, 3090 .test = alg_test_hash,
3498 .fips_allowed = 1, 3091 .fips_allowed = 1,
3499 .suite = { 3092 .suite = {
3500 .hash = { 3093 .hash = __VECS(hmac_sha512_tv_template)
3501 .vecs = hmac_sha512_tv_template,
3502 .count = HMAC_SHA512_TEST_VECTORS
3503 }
3504 } 3094 }
3505 }, { 3095 }, {
3506 .alg = "jitterentropy_rng", 3096 .alg = "jitterentropy_rng",
@@ -3512,14 +3102,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3512 .fips_allowed = 1, 3102 .fips_allowed = 1,
3513 .suite = { 3103 .suite = {
3514 .cipher = { 3104 .cipher = {
3515 .enc = { 3105 .enc = __VECS(aes_kw_enc_tv_template),
3516 .vecs = aes_kw_enc_tv_template, 3106 .dec = __VECS(aes_kw_dec_tv_template)
3517 .count = ARRAY_SIZE(aes_kw_enc_tv_template)
3518 },
3519 .dec = {
3520 .vecs = aes_kw_dec_tv_template,
3521 .count = ARRAY_SIZE(aes_kw_dec_tv_template)
3522 }
3523 } 3107 }
3524 } 3108 }
3525 }, { 3109 }, {
@@ -3527,14 +3111,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3527 .test = alg_test_skcipher, 3111 .test = alg_test_skcipher,
3528 .suite = { 3112 .suite = {
3529 .cipher = { 3113 .cipher = {
3530 .enc = { 3114 .enc = __VECS(aes_lrw_enc_tv_template),
3531 .vecs = aes_lrw_enc_tv_template, 3115 .dec = __VECS(aes_lrw_dec_tv_template)
3532 .count = AES_LRW_ENC_TEST_VECTORS
3533 },
3534 .dec = {
3535 .vecs = aes_lrw_dec_tv_template,
3536 .count = AES_LRW_DEC_TEST_VECTORS
3537 }
3538 } 3116 }
3539 } 3117 }
3540 }, { 3118 }, {
@@ -3542,14 +3120,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3542 .test = alg_test_skcipher, 3120 .test = alg_test_skcipher,
3543 .suite = { 3121 .suite = {
3544 .cipher = { 3122 .cipher = {
3545 .enc = { 3123 .enc = __VECS(camellia_lrw_enc_tv_template),
3546 .vecs = camellia_lrw_enc_tv_template, 3124 .dec = __VECS(camellia_lrw_dec_tv_template)
3547 .count = CAMELLIA_LRW_ENC_TEST_VECTORS
3548 },
3549 .dec = {
3550 .vecs = camellia_lrw_dec_tv_template,
3551 .count = CAMELLIA_LRW_DEC_TEST_VECTORS
3552 }
3553 } 3125 }
3554 } 3126 }
3555 }, { 3127 }, {
@@ -3557,14 +3129,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3557 .test = alg_test_skcipher, 3129 .test = alg_test_skcipher,
3558 .suite = { 3130 .suite = {
3559 .cipher = { 3131 .cipher = {
3560 .enc = { 3132 .enc = __VECS(cast6_lrw_enc_tv_template),
3561 .vecs = cast6_lrw_enc_tv_template, 3133 .dec = __VECS(cast6_lrw_dec_tv_template)
3562 .count = CAST6_LRW_ENC_TEST_VECTORS
3563 },
3564 .dec = {
3565 .vecs = cast6_lrw_dec_tv_template,
3566 .count = CAST6_LRW_DEC_TEST_VECTORS
3567 }
3568 } 3134 }
3569 } 3135 }
3570 }, { 3136 }, {
@@ -3572,14 +3138,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3572 .test = alg_test_skcipher, 3138 .test = alg_test_skcipher,
3573 .suite = { 3139 .suite = {
3574 .cipher = { 3140 .cipher = {
3575 .enc = { 3141 .enc = __VECS(serpent_lrw_enc_tv_template),
3576 .vecs = serpent_lrw_enc_tv_template, 3142 .dec = __VECS(serpent_lrw_dec_tv_template)
3577 .count = SERPENT_LRW_ENC_TEST_VECTORS
3578 },
3579 .dec = {
3580 .vecs = serpent_lrw_dec_tv_template,
3581 .count = SERPENT_LRW_DEC_TEST_VECTORS
3582 }
3583 } 3143 }
3584 } 3144 }
3585 }, { 3145 }, {
@@ -3587,14 +3147,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3587 .test = alg_test_skcipher, 3147 .test = alg_test_skcipher,
3588 .suite = { 3148 .suite = {
3589 .cipher = { 3149 .cipher = {
3590 .enc = { 3150 .enc = __VECS(tf_lrw_enc_tv_template),
3591 .vecs = tf_lrw_enc_tv_template, 3151 .dec = __VECS(tf_lrw_dec_tv_template)
3592 .count = TF_LRW_ENC_TEST_VECTORS
3593 },
3594 .dec = {
3595 .vecs = tf_lrw_dec_tv_template,
3596 .count = TF_LRW_DEC_TEST_VECTORS
3597 }
3598 } 3152 }
3599 } 3153 }
3600 }, { 3154 }, {
@@ -3603,14 +3157,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3603 .fips_allowed = 1, 3157 .fips_allowed = 1,
3604 .suite = { 3158 .suite = {
3605 .comp = { 3159 .comp = {
3606 .comp = { 3160 .comp = __VECS(lz4_comp_tv_template),
3607 .vecs = lz4_comp_tv_template, 3161 .decomp = __VECS(lz4_decomp_tv_template)
3608 .count = LZ4_COMP_TEST_VECTORS
3609 },
3610 .decomp = {
3611 .vecs = lz4_decomp_tv_template,
3612 .count = LZ4_DECOMP_TEST_VECTORS
3613 }
3614 } 3162 }
3615 } 3163 }
3616 }, { 3164 }, {
@@ -3619,14 +3167,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3619 .fips_allowed = 1, 3167 .fips_allowed = 1,
3620 .suite = { 3168 .suite = {
3621 .comp = { 3169 .comp = {
3622 .comp = { 3170 .comp = __VECS(lz4hc_comp_tv_template),
3623 .vecs = lz4hc_comp_tv_template, 3171 .decomp = __VECS(lz4hc_decomp_tv_template)
3624 .count = LZ4HC_COMP_TEST_VECTORS
3625 },
3626 .decomp = {
3627 .vecs = lz4hc_decomp_tv_template,
3628 .count = LZ4HC_DECOMP_TEST_VECTORS
3629 }
3630 } 3172 }
3631 } 3173 }
3632 }, { 3174 }, {
@@ -3635,42 +3177,27 @@ static const struct alg_test_desc alg_test_descs[] = {
3635 .fips_allowed = 1, 3177 .fips_allowed = 1,
3636 .suite = { 3178 .suite = {
3637 .comp = { 3179 .comp = {
3638 .comp = { 3180 .comp = __VECS(lzo_comp_tv_template),
3639 .vecs = lzo_comp_tv_template, 3181 .decomp = __VECS(lzo_decomp_tv_template)
3640 .count = LZO_COMP_TEST_VECTORS
3641 },
3642 .decomp = {
3643 .vecs = lzo_decomp_tv_template,
3644 .count = LZO_DECOMP_TEST_VECTORS
3645 }
3646 } 3182 }
3647 } 3183 }
3648 }, { 3184 }, {
3649 .alg = "md4", 3185 .alg = "md4",
3650 .test = alg_test_hash, 3186 .test = alg_test_hash,
3651 .suite = { 3187 .suite = {
3652 .hash = { 3188 .hash = __VECS(md4_tv_template)
3653 .vecs = md4_tv_template,
3654 .count = MD4_TEST_VECTORS
3655 }
3656 } 3189 }
3657 }, { 3190 }, {
3658 .alg = "md5", 3191 .alg = "md5",
3659 .test = alg_test_hash, 3192 .test = alg_test_hash,
3660 .suite = { 3193 .suite = {
3661 .hash = { 3194 .hash = __VECS(md5_tv_template)
3662 .vecs = md5_tv_template,
3663 .count = MD5_TEST_VECTORS
3664 }
3665 } 3195 }
3666 }, { 3196 }, {
3667 .alg = "michael_mic", 3197 .alg = "michael_mic",
3668 .test = alg_test_hash, 3198 .test = alg_test_hash,
3669 .suite = { 3199 .suite = {
3670 .hash = { 3200 .hash = __VECS(michael_mic_tv_template)
3671 .vecs = michael_mic_tv_template,
3672 .count = MICHAEL_MIC_TEST_VECTORS
3673 }
3674 } 3201 }
3675 }, { 3202 }, {
3676 .alg = "ofb(aes)", 3203 .alg = "ofb(aes)",
@@ -3678,14 +3205,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3678 .fips_allowed = 1, 3205 .fips_allowed = 1,
3679 .suite = { 3206 .suite = {
3680 .cipher = { 3207 .cipher = {
3681 .enc = { 3208 .enc = __VECS(aes_ofb_enc_tv_template),
3682 .vecs = aes_ofb_enc_tv_template, 3209 .dec = __VECS(aes_ofb_dec_tv_template)
3683 .count = AES_OFB_ENC_TEST_VECTORS
3684 },
3685 .dec = {
3686 .vecs = aes_ofb_dec_tv_template,
3687 .count = AES_OFB_DEC_TEST_VECTORS
3688 }
3689 } 3210 }
3690 } 3211 }
3691 }, { 3212 }, {
@@ -3693,24 +3214,15 @@ static const struct alg_test_desc alg_test_descs[] = {
3693 .test = alg_test_skcipher, 3214 .test = alg_test_skcipher,
3694 .suite = { 3215 .suite = {
3695 .cipher = { 3216 .cipher = {
3696 .enc = { 3217 .enc = __VECS(fcrypt_pcbc_enc_tv_template),
3697 .vecs = fcrypt_pcbc_enc_tv_template, 3218 .dec = __VECS(fcrypt_pcbc_dec_tv_template)
3698 .count = FCRYPT_ENC_TEST_VECTORS
3699 },
3700 .dec = {
3701 .vecs = fcrypt_pcbc_dec_tv_template,
3702 .count = FCRYPT_DEC_TEST_VECTORS
3703 }
3704 } 3219 }
3705 } 3220 }
3706 }, { 3221 }, {
3707 .alg = "poly1305", 3222 .alg = "poly1305",
3708 .test = alg_test_hash, 3223 .test = alg_test_hash,
3709 .suite = { 3224 .suite = {
3710 .hash = { 3225 .hash = __VECS(poly1305_tv_template)
3711 .vecs = poly1305_tv_template,
3712 .count = POLY1305_TEST_VECTORS
3713 }
3714 } 3226 }
3715 }, { 3227 }, {
3716 .alg = "rfc3686(ctr(aes))", 3228 .alg = "rfc3686(ctr(aes))",
@@ -3718,14 +3230,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3718 .fips_allowed = 1, 3230 .fips_allowed = 1,
3719 .suite = { 3231 .suite = {
3720 .cipher = { 3232 .cipher = {
3721 .enc = { 3233 .enc = __VECS(aes_ctr_rfc3686_enc_tv_template),
3722 .vecs = aes_ctr_rfc3686_enc_tv_template, 3234 .dec = __VECS(aes_ctr_rfc3686_dec_tv_template)
3723 .count = AES_CTR_3686_ENC_TEST_VECTORS
3724 },
3725 .dec = {
3726 .vecs = aes_ctr_rfc3686_dec_tv_template,
3727 .count = AES_CTR_3686_DEC_TEST_VECTORS
3728 }
3729 } 3235 }
3730 } 3236 }
3731 }, { 3237 }, {
@@ -3734,14 +3240,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3734 .fips_allowed = 1, 3240 .fips_allowed = 1,
3735 .suite = { 3241 .suite = {
3736 .aead = { 3242 .aead = {
3737 .enc = { 3243 .enc = __VECS(aes_gcm_rfc4106_enc_tv_template),
3738 .vecs = aes_gcm_rfc4106_enc_tv_template, 3244 .dec = __VECS(aes_gcm_rfc4106_dec_tv_template)
3739 .count = AES_GCM_4106_ENC_TEST_VECTORS
3740 },
3741 .dec = {
3742 .vecs = aes_gcm_rfc4106_dec_tv_template,
3743 .count = AES_GCM_4106_DEC_TEST_VECTORS
3744 }
3745 } 3245 }
3746 } 3246 }
3747 }, { 3247 }, {
@@ -3750,14 +3250,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3750 .fips_allowed = 1, 3250 .fips_allowed = 1,
3751 .suite = { 3251 .suite = {
3752 .aead = { 3252 .aead = {
3753 .enc = { 3253 .enc = __VECS(aes_ccm_rfc4309_enc_tv_template),
3754 .vecs = aes_ccm_rfc4309_enc_tv_template, 3254 .dec = __VECS(aes_ccm_rfc4309_dec_tv_template)
3755 .count = AES_CCM_4309_ENC_TEST_VECTORS
3756 },
3757 .dec = {
3758 .vecs = aes_ccm_rfc4309_dec_tv_template,
3759 .count = AES_CCM_4309_DEC_TEST_VECTORS
3760 }
3761 } 3255 }
3762 } 3256 }
3763 }, { 3257 }, {
@@ -3765,14 +3259,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3765 .test = alg_test_aead, 3259 .test = alg_test_aead,
3766 .suite = { 3260 .suite = {
3767 .aead = { 3261 .aead = {
3768 .enc = { 3262 .enc = __VECS(aes_gcm_rfc4543_enc_tv_template),
3769 .vecs = aes_gcm_rfc4543_enc_tv_template, 3263 .dec = __VECS(aes_gcm_rfc4543_dec_tv_template),
3770 .count = AES_GCM_4543_ENC_TEST_VECTORS
3771 },
3772 .dec = {
3773 .vecs = aes_gcm_rfc4543_dec_tv_template,
3774 .count = AES_GCM_4543_DEC_TEST_VECTORS
3775 },
3776 } 3264 }
3777 } 3265 }
3778 }, { 3266 }, {
@@ -3780,14 +3268,8 @@ static const struct alg_test_desc alg_test_descs[] = {
3780 .test = alg_test_aead, 3268 .test = alg_test_aead,
3781 .suite = { 3269 .suite = {
3782 .aead = { 3270 .aead = {
3783 .enc = { 3271 .enc = __VECS(rfc7539_enc_tv_template),
3784 .vecs = rfc7539_enc_tv_template, 3272 .dec = __VECS(rfc7539_dec_tv_template),
3785 .count = RFC7539_ENC_TEST_VECTORS
3786 },
3787 .dec = {
3788 .vecs = rfc7539_dec_tv_template,
3789 .count = RFC7539_DEC_TEST_VECTORS
3790 },
3791 } 3273 }
3792 } 3274 }
3793 }, { 3275 }, {
@@ -3795,71 +3277,47 @@ static const struct alg_test_desc alg_test_descs[] = {
3795 .test = alg_test_aead, 3277 .test = alg_test_aead,
3796 .suite = { 3278 .suite = {
3797 .aead = { 3279 .aead = {
3798 .enc = { 3280 .enc = __VECS(rfc7539esp_enc_tv_template),
3799 .vecs = rfc7539esp_enc_tv_template, 3281 .dec = __VECS(rfc7539esp_dec_tv_template),
3800 .count = RFC7539ESP_ENC_TEST_VECTORS
3801 },
3802 .dec = {
3803 .vecs = rfc7539esp_dec_tv_template,
3804 .count = RFC7539ESP_DEC_TEST_VECTORS
3805 },
3806 } 3282 }
3807 } 3283 }
3808 }, { 3284 }, {
3809 .alg = "rmd128", 3285 .alg = "rmd128",
3810 .test = alg_test_hash, 3286 .test = alg_test_hash,
3811 .suite = { 3287 .suite = {
3812 .hash = { 3288 .hash = __VECS(rmd128_tv_template)
3813 .vecs = rmd128_tv_template,
3814 .count = RMD128_TEST_VECTORS
3815 }
3816 } 3289 }
3817 }, { 3290 }, {
3818 .alg = "rmd160", 3291 .alg = "rmd160",
3819 .test = alg_test_hash, 3292 .test = alg_test_hash,
3820 .suite = { 3293 .suite = {
3821 .hash = { 3294 .hash = __VECS(rmd160_tv_template)
3822 .vecs = rmd160_tv_template,
3823 .count = RMD160_TEST_VECTORS
3824 }
3825 } 3295 }
3826 }, { 3296 }, {
3827 .alg = "rmd256", 3297 .alg = "rmd256",
3828 .test = alg_test_hash, 3298 .test = alg_test_hash,
3829 .suite = { 3299 .suite = {
3830 .hash = { 3300 .hash = __VECS(rmd256_tv_template)
3831 .vecs = rmd256_tv_template,
3832 .count = RMD256_TEST_VECTORS
3833 }
3834 } 3301 }
3835 }, { 3302 }, {
3836 .alg = "rmd320", 3303 .alg = "rmd320",
3837 .test = alg_test_hash, 3304 .test = alg_test_hash,
3838 .suite = { 3305 .suite = {
3839 .hash = { 3306 .hash = __VECS(rmd320_tv_template)
3840 .vecs = rmd320_tv_template,
3841 .count = RMD320_TEST_VECTORS
3842 }
3843 } 3307 }
3844 }, { 3308 }, {
3845 .alg = "rsa", 3309 .alg = "rsa",
3846 .test = alg_test_akcipher, 3310 .test = alg_test_akcipher,
3847 .fips_allowed = 1, 3311 .fips_allowed = 1,
3848 .suite = { 3312 .suite = {
3849 .akcipher = { 3313 .akcipher = __VECS(rsa_tv_template)
3850 .vecs = rsa_tv_template,
3851 .count = RSA_TEST_VECTORS
3852 }
3853 } 3314 }
3854 }, { 3315 }, {
3855 .alg = "salsa20", 3316 .alg = "salsa20",
3856 .test = alg_test_skcipher, 3317 .test = alg_test_skcipher,
3857 .suite = { 3318 .suite = {
3858 .cipher = { 3319 .cipher = {
3859 .enc = { 3320 .enc = __VECS(salsa20_stream_enc_tv_template)
3860 .vecs = salsa20_stream_enc_tv_template,
3861 .count = SALSA20_STREAM_ENC_TEST_VECTORS
3862 }
3863 } 3321 }
3864 } 3322 }
3865 }, { 3323 }, {
@@ -3867,162 +3325,111 @@ static const struct alg_test_desc alg_test_descs[] = {
3867 .test = alg_test_hash, 3325 .test = alg_test_hash,
3868 .fips_allowed = 1, 3326 .fips_allowed = 1,
3869 .suite = { 3327 .suite = {
3870 .hash = { 3328 .hash = __VECS(sha1_tv_template)
3871 .vecs = sha1_tv_template,
3872 .count = SHA1_TEST_VECTORS
3873 }
3874 } 3329 }
3875 }, { 3330 }, {
3876 .alg = "sha224", 3331 .alg = "sha224",
3877 .test = alg_test_hash, 3332 .test = alg_test_hash,
3878 .fips_allowed = 1, 3333 .fips_allowed = 1,
3879 .suite = { 3334 .suite = {
3880 .hash = { 3335 .hash = __VECS(sha224_tv_template)
3881 .vecs = sha224_tv_template,
3882 .count = SHA224_TEST_VECTORS
3883 }
3884 } 3336 }
3885 }, { 3337 }, {
3886 .alg = "sha256", 3338 .alg = "sha256",
3887 .test = alg_test_hash, 3339 .test = alg_test_hash,
3888 .fips_allowed = 1, 3340 .fips_allowed = 1,
3889 .suite = { 3341 .suite = {
3890 .hash = { 3342 .hash = __VECS(sha256_tv_template)
3891 .vecs = sha256_tv_template,
3892 .count = SHA256_TEST_VECTORS
3893 }
3894 } 3343 }
3895 }, { 3344 }, {
3896 .alg = "sha3-224", 3345 .alg = "sha3-224",
3897 .test = alg_test_hash, 3346 .test = alg_test_hash,
3898 .fips_allowed = 1, 3347 .fips_allowed = 1,
3899 .suite = { 3348 .suite = {
3900 .hash = { 3349 .hash = __VECS(sha3_224_tv_template)
3901 .vecs = sha3_224_tv_template,
3902 .count = SHA3_224_TEST_VECTORS
3903 }
3904 } 3350 }
3905 }, { 3351 }, {
3906 .alg = "sha3-256", 3352 .alg = "sha3-256",
3907 .test = alg_test_hash, 3353 .test = alg_test_hash,
3908 .fips_allowed = 1, 3354 .fips_allowed = 1,
3909 .suite = { 3355 .suite = {
3910 .hash = { 3356 .hash = __VECS(sha3_256_tv_template)
3911 .vecs = sha3_256_tv_template,
3912 .count = SHA3_256_TEST_VECTORS
3913 }
3914 } 3357 }
3915 }, { 3358 }, {
3916 .alg = "sha3-384", 3359 .alg = "sha3-384",
3917 .test = alg_test_hash, 3360 .test = alg_test_hash,
3918 .fips_allowed = 1, 3361 .fips_allowed = 1,
3919 .suite = { 3362 .suite = {
3920 .hash = { 3363 .hash = __VECS(sha3_384_tv_template)
3921 .vecs = sha3_384_tv_template,
3922 .count = SHA3_384_TEST_VECTORS
3923 }
3924 } 3364 }
3925 }, { 3365 }, {
3926 .alg = "sha3-512", 3366 .alg = "sha3-512",
3927 .test = alg_test_hash, 3367 .test = alg_test_hash,
3928 .fips_allowed = 1, 3368 .fips_allowed = 1,
3929 .suite = { 3369 .suite = {
3930 .hash = { 3370 .hash = __VECS(sha3_512_tv_template)
3931 .vecs = sha3_512_tv_template,
3932 .count = SHA3_512_TEST_VECTORS
3933 }
3934 } 3371 }
3935 }, { 3372 }, {
3936 .alg = "sha384", 3373 .alg = "sha384",
3937 .test = alg_test_hash, 3374 .test = alg_test_hash,
3938 .fips_allowed = 1, 3375 .fips_allowed = 1,
3939 .suite = { 3376 .suite = {
3940 .hash = { 3377 .hash = __VECS(sha384_tv_template)
3941 .vecs = sha384_tv_template,
3942 .count = SHA384_TEST_VECTORS
3943 }
3944 } 3378 }
3945 }, { 3379 }, {
3946 .alg = "sha512", 3380 .alg = "sha512",
3947 .test = alg_test_hash, 3381 .test = alg_test_hash,
3948 .fips_allowed = 1, 3382 .fips_allowed = 1,
3949 .suite = { 3383 .suite = {
3950 .hash = { 3384 .hash = __VECS(sha512_tv_template)
3951 .vecs = sha512_tv_template,
3952 .count = SHA512_TEST_VECTORS
3953 }
3954 } 3385 }
3955 }, { 3386 }, {
3956 .alg = "tgr128", 3387 .alg = "tgr128",
3957 .test = alg_test_hash, 3388 .test = alg_test_hash,
3958 .suite = { 3389 .suite = {
3959 .hash = { 3390 .hash = __VECS(tgr128_tv_template)
3960 .vecs = tgr128_tv_template,
3961 .count = TGR128_TEST_VECTORS
3962 }
3963 } 3391 }
3964 }, { 3392 }, {
3965 .alg = "tgr160", 3393 .alg = "tgr160",
3966 .test = alg_test_hash, 3394 .test = alg_test_hash,
3967 .suite = { 3395 .suite = {
3968 .hash = { 3396 .hash = __VECS(tgr160_tv_template)
3969 .vecs = tgr160_tv_template,
3970 .count = TGR160_TEST_VECTORS
3971 }
3972 } 3397 }
3973 }, { 3398 }, {
3974 .alg = "tgr192", 3399 .alg = "tgr192",
3975 .test = alg_test_hash, 3400 .test = alg_test_hash,
3976 .suite = { 3401 .suite = {
3977 .hash = { 3402 .hash = __VECS(tgr192_tv_template)
3978 .vecs = tgr192_tv_template,
3979 .count = TGR192_TEST_VECTORS
3980 }
3981 } 3403 }
3982 }, { 3404 }, {
3983 .alg = "vmac(aes)", 3405 .alg = "vmac(aes)",
3984 .test = alg_test_hash, 3406 .test = alg_test_hash,
3985 .suite = { 3407 .suite = {
3986 .hash = { 3408 .hash = __VECS(aes_vmac128_tv_template)
3987 .vecs = aes_vmac128_tv_template,
3988 .count = VMAC_AES_TEST_VECTORS
3989 }
3990 } 3409 }
3991 }, { 3410 }, {
3992 .alg = "wp256", 3411 .alg = "wp256",
3993 .test = alg_test_hash, 3412 .test = alg_test_hash,
3994 .suite = { 3413 .suite = {
3995 .hash = { 3414 .hash = __VECS(wp256_tv_template)
3996 .vecs = wp256_tv_template,
3997 .count = WP256_TEST_VECTORS
3998 }
3999 } 3415 }
4000 }, { 3416 }, {
4001 .alg = "wp384", 3417 .alg = "wp384",
4002 .test = alg_test_hash, 3418 .test = alg_test_hash,
4003 .suite = { 3419 .suite = {
4004 .hash = { 3420 .hash = __VECS(wp384_tv_template)
4005 .vecs = wp384_tv_template,
4006 .count = WP384_TEST_VECTORS
4007 }
4008 } 3421 }
4009 }, { 3422 }, {
4010 .alg = "wp512", 3423 .alg = "wp512",
4011 .test = alg_test_hash, 3424 .test = alg_test_hash,
4012 .suite = { 3425 .suite = {
4013 .hash = { 3426 .hash = __VECS(wp512_tv_template)
4014 .vecs = wp512_tv_template,
4015 .count = WP512_TEST_VECTORS
4016 }
4017 } 3427 }
4018 }, { 3428 }, {
4019 .alg = "xcbc(aes)", 3429 .alg = "xcbc(aes)",
4020 .test = alg_test_hash, 3430 .test = alg_test_hash,
4021 .suite = { 3431 .suite = {
4022 .hash = { 3432 .hash = __VECS(aes_xcbc128_tv_template)
4023 .vecs = aes_xcbc128_tv_template,
4024 .count = XCBC_AES_TEST_VECTORS
4025 }
4026 } 3433 }
4027 }, { 3434 }, {
4028 .alg = "xts(aes)", 3435 .alg = "xts(aes)",
@@ -4030,14 +3437,8 @@ static const struct alg_test_desc alg_test_descs[] = {
4030 .fips_allowed = 1, 3437 .fips_allowed = 1,
4031 .suite = { 3438 .suite = {
4032 .cipher = { 3439 .cipher = {
4033 .enc = { 3440 .enc = __VECS(aes_xts_enc_tv_template),
4034 .vecs = aes_xts_enc_tv_template, 3441 .dec = __VECS(aes_xts_dec_tv_template)
4035 .count = AES_XTS_ENC_TEST_VECTORS
4036 },
4037 .dec = {
4038 .vecs = aes_xts_dec_tv_template,
4039 .count = AES_XTS_DEC_TEST_VECTORS
4040 }
4041 } 3442 }
4042 } 3443 }
4043 }, { 3444 }, {
@@ -4045,14 +3446,8 @@ static const struct alg_test_desc alg_test_descs[] = {
4045 .test = alg_test_skcipher, 3446 .test = alg_test_skcipher,
4046 .suite = { 3447 .suite = {
4047 .cipher = { 3448 .cipher = {
4048 .enc = { 3449 .enc = __VECS(camellia_xts_enc_tv_template),
4049 .vecs = camellia_xts_enc_tv_template, 3450 .dec = __VECS(camellia_xts_dec_tv_template)
4050 .count = CAMELLIA_XTS_ENC_TEST_VECTORS
4051 },
4052 .dec = {
4053 .vecs = camellia_xts_dec_tv_template,
4054 .count = CAMELLIA_XTS_DEC_TEST_VECTORS
4055 }
4056 } 3451 }
4057 } 3452 }
4058 }, { 3453 }, {
@@ -4060,14 +3455,8 @@ static const struct alg_test_desc alg_test_descs[] = {
4060 .test = alg_test_skcipher, 3455 .test = alg_test_skcipher,
4061 .suite = { 3456 .suite = {
4062 .cipher = { 3457 .cipher = {
4063 .enc = { 3458 .enc = __VECS(cast6_xts_enc_tv_template),
4064 .vecs = cast6_xts_enc_tv_template, 3459 .dec = __VECS(cast6_xts_dec_tv_template)
4065 .count = CAST6_XTS_ENC_TEST_VECTORS
4066 },
4067 .dec = {
4068 .vecs = cast6_xts_dec_tv_template,
4069 .count = CAST6_XTS_DEC_TEST_VECTORS
4070 }
4071 } 3460 }
4072 } 3461 }
4073 }, { 3462 }, {
@@ -4075,14 +3464,8 @@ static const struct alg_test_desc alg_test_descs[] = {
4075 .test = alg_test_skcipher, 3464 .test = alg_test_skcipher,
4076 .suite = { 3465 .suite = {
4077 .cipher = { 3466 .cipher = {
4078 .enc = { 3467 .enc = __VECS(serpent_xts_enc_tv_template),
4079 .vecs = serpent_xts_enc_tv_template, 3468 .dec = __VECS(serpent_xts_dec_tv_template)
4080 .count = SERPENT_XTS_ENC_TEST_VECTORS
4081 },
4082 .dec = {
4083 .vecs = serpent_xts_dec_tv_template,
4084 .count = SERPENT_XTS_DEC_TEST_VECTORS
4085 }
4086 } 3469 }
4087 } 3470 }
4088 }, { 3471 }, {
@@ -4090,14 +3473,8 @@ static const struct alg_test_desc alg_test_descs[] = {
4090 .test = alg_test_skcipher, 3473 .test = alg_test_skcipher,
4091 .suite = { 3474 .suite = {
4092 .cipher = { 3475 .cipher = {
4093 .enc = { 3476 .enc = __VECS(tf_xts_enc_tv_template),
4094 .vecs = tf_xts_enc_tv_template, 3477 .dec = __VECS(tf_xts_dec_tv_template)
4095 .count = TF_XTS_ENC_TEST_VECTORS
4096 },
4097 .dec = {
4098 .vecs = tf_xts_dec_tv_template,
4099 .count = TF_XTS_DEC_TEST_VECTORS
4100 }
4101 } 3478 }
4102 } 3479 }
4103 } 3480 }
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 9b656be7f52f..f85e51cf7dcc 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -151,11 +151,6 @@ static char zeroed_string[48];
151/* 151/*
152 * RSA test vectors. Borrowed from openSSL. 152 * RSA test vectors. Borrowed from openSSL.
153 */ 153 */
154#ifdef CONFIG_CRYPTO_FIPS
155#define RSA_TEST_VECTORS 2
156#else
157#define RSA_TEST_VECTORS 5
158#endif
159static struct akcipher_testvec rsa_tv_template[] = { 154static struct akcipher_testvec rsa_tv_template[] = {
160 { 155 {
161#ifndef CONFIG_CRYPTO_FIPS 156#ifndef CONFIG_CRYPTO_FIPS
@@ -340,6 +335,7 @@ static struct akcipher_testvec rsa_tv_template[] = {
340 .m_size = 8, 335 .m_size = 8,
341 .c_size = 256, 336 .c_size = 256,
342 .public_key_vec = true, 337 .public_key_vec = true,
338#ifndef CONFIG_CRYPTO_FIPS
343 }, { 339 }, {
344 .key = 340 .key =
345 "\x30\x82\x09\x29" /* sequence of 2345 bytes */ 341 "\x30\x82\x09\x29" /* sequence of 2345 bytes */
@@ -538,11 +534,10 @@ static struct akcipher_testvec rsa_tv_template[] = {
538 .key_len = 2349, 534 .key_len = 2349,
539 .m_size = 8, 535 .m_size = 8,
540 .c_size = 512, 536 .c_size = 512,
537#endif
541 } 538 }
542}; 539};
543 540
544#define DH_TEST_VECTORS 2
545
546struct kpp_testvec dh_tv_template[] = { 541struct kpp_testvec dh_tv_template[] = {
547 { 542 {
548 .secret = 543 .secret =
@@ -760,11 +755,6 @@ struct kpp_testvec dh_tv_template[] = {
760 } 755 }
761}; 756};
762 757
763#ifdef CONFIG_CRYPTO_FIPS
764#define ECDH_TEST_VECTORS 1
765#else
766#define ECDH_TEST_VECTORS 2
767#endif
768struct kpp_testvec ecdh_tv_template[] = { 758struct kpp_testvec ecdh_tv_template[] = {
769 { 759 {
770#ifndef CONFIG_CRYPTO_FIPS 760#ifndef CONFIG_CRYPTO_FIPS
@@ -856,8 +846,6 @@ struct kpp_testvec ecdh_tv_template[] = {
856/* 846/*
857 * MD4 test vectors from RFC1320 847 * MD4 test vectors from RFC1320
858 */ 848 */
859#define MD4_TEST_VECTORS 7
860
861static struct hash_testvec md4_tv_template [] = { 849static struct hash_testvec md4_tv_template [] = {
862 { 850 {
863 .plaintext = "", 851 .plaintext = "",
@@ -899,7 +887,6 @@ static struct hash_testvec md4_tv_template [] = {
899 }, 887 },
900}; 888};
901 889
902#define SHA3_224_TEST_VECTORS 3
903static struct hash_testvec sha3_224_tv_template[] = { 890static struct hash_testvec sha3_224_tv_template[] = {
904 { 891 {
905 .plaintext = "", 892 .plaintext = "",
@@ -925,7 +912,6 @@ static struct hash_testvec sha3_224_tv_template[] = {
925 }, 912 },
926}; 913};
927 914
928#define SHA3_256_TEST_VECTORS 3
929static struct hash_testvec sha3_256_tv_template[] = { 915static struct hash_testvec sha3_256_tv_template[] = {
930 { 916 {
931 .plaintext = "", 917 .plaintext = "",
@@ -952,7 +938,6 @@ static struct hash_testvec sha3_256_tv_template[] = {
952}; 938};
953 939
954 940
955#define SHA3_384_TEST_VECTORS 3
956static struct hash_testvec sha3_384_tv_template[] = { 941static struct hash_testvec sha3_384_tv_template[] = {
957 { 942 {
958 .plaintext = "", 943 .plaintext = "",
@@ -985,7 +970,6 @@ static struct hash_testvec sha3_384_tv_template[] = {
985}; 970};
986 971
987 972
988#define SHA3_512_TEST_VECTORS 3
989static struct hash_testvec sha3_512_tv_template[] = { 973static struct hash_testvec sha3_512_tv_template[] = {
990 { 974 {
991 .plaintext = "", 975 .plaintext = "",
@@ -1027,8 +1011,6 @@ static struct hash_testvec sha3_512_tv_template[] = {
1027/* 1011/*
1028 * MD5 test vectors from RFC1321 1012 * MD5 test vectors from RFC1321
1029 */ 1013 */
1030#define MD5_TEST_VECTORS 7
1031
1032static struct hash_testvec md5_tv_template[] = { 1014static struct hash_testvec md5_tv_template[] = {
1033 { 1015 {
1034 .digest = "\xd4\x1d\x8c\xd9\x8f\x00\xb2\x04" 1016 .digest = "\xd4\x1d\x8c\xd9\x8f\x00\xb2\x04"
@@ -1073,8 +1055,6 @@ static struct hash_testvec md5_tv_template[] = {
1073/* 1055/*
1074 * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E) 1056 * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E)
1075 */ 1057 */
1076#define RMD128_TEST_VECTORS 10
1077
1078static struct hash_testvec rmd128_tv_template[] = { 1058static struct hash_testvec rmd128_tv_template[] = {
1079 { 1059 {
1080 .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e" 1060 .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e"
@@ -1137,8 +1117,6 @@ static struct hash_testvec rmd128_tv_template[] = {
1137/* 1117/*
1138 * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E) 1118 * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E)
1139 */ 1119 */
1140#define RMD160_TEST_VECTORS 10
1141
1142static struct hash_testvec rmd160_tv_template[] = { 1120static struct hash_testvec rmd160_tv_template[] = {
1143 { 1121 {
1144 .digest = "\x9c\x11\x85\xa5\xc5\xe9\xfc\x54\x61\x28" 1122 .digest = "\x9c\x11\x85\xa5\xc5\xe9\xfc\x54\x61\x28"
@@ -1201,8 +1179,6 @@ static struct hash_testvec rmd160_tv_template[] = {
1201/* 1179/*
1202 * RIPEMD-256 test vectors 1180 * RIPEMD-256 test vectors
1203 */ 1181 */
1204#define RMD256_TEST_VECTORS 8
1205
1206static struct hash_testvec rmd256_tv_template[] = { 1182static struct hash_testvec rmd256_tv_template[] = {
1207 { 1183 {
1208 .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18" 1184 .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18"
@@ -1269,8 +1245,6 @@ static struct hash_testvec rmd256_tv_template[] = {
1269/* 1245/*
1270 * RIPEMD-320 test vectors 1246 * RIPEMD-320 test vectors
1271 */ 1247 */
1272#define RMD320_TEST_VECTORS 8
1273
1274static struct hash_testvec rmd320_tv_template[] = { 1248static struct hash_testvec rmd320_tv_template[] = {
1275 { 1249 {
1276 .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1" 1250 .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1"
@@ -1334,7 +1308,6 @@ static struct hash_testvec rmd320_tv_template[] = {
1334 } 1308 }
1335}; 1309};
1336 1310
1337#define CRCT10DIF_TEST_VECTORS ARRAY_SIZE(crct10dif_tv_template)
1338static struct hash_testvec crct10dif_tv_template[] = { 1311static struct hash_testvec crct10dif_tv_template[] = {
1339 { 1312 {
1340 .plaintext = "abc", 1313 .plaintext = "abc",
@@ -1385,8 +1358,6 @@ static struct hash_testvec crct10dif_tv_template[] = {
1385 * SHA1 test vectors from from FIPS PUB 180-1 1358 * SHA1 test vectors from from FIPS PUB 180-1
1386 * Long vector from CAVS 5.0 1359 * Long vector from CAVS 5.0
1387 */ 1360 */
1388#define SHA1_TEST_VECTORS 6
1389
1390static struct hash_testvec sha1_tv_template[] = { 1361static struct hash_testvec sha1_tv_template[] = {
1391 { 1362 {
1392 .plaintext = "", 1363 .plaintext = "",
@@ -1577,8 +1548,6 @@ static struct hash_testvec sha1_tv_template[] = {
1577/* 1548/*
1578 * SHA224 test vectors from from FIPS PUB 180-2 1549 * SHA224 test vectors from from FIPS PUB 180-2
1579 */ 1550 */
1580#define SHA224_TEST_VECTORS 5
1581
1582static struct hash_testvec sha224_tv_template[] = { 1551static struct hash_testvec sha224_tv_template[] = {
1583 { 1552 {
1584 .plaintext = "", 1553 .plaintext = "",
@@ -1751,8 +1720,6 @@ static struct hash_testvec sha224_tv_template[] = {
1751/* 1720/*
1752 * SHA256 test vectors from from NIST 1721 * SHA256 test vectors from from NIST
1753 */ 1722 */
1754#define SHA256_TEST_VECTORS 5
1755
1756static struct hash_testvec sha256_tv_template[] = { 1723static struct hash_testvec sha256_tv_template[] = {
1757 { 1724 {
1758 .plaintext = "", 1725 .plaintext = "",
@@ -1924,8 +1891,6 @@ static struct hash_testvec sha256_tv_template[] = {
1924/* 1891/*
1925 * SHA384 test vectors from from NIST and kerneli 1892 * SHA384 test vectors from from NIST and kerneli
1926 */ 1893 */
1927#define SHA384_TEST_VECTORS 6
1928
1929static struct hash_testvec sha384_tv_template[] = { 1894static struct hash_testvec sha384_tv_template[] = {
1930 { 1895 {
1931 .plaintext = "", 1896 .plaintext = "",
@@ -2118,8 +2083,6 @@ static struct hash_testvec sha384_tv_template[] = {
2118/* 2083/*
2119 * SHA512 test vectors from from NIST and kerneli 2084 * SHA512 test vectors from from NIST and kerneli
2120 */ 2085 */
2121#define SHA512_TEST_VECTORS 6
2122
2123static struct hash_testvec sha512_tv_template[] = { 2086static struct hash_testvec sha512_tv_template[] = {
2124 { 2087 {
2125 .plaintext = "", 2088 .plaintext = "",
@@ -2327,8 +2290,6 @@ static struct hash_testvec sha512_tv_template[] = {
2327 * by Vincent Rijmen and Paulo S. L. M. Barreto as part of the NESSIE 2290 * by Vincent Rijmen and Paulo S. L. M. Barreto as part of the NESSIE
2328 * submission 2291 * submission
2329 */ 2292 */
2330#define WP512_TEST_VECTORS 8
2331
2332static struct hash_testvec wp512_tv_template[] = { 2293static struct hash_testvec wp512_tv_template[] = {
2333 { 2294 {
2334 .plaintext = "", 2295 .plaintext = "",
@@ -2425,8 +2386,6 @@ static struct hash_testvec wp512_tv_template[] = {
2425 }, 2386 },
2426}; 2387};
2427 2388
2428#define WP384_TEST_VECTORS 8
2429
2430static struct hash_testvec wp384_tv_template[] = { 2389static struct hash_testvec wp384_tv_template[] = {
2431 { 2390 {
2432 .plaintext = "", 2391 .plaintext = "",
@@ -2507,8 +2466,6 @@ static struct hash_testvec wp384_tv_template[] = {
2507 }, 2466 },
2508}; 2467};
2509 2468
2510#define WP256_TEST_VECTORS 8
2511
2512static struct hash_testvec wp256_tv_template[] = { 2469static struct hash_testvec wp256_tv_template[] = {
2513 { 2470 {
2514 .plaintext = "", 2471 .plaintext = "",
@@ -2576,8 +2533,6 @@ static struct hash_testvec wp256_tv_template[] = {
2576/* 2533/*
2577 * TIGER test vectors from Tiger website 2534 * TIGER test vectors from Tiger website
2578 */ 2535 */
2579#define TGR192_TEST_VECTORS 6
2580
2581static struct hash_testvec tgr192_tv_template[] = { 2536static struct hash_testvec tgr192_tv_template[] = {
2582 { 2537 {
2583 .plaintext = "", 2538 .plaintext = "",
@@ -2621,8 +2576,6 @@ static struct hash_testvec tgr192_tv_template[] = {
2621 }, 2576 },
2622}; 2577};
2623 2578
2624#define TGR160_TEST_VECTORS 6
2625
2626static struct hash_testvec tgr160_tv_template[] = { 2579static struct hash_testvec tgr160_tv_template[] = {
2627 { 2580 {
2628 .plaintext = "", 2581 .plaintext = "",
@@ -2666,8 +2619,6 @@ static struct hash_testvec tgr160_tv_template[] = {
2666 }, 2619 },
2667}; 2620};
2668 2621
2669#define TGR128_TEST_VECTORS 6
2670
2671static struct hash_testvec tgr128_tv_template[] = { 2622static struct hash_testvec tgr128_tv_template[] = {
2672 { 2623 {
2673 .plaintext = "", 2624 .plaintext = "",
@@ -2705,8 +2656,6 @@ static struct hash_testvec tgr128_tv_template[] = {
2705 }, 2656 },
2706}; 2657};
2707 2658
2708#define GHASH_TEST_VECTORS 6
2709
2710static struct hash_testvec ghash_tv_template[] = 2659static struct hash_testvec ghash_tv_template[] =
2711{ 2660{
2712 { 2661 {
@@ -2822,8 +2771,6 @@ static struct hash_testvec ghash_tv_template[] =
2822 * HMAC-MD5 test vectors from RFC2202 2771 * HMAC-MD5 test vectors from RFC2202
2823 * (These need to be fixed to not use strlen). 2772 * (These need to be fixed to not use strlen).
2824 */ 2773 */
2825#define HMAC_MD5_TEST_VECTORS 7
2826
2827static struct hash_testvec hmac_md5_tv_template[] = 2774static struct hash_testvec hmac_md5_tv_template[] =
2828{ 2775{
2829 { 2776 {
@@ -2904,8 +2851,6 @@ static struct hash_testvec hmac_md5_tv_template[] =
2904/* 2851/*
2905 * HMAC-RIPEMD128 test vectors from RFC2286 2852 * HMAC-RIPEMD128 test vectors from RFC2286
2906 */ 2853 */
2907#define HMAC_RMD128_TEST_VECTORS 7
2908
2909static struct hash_testvec hmac_rmd128_tv_template[] = { 2854static struct hash_testvec hmac_rmd128_tv_template[] = {
2910 { 2855 {
2911 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", 2856 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b",
@@ -2985,8 +2930,6 @@ static struct hash_testvec hmac_rmd128_tv_template[] = {
2985/* 2930/*
2986 * HMAC-RIPEMD160 test vectors from RFC2286 2931 * HMAC-RIPEMD160 test vectors from RFC2286
2987 */ 2932 */
2988#define HMAC_RMD160_TEST_VECTORS 7
2989
2990static struct hash_testvec hmac_rmd160_tv_template[] = { 2933static struct hash_testvec hmac_rmd160_tv_template[] = {
2991 { 2934 {
2992 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", 2935 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b",
@@ -3066,8 +3009,6 @@ static struct hash_testvec hmac_rmd160_tv_template[] = {
3066/* 3009/*
3067 * HMAC-SHA1 test vectors from RFC2202 3010 * HMAC-SHA1 test vectors from RFC2202
3068 */ 3011 */
3069#define HMAC_SHA1_TEST_VECTORS 7
3070
3071static struct hash_testvec hmac_sha1_tv_template[] = { 3012static struct hash_testvec hmac_sha1_tv_template[] = {
3072 { 3013 {
3073 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", 3014 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b",
@@ -3149,8 +3090,6 @@ static struct hash_testvec hmac_sha1_tv_template[] = {
3149/* 3090/*
3150 * SHA224 HMAC test vectors from RFC4231 3091 * SHA224 HMAC test vectors from RFC4231
3151 */ 3092 */
3152#define HMAC_SHA224_TEST_VECTORS 4
3153
3154static struct hash_testvec hmac_sha224_tv_template[] = { 3093static struct hash_testvec hmac_sha224_tv_template[] = {
3155 { 3094 {
3156 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 3095 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -3264,8 +3203,6 @@ static struct hash_testvec hmac_sha224_tv_template[] = {
3264 * HMAC-SHA256 test vectors from 3203 * HMAC-SHA256 test vectors from
3265 * draft-ietf-ipsec-ciph-sha-256-01.txt 3204 * draft-ietf-ipsec-ciph-sha-256-01.txt
3266 */ 3205 */
3267#define HMAC_SHA256_TEST_VECTORS 10
3268
3269static struct hash_testvec hmac_sha256_tv_template[] = { 3206static struct hash_testvec hmac_sha256_tv_template[] = {
3270 { 3207 {
3271 .key = "\x01\x02\x03\x04\x05\x06\x07\x08" 3208 .key = "\x01\x02\x03\x04\x05\x06\x07\x08"
@@ -3401,8 +3338,6 @@ static struct hash_testvec hmac_sha256_tv_template[] = {
3401 }, 3338 },
3402}; 3339};
3403 3340
3404#define CMAC_AES_TEST_VECTORS 6
3405
3406static struct hash_testvec aes_cmac128_tv_template[] = { 3341static struct hash_testvec aes_cmac128_tv_template[] = {
3407 { /* From NIST Special Publication 800-38B, AES-128 */ 3342 { /* From NIST Special Publication 800-38B, AES-128 */
3408 .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" 3343 .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
@@ -3478,7 +3413,65 @@ static struct hash_testvec aes_cmac128_tv_template[] = {
3478 } 3413 }
3479}; 3414};
3480 3415
3481#define CMAC_DES3_EDE_TEST_VECTORS 4 3416static struct hash_testvec aes_cbcmac_tv_template[] = {
3417 {
3418 .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
3419 "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
3420 .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
3421 "\xe9\x3d\x7e\x11\x73\x93\x17\x2a",
3422 .digest = "\x3a\xd7\x7b\xb4\x0d\x7a\x36\x60"
3423 "\xa8\x9e\xca\xf3\x24\x66\xef\x97",
3424 .psize = 16,
3425 .ksize = 16,
3426 }, {
3427 .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
3428 "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
3429 .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
3430 "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
3431 "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
3432 "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
3433 "\x30",
3434 .digest = "\x9d\x0d\xd0\x63\xfb\xcb\x24\x43"
3435 "\xf8\xf2\x76\x03\xac\x39\xb0\x9d",
3436 .psize = 33,
3437 .ksize = 16,
3438 .np = 2,
3439 .tap = { 7, 26 },
3440 }, {
3441 .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
3442 "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
3443 .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
3444 "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
3445 "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
3446 "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
3447 "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
3448 "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
3449 "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
3450 "\xad\x2b\x41\x7b\xe6\x6c\x37",
3451 .digest = "\xc0\x71\x73\xb8\xa0\x2c\x11\x7c"
3452 "\xaf\xdc\xb2\xf8\x89\x32\xa3\x3a",
3453 .psize = 63,
3454 .ksize = 16,
3455 }, {
3456 .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
3457 "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
3458 "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
3459 "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
3460 .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
3461 "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
3462 "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
3463 "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
3464 "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
3465 "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
3466 "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
3467 "\xad\x2b\x41\x7b\xe6\x6c\x37\x10"
3468 "\x1c",
3469 .digest = "\x6a\x4e\xdb\x21\x47\x51\xdf\x4f"
3470 "\xa8\x4d\x4c\x10\x3b\x72\x7d\xd6",
3471 .psize = 65,
3472 .ksize = 32,
3473 }
3474};
3482 3475
3483static struct hash_testvec des3_ede_cmac64_tv_template[] = { 3476static struct hash_testvec des3_ede_cmac64_tv_template[] = {
3484/* 3477/*
@@ -3526,8 +3519,6 @@ static struct hash_testvec des3_ede_cmac64_tv_template[] = {
3526 } 3519 }
3527}; 3520};
3528 3521
3529#define XCBC_AES_TEST_VECTORS 6
3530
3531static struct hash_testvec aes_xcbc128_tv_template[] = { 3522static struct hash_testvec aes_xcbc128_tv_template[] = {
3532 { 3523 {
3533 .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 3524 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
@@ -3594,7 +3585,6 @@ static struct hash_testvec aes_xcbc128_tv_template[] = {
3594 } 3585 }
3595}; 3586};
3596 3587
3597#define VMAC_AES_TEST_VECTORS 11
3598static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', 3588static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01',
3599 '\x02', '\x03', '\x02', '\x02', 3589 '\x02', '\x03', '\x02', '\x02',
3600 '\x02', '\x04', '\x01', '\x07', 3590 '\x02', '\x04', '\x01', '\x07',
@@ -3701,8 +3691,6 @@ static struct hash_testvec aes_vmac128_tv_template[] = {
3701 * SHA384 HMAC test vectors from RFC4231 3691 * SHA384 HMAC test vectors from RFC4231
3702 */ 3692 */
3703 3693
3704#define HMAC_SHA384_TEST_VECTORS 4
3705
3706static struct hash_testvec hmac_sha384_tv_template[] = { 3694static struct hash_testvec hmac_sha384_tv_template[] = {
3707 { 3695 {
3708 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 3696 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -3801,8 +3789,6 @@ static struct hash_testvec hmac_sha384_tv_template[] = {
3801 * SHA512 HMAC test vectors from RFC4231 3789 * SHA512 HMAC test vectors from RFC4231
3802 */ 3790 */
3803 3791
3804#define HMAC_SHA512_TEST_VECTORS 4
3805
3806static struct hash_testvec hmac_sha512_tv_template[] = { 3792static struct hash_testvec hmac_sha512_tv_template[] = {
3807 { 3793 {
3808 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 3794 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -3908,8 +3894,6 @@ static struct hash_testvec hmac_sha512_tv_template[] = {
3908 }, 3894 },
3909}; 3895};
3910 3896
3911#define HMAC_SHA3_224_TEST_VECTORS 4
3912
3913static struct hash_testvec hmac_sha3_224_tv_template[] = { 3897static struct hash_testvec hmac_sha3_224_tv_template[] = {
3914 { 3898 {
3915 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 3899 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -3999,8 +3983,6 @@ static struct hash_testvec hmac_sha3_224_tv_template[] = {
3999 }, 3983 },
4000}; 3984};
4001 3985
4002#define HMAC_SHA3_256_TEST_VECTORS 4
4003
4004static struct hash_testvec hmac_sha3_256_tv_template[] = { 3986static struct hash_testvec hmac_sha3_256_tv_template[] = {
4005 { 3987 {
4006 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 3988 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -4090,8 +4072,6 @@ static struct hash_testvec hmac_sha3_256_tv_template[] = {
4090 }, 4072 },
4091}; 4073};
4092 4074
4093#define HMAC_SHA3_384_TEST_VECTORS 4
4094
4095static struct hash_testvec hmac_sha3_384_tv_template[] = { 4075static struct hash_testvec hmac_sha3_384_tv_template[] = {
4096 { 4076 {
4097 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 4077 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -4189,8 +4169,6 @@ static struct hash_testvec hmac_sha3_384_tv_template[] = {
4189 }, 4169 },
4190}; 4170};
4191 4171
4192#define HMAC_SHA3_512_TEST_VECTORS 4
4193
4194static struct hash_testvec hmac_sha3_512_tv_template[] = { 4172static struct hash_testvec hmac_sha3_512_tv_template[] = {
4195 { 4173 {
4196 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" 4174 .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
@@ -4300,8 +4278,6 @@ static struct hash_testvec hmac_sha3_512_tv_template[] = {
4300 * Poly1305 test vectors from RFC7539 A.3. 4278 * Poly1305 test vectors from RFC7539 A.3.
4301 */ 4279 */
4302 4280
4303#define POLY1305_TEST_VECTORS 11
4304
4305static struct hash_testvec poly1305_tv_template[] = { 4281static struct hash_testvec poly1305_tv_template[] = {
4306 { /* Test Vector #1 */ 4282 { /* Test Vector #1 */
4307 .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" 4283 .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -4547,19 +4523,6 @@ static struct hash_testvec poly1305_tv_template[] = {
4547/* 4523/*
4548 * DES test vectors. 4524 * DES test vectors.
4549 */ 4525 */
4550#define DES_ENC_TEST_VECTORS 11
4551#define DES_DEC_TEST_VECTORS 5
4552#define DES_CBC_ENC_TEST_VECTORS 6
4553#define DES_CBC_DEC_TEST_VECTORS 5
4554#define DES_CTR_ENC_TEST_VECTORS 2
4555#define DES_CTR_DEC_TEST_VECTORS 2
4556#define DES3_EDE_ENC_TEST_VECTORS 4
4557#define DES3_EDE_DEC_TEST_VECTORS 4
4558#define DES3_EDE_CBC_ENC_TEST_VECTORS 2
4559#define DES3_EDE_CBC_DEC_TEST_VECTORS 2
4560#define DES3_EDE_CTR_ENC_TEST_VECTORS 2
4561#define DES3_EDE_CTR_DEC_TEST_VECTORS 2
4562
4563static struct cipher_testvec des_enc_tv_template[] = { 4526static struct cipher_testvec des_enc_tv_template[] = {
4564 { /* From Applied Cryptography */ 4527 { /* From Applied Cryptography */
4565 .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", 4528 .key = "\x01\x23\x45\x67\x89\xab\xcd\xef",
@@ -6620,13 +6583,6 @@ static struct cipher_testvec des3_ede_ctr_dec_tv_template[] = {
6620/* 6583/*
6621 * Blowfish test vectors. 6584 * Blowfish test vectors.
6622 */ 6585 */
6623#define BF_ENC_TEST_VECTORS 7
6624#define BF_DEC_TEST_VECTORS 7
6625#define BF_CBC_ENC_TEST_VECTORS 2
6626#define BF_CBC_DEC_TEST_VECTORS 2
6627#define BF_CTR_ENC_TEST_VECTORS 2
6628#define BF_CTR_DEC_TEST_VECTORS 2
6629
6630static struct cipher_testvec bf_enc_tv_template[] = { 6586static struct cipher_testvec bf_enc_tv_template[] = {
6631 { /* DES test vectors from OpenSSL */ 6587 { /* DES test vectors from OpenSSL */
6632 .key = "\x00\x00\x00\x00\x00\x00\x00\x00", 6588 .key = "\x00\x00\x00\x00\x00\x00\x00\x00",
@@ -8152,17 +8108,6 @@ static struct cipher_testvec bf_ctr_dec_tv_template[] = {
8152/* 8108/*
8153 * Twofish test vectors. 8109 * Twofish test vectors.
8154 */ 8110 */
8155#define TF_ENC_TEST_VECTORS 4
8156#define TF_DEC_TEST_VECTORS 4
8157#define TF_CBC_ENC_TEST_VECTORS 5
8158#define TF_CBC_DEC_TEST_VECTORS 5
8159#define TF_CTR_ENC_TEST_VECTORS 2
8160#define TF_CTR_DEC_TEST_VECTORS 2
8161#define TF_LRW_ENC_TEST_VECTORS 8
8162#define TF_LRW_DEC_TEST_VECTORS 8
8163#define TF_XTS_ENC_TEST_VECTORS 5
8164#define TF_XTS_DEC_TEST_VECTORS 5
8165
8166static struct cipher_testvec tf_enc_tv_template[] = { 8111static struct cipher_testvec tf_enc_tv_template[] = {
8167 { 8112 {
8168 .key = zeroed_string, 8113 .key = zeroed_string,
@@ -10881,24 +10826,6 @@ static struct cipher_testvec tf_xts_dec_tv_template[] = {
10881 * Serpent test vectors. These are backwards because Serpent writes 10826 * Serpent test vectors. These are backwards because Serpent writes
10882 * octet sequences in right-to-left mode. 10827 * octet sequences in right-to-left mode.
10883 */ 10828 */
10884#define SERPENT_ENC_TEST_VECTORS 5
10885#define SERPENT_DEC_TEST_VECTORS 5
10886
10887#define TNEPRES_ENC_TEST_VECTORS 4
10888#define TNEPRES_DEC_TEST_VECTORS 4
10889
10890#define SERPENT_CBC_ENC_TEST_VECTORS 1
10891#define SERPENT_CBC_DEC_TEST_VECTORS 1
10892
10893#define SERPENT_CTR_ENC_TEST_VECTORS 2
10894#define SERPENT_CTR_DEC_TEST_VECTORS 2
10895
10896#define SERPENT_LRW_ENC_TEST_VECTORS 8
10897#define SERPENT_LRW_DEC_TEST_VECTORS 8
10898
10899#define SERPENT_XTS_ENC_TEST_VECTORS 5
10900#define SERPENT_XTS_DEC_TEST_VECTORS 5
10901
10902static struct cipher_testvec serpent_enc_tv_template[] = { 10829static struct cipher_testvec serpent_enc_tv_template[] = {
10903 { 10830 {
10904 .input = "\x00\x01\x02\x03\x04\x05\x06\x07" 10831 .input = "\x00\x01\x02\x03\x04\x05\x06\x07"
@@ -13637,17 +13564,6 @@ static struct cipher_testvec serpent_xts_dec_tv_template[] = {
13637}; 13564};
13638 13565
13639/* Cast6 test vectors from RFC 2612 */ 13566/* Cast6 test vectors from RFC 2612 */
13640#define CAST6_ENC_TEST_VECTORS 4
13641#define CAST6_DEC_TEST_VECTORS 4
13642#define CAST6_CBC_ENC_TEST_VECTORS 1
13643#define CAST6_CBC_DEC_TEST_VECTORS 1
13644#define CAST6_CTR_ENC_TEST_VECTORS 2
13645#define CAST6_CTR_DEC_TEST_VECTORS 2
13646#define CAST6_LRW_ENC_TEST_VECTORS 1
13647#define CAST6_LRW_DEC_TEST_VECTORS 1
13648#define CAST6_XTS_ENC_TEST_VECTORS 1
13649#define CAST6_XTS_DEC_TEST_VECTORS 1
13650
13651static struct cipher_testvec cast6_enc_tv_template[] = { 13567static struct cipher_testvec cast6_enc_tv_template[] = {
13652 { 13568 {
13653 .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" 13569 .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
@@ -15182,38 +15098,6 @@ static struct cipher_testvec cast6_xts_dec_tv_template[] = {
15182/* 15098/*
15183 * AES test vectors. 15099 * AES test vectors.
15184 */ 15100 */
15185#define AES_ENC_TEST_VECTORS 4
15186#define AES_DEC_TEST_VECTORS 4
15187#define AES_CBC_ENC_TEST_VECTORS 5
15188#define AES_CBC_DEC_TEST_VECTORS 5
15189#define HMAC_MD5_ECB_CIPHER_NULL_ENC_TEST_VECTORS 2
15190#define HMAC_MD5_ECB_CIPHER_NULL_DEC_TEST_VECTORS 2
15191#define HMAC_SHA1_ECB_CIPHER_NULL_ENC_TEST_VEC 2
15192#define HMAC_SHA1_ECB_CIPHER_NULL_DEC_TEST_VEC 2
15193#define HMAC_SHA1_AES_CBC_ENC_TEST_VEC 7
15194#define HMAC_SHA256_AES_CBC_ENC_TEST_VEC 7
15195#define HMAC_SHA512_AES_CBC_ENC_TEST_VEC 7
15196#define AES_LRW_ENC_TEST_VECTORS 8
15197#define AES_LRW_DEC_TEST_VECTORS 8
15198#define AES_XTS_ENC_TEST_VECTORS 5
15199#define AES_XTS_DEC_TEST_VECTORS 5
15200#define AES_CTR_ENC_TEST_VECTORS 5
15201#define AES_CTR_DEC_TEST_VECTORS 5
15202#define AES_OFB_ENC_TEST_VECTORS 1
15203#define AES_OFB_DEC_TEST_VECTORS 1
15204#define AES_CTR_3686_ENC_TEST_VECTORS 7
15205#define AES_CTR_3686_DEC_TEST_VECTORS 6
15206#define AES_GCM_ENC_TEST_VECTORS 9
15207#define AES_GCM_DEC_TEST_VECTORS 8
15208#define AES_GCM_4106_ENC_TEST_VECTORS 23
15209#define AES_GCM_4106_DEC_TEST_VECTORS 23
15210#define AES_GCM_4543_ENC_TEST_VECTORS 1
15211#define AES_GCM_4543_DEC_TEST_VECTORS 2
15212#define AES_CCM_ENC_TEST_VECTORS 8
15213#define AES_CCM_DEC_TEST_VECTORS 7
15214#define AES_CCM_4309_ENC_TEST_VECTORS 7
15215#define AES_CCM_4309_DEC_TEST_VECTORS 10
15216
15217static struct cipher_testvec aes_enc_tv_template[] = { 15101static struct cipher_testvec aes_enc_tv_template[] = {
15218 { /* From FIPS-197 */ 15102 { /* From FIPS-197 */
15219 .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 15103 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
@@ -17069,8 +16953,6 @@ static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = {
17069 }, 16953 },
17070}; 16954};
17071 16955
17072#define HMAC_SHA1_DES_CBC_ENC_TEST_VEC 1
17073
17074static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { 16956static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = {
17075 { /*Generated with cryptopp*/ 16957 { /*Generated with cryptopp*/
17076#ifdef __LITTLE_ENDIAN 16958#ifdef __LITTLE_ENDIAN
@@ -17130,8 +17012,6 @@ static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = {
17130 }, 17012 },
17131}; 17013};
17132 17014
17133#define HMAC_SHA224_DES_CBC_ENC_TEST_VEC 1
17134
17135static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { 17015static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = {
17136 { /*Generated with cryptopp*/ 17016 { /*Generated with cryptopp*/
17137#ifdef __LITTLE_ENDIAN 17017#ifdef __LITTLE_ENDIAN
@@ -17191,8 +17071,6 @@ static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = {
17191 }, 17071 },
17192}; 17072};
17193 17073
17194#define HMAC_SHA256_DES_CBC_ENC_TEST_VEC 1
17195
17196static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { 17074static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = {
17197 { /*Generated with cryptopp*/ 17075 { /*Generated with cryptopp*/
17198#ifdef __LITTLE_ENDIAN 17076#ifdef __LITTLE_ENDIAN
@@ -17254,8 +17132,6 @@ static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = {
17254 }, 17132 },
17255}; 17133};
17256 17134
17257#define HMAC_SHA384_DES_CBC_ENC_TEST_VEC 1
17258
17259static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { 17135static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = {
17260 { /*Generated with cryptopp*/ 17136 { /*Generated with cryptopp*/
17261#ifdef __LITTLE_ENDIAN 17137#ifdef __LITTLE_ENDIAN
@@ -17321,8 +17197,6 @@ static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = {
17321 }, 17197 },
17322}; 17198};
17323 17199
17324#define HMAC_SHA512_DES_CBC_ENC_TEST_VEC 1
17325
17326static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { 17200static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = {
17327 { /*Generated with cryptopp*/ 17201 { /*Generated with cryptopp*/
17328#ifdef __LITTLE_ENDIAN 17202#ifdef __LITTLE_ENDIAN
@@ -17392,8 +17266,6 @@ static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = {
17392 }, 17266 },
17393}; 17267};
17394 17268
17395#define HMAC_SHA1_DES3_EDE_CBC_ENC_TEST_VEC 1
17396
17397static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { 17269static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = {
17398 { /*Generated with cryptopp*/ 17270 { /*Generated with cryptopp*/
17399#ifdef __LITTLE_ENDIAN 17271#ifdef __LITTLE_ENDIAN
@@ -17455,8 +17327,6 @@ static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = {
17455 }, 17327 },
17456}; 17328};
17457 17329
17458#define HMAC_SHA224_DES3_EDE_CBC_ENC_TEST_VEC 1
17459
17460static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { 17330static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = {
17461 { /*Generated with cryptopp*/ 17331 { /*Generated with cryptopp*/
17462#ifdef __LITTLE_ENDIAN 17332#ifdef __LITTLE_ENDIAN
@@ -17518,8 +17388,6 @@ static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = {
17518 }, 17388 },
17519}; 17389};
17520 17390
17521#define HMAC_SHA256_DES3_EDE_CBC_ENC_TEST_VEC 1
17522
17523static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { 17391static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = {
17524 { /*Generated with cryptopp*/ 17392 { /*Generated with cryptopp*/
17525#ifdef __LITTLE_ENDIAN 17393#ifdef __LITTLE_ENDIAN
@@ -17583,8 +17451,6 @@ static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = {
17583 }, 17451 },
17584}; 17452};
17585 17453
17586#define HMAC_SHA384_DES3_EDE_CBC_ENC_TEST_VEC 1
17587
17588static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { 17454static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = {
17589 { /*Generated with cryptopp*/ 17455 { /*Generated with cryptopp*/
17590#ifdef __LITTLE_ENDIAN 17456#ifdef __LITTLE_ENDIAN
@@ -17652,8 +17518,6 @@ static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = {
17652 }, 17518 },
17653}; 17519};
17654 17520
17655#define HMAC_SHA512_DES3_EDE_CBC_ENC_TEST_VEC 1
17656
17657static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { 17521static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = {
17658 { /*Generated with cryptopp*/ 17522 { /*Generated with cryptopp*/
17659#ifdef __LITTLE_ENDIAN 17523#ifdef __LITTLE_ENDIAN
@@ -24434,8 +24298,6 @@ static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = {
24434/* 24298/*
24435 * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5. 24299 * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5.
24436 */ 24300 */
24437#define RFC7539_ENC_TEST_VECTORS 2
24438#define RFC7539_DEC_TEST_VECTORS 2
24439static struct aead_testvec rfc7539_enc_tv_template[] = { 24301static struct aead_testvec rfc7539_enc_tv_template[] = {
24440 { 24302 {
24441 .key = "\x80\x81\x82\x83\x84\x85\x86\x87" 24303 .key = "\x80\x81\x82\x83\x84\x85\x86\x87"
@@ -24703,8 +24565,6 @@ static struct aead_testvec rfc7539_dec_tv_template[] = {
24703/* 24565/*
24704 * draft-irtf-cfrg-chacha20-poly1305 24566 * draft-irtf-cfrg-chacha20-poly1305
24705 */ 24567 */
24706#define RFC7539ESP_DEC_TEST_VECTORS 1
24707#define RFC7539ESP_ENC_TEST_VECTORS 1
24708static struct aead_testvec rfc7539esp_enc_tv_template[] = { 24568static struct aead_testvec rfc7539esp_enc_tv_template[] = {
24709 { 24569 {
24710 .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" 24570 .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
@@ -24927,8 +24787,6 @@ static struct cipher_testvec aes_kw_dec_tv_template[] = {
24927 * http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf 24787 * http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf
24928 * Only AES-128 is supported at this time. 24788 * Only AES-128 is supported at this time.
24929 */ 24789 */
24930#define ANSI_CPRNG_AES_TEST_VECTORS 6
24931
24932static struct cprng_testvec ansi_cprng_aes_tv_template[] = { 24790static struct cprng_testvec ansi_cprng_aes_tv_template[] = {
24933 { 24791 {
24934 .key = "\xf3\xb1\x66\x6d\x13\x60\x72\x42" 24792 .key = "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
@@ -25846,13 +25704,6 @@ static struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = {
25846}; 25704};
25847 25705
25848/* Cast5 test vectors from RFC 2144 */ 25706/* Cast5 test vectors from RFC 2144 */
25849#define CAST5_ENC_TEST_VECTORS 4
25850#define CAST5_DEC_TEST_VECTORS 4
25851#define CAST5_CBC_ENC_TEST_VECTORS 1
25852#define CAST5_CBC_DEC_TEST_VECTORS 1
25853#define CAST5_CTR_ENC_TEST_VECTORS 2
25854#define CAST5_CTR_DEC_TEST_VECTORS 2
25855
25856static struct cipher_testvec cast5_enc_tv_template[] = { 25707static struct cipher_testvec cast5_enc_tv_template[] = {
25857 { 25708 {
25858 .key = "\x01\x23\x45\x67\x12\x34\x56\x78" 25709 .key = "\x01\x23\x45\x67\x12\x34\x56\x78"
@@ -26756,9 +26607,6 @@ static struct cipher_testvec cast5_ctr_dec_tv_template[] = {
26756/* 26607/*
26757 * ARC4 test vectors from OpenSSL 26608 * ARC4 test vectors from OpenSSL
26758 */ 26609 */
26759#define ARC4_ENC_TEST_VECTORS 7
26760#define ARC4_DEC_TEST_VECTORS 7
26761
26762static struct cipher_testvec arc4_enc_tv_template[] = { 26610static struct cipher_testvec arc4_enc_tv_template[] = {
26763 { 26611 {
26764 .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", 26612 .key = "\x01\x23\x45\x67\x89\xab\xcd\xef",
@@ -26894,9 +26742,6 @@ static struct cipher_testvec arc4_dec_tv_template[] = {
26894/* 26742/*
26895 * TEA test vectors 26743 * TEA test vectors
26896 */ 26744 */
26897#define TEA_ENC_TEST_VECTORS 4
26898#define TEA_DEC_TEST_VECTORS 4
26899
26900static struct cipher_testvec tea_enc_tv_template[] = { 26745static struct cipher_testvec tea_enc_tv_template[] = {
26901 { 26746 {
26902 .key = zeroed_string, 26747 .key = zeroed_string,
@@ -26986,9 +26831,6 @@ static struct cipher_testvec tea_dec_tv_template[] = {
26986/* 26831/*
26987 * XTEA test vectors 26832 * XTEA test vectors
26988 */ 26833 */
26989#define XTEA_ENC_TEST_VECTORS 4
26990#define XTEA_DEC_TEST_VECTORS 4
26991
26992static struct cipher_testvec xtea_enc_tv_template[] = { 26834static struct cipher_testvec xtea_enc_tv_template[] = {
26993 { 26835 {
26994 .key = zeroed_string, 26836 .key = zeroed_string,
@@ -27078,9 +26920,6 @@ static struct cipher_testvec xtea_dec_tv_template[] = {
27078/* 26920/*
27079 * KHAZAD test vectors. 26921 * KHAZAD test vectors.
27080 */ 26922 */
27081#define KHAZAD_ENC_TEST_VECTORS 5
27082#define KHAZAD_DEC_TEST_VECTORS 5
27083
27084static struct cipher_testvec khazad_enc_tv_template[] = { 26923static struct cipher_testvec khazad_enc_tv_template[] = {
27085 { 26924 {
27086 .key = "\x80\x00\x00\x00\x00\x00\x00\x00" 26925 .key = "\x80\x00\x00\x00\x00\x00\x00\x00"
@@ -27177,11 +27016,6 @@ static struct cipher_testvec khazad_dec_tv_template[] = {
27177 * Anubis test vectors. 27016 * Anubis test vectors.
27178 */ 27017 */
27179 27018
27180#define ANUBIS_ENC_TEST_VECTORS 5
27181#define ANUBIS_DEC_TEST_VECTORS 5
27182#define ANUBIS_CBC_ENC_TEST_VECTORS 2
27183#define ANUBIS_CBC_DEC_TEST_VECTORS 2
27184
27185static struct cipher_testvec anubis_enc_tv_template[] = { 27019static struct cipher_testvec anubis_enc_tv_template[] = {
27186 { 27020 {
27187 .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" 27021 .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
@@ -27381,9 +27215,6 @@ static struct cipher_testvec anubis_cbc_dec_tv_template[] = {
27381/* 27215/*
27382 * XETA test vectors 27216 * XETA test vectors
27383 */ 27217 */
27384#define XETA_ENC_TEST_VECTORS 4
27385#define XETA_DEC_TEST_VECTORS 4
27386
27387static struct cipher_testvec xeta_enc_tv_template[] = { 27218static struct cipher_testvec xeta_enc_tv_template[] = {
27388 { 27219 {
27389 .key = zeroed_string, 27220 .key = zeroed_string,
@@ -27473,9 +27304,6 @@ static struct cipher_testvec xeta_dec_tv_template[] = {
27473/* 27304/*
27474 * FCrypt test vectors 27305 * FCrypt test vectors
27475 */ 27306 */
27476#define FCRYPT_ENC_TEST_VECTORS ARRAY_SIZE(fcrypt_pcbc_enc_tv_template)
27477#define FCRYPT_DEC_TEST_VECTORS ARRAY_SIZE(fcrypt_pcbc_dec_tv_template)
27478
27479static struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { 27307static struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = {
27480 { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ 27308 { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */
27481 .key = "\x00\x00\x00\x00\x00\x00\x00\x00", 27309 .key = "\x00\x00\x00\x00\x00\x00\x00\x00",
@@ -27601,17 +27429,6 @@ static struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = {
27601/* 27429/*
27602 * CAMELLIA test vectors. 27430 * CAMELLIA test vectors.
27603 */ 27431 */
27604#define CAMELLIA_ENC_TEST_VECTORS 4
27605#define CAMELLIA_DEC_TEST_VECTORS 4
27606#define CAMELLIA_CBC_ENC_TEST_VECTORS 3
27607#define CAMELLIA_CBC_DEC_TEST_VECTORS 3
27608#define CAMELLIA_CTR_ENC_TEST_VECTORS 2
27609#define CAMELLIA_CTR_DEC_TEST_VECTORS 2
27610#define CAMELLIA_LRW_ENC_TEST_VECTORS 8
27611#define CAMELLIA_LRW_DEC_TEST_VECTORS 8
27612#define CAMELLIA_XTS_ENC_TEST_VECTORS 5
27613#define CAMELLIA_XTS_DEC_TEST_VECTORS 5
27614
27615static struct cipher_testvec camellia_enc_tv_template[] = { 27432static struct cipher_testvec camellia_enc_tv_template[] = {
27616 { 27433 {
27617 .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" 27434 .key = "\x01\x23\x45\x67\x89\xab\xcd\xef"
@@ -31331,9 +31148,6 @@ static struct cipher_testvec camellia_xts_dec_tv_template[] = {
31331/* 31148/*
31332 * SEED test vectors 31149 * SEED test vectors
31333 */ 31150 */
31334#define SEED_ENC_TEST_VECTORS 4
31335#define SEED_DEC_TEST_VECTORS 4
31336
31337static struct cipher_testvec seed_enc_tv_template[] = { 31151static struct cipher_testvec seed_enc_tv_template[] = {
31338 { 31152 {
31339 .key = zeroed_string, 31153 .key = zeroed_string,
@@ -31418,7 +31232,6 @@ static struct cipher_testvec seed_dec_tv_template[] = {
31418 } 31232 }
31419}; 31233};
31420 31234
31421#define SALSA20_STREAM_ENC_TEST_VECTORS 5
31422static struct cipher_testvec salsa20_stream_enc_tv_template[] = { 31235static struct cipher_testvec salsa20_stream_enc_tv_template[] = {
31423 /* 31236 /*
31424 * Testvectors from verified.test-vectors submitted to ECRYPT. 31237 * Testvectors from verified.test-vectors submitted to ECRYPT.
@@ -32588,7 +32401,6 @@ static struct cipher_testvec salsa20_stream_enc_tv_template[] = {
32588 }, 32401 },
32589}; 32402};
32590 32403
32591#define CHACHA20_ENC_TEST_VECTORS 4
32592static struct cipher_testvec chacha20_enc_tv_template[] = { 32404static struct cipher_testvec chacha20_enc_tv_template[] = {
32593 { /* RFC7539 A.2. Test Vector #1 */ 32405 { /* RFC7539 A.2. Test Vector #1 */
32594 .key = "\x00\x00\x00\x00\x00\x00\x00\x00" 32406 .key = "\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -33100,8 +32912,6 @@ static struct cipher_testvec chacha20_enc_tv_template[] = {
33100/* 32912/*
33101 * CTS (Cipher Text Stealing) mode tests 32913 * CTS (Cipher Text Stealing) mode tests
33102 */ 32914 */
33103#define CTS_MODE_ENC_TEST_VECTORS 6
33104#define CTS_MODE_DEC_TEST_VECTORS 6
33105static struct cipher_testvec cts_mode_enc_tv_template[] = { 32915static struct cipher_testvec cts_mode_enc_tv_template[] = {
33106 { /* from rfc3962 */ 32916 { /* from rfc3962 */
33107 .klen = 16, 32917 .klen = 16,
@@ -33322,9 +33132,6 @@ struct comp_testvec {
33322 * Params: winbits=-11, Z_DEFAULT_COMPRESSION, MAX_MEM_LEVEL. 33132 * Params: winbits=-11, Z_DEFAULT_COMPRESSION, MAX_MEM_LEVEL.
33323 */ 33133 */
33324 33134
33325#define DEFLATE_COMP_TEST_VECTORS 2
33326#define DEFLATE_DECOMP_TEST_VECTORS 2
33327
33328static struct comp_testvec deflate_comp_tv_template[] = { 33135static struct comp_testvec deflate_comp_tv_template[] = {
33329 { 33136 {
33330 .inlen = 70, 33137 .inlen = 70,
@@ -33400,9 +33207,6 @@ static struct comp_testvec deflate_decomp_tv_template[] = {
33400/* 33207/*
33401 * LZO test vectors (null-terminated strings). 33208 * LZO test vectors (null-terminated strings).
33402 */ 33209 */
33403#define LZO_COMP_TEST_VECTORS 2
33404#define LZO_DECOMP_TEST_VECTORS 2
33405
33406static struct comp_testvec lzo_comp_tv_template[] = { 33210static struct comp_testvec lzo_comp_tv_template[] = {
33407 { 33211 {
33408 .inlen = 70, 33212 .inlen = 70,
@@ -33534,8 +33338,6 @@ static struct hash_testvec michael_mic_tv_template[] = {
33534/* 33338/*
33535 * CRC32 test vectors 33339 * CRC32 test vectors
33536 */ 33340 */
33537#define CRC32_TEST_VECTORS 14
33538
33539static struct hash_testvec crc32_tv_template[] = { 33341static struct hash_testvec crc32_tv_template[] = {
33540 { 33342 {
33541 .key = "\x87\xa9\xcb\xed", 33343 .key = "\x87\xa9\xcb\xed",
@@ -33968,8 +33770,6 @@ static struct hash_testvec crc32_tv_template[] = {
33968/* 33770/*
33969 * CRC32C test vectors 33771 * CRC32C test vectors
33970 */ 33772 */
33971#define CRC32C_TEST_VECTORS 15
33972
33973static struct hash_testvec crc32c_tv_template[] = { 33773static struct hash_testvec crc32c_tv_template[] = {
33974 { 33774 {
33975 .psize = 0, 33775 .psize = 0,
@@ -34406,8 +34206,6 @@ static struct hash_testvec crc32c_tv_template[] = {
34406/* 34206/*
34407 * Blakcifn CRC test vectors 34207 * Blakcifn CRC test vectors
34408 */ 34208 */
34409#define BFIN_CRC_TEST_VECTORS 6
34410
34411static struct hash_testvec bfin_crc_tv_template[] = { 34209static struct hash_testvec bfin_crc_tv_template[] = {
34412 { 34210 {
34413 .psize = 0, 34211 .psize = 0,
@@ -34493,9 +34291,6 @@ static struct hash_testvec bfin_crc_tv_template[] = {
34493 34291
34494}; 34292};
34495 34293
34496#define LZ4_COMP_TEST_VECTORS 1
34497#define LZ4_DECOMP_TEST_VECTORS 1
34498
34499static struct comp_testvec lz4_comp_tv_template[] = { 34294static struct comp_testvec lz4_comp_tv_template[] = {
34500 { 34295 {
34501 .inlen = 70, 34296 .inlen = 70,
@@ -34526,9 +34321,6 @@ static struct comp_testvec lz4_decomp_tv_template[] = {
34526 }, 34321 },
34527}; 34322};
34528 34323
34529#define LZ4HC_COMP_TEST_VECTORS 1
34530#define LZ4HC_DECOMP_TEST_VECTORS 1
34531
34532static struct comp_testvec lz4hc_comp_tv_template[] = { 34324static struct comp_testvec lz4hc_comp_tv_template[] = {
34533 { 34325 {
34534 .inlen = 70, 34326 .inlen = 70,
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index ceff2fc524b1..0cafe08919c9 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -172,8 +172,8 @@ config HW_RANDOM_OMAP
172 default HW_RANDOM 172 default HW_RANDOM
173 ---help--- 173 ---help---
174 This driver provides kernel-side support for the Random Number 174 This driver provides kernel-side support for the Random Number
175 Generator hardware found on OMAP16xx, OMAP2/3/4/5 and AM33xx/AM43xx 175 Generator hardware found on OMAP16xx, OMAP2/3/4/5, AM33xx/AM43xx
176 multimedia processors. 176 multimedia processors, and Marvell Armada 7k/8k SoCs.
177 177
178 To compile this driver as a module, choose M here: the 178 To compile this driver as a module, choose M here: the
179 module will be called omap-rng. 179 module will be called omap-rng.
diff --git a/drivers/char/hw_random/cavium-rng-vf.c b/drivers/char/hw_random/cavium-rng-vf.c
index 066ae0e78d63..dd1007aecb10 100644
--- a/drivers/char/hw_random/cavium-rng-vf.c
+++ b/drivers/char/hw_random/cavium-rng-vf.c
@@ -57,7 +57,11 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
57 return -ENOMEM; 57 return -ENOMEM;
58 } 58 }
59 59
60 rng->ops.name = "cavium rng"; 60 rng->ops.name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
61 "cavium-rng-%s", dev_name(&pdev->dev));
62 if (!rng->ops.name)
63 return -ENOMEM;
64
61 rng->ops.read = cavium_rng_read; 65 rng->ops.read = cavium_rng_read;
62 rng->ops.quality = 1000; 66 rng->ops.quality = 1000;
63 67
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 87fba424817e..5c654b5d4adf 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -1,55 +1,30 @@
1/* 1/*
2 Added support for the AMD Geode LX RNG 2 * hw_random/core.c: HWRNG core API
3 (c) Copyright 2004-2005 Advanced Micro Devices, Inc. 3 *
4 4 * Copyright 2006 Michael Buesch <m@bues.ch>
5 derived from 5 * Copyright 2005 (c) MontaVista Software, Inc.
6 6 *
7 Hardware driver for the Intel/AMD/VIA Random Number Generators (RNG) 7 * Please read Documentation/hw_random.txt for details on use.
8 (c) Copyright 2003 Red Hat Inc <jgarzik@redhat.com> 8 *
9 9 * This software may be used and distributed according to the terms
10 derived from 10 * of the GNU General Public License, incorporated herein by reference.
11
12 Hardware driver for the AMD 768 Random Number Generator (RNG)
13 (c) Copyright 2001 Red Hat Inc <alan@redhat.com>
14
15 derived from
16
17 Hardware driver for Intel i810 Random Number Generator (RNG)
18 Copyright 2000,2001 Jeff Garzik <jgarzik@pobox.com>
19 Copyright 2000,2001 Philipp Rumpf <prumpf@mandrakesoft.com>
20
21 Added generic RNG API
22 Copyright 2006 Michael Buesch <m@bues.ch>
23 Copyright 2005 (c) MontaVista Software, Inc.
24
25 Please read Documentation/hw_random.txt for details on use.
26
27 ----------------------------------------------------------
28 This software may be used and distributed according to the terms
29 of the GNU General Public License, incorporated herein by reference.
30
31 */ 11 */
32 12
33 13#include <linux/delay.h>
34#include <linux/device.h> 14#include <linux/device.h>
15#include <linux/err.h>
16#include <linux/fs.h>
35#include <linux/hw_random.h> 17#include <linux/hw_random.h>
36#include <linux/module.h>
37#include <linux/kernel.h> 18#include <linux/kernel.h>
38#include <linux/fs.h>
39#include <linux/sched.h>
40#include <linux/miscdevice.h>
41#include <linux/kthread.h> 19#include <linux/kthread.h>
42#include <linux/delay.h> 20#include <linux/miscdevice.h>
43#include <linux/slab.h> 21#include <linux/module.h>
44#include <linux/random.h> 22#include <linux/random.h>
45#include <linux/err.h> 23#include <linux/sched.h>
24#include <linux/slab.h>
46#include <linux/uaccess.h> 25#include <linux/uaccess.h>
47 26
48
49#define RNG_MODULE_NAME "hw_random" 27#define RNG_MODULE_NAME "hw_random"
50#define PFX RNG_MODULE_NAME ": "
51#define RNG_MISCDEV_MINOR 183 /* official */
52
53 28
54static struct hwrng *current_rng; 29static struct hwrng *current_rng;
55static struct task_struct *hwrng_fill; 30static struct task_struct *hwrng_fill;
@@ -296,7 +271,6 @@ out_put:
296 goto out; 271 goto out;
297} 272}
298 273
299
300static const struct file_operations rng_chrdev_ops = { 274static const struct file_operations rng_chrdev_ops = {
301 .owner = THIS_MODULE, 275 .owner = THIS_MODULE,
302 .open = rng_dev_open, 276 .open = rng_dev_open,
@@ -307,14 +281,13 @@ static const struct file_operations rng_chrdev_ops = {
307static const struct attribute_group *rng_dev_groups[]; 281static const struct attribute_group *rng_dev_groups[];
308 282
309static struct miscdevice rng_miscdev = { 283static struct miscdevice rng_miscdev = {
310 .minor = RNG_MISCDEV_MINOR, 284 .minor = HWRNG_MINOR,
311 .name = RNG_MODULE_NAME, 285 .name = RNG_MODULE_NAME,
312 .nodename = "hwrng", 286 .nodename = "hwrng",
313 .fops = &rng_chrdev_ops, 287 .fops = &rng_chrdev_ops,
314 .groups = rng_dev_groups, 288 .groups = rng_dev_groups,
315}; 289};
316 290
317
318static ssize_t hwrng_attr_current_store(struct device *dev, 291static ssize_t hwrng_attr_current_store(struct device *dev,
319 struct device_attribute *attr, 292 struct device_attribute *attr,
320 const char *buf, size_t len) 293 const char *buf, size_t len)
@@ -444,8 +417,7 @@ int hwrng_register(struct hwrng *rng)
444 int err = -EINVAL; 417 int err = -EINVAL;
445 struct hwrng *old_rng, *tmp; 418 struct hwrng *old_rng, *tmp;
446 419
447 if (rng->name == NULL || 420 if (!rng->name || (!rng->data_read && !rng->read))
448 (rng->data_read == NULL && rng->read == NULL))
449 goto out; 421 goto out;
450 422
451 mutex_lock(&rng_mutex); 423 mutex_lock(&rng_mutex);
diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c
index 3b06c1d6cfb2..31cbdbbaebfc 100644
--- a/drivers/char/hw_random/n2-drv.c
+++ b/drivers/char/hw_random/n2-drv.c
@@ -21,11 +21,11 @@
21 21
22#define DRV_MODULE_NAME "n2rng" 22#define DRV_MODULE_NAME "n2rng"
23#define PFX DRV_MODULE_NAME ": " 23#define PFX DRV_MODULE_NAME ": "
24#define DRV_MODULE_VERSION "0.2" 24#define DRV_MODULE_VERSION "0.3"
25#define DRV_MODULE_RELDATE "July 27, 2011" 25#define DRV_MODULE_RELDATE "Jan 7, 2017"
26 26
27static char version[] = 27static char version[] =
28 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 28 DRV_MODULE_NAME " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
29 29
30MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 30MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
31MODULE_DESCRIPTION("Niagara2 RNG driver"); 31MODULE_DESCRIPTION("Niagara2 RNG driver");
@@ -302,26 +302,57 @@ static int n2rng_try_read_ctl(struct n2rng *np)
302 return n2rng_hv_err_trans(hv_err); 302 return n2rng_hv_err_trans(hv_err);
303} 303}
304 304
305#define CONTROL_DEFAULT_BASE \ 305static u64 n2rng_control_default(struct n2rng *np, int ctl)
306 ((2 << RNG_CTL_ASEL_SHIFT) | \ 306{
307 (N2RNG_ACCUM_CYCLES_DEFAULT << RNG_CTL_WAIT_SHIFT) | \ 307 u64 val = 0;
308 RNG_CTL_LFSR) 308
309 309 if (np->data->chip_version == 1) {
310#define CONTROL_DEFAULT_0 \ 310 val = ((2 << RNG_v1_CTL_ASEL_SHIFT) |
311 (CONTROL_DEFAULT_BASE | \ 311 (N2RNG_ACCUM_CYCLES_DEFAULT << RNG_v1_CTL_WAIT_SHIFT) |
312 (1 << RNG_CTL_VCO_SHIFT) | \ 312 RNG_CTL_LFSR);
313 RNG_CTL_ES1) 313
314#define CONTROL_DEFAULT_1 \ 314 switch (ctl) {
315 (CONTROL_DEFAULT_BASE | \ 315 case 0:
316 (2 << RNG_CTL_VCO_SHIFT) | \ 316 val |= (1 << RNG_v1_CTL_VCO_SHIFT) | RNG_CTL_ES1;
317 RNG_CTL_ES2) 317 break;
318#define CONTROL_DEFAULT_2 \ 318 case 1:
319 (CONTROL_DEFAULT_BASE | \ 319 val |= (2 << RNG_v1_CTL_VCO_SHIFT) | RNG_CTL_ES2;
320 (3 << RNG_CTL_VCO_SHIFT) | \ 320 break;
321 RNG_CTL_ES3) 321 case 2:
322#define CONTROL_DEFAULT_3 \ 322 val |= (3 << RNG_v1_CTL_VCO_SHIFT) | RNG_CTL_ES3;
323 (CONTROL_DEFAULT_BASE | \ 323 break;
324 RNG_CTL_ES1 | RNG_CTL_ES2 | RNG_CTL_ES3) 324 case 3:
325 val |= RNG_CTL_ES1 | RNG_CTL_ES2 | RNG_CTL_ES3;
326 break;
327 default:
328 break;
329 }
330
331 } else {
332 val = ((2 << RNG_v2_CTL_ASEL_SHIFT) |
333 (N2RNG_ACCUM_CYCLES_DEFAULT << RNG_v2_CTL_WAIT_SHIFT) |
334 RNG_CTL_LFSR);
335
336 switch (ctl) {
337 case 0:
338 val |= (1 << RNG_v2_CTL_VCO_SHIFT) | RNG_CTL_ES1;
339 break;
340 case 1:
341 val |= (2 << RNG_v2_CTL_VCO_SHIFT) | RNG_CTL_ES2;
342 break;
343 case 2:
344 val |= (3 << RNG_v2_CTL_VCO_SHIFT) | RNG_CTL_ES3;
345 break;
346 case 3:
347 val |= RNG_CTL_ES1 | RNG_CTL_ES2 | RNG_CTL_ES3;
348 break;
349 default:
350 break;
351 }
352 }
353
354 return val;
355}
325 356
326static void n2rng_control_swstate_init(struct n2rng *np) 357static void n2rng_control_swstate_init(struct n2rng *np)
327{ 358{
@@ -336,10 +367,10 @@ static void n2rng_control_swstate_init(struct n2rng *np)
336 for (i = 0; i < np->num_units; i++) { 367 for (i = 0; i < np->num_units; i++) {
337 struct n2rng_unit *up = &np->units[i]; 368 struct n2rng_unit *up = &np->units[i];
338 369
339 up->control[0] = CONTROL_DEFAULT_0; 370 up->control[0] = n2rng_control_default(np, 0);
340 up->control[1] = CONTROL_DEFAULT_1; 371 up->control[1] = n2rng_control_default(np, 1);
341 up->control[2] = CONTROL_DEFAULT_2; 372 up->control[2] = n2rng_control_default(np, 2);
342 up->control[3] = CONTROL_DEFAULT_3; 373 up->control[3] = n2rng_control_default(np, 3);
343 } 374 }
344 375
345 np->hv_state = HV_RNG_STATE_UNCONFIGURED; 376 np->hv_state = HV_RNG_STATE_UNCONFIGURED;
@@ -399,6 +430,7 @@ static int n2rng_data_read(struct hwrng *rng, u32 *data)
399 } else { 430 } else {
400 int err = n2rng_generic_read_data(ra); 431 int err = n2rng_generic_read_data(ra);
401 if (!err) { 432 if (!err) {
433 np->flags |= N2RNG_FLAG_BUFFER_VALID;
402 np->buffer = np->test_data >> 32; 434 np->buffer = np->test_data >> 32;
403 *data = np->test_data & 0xffffffff; 435 *data = np->test_data & 0xffffffff;
404 len = 4; 436 len = 4;
@@ -487,9 +519,21 @@ static void n2rng_dump_test_buffer(struct n2rng *np)
487 519
488static int n2rng_check_selftest_buffer(struct n2rng *np, unsigned long unit) 520static int n2rng_check_selftest_buffer(struct n2rng *np, unsigned long unit)
489{ 521{
490 u64 val = SELFTEST_VAL; 522 u64 val;
491 int err, matches, limit; 523 int err, matches, limit;
492 524
525 switch (np->data->id) {
526 case N2_n2_rng:
527 case N2_vf_rng:
528 case N2_kt_rng:
529 case N2_m4_rng: /* yes, m4 uses the old value */
530 val = RNG_v1_SELFTEST_VAL;
531 break;
532 default:
533 val = RNG_v2_SELFTEST_VAL;
534 break;
535 }
536
493 matches = 0; 537 matches = 0;
494 for (limit = 0; limit < SELFTEST_LOOPS_MAX; limit++) { 538 for (limit = 0; limit < SELFTEST_LOOPS_MAX; limit++) {
495 matches += n2rng_test_buffer_find(np, val); 539 matches += n2rng_test_buffer_find(np, val);
@@ -512,14 +556,32 @@ static int n2rng_check_selftest_buffer(struct n2rng *np, unsigned long unit)
512static int n2rng_control_selftest(struct n2rng *np, unsigned long unit) 556static int n2rng_control_selftest(struct n2rng *np, unsigned long unit)
513{ 557{
514 int err; 558 int err;
559 u64 base, base3;
560
561 switch (np->data->id) {
562 case N2_n2_rng:
563 case N2_vf_rng:
564 case N2_kt_rng:
565 base = RNG_v1_CTL_ASEL_NOOUT << RNG_v1_CTL_ASEL_SHIFT;
566 base3 = base | RNG_CTL_LFSR |
567 ((RNG_v1_SELFTEST_TICKS - 2) << RNG_v1_CTL_WAIT_SHIFT);
568 break;
569 case N2_m4_rng:
570 base = RNG_v2_CTL_ASEL_NOOUT << RNG_v2_CTL_ASEL_SHIFT;
571 base3 = base | RNG_CTL_LFSR |
572 ((RNG_v1_SELFTEST_TICKS - 2) << RNG_v2_CTL_WAIT_SHIFT);
573 break;
574 default:
575 base = RNG_v2_CTL_ASEL_NOOUT << RNG_v2_CTL_ASEL_SHIFT;
576 base3 = base | RNG_CTL_LFSR |
577 (RNG_v2_SELFTEST_TICKS << RNG_v2_CTL_WAIT_SHIFT);
578 break;
579 }
515 580
516 np->test_control[0] = (0x2 << RNG_CTL_ASEL_SHIFT); 581 np->test_control[0] = base;
517 np->test_control[1] = (0x2 << RNG_CTL_ASEL_SHIFT); 582 np->test_control[1] = base;
518 np->test_control[2] = (0x2 << RNG_CTL_ASEL_SHIFT); 583 np->test_control[2] = base;
519 np->test_control[3] = ((0x2 << RNG_CTL_ASEL_SHIFT) | 584 np->test_control[3] = base3;
520 RNG_CTL_LFSR |
521 ((SELFTEST_TICKS - 2) << RNG_CTL_WAIT_SHIFT));
522
523 585
524 err = n2rng_entropy_diag_read(np, unit, np->test_control, 586 err = n2rng_entropy_diag_read(np, unit, np->test_control,
525 HV_RNG_STATE_HEALTHCHECK, 587 HV_RNG_STATE_HEALTHCHECK,
@@ -557,11 +619,19 @@ static int n2rng_control_configure_units(struct n2rng *np)
557 struct n2rng_unit *up = &np->units[unit]; 619 struct n2rng_unit *up = &np->units[unit];
558 unsigned long ctl_ra = __pa(&up->control[0]); 620 unsigned long ctl_ra = __pa(&up->control[0]);
559 int esrc; 621 int esrc;
560 u64 base; 622 u64 base, shift;
561 623
562 base = ((np->accum_cycles << RNG_CTL_WAIT_SHIFT) | 624 if (np->data->chip_version == 1) {
563 (2 << RNG_CTL_ASEL_SHIFT) | 625 base = ((np->accum_cycles << RNG_v1_CTL_WAIT_SHIFT) |
564 RNG_CTL_LFSR); 626 (RNG_v1_CTL_ASEL_NOOUT << RNG_v1_CTL_ASEL_SHIFT) |
627 RNG_CTL_LFSR);
628 shift = RNG_v1_CTL_VCO_SHIFT;
629 } else {
630 base = ((np->accum_cycles << RNG_v2_CTL_WAIT_SHIFT) |
631 (RNG_v2_CTL_ASEL_NOOUT << RNG_v2_CTL_ASEL_SHIFT) |
632 RNG_CTL_LFSR);
633 shift = RNG_v2_CTL_VCO_SHIFT;
634 }
565 635
566 /* XXX This isn't the best. We should fetch a bunch 636 /* XXX This isn't the best. We should fetch a bunch
567 * XXX of words using each entropy source combined XXX 637 * XXX of words using each entropy source combined XXX
@@ -570,7 +640,7 @@ static int n2rng_control_configure_units(struct n2rng *np)
570 */ 640 */
571 for (esrc = 0; esrc < 3; esrc++) 641 for (esrc = 0; esrc < 3; esrc++)
572 up->control[esrc] = base | 642 up->control[esrc] = base |
573 (esrc << RNG_CTL_VCO_SHIFT) | 643 (esrc << shift) |
574 (RNG_CTL_ES1 << esrc); 644 (RNG_CTL_ES1 << esrc);
575 645
576 up->control[3] = base | 646 up->control[3] = base |
@@ -589,6 +659,7 @@ static void n2rng_work(struct work_struct *work)
589{ 659{
590 struct n2rng *np = container_of(work, struct n2rng, work.work); 660 struct n2rng *np = container_of(work, struct n2rng, work.work);
591 int err = 0; 661 int err = 0;
662 static int retries = 4;
592 663
593 if (!(np->flags & N2RNG_FLAG_CONTROL)) { 664 if (!(np->flags & N2RNG_FLAG_CONTROL)) {
594 err = n2rng_guest_check(np); 665 err = n2rng_guest_check(np);
@@ -606,7 +677,9 @@ static void n2rng_work(struct work_struct *work)
606 dev_info(&np->op->dev, "RNG ready\n"); 677 dev_info(&np->op->dev, "RNG ready\n");
607 } 678 }
608 679
609 if (err && !(np->flags & N2RNG_FLAG_SHUTDOWN)) 680 if (--retries == 0)
681 dev_err(&np->op->dev, "Self-test retries failed, RNG not ready\n");
682 else if (err && !(np->flags & N2RNG_FLAG_SHUTDOWN))
610 schedule_delayed_work(&np->work, HZ * 2); 683 schedule_delayed_work(&np->work, HZ * 2);
611} 684}
612 685
@@ -622,24 +695,23 @@ static const struct of_device_id n2rng_match[];
622static int n2rng_probe(struct platform_device *op) 695static int n2rng_probe(struct platform_device *op)
623{ 696{
624 const struct of_device_id *match; 697 const struct of_device_id *match;
625 int multi_capable;
626 int err = -ENOMEM; 698 int err = -ENOMEM;
627 struct n2rng *np; 699 struct n2rng *np;
628 700
629 match = of_match_device(n2rng_match, &op->dev); 701 match = of_match_device(n2rng_match, &op->dev);
630 if (!match) 702 if (!match)
631 return -EINVAL; 703 return -EINVAL;
632 multi_capable = (match->data != NULL);
633 704
634 n2rng_driver_version(); 705 n2rng_driver_version();
635 np = devm_kzalloc(&op->dev, sizeof(*np), GFP_KERNEL); 706 np = devm_kzalloc(&op->dev, sizeof(*np), GFP_KERNEL);
636 if (!np) 707 if (!np)
637 goto out; 708 goto out;
638 np->op = op; 709 np->op = op;
710 np->data = (struct n2rng_template *)match->data;
639 711
640 INIT_DELAYED_WORK(&np->work, n2rng_work); 712 INIT_DELAYED_WORK(&np->work, n2rng_work);
641 713
642 if (multi_capable) 714 if (np->data->multi_capable)
643 np->flags |= N2RNG_FLAG_MULTI; 715 np->flags |= N2RNG_FLAG_MULTI;
644 716
645 err = -ENODEV; 717 err = -ENODEV;
@@ -670,8 +742,9 @@ static int n2rng_probe(struct platform_device *op)
670 dev_err(&op->dev, "VF RNG lacks rng-#units property\n"); 742 dev_err(&op->dev, "VF RNG lacks rng-#units property\n");
671 goto out_hvapi_unregister; 743 goto out_hvapi_unregister;
672 } 744 }
673 } else 745 } else {
674 np->num_units = 1; 746 np->num_units = 1;
747 }
675 748
676 dev_info(&op->dev, "Registered RNG HVAPI major %lu minor %lu\n", 749 dev_info(&op->dev, "Registered RNG HVAPI major %lu minor %lu\n",
677 np->hvapi_major, np->hvapi_minor); 750 np->hvapi_major, np->hvapi_minor);
@@ -692,7 +765,7 @@ static int n2rng_probe(struct platform_device *op)
692 "multi-unit-capable" : "single-unit"), 765 "multi-unit-capable" : "single-unit"),
693 np->num_units); 766 np->num_units);
694 767
695 np->hwrng.name = "n2rng"; 768 np->hwrng.name = DRV_MODULE_NAME;
696 np->hwrng.data_read = n2rng_data_read; 769 np->hwrng.data_read = n2rng_data_read;
697 np->hwrng.priv = (unsigned long) np; 770 np->hwrng.priv = (unsigned long) np;
698 771
@@ -728,30 +801,61 @@ static int n2rng_remove(struct platform_device *op)
728 return 0; 801 return 0;
729} 802}
730 803
804static struct n2rng_template n2_template = {
805 .id = N2_n2_rng,
806 .multi_capable = 0,
807 .chip_version = 1,
808};
809
810static struct n2rng_template vf_template = {
811 .id = N2_vf_rng,
812 .multi_capable = 1,
813 .chip_version = 1,
814};
815
816static struct n2rng_template kt_template = {
817 .id = N2_kt_rng,
818 .multi_capable = 1,
819 .chip_version = 1,
820};
821
822static struct n2rng_template m4_template = {
823 .id = N2_m4_rng,
824 .multi_capable = 1,
825 .chip_version = 2,
826};
827
828static struct n2rng_template m7_template = {
829 .id = N2_m7_rng,
830 .multi_capable = 1,
831 .chip_version = 2,
832};
833
731static const struct of_device_id n2rng_match[] = { 834static const struct of_device_id n2rng_match[] = {
732 { 835 {
733 .name = "random-number-generator", 836 .name = "random-number-generator",
734 .compatible = "SUNW,n2-rng", 837 .compatible = "SUNW,n2-rng",
838 .data = &n2_template,
735 }, 839 },
736 { 840 {
737 .name = "random-number-generator", 841 .name = "random-number-generator",
738 .compatible = "SUNW,vf-rng", 842 .compatible = "SUNW,vf-rng",
739 .data = (void *) 1, 843 .data = &vf_template,
740 }, 844 },
741 { 845 {
742 .name = "random-number-generator", 846 .name = "random-number-generator",
743 .compatible = "SUNW,kt-rng", 847 .compatible = "SUNW,kt-rng",
744 .data = (void *) 1, 848 .data = &kt_template,
745 }, 849 },
746 { 850 {
747 .name = "random-number-generator", 851 .name = "random-number-generator",
748 .compatible = "ORCL,m4-rng", 852 .compatible = "ORCL,m4-rng",
749 .data = (void *) 1, 853 .data = &m4_template,
750 }, 854 },
751 { 855 {
752 .name = "random-number-generator", 856 .name = "random-number-generator",
753 .compatible = "ORCL,m7-rng", 857 .compatible = "ORCL,m7-rng",
754 .data = (void *) 1, 858 .data = &m7_template,
755 }, 859 },
756 {}, 860 {},
757}; 861};
diff --git a/drivers/char/hw_random/n2rng.h b/drivers/char/hw_random/n2rng.h
index f244ac89087f..6bad6cc634e8 100644
--- a/drivers/char/hw_random/n2rng.h
+++ b/drivers/char/hw_random/n2rng.h
@@ -6,18 +6,34 @@
6#ifndef _N2RNG_H 6#ifndef _N2RNG_H
7#define _N2RNG_H 7#define _N2RNG_H
8 8
9#define RNG_CTL_WAIT 0x0000000001fffe00ULL /* Minimum wait time */ 9/* ver1 devices - n2-rng, vf-rng, kt-rng */
10#define RNG_CTL_WAIT_SHIFT 9 10#define RNG_v1_CTL_WAIT 0x0000000001fffe00ULL /* Minimum wait time */
11#define RNG_CTL_BYPASS 0x0000000000000100ULL /* VCO voltage source */ 11#define RNG_v1_CTL_WAIT_SHIFT 9
12#define RNG_CTL_VCO 0x00000000000000c0ULL /* VCO rate control */ 12#define RNG_v1_CTL_BYPASS 0x0000000000000100ULL /* VCO voltage source */
13#define RNG_CTL_VCO_SHIFT 6 13#define RNG_v1_CTL_VCO 0x00000000000000c0ULL /* VCO rate control */
14#define RNG_CTL_ASEL 0x0000000000000030ULL /* Analog MUX select */ 14#define RNG_v1_CTL_VCO_SHIFT 6
15#define RNG_CTL_ASEL_SHIFT 4 15#define RNG_v1_CTL_ASEL 0x0000000000000030ULL /* Analog MUX select */
16#define RNG_v1_CTL_ASEL_SHIFT 4
17#define RNG_v1_CTL_ASEL_NOOUT 2
18
19/* these are the same in v2 as in v1 */
16#define RNG_CTL_LFSR 0x0000000000000008ULL /* Use LFSR or plain shift */ 20#define RNG_CTL_LFSR 0x0000000000000008ULL /* Use LFSR or plain shift */
17#define RNG_CTL_ES3 0x0000000000000004ULL /* Enable entropy source 3 */ 21#define RNG_CTL_ES3 0x0000000000000004ULL /* Enable entropy source 3 */
18#define RNG_CTL_ES2 0x0000000000000002ULL /* Enable entropy source 2 */ 22#define RNG_CTL_ES2 0x0000000000000002ULL /* Enable entropy source 2 */
19#define RNG_CTL_ES1 0x0000000000000001ULL /* Enable entropy source 1 */ 23#define RNG_CTL_ES1 0x0000000000000001ULL /* Enable entropy source 1 */
20 24
25/* ver2 devices - m4-rng, m7-rng */
26#define RNG_v2_CTL_WAIT 0x0000000007fff800ULL /* Minimum wait time */
27#define RNG_v2_CTL_WAIT_SHIFT 12
28#define RNG_v2_CTL_BYPASS 0x0000000000000400ULL /* VCO voltage source */
29#define RNG_v2_CTL_VCO 0x0000000000000300ULL /* VCO rate control */
30#define RNG_v2_CTL_VCO_SHIFT 9
31#define RNG_v2_CTL_PERF 0x0000000000000180ULL /* Perf */
32#define RNG_v2_CTL_ASEL 0x0000000000000070ULL /* Analog MUX select */
33#define RNG_v2_CTL_ASEL_SHIFT 4
34#define RNG_v2_CTL_ASEL_NOOUT 7
35
36
21#define HV_FAST_RNG_GET_DIAG_CTL 0x130 37#define HV_FAST_RNG_GET_DIAG_CTL 0x130
22#define HV_FAST_RNG_CTL_READ 0x131 38#define HV_FAST_RNG_CTL_READ 0x131
23#define HV_FAST_RNG_CTL_WRITE 0x132 39#define HV_FAST_RNG_CTL_WRITE 0x132
@@ -60,6 +76,20 @@ extern unsigned long sun4v_rng_data_read_diag_v2(unsigned long data_ra,
60extern unsigned long sun4v_rng_data_read(unsigned long data_ra, 76extern unsigned long sun4v_rng_data_read(unsigned long data_ra,
61 unsigned long *tick_delta); 77 unsigned long *tick_delta);
62 78
79enum n2rng_compat_id {
80 N2_n2_rng,
81 N2_vf_rng,
82 N2_kt_rng,
83 N2_m4_rng,
84 N2_m7_rng,
85};
86
87struct n2rng_template {
88 enum n2rng_compat_id id;
89 int multi_capable;
90 int chip_version;
91};
92
63struct n2rng_unit { 93struct n2rng_unit {
64 u64 control[HV_RNG_NUM_CONTROL]; 94 u64 control[HV_RNG_NUM_CONTROL];
65}; 95};
@@ -74,6 +104,7 @@ struct n2rng {
74#define N2RNG_FLAG_SHUTDOWN 0x00000010 /* Driver unregistering */ 104#define N2RNG_FLAG_SHUTDOWN 0x00000010 /* Driver unregistering */
75#define N2RNG_FLAG_BUFFER_VALID 0x00000020 /* u32 buffer holds valid data */ 105#define N2RNG_FLAG_BUFFER_VALID 0x00000020 /* u32 buffer holds valid data */
76 106
107 struct n2rng_template *data;
77 int num_units; 108 int num_units;
78 struct n2rng_unit *units; 109 struct n2rng_unit *units;
79 110
@@ -97,8 +128,10 @@ struct n2rng {
97 128
98 u64 scratch_control[HV_RNG_NUM_CONTROL]; 129 u64 scratch_control[HV_RNG_NUM_CONTROL];
99 130
100#define SELFTEST_TICKS 38859 131#define RNG_v1_SELFTEST_TICKS 38859
101#define SELFTEST_VAL ((u64)0xB8820C7BD387E32C) 132#define RNG_v1_SELFTEST_VAL ((u64)0xB8820C7BD387E32C)
133#define RNG_v2_SELFTEST_TICKS 64
134#define RNG_v2_SELFTEST_VAL ((u64)0xffffffffffffffff)
102#define SELFTEST_POLY ((u64)0x231DCEE91262B8A3) 135#define SELFTEST_POLY ((u64)0x231DCEE91262B8A3)
103#define SELFTEST_MATCH_GOAL 6 136#define SELFTEST_MATCH_GOAL 6
104#define SELFTEST_LOOPS_MAX 40000 137#define SELFTEST_LOOPS_MAX 40000
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 79564785ae30..2cac445b02fd 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -339,7 +339,7 @@ config CRYPTO_DEV_OMAP_DES
339 339
340config CRYPTO_DEV_PICOXCELL 340config CRYPTO_DEV_PICOXCELL
341 tristate "Support for picoXcell IPSEC and Layer2 crypto engines" 341 tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
342 depends on ARCH_PICOXCELL && HAVE_CLK 342 depends on (ARCH_PICOXCELL || COMPILE_TEST) && HAVE_CLK
343 select CRYPTO_AEAD 343 select CRYPTO_AEAD
344 select CRYPTO_AES 344 select CRYPTO_AES
345 select CRYPTO_AUTHENC 345 select CRYPTO_AUTHENC
@@ -415,10 +415,23 @@ config CRYPTO_DEV_BFIN_CRC
415 Newer Blackfin processors have CRC hardware. Select this if you 415 Newer Blackfin processors have CRC hardware. Select this if you
416 want to use the Blackfin CRC module. 416 want to use the Blackfin CRC module.
417 417
418config CRYPTO_DEV_ATMEL_AUTHENC
419 tristate "Support for Atmel IPSEC/SSL hw accelerator"
420 depends on HAS_DMA
421 depends on ARCH_AT91 || COMPILE_TEST
422 select CRYPTO_AUTHENC
423 select CRYPTO_DEV_ATMEL_AES
424 select CRYPTO_DEV_ATMEL_SHA
425 help
426 Some Atmel processors can combine the AES and SHA hw accelerators
427 to enhance support of IPSEC/SSL.
428 Select this if you want to use the Atmel modules for
429 authenc(hmac(shaX),Y(cbc)) algorithms.
430
418config CRYPTO_DEV_ATMEL_AES 431config CRYPTO_DEV_ATMEL_AES
419 tristate "Support for Atmel AES hw accelerator" 432 tristate "Support for Atmel AES hw accelerator"
420 depends on HAS_DMA 433 depends on HAS_DMA
421 depends on AT_XDMAC || AT_HDMAC || COMPILE_TEST 434 depends on ARCH_AT91 || COMPILE_TEST
422 select CRYPTO_AES 435 select CRYPTO_AES
423 select CRYPTO_AEAD 436 select CRYPTO_AEAD
424 select CRYPTO_BLKCIPHER 437 select CRYPTO_BLKCIPHER
@@ -432,7 +445,7 @@ config CRYPTO_DEV_ATMEL_AES
432 445
433config CRYPTO_DEV_ATMEL_TDES 446config CRYPTO_DEV_ATMEL_TDES
434 tristate "Support for Atmel DES/TDES hw accelerator" 447 tristate "Support for Atmel DES/TDES hw accelerator"
435 depends on ARCH_AT91 448 depends on ARCH_AT91 || COMPILE_TEST
436 select CRYPTO_DES 449 select CRYPTO_DES
437 select CRYPTO_BLKCIPHER 450 select CRYPTO_BLKCIPHER
438 help 451 help
@@ -445,7 +458,7 @@ config CRYPTO_DEV_ATMEL_TDES
445 458
446config CRYPTO_DEV_ATMEL_SHA 459config CRYPTO_DEV_ATMEL_SHA
447 tristate "Support for Atmel SHA hw accelerator" 460 tristate "Support for Atmel SHA hw accelerator"
448 depends on ARCH_AT91 461 depends on ARCH_AT91 || COMPILE_TEST
449 select CRYPTO_HASH 462 select CRYPTO_HASH
450 help 463 help
451 Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512 464 Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512
@@ -484,6 +497,7 @@ config CRYPTO_DEV_MXS_DCP
484 will be called mxs-dcp. 497 will be called mxs-dcp.
485 498
486source "drivers/crypto/qat/Kconfig" 499source "drivers/crypto/qat/Kconfig"
500source "drivers/crypto/cavium/cpt/Kconfig"
487 501
488config CRYPTO_DEV_QCE 502config CRYPTO_DEV_QCE
489 tristate "Qualcomm crypto engine accelerator" 503 tristate "Qualcomm crypto engine accelerator"
@@ -553,8 +567,39 @@ config CRYPTO_DEV_ROCKCHIP
553 This driver interfaces with the hardware crypto accelerator. 567 This driver interfaces with the hardware crypto accelerator.
554 Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode. 568 Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode.
555 569
570config CRYPTO_DEV_MEDIATEK
571 tristate "MediaTek's EIP97 Cryptographic Engine driver"
572 depends on (ARM && ARCH_MEDIATEK) || COMPILE_TEST
573 select CRYPTO_AES
574 select CRYPTO_AEAD
575 select CRYPTO_BLKCIPHER
576 select CRYPTO_CTR
577 select CRYPTO_SHA1
578 select CRYPTO_SHA256
579 select CRYPTO_SHA512
580 select CRYPTO_HMAC
581 help
582 This driver allows you to utilize the hardware crypto accelerator
583 EIP97 which can be found on the MT7623 MT2701, MT8521p, etc ....
584 Select this if you want to use it for AES/SHA1/SHA2 algorithms.
585
556source "drivers/crypto/chelsio/Kconfig" 586source "drivers/crypto/chelsio/Kconfig"
557 587
558source "drivers/crypto/virtio/Kconfig" 588source "drivers/crypto/virtio/Kconfig"
559 589
590config CRYPTO_DEV_BCM_SPU
591 tristate "Broadcom symmetric crypto/hash acceleration support"
592 depends on ARCH_BCM_IPROC
593 depends on BCM_PDC_MBOX
594 default m
595 select CRYPTO_DES
596 select CRYPTO_MD5
597 select CRYPTO_SHA1
598 select CRYPTO_SHA256
599 select CRYPTO_SHA512
600 help
601 This driver provides support for Broadcom crypto acceleration using the
602 Secure Processing Unit (SPU). The SPU driver registers ablkcipher,
603 ahash, and aead algorithms with the kernel cryptographic API.
604
560endif # CRYPTO_HW 605endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index bc53cb833a06..739609471169 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -3,6 +3,8 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
3obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o 3obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
4obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o 4obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o
5obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ 5obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
6obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
7obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/
6obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ 8obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
7obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o 9obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
8obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o 10obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
@@ -10,7 +12,9 @@ obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
10obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o 12obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
11obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o 13obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
12obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell/ 14obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell/
15obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mediatek/
13obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o 16obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o
17obj-$(CONFIG_CRYPTO_DEV_MXC_SCC) += mxc-scc.o
14obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o 18obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
15n2_crypto-y := n2_core.o n2_asm.o 19n2_crypto-y := n2_core.o n2_asm.o
16obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ 20obj-$(CONFIG_CRYPTO_DEV_NX) += nx/
@@ -21,15 +25,14 @@ obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
21obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o 25obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
22obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o 26obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o
23obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ 27obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
28obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
29obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
30obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
24obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o 31obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
25obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o 32obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
26obj-$(CONFIG_CRYPTO_DEV_MXC_SCC) += mxc-scc.o 33obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
27obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o 34obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
28obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ 35obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
29obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
30obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
31obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
32obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
33obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
34obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
35obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ 36obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
37obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
38obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
diff --git a/drivers/crypto/atmel-aes-regs.h b/drivers/crypto/atmel-aes-regs.h
index 0ec04407b533..7694679802b3 100644
--- a/drivers/crypto/atmel-aes-regs.h
+++ b/drivers/crypto/atmel-aes-regs.h
@@ -68,6 +68,22 @@
68#define AES_CTRR 0x98 68#define AES_CTRR 0x98
69#define AES_GCMHR(x) (0x9c + ((x) * 0x04)) 69#define AES_GCMHR(x) (0x9c + ((x) * 0x04))
70 70
71#define AES_EMR 0xb0
72#define AES_EMR_APEN BIT(0) /* Auto Padding Enable */
73#define AES_EMR_APM BIT(1) /* Auto Padding Mode */
74#define AES_EMR_APM_IPSEC 0x0
75#define AES_EMR_APM_SSL BIT(1)
76#define AES_EMR_PLIPEN BIT(4) /* PLIP Enable */
77#define AES_EMR_PLIPD BIT(5) /* PLIP Decipher */
78#define AES_EMR_PADLEN_MASK (0xFu << 8)
79#define AES_EMR_PADLEN_OFFSET 8
80#define AES_EMR_PADLEN(padlen) (((padlen) << AES_EMR_PADLEN_OFFSET) &\
81 AES_EMR_PADLEN_MASK)
82#define AES_EMR_NHEAD_MASK (0xFu << 16)
83#define AES_EMR_NHEAD_OFFSET 16
84#define AES_EMR_NHEAD(nhead) (((nhead) << AES_EMR_NHEAD_OFFSET) &\
85 AES_EMR_NHEAD_MASK)
86
71#define AES_TWR(x) (0xc0 + ((x) * 0x04)) 87#define AES_TWR(x) (0xc0 + ((x) * 0x04))
72#define AES_ALPHAR(x) (0xd0 + ((x) * 0x04)) 88#define AES_ALPHAR(x) (0xd0 + ((x) * 0x04))
73 89
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 0e3d0d655b96..29e20c37f3a6 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -41,6 +41,7 @@
41#include <linux/platform_data/crypto-atmel.h> 41#include <linux/platform_data/crypto-atmel.h>
42#include <dt-bindings/dma/at91.h> 42#include <dt-bindings/dma/at91.h>
43#include "atmel-aes-regs.h" 43#include "atmel-aes-regs.h"
44#include "atmel-authenc.h"
44 45
45#define ATMEL_AES_PRIORITY 300 46#define ATMEL_AES_PRIORITY 300
46 47
@@ -78,6 +79,7 @@
78#define AES_FLAGS_INIT BIT(2) 79#define AES_FLAGS_INIT BIT(2)
79#define AES_FLAGS_BUSY BIT(3) 80#define AES_FLAGS_BUSY BIT(3)
80#define AES_FLAGS_DUMP_REG BIT(4) 81#define AES_FLAGS_DUMP_REG BIT(4)
82#define AES_FLAGS_OWN_SHA BIT(5)
81 83
82#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY) 84#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY)
83 85
@@ -92,6 +94,7 @@ struct atmel_aes_caps {
92 bool has_ctr32; 94 bool has_ctr32;
93 bool has_gcm; 95 bool has_gcm;
94 bool has_xts; 96 bool has_xts;
97 bool has_authenc;
95 u32 max_burst_size; 98 u32 max_burst_size;
96}; 99};
97 100
@@ -144,10 +147,31 @@ struct atmel_aes_xts_ctx {
144 u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; 147 u32 key2[AES_KEYSIZE_256 / sizeof(u32)];
145}; 148};
146 149
150#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
151struct atmel_aes_authenc_ctx {
152 struct atmel_aes_base_ctx base;
153 struct atmel_sha_authenc_ctx *auth;
154};
155#endif
156
147struct atmel_aes_reqctx { 157struct atmel_aes_reqctx {
148 unsigned long mode; 158 unsigned long mode;
149}; 159};
150 160
161#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
162struct atmel_aes_authenc_reqctx {
163 struct atmel_aes_reqctx base;
164
165 struct scatterlist src[2];
166 struct scatterlist dst[2];
167 size_t textlen;
168 u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)];
169
170 /* auth_req MUST be place last. */
171 struct ahash_request auth_req;
172};
173#endif
174
151struct atmel_aes_dma { 175struct atmel_aes_dma {
152 struct dma_chan *chan; 176 struct dma_chan *chan;
153 struct scatterlist *sg; 177 struct scatterlist *sg;
@@ -291,6 +315,9 @@ static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
291 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); 315 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
292 break; 316 break;
293 317
318 case AES_EMR:
319 return "EMR";
320
294 case AES_TWR(0): 321 case AES_TWR(0):
295 case AES_TWR(1): 322 case AES_TWR(1):
296 case AES_TWR(2): 323 case AES_TWR(2):
@@ -463,8 +490,16 @@ static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)
463 return (dd->flags & AES_FLAGS_ENCRYPT); 490 return (dd->flags & AES_FLAGS_ENCRYPT);
464} 491}
465 492
493#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
494static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
495#endif
496
466static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) 497static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)
467{ 498{
499#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
500 atmel_aes_authenc_complete(dd, err);
501#endif
502
468 clk_disable(dd->iclk); 503 clk_disable(dd->iclk);
469 dd->flags &= ~AES_FLAGS_BUSY; 504 dd->flags &= ~AES_FLAGS_BUSY;
470 505
@@ -879,6 +914,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
879 struct crypto_async_request *areq, *backlog; 914 struct crypto_async_request *areq, *backlog;
880 struct atmel_aes_base_ctx *ctx; 915 struct atmel_aes_base_ctx *ctx;
881 unsigned long flags; 916 unsigned long flags;
917 bool start_async;
882 int err, ret = 0; 918 int err, ret = 0;
883 919
884 spin_lock_irqsave(&dd->lock, flags); 920 spin_lock_irqsave(&dd->lock, flags);
@@ -904,10 +940,12 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
904 940
905 dd->areq = areq; 941 dd->areq = areq;
906 dd->ctx = ctx; 942 dd->ctx = ctx;
907 dd->is_async = (areq != new_areq); 943 start_async = (areq != new_areq);
944 dd->is_async = start_async;
908 945
946 /* WARNING: ctx->start() MAY change dd->is_async. */
909 err = ctx->start(dd); 947 err = ctx->start(dd);
910 return (dd->is_async) ? ret : err; 948 return (start_async) ? ret : err;
911} 949}
912 950
913 951
@@ -1928,6 +1966,384 @@ static struct crypto_alg aes_xts_alg = {
1928 } 1966 }
1929}; 1967};
1930 1968
1969#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
1970/* authenc aead functions */
1971
1972static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
1973static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
1974 bool is_async);
1975static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
1976 bool is_async);
1977static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
1978static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
1979 bool is_async);
1980
1981static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)
1982{
1983 struct aead_request *req = aead_request_cast(dd->areq);
1984 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
1985
1986 if (err && (dd->flags & AES_FLAGS_OWN_SHA))
1987 atmel_sha_authenc_abort(&rctx->auth_req);
1988 dd->flags &= ~AES_FLAGS_OWN_SHA;
1989}
1990
1991static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)
1992{
1993 struct aead_request *req = aead_request_cast(dd->areq);
1994 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
1995 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1996 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
1997 int err;
1998
1999 atmel_aes_set_mode(dd, &rctx->base);
2000
2001 err = atmel_aes_hw_init(dd);
2002 if (err)
2003 return atmel_aes_complete(dd, err);
2004
2005 return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth,
2006 atmel_aes_authenc_init, dd);
2007}
2008
2009static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
2010 bool is_async)
2011{
2012 struct aead_request *req = aead_request_cast(dd->areq);
2013 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2014
2015 if (is_async)
2016 dd->is_async = true;
2017 if (err)
2018 return atmel_aes_complete(dd, err);
2019
2020 /* If here, we've got the ownership of the SHA device. */
2021 dd->flags |= AES_FLAGS_OWN_SHA;
2022
2023 /* Configure the SHA device. */
2024 return atmel_sha_authenc_init(&rctx->auth_req,
2025 req->src, req->assoclen,
2026 rctx->textlen,
2027 atmel_aes_authenc_transfer, dd);
2028}
2029
2030static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
2031 bool is_async)
2032{
2033 struct aead_request *req = aead_request_cast(dd->areq);
2034 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2035 bool enc = atmel_aes_is_encrypt(dd);
2036 struct scatterlist *src, *dst;
2037 u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
2038 u32 emr;
2039
2040 if (is_async)
2041 dd->is_async = true;
2042 if (err)
2043 return atmel_aes_complete(dd, err);
2044
2045 /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */
2046 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
2047 dst = src;
2048
2049 if (req->src != req->dst)
2050 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
2051
2052 /* Configure the AES device. */
2053 memcpy(iv, req->iv, sizeof(iv));
2054
2055 /*
2056 * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to
2057 * 'true' even if the data transfer is actually performed by the CPU (so
2058 * not by the DMA) because we must force the AES_MR_SMOD bitfield to the
2059 * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD
2060 * must be set to *_MR_SMOD_IDATAR0.
2061 */
2062 atmel_aes_write_ctrl(dd, true, iv);
2063 emr = AES_EMR_PLIPEN;
2064 if (!enc)
2065 emr |= AES_EMR_PLIPD;
2066 atmel_aes_write(dd, AES_EMR, emr);
2067
2068 /* Transfer data. */
2069 return atmel_aes_dma_start(dd, src, dst, rctx->textlen,
2070 atmel_aes_authenc_digest);
2071}
2072
2073static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)
2074{
2075 struct aead_request *req = aead_request_cast(dd->areq);
2076 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2077
2078 /* atmel_sha_authenc_final() releases the SHA device. */
2079 dd->flags &= ~AES_FLAGS_OWN_SHA;
2080 return atmel_sha_authenc_final(&rctx->auth_req,
2081 rctx->digest, sizeof(rctx->digest),
2082 atmel_aes_authenc_final, dd);
2083}
2084
2085static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
2086 bool is_async)
2087{
2088 struct aead_request *req = aead_request_cast(dd->areq);
2089 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2090 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2091 bool enc = atmel_aes_is_encrypt(dd);
2092 u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest;
2093 u32 offs, authsize;
2094
2095 if (is_async)
2096 dd->is_async = true;
2097 if (err)
2098 goto complete;
2099
2100 offs = req->assoclen + rctx->textlen;
2101 authsize = crypto_aead_authsize(tfm);
2102 if (enc) {
2103 scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1);
2104 } else {
2105 scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0);
2106 if (crypto_memneq(idigest, odigest, authsize))
2107 err = -EBADMSG;
2108 }
2109
2110complete:
2111 return atmel_aes_complete(dd, err);
2112}
2113
2114static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key,
2115 unsigned int keylen)
2116{
2117 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2118 struct crypto_authenc_keys keys;
2119 u32 flags;
2120 int err;
2121
2122 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
2123 goto badkey;
2124
2125 if (keys.enckeylen > sizeof(ctx->base.key))
2126 goto badkey;
2127
2128 /* Save auth key. */
2129 flags = crypto_aead_get_flags(tfm);
2130 err = atmel_sha_authenc_setkey(ctx->auth,
2131 keys.authkey, keys.authkeylen,
2132 &flags);
2133 crypto_aead_set_flags(tfm, flags & CRYPTO_TFM_RES_MASK);
2134 if (err) {
2135 memzero_explicit(&keys, sizeof(keys));
2136 return err;
2137 }
2138
2139 /* Save enc key. */
2140 ctx->base.keylen = keys.enckeylen;
2141 memcpy(ctx->base.key, keys.enckey, keys.enckeylen);
2142
2143 memzero_explicit(&keys, sizeof(keys));
2144 return 0;
2145
2146badkey:
2147 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2148 memzero_explicit(&key, sizeof(keys));
2149 return -EINVAL;
2150}
2151
2152static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
2153 unsigned long auth_mode)
2154{
2155 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2156 unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize();
2157
2158 ctx->auth = atmel_sha_authenc_spawn(auth_mode);
2159 if (IS_ERR(ctx->auth))
2160 return PTR_ERR(ctx->auth);
2161
2162 crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
2163 auth_reqsize));
2164 ctx->base.start = atmel_aes_authenc_start;
2165
2166 return 0;
2167}
2168
2169static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm)
2170{
2171 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1);
2172}
2173
2174static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm)
2175{
2176 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224);
2177}
2178
2179static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm)
2180{
2181 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256);
2182}
2183
2184static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm)
2185{
2186 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384);
2187}
2188
2189static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm)
2190{
2191 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512);
2192}
2193
2194static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm)
2195{
2196 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2197
2198 atmel_sha_authenc_free(ctx->auth);
2199}
2200
2201static int atmel_aes_authenc_crypt(struct aead_request *req,
2202 unsigned long mode)
2203{
2204 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2205 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2206 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
2207 u32 authsize = crypto_aead_authsize(tfm);
2208 bool enc = (mode & AES_FLAGS_ENCRYPT);
2209 struct atmel_aes_dev *dd;
2210
2211 /* Compute text length. */
2212 if (!enc && req->cryptlen < authsize)
2213 return -EINVAL;
2214 rctx->textlen = req->cryptlen - (enc ? 0 : authsize);
2215
2216 /*
2217 * Currently, empty messages are not supported yet:
2218 * the SHA auto-padding can be used only on non-empty messages.
2219 * Hence a special case needs to be implemented for empty message.
2220 */
2221 if (!rctx->textlen && !req->assoclen)
2222 return -EINVAL;
2223
2224 rctx->base.mode = mode;
2225 ctx->block_size = AES_BLOCK_SIZE;
2226
2227 dd = atmel_aes_find_dev(ctx);
2228 if (!dd)
2229 return -ENODEV;
2230
2231 return atmel_aes_handle_queue(dd, &req->base);
2232}
2233
2234static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req)
2235{
2236 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
2237}
2238
2239static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req)
2240{
2241 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC);
2242}
2243
2244static struct aead_alg aes_authenc_algs[] = {
2245{
2246 .setkey = atmel_aes_authenc_setkey,
2247 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2248 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2249 .init = atmel_aes_authenc_hmac_sha1_init_tfm,
2250 .exit = atmel_aes_authenc_exit_tfm,
2251 .ivsize = AES_BLOCK_SIZE,
2252 .maxauthsize = SHA1_DIGEST_SIZE,
2253
2254 .base = {
2255 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2256 .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes",
2257 .cra_priority = ATMEL_AES_PRIORITY,
2258 .cra_flags = CRYPTO_ALG_ASYNC,
2259 .cra_blocksize = AES_BLOCK_SIZE,
2260 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2261 .cra_alignmask = 0xf,
2262 .cra_module = THIS_MODULE,
2263 },
2264},
2265{
2266 .setkey = atmel_aes_authenc_setkey,
2267 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2268 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2269 .init = atmel_aes_authenc_hmac_sha224_init_tfm,
2270 .exit = atmel_aes_authenc_exit_tfm,
2271 .ivsize = AES_BLOCK_SIZE,
2272 .maxauthsize = SHA224_DIGEST_SIZE,
2273
2274 .base = {
2275 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2276 .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes",
2277 .cra_priority = ATMEL_AES_PRIORITY,
2278 .cra_flags = CRYPTO_ALG_ASYNC,
2279 .cra_blocksize = AES_BLOCK_SIZE,
2280 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2281 .cra_alignmask = 0xf,
2282 .cra_module = THIS_MODULE,
2283 },
2284},
2285{
2286 .setkey = atmel_aes_authenc_setkey,
2287 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2288 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2289 .init = atmel_aes_authenc_hmac_sha256_init_tfm,
2290 .exit = atmel_aes_authenc_exit_tfm,
2291 .ivsize = AES_BLOCK_SIZE,
2292 .maxauthsize = SHA256_DIGEST_SIZE,
2293
2294 .base = {
2295 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2296 .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes",
2297 .cra_priority = ATMEL_AES_PRIORITY,
2298 .cra_flags = CRYPTO_ALG_ASYNC,
2299 .cra_blocksize = AES_BLOCK_SIZE,
2300 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2301 .cra_alignmask = 0xf,
2302 .cra_module = THIS_MODULE,
2303 },
2304},
2305{
2306 .setkey = atmel_aes_authenc_setkey,
2307 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2308 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2309 .init = atmel_aes_authenc_hmac_sha384_init_tfm,
2310 .exit = atmel_aes_authenc_exit_tfm,
2311 .ivsize = AES_BLOCK_SIZE,
2312 .maxauthsize = SHA384_DIGEST_SIZE,
2313
2314 .base = {
2315 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2316 .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes",
2317 .cra_priority = ATMEL_AES_PRIORITY,
2318 .cra_flags = CRYPTO_ALG_ASYNC,
2319 .cra_blocksize = AES_BLOCK_SIZE,
2320 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2321 .cra_alignmask = 0xf,
2322 .cra_module = THIS_MODULE,
2323 },
2324},
2325{
2326 .setkey = atmel_aes_authenc_setkey,
2327 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2328 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2329 .init = atmel_aes_authenc_hmac_sha512_init_tfm,
2330 .exit = atmel_aes_authenc_exit_tfm,
2331 .ivsize = AES_BLOCK_SIZE,
2332 .maxauthsize = SHA512_DIGEST_SIZE,
2333
2334 .base = {
2335 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2336 .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes",
2337 .cra_priority = ATMEL_AES_PRIORITY,
2338 .cra_flags = CRYPTO_ALG_ASYNC,
2339 .cra_blocksize = AES_BLOCK_SIZE,
2340 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2341 .cra_alignmask = 0xf,
2342 .cra_module = THIS_MODULE,
2343 },
2344},
2345};
2346#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
1931 2347
1932/* Probe functions */ 2348/* Probe functions */
1933 2349
@@ -2037,6 +2453,12 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)
2037{ 2453{
2038 int i; 2454 int i;
2039 2455
2456#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2457 if (dd->caps.has_authenc)
2458 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++)
2459 crypto_unregister_aead(&aes_authenc_algs[i]);
2460#endif
2461
2040 if (dd->caps.has_xts) 2462 if (dd->caps.has_xts)
2041 crypto_unregister_alg(&aes_xts_alg); 2463 crypto_unregister_alg(&aes_xts_alg);
2042 2464
@@ -2078,8 +2500,25 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
2078 goto err_aes_xts_alg; 2500 goto err_aes_xts_alg;
2079 } 2501 }
2080 2502
2503#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2504 if (dd->caps.has_authenc) {
2505 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) {
2506 err = crypto_register_aead(&aes_authenc_algs[i]);
2507 if (err)
2508 goto err_aes_authenc_alg;
2509 }
2510 }
2511#endif
2512
2081 return 0; 2513 return 0;
2082 2514
2515#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2516 /* i = ARRAY_SIZE(aes_authenc_algs); */
2517err_aes_authenc_alg:
2518 for (j = 0; j < i; j++)
2519 crypto_unregister_aead(&aes_authenc_algs[j]);
2520 crypto_unregister_alg(&aes_xts_alg);
2521#endif
2083err_aes_xts_alg: 2522err_aes_xts_alg:
2084 crypto_unregister_aead(&aes_gcm_alg); 2523 crypto_unregister_aead(&aes_gcm_alg);
2085err_aes_gcm_alg: 2524err_aes_gcm_alg:
@@ -2100,6 +2539,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
2100 dd->caps.has_ctr32 = 0; 2539 dd->caps.has_ctr32 = 0;
2101 dd->caps.has_gcm = 0; 2540 dd->caps.has_gcm = 0;
2102 dd->caps.has_xts = 0; 2541 dd->caps.has_xts = 0;
2542 dd->caps.has_authenc = 0;
2103 dd->caps.max_burst_size = 1; 2543 dd->caps.max_burst_size = 1;
2104 2544
2105 /* keep only major version number */ 2545 /* keep only major version number */
@@ -2110,6 +2550,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
2110 dd->caps.has_ctr32 = 1; 2550 dd->caps.has_ctr32 = 1;
2111 dd->caps.has_gcm = 1; 2551 dd->caps.has_gcm = 1;
2112 dd->caps.has_xts = 1; 2552 dd->caps.has_xts = 1;
2553 dd->caps.has_authenc = 1;
2113 dd->caps.max_burst_size = 4; 2554 dd->caps.max_burst_size = 4;
2114 break; 2555 break;
2115 case 0x200: 2556 case 0x200:
@@ -2268,6 +2709,13 @@ static int atmel_aes_probe(struct platform_device *pdev)
2268 2709
2269 atmel_aes_get_cap(aes_dd); 2710 atmel_aes_get_cap(aes_dd);
2270 2711
2712#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2713 if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) {
2714 err = -EPROBE_DEFER;
2715 goto iclk_unprepare;
2716 }
2717#endif
2718
2271 err = atmel_aes_buff_init(aes_dd); 2719 err = atmel_aes_buff_init(aes_dd);
2272 if (err) 2720 if (err)
2273 goto err_aes_buff; 2721 goto err_aes_buff;
@@ -2304,7 +2752,8 @@ res_err:
2304 tasklet_kill(&aes_dd->done_task); 2752 tasklet_kill(&aes_dd->done_task);
2305 tasklet_kill(&aes_dd->queue_task); 2753 tasklet_kill(&aes_dd->queue_task);
2306aes_dd_err: 2754aes_dd_err:
2307 dev_err(dev, "initialization failed.\n"); 2755 if (err != -EPROBE_DEFER)
2756 dev_err(dev, "initialization failed.\n");
2308 2757
2309 return err; 2758 return err;
2310} 2759}
diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h
new file mode 100644
index 000000000000..2a60d1224143
--- /dev/null
+++ b/drivers/crypto/atmel-authenc.h
@@ -0,0 +1,64 @@
1/*
2 * API for Atmel Secure Protocol Layers Improved Performances (SPLIP)
3 *
4 * Copyright (C) 2016 Atmel Corporation
5 *
6 * Author: Cyrille Pitchen <cyrille.pitchen@atmel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
17 * You should have received a copy of the GNU General Public License along with
18 * this program. If not, see <http://www.gnu.org/licenses/>.
19 *
20 * This driver is based on drivers/mtd/spi-nor/fsl-quadspi.c from Freescale.
21 */
22
23#ifndef __ATMEL_AUTHENC_H__
24#define __ATMEL_AUTHENC_H__
25
26#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
27
28#include <crypto/authenc.h>
29#include <crypto/hash.h>
30#include <crypto/sha.h>
31#include "atmel-sha-regs.h"
32
33struct atmel_aes_dev;
34typedef int (*atmel_aes_authenc_fn_t)(struct atmel_aes_dev *, int, bool);
35
36struct atmel_sha_authenc_ctx;
37
38bool atmel_sha_authenc_is_ready(void);
39unsigned int atmel_sha_authenc_get_reqsize(void);
40
41struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode);
42void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth);
43int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth,
44 const u8 *key, unsigned int keylen,
45 u32 *flags);
46
47int atmel_sha_authenc_schedule(struct ahash_request *req,
48 struct atmel_sha_authenc_ctx *auth,
49 atmel_aes_authenc_fn_t cb,
50 struct atmel_aes_dev *dd);
51int atmel_sha_authenc_init(struct ahash_request *req,
52 struct scatterlist *assoc, unsigned int assoclen,
53 unsigned int textlen,
54 atmel_aes_authenc_fn_t cb,
55 struct atmel_aes_dev *dd);
56int atmel_sha_authenc_final(struct ahash_request *req,
57 u32 *digest, unsigned int digestlen,
58 atmel_aes_authenc_fn_t cb,
59 struct atmel_aes_dev *dd);
60void atmel_sha_authenc_abort(struct ahash_request *req);
61
62#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
63
64#endif /* __ATMEL_AUTHENC_H__ */
diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h
index e08897109cab..1b0eba4a2706 100644
--- a/drivers/crypto/atmel-sha-regs.h
+++ b/drivers/crypto/atmel-sha-regs.h
@@ -16,16 +16,33 @@
16#define SHA_MR_MODE_MANUAL 0x0 16#define SHA_MR_MODE_MANUAL 0x0
17#define SHA_MR_MODE_AUTO 0x1 17#define SHA_MR_MODE_AUTO 0x1
18#define SHA_MR_MODE_PDC 0x2 18#define SHA_MR_MODE_PDC 0x2
19#define SHA_MR_MODE_IDATAR0 0x2
19#define SHA_MR_PROCDLY (1 << 4) 20#define SHA_MR_PROCDLY (1 << 4)
20#define SHA_MR_UIHV (1 << 5) 21#define SHA_MR_UIHV (1 << 5)
21#define SHA_MR_UIEHV (1 << 6) 22#define SHA_MR_UIEHV (1 << 6)
23#define SHA_MR_ALGO_MASK GENMASK(10, 8)
22#define SHA_MR_ALGO_SHA1 (0 << 8) 24#define SHA_MR_ALGO_SHA1 (0 << 8)
23#define SHA_MR_ALGO_SHA256 (1 << 8) 25#define SHA_MR_ALGO_SHA256 (1 << 8)
24#define SHA_MR_ALGO_SHA384 (2 << 8) 26#define SHA_MR_ALGO_SHA384 (2 << 8)
25#define SHA_MR_ALGO_SHA512 (3 << 8) 27#define SHA_MR_ALGO_SHA512 (3 << 8)
26#define SHA_MR_ALGO_SHA224 (4 << 8) 28#define SHA_MR_ALGO_SHA224 (4 << 8)
29#define SHA_MR_HMAC (1 << 11)
27#define SHA_MR_DUALBUFF (1 << 16) 30#define SHA_MR_DUALBUFF (1 << 16)
28 31
32#define SHA_FLAGS_ALGO_MASK SHA_MR_ALGO_MASK
33#define SHA_FLAGS_SHA1 SHA_MR_ALGO_SHA1
34#define SHA_FLAGS_SHA256 SHA_MR_ALGO_SHA256
35#define SHA_FLAGS_SHA384 SHA_MR_ALGO_SHA384
36#define SHA_FLAGS_SHA512 SHA_MR_ALGO_SHA512
37#define SHA_FLAGS_SHA224 SHA_MR_ALGO_SHA224
38#define SHA_FLAGS_HMAC SHA_MR_HMAC
39#define SHA_FLAGS_HMAC_SHA1 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA1)
40#define SHA_FLAGS_HMAC_SHA256 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA256)
41#define SHA_FLAGS_HMAC_SHA384 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA384)
42#define SHA_FLAGS_HMAC_SHA512 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA512)
43#define SHA_FLAGS_HMAC_SHA224 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA224)
44#define SHA_FLAGS_MODE_MASK (SHA_FLAGS_HMAC | SHA_FLAGS_ALGO_MASK)
45
29#define SHA_IER 0x10 46#define SHA_IER 0x10
30#define SHA_IDR 0x14 47#define SHA_IDR 0x14
31#define SHA_IMR 0x18 48#define SHA_IMR 0x18
@@ -40,6 +57,9 @@
40#define SHA_ISR_URAT_MR (0x2 << 12) 57#define SHA_ISR_URAT_MR (0x2 << 12)
41#define SHA_ISR_URAT_WO (0x5 << 12) 58#define SHA_ISR_URAT_WO (0x5 << 12)
42 59
60#define SHA_MSR 0x20
61#define SHA_BCR 0x30
62
43#define SHA_HW_VERSION 0xFC 63#define SHA_HW_VERSION 0xFC
44 64
45#define SHA_TPR 0x108 65#define SHA_TPR 0x108
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 97e34799e077..a9482023d7d3 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -41,6 +41,7 @@
41#include <crypto/internal/hash.h> 41#include <crypto/internal/hash.h>
42#include <linux/platform_data/crypto-atmel.h> 42#include <linux/platform_data/crypto-atmel.h>
43#include "atmel-sha-regs.h" 43#include "atmel-sha-regs.h"
44#include "atmel-authenc.h"
44 45
45/* SHA flags */ 46/* SHA flags */
46#define SHA_FLAGS_BUSY BIT(0) 47#define SHA_FLAGS_BUSY BIT(0)
@@ -50,21 +51,22 @@
50#define SHA_FLAGS_INIT BIT(4) 51#define SHA_FLAGS_INIT BIT(4)
51#define SHA_FLAGS_CPU BIT(5) 52#define SHA_FLAGS_CPU BIT(5)
52#define SHA_FLAGS_DMA_READY BIT(6) 53#define SHA_FLAGS_DMA_READY BIT(6)
54#define SHA_FLAGS_DUMP_REG BIT(7)
55
56/* bits[11:8] are reserved. */
53 57
54#define SHA_FLAGS_FINUP BIT(16) 58#define SHA_FLAGS_FINUP BIT(16)
55#define SHA_FLAGS_SG BIT(17) 59#define SHA_FLAGS_SG BIT(17)
56#define SHA_FLAGS_ALGO_MASK GENMASK(22, 18)
57#define SHA_FLAGS_SHA1 BIT(18)
58#define SHA_FLAGS_SHA224 BIT(19)
59#define SHA_FLAGS_SHA256 BIT(20)
60#define SHA_FLAGS_SHA384 BIT(21)
61#define SHA_FLAGS_SHA512 BIT(22)
62#define SHA_FLAGS_ERROR BIT(23) 60#define SHA_FLAGS_ERROR BIT(23)
63#define SHA_FLAGS_PAD BIT(24) 61#define SHA_FLAGS_PAD BIT(24)
64#define SHA_FLAGS_RESTORE BIT(25) 62#define SHA_FLAGS_RESTORE BIT(25)
63#define SHA_FLAGS_IDATAR0 BIT(26)
64#define SHA_FLAGS_WAIT_DATARDY BIT(27)
65 65
66#define SHA_OP_INIT 0
66#define SHA_OP_UPDATE 1 67#define SHA_OP_UPDATE 1
67#define SHA_OP_FINAL 2 68#define SHA_OP_FINAL 2
69#define SHA_OP_DIGEST 3
68 70
69#define SHA_BUFFER_LEN (PAGE_SIZE / 16) 71#define SHA_BUFFER_LEN (PAGE_SIZE / 16)
70 72
@@ -76,6 +78,7 @@ struct atmel_sha_caps {
76 bool has_sha224; 78 bool has_sha224;
77 bool has_sha_384_512; 79 bool has_sha_384_512;
78 bool has_uihv; 80 bool has_uihv;
81 bool has_hmac;
79}; 82};
80 83
81struct atmel_sha_dev; 84struct atmel_sha_dev;
@@ -101,12 +104,16 @@ struct atmel_sha_reqctx {
101 unsigned int total; /* total request */ 104 unsigned int total; /* total request */
102 105
103 size_t block_size; 106 size_t block_size;
107 size_t hash_size;
104 108
105 u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); 109 u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
106}; 110};
107 111
112typedef int (*atmel_sha_fn_t)(struct atmel_sha_dev *);
113
108struct atmel_sha_ctx { 114struct atmel_sha_ctx {
109 struct atmel_sha_dev *dd; 115 struct atmel_sha_dev *dd;
116 atmel_sha_fn_t start;
110 117
111 unsigned long flags; 118 unsigned long flags;
112}; 119};
@@ -116,6 +123,9 @@ struct atmel_sha_ctx {
116struct atmel_sha_dma { 123struct atmel_sha_dma {
117 struct dma_chan *chan; 124 struct dma_chan *chan;
118 struct dma_slave_config dma_conf; 125 struct dma_slave_config dma_conf;
126 struct scatterlist *sg;
127 int nents;
128 unsigned int last_sg_length;
119}; 129};
120 130
121struct atmel_sha_dev { 131struct atmel_sha_dev {
@@ -134,11 +144,17 @@ struct atmel_sha_dev {
134 unsigned long flags; 144 unsigned long flags;
135 struct crypto_queue queue; 145 struct crypto_queue queue;
136 struct ahash_request *req; 146 struct ahash_request *req;
147 bool is_async;
148 bool force_complete;
149 atmel_sha_fn_t resume;
150 atmel_sha_fn_t cpu_transfer_complete;
137 151
138 struct atmel_sha_dma dma_lch_in; 152 struct atmel_sha_dma dma_lch_in;
139 153
140 struct atmel_sha_caps caps; 154 struct atmel_sha_caps caps;
141 155
156 struct scatterlist tmp;
157
142 u32 hw_version; 158 u32 hw_version;
143}; 159};
144 160
@@ -152,17 +168,140 @@ static struct atmel_sha_drv atmel_sha = {
152 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock), 168 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock),
153}; 169};
154 170
171#ifdef VERBOSE_DEBUG
172static const char *atmel_sha_reg_name(u32 offset, char *tmp, size_t sz, bool wr)
173{
174 switch (offset) {
175 case SHA_CR:
176 return "CR";
177
178 case SHA_MR:
179 return "MR";
180
181 case SHA_IER:
182 return "IER";
183
184 case SHA_IDR:
185 return "IDR";
186
187 case SHA_IMR:
188 return "IMR";
189
190 case SHA_ISR:
191 return "ISR";
192
193 case SHA_MSR:
194 return "MSR";
195
196 case SHA_BCR:
197 return "BCR";
198
199 case SHA_REG_DIN(0):
200 case SHA_REG_DIN(1):
201 case SHA_REG_DIN(2):
202 case SHA_REG_DIN(3):
203 case SHA_REG_DIN(4):
204 case SHA_REG_DIN(5):
205 case SHA_REG_DIN(6):
206 case SHA_REG_DIN(7):
207 case SHA_REG_DIN(8):
208 case SHA_REG_DIN(9):
209 case SHA_REG_DIN(10):
210 case SHA_REG_DIN(11):
211 case SHA_REG_DIN(12):
212 case SHA_REG_DIN(13):
213 case SHA_REG_DIN(14):
214 case SHA_REG_DIN(15):
215 snprintf(tmp, sz, "IDATAR[%u]", (offset - SHA_REG_DIN(0)) >> 2);
216 break;
217
218 case SHA_REG_DIGEST(0):
219 case SHA_REG_DIGEST(1):
220 case SHA_REG_DIGEST(2):
221 case SHA_REG_DIGEST(3):
222 case SHA_REG_DIGEST(4):
223 case SHA_REG_DIGEST(5):
224 case SHA_REG_DIGEST(6):
225 case SHA_REG_DIGEST(7):
226 case SHA_REG_DIGEST(8):
227 case SHA_REG_DIGEST(9):
228 case SHA_REG_DIGEST(10):
229 case SHA_REG_DIGEST(11):
230 case SHA_REG_DIGEST(12):
231 case SHA_REG_DIGEST(13):
232 case SHA_REG_DIGEST(14):
233 case SHA_REG_DIGEST(15):
234 if (wr)
235 snprintf(tmp, sz, "IDATAR[%u]",
236 16u + ((offset - SHA_REG_DIGEST(0)) >> 2));
237 else
238 snprintf(tmp, sz, "ODATAR[%u]",
239 (offset - SHA_REG_DIGEST(0)) >> 2);
240 break;
241
242 case SHA_HW_VERSION:
243 return "HWVER";
244
245 default:
246 snprintf(tmp, sz, "0x%02x", offset);
247 break;
248 }
249
250 return tmp;
251}
252
253#endif /* VERBOSE_DEBUG */
254
155static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset) 255static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset)
156{ 256{
157 return readl_relaxed(dd->io_base + offset); 257 u32 value = readl_relaxed(dd->io_base + offset);
258
259#ifdef VERBOSE_DEBUG
260 if (dd->flags & SHA_FLAGS_DUMP_REG) {
261 char tmp[16];
262
263 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,
264 atmel_sha_reg_name(offset, tmp, sizeof(tmp), false));
265 }
266#endif /* VERBOSE_DEBUG */
267
268 return value;
158} 269}
159 270
160static inline void atmel_sha_write(struct atmel_sha_dev *dd, 271static inline void atmel_sha_write(struct atmel_sha_dev *dd,
161 u32 offset, u32 value) 272 u32 offset, u32 value)
162{ 273{
274#ifdef VERBOSE_DEBUG
275 if (dd->flags & SHA_FLAGS_DUMP_REG) {
276 char tmp[16];
277
278 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,
279 atmel_sha_reg_name(offset, tmp, sizeof(tmp), true));
280 }
281#endif /* VERBOSE_DEBUG */
282
163 writel_relaxed(value, dd->io_base + offset); 283 writel_relaxed(value, dd->io_base + offset);
164} 284}
165 285
286static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err)
287{
288 struct ahash_request *req = dd->req;
289
290 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
291 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY |
292 SHA_FLAGS_DUMP_REG);
293
294 clk_disable(dd->iclk);
295
296 if ((dd->is_async || dd->force_complete) && req->base.complete)
297 req->base.complete(&req->base, err);
298
299 /* handle new request */
300 tasklet_schedule(&dd->queue_task);
301
302 return err;
303}
304
166static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) 305static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)
167{ 306{
168 size_t count; 307 size_t count;
@@ -241,7 +380,9 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
241 bits[1] = cpu_to_be64(size[0] << 3); 380 bits[1] = cpu_to_be64(size[0] << 3);
242 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); 381 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61);
243 382
244 if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { 383 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
384 case SHA_FLAGS_SHA384:
385 case SHA_FLAGS_SHA512:
245 index = ctx->bufcnt & 0x7f; 386 index = ctx->bufcnt & 0x7f;
246 padlen = (index < 112) ? (112 - index) : ((128+112) - index); 387 padlen = (index < 112) ? (112 - index) : ((128+112) - index);
247 *(ctx->buffer + ctx->bufcnt) = 0x80; 388 *(ctx->buffer + ctx->bufcnt) = 0x80;
@@ -249,7 +390,9 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
249 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); 390 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
250 ctx->bufcnt += padlen + 16; 391 ctx->bufcnt += padlen + 16;
251 ctx->flags |= SHA_FLAGS_PAD; 392 ctx->flags |= SHA_FLAGS_PAD;
252 } else { 393 break;
394
395 default:
253 index = ctx->bufcnt & 0x3f; 396 index = ctx->bufcnt & 0x3f;
254 padlen = (index < 56) ? (56 - index) : ((64+56) - index); 397 padlen = (index < 56) ? (56 - index) : ((64+56) - index);
255 *(ctx->buffer + ctx->bufcnt) = 0x80; 398 *(ctx->buffer + ctx->bufcnt) = 0x80;
@@ -257,14 +400,12 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
257 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); 400 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
258 ctx->bufcnt += padlen + 8; 401 ctx->bufcnt += padlen + 8;
259 ctx->flags |= SHA_FLAGS_PAD; 402 ctx->flags |= SHA_FLAGS_PAD;
403 break;
260 } 404 }
261} 405}
262 406
263static int atmel_sha_init(struct ahash_request *req) 407static struct atmel_sha_dev *atmel_sha_find_dev(struct atmel_sha_ctx *tctx)
264{ 408{
265 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
266 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
267 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
268 struct atmel_sha_dev *dd = NULL; 409 struct atmel_sha_dev *dd = NULL;
269 struct atmel_sha_dev *tmp; 410 struct atmel_sha_dev *tmp;
270 411
@@ -281,6 +422,16 @@ static int atmel_sha_init(struct ahash_request *req)
281 422
282 spin_unlock_bh(&atmel_sha.lock); 423 spin_unlock_bh(&atmel_sha.lock);
283 424
425 return dd;
426}
427
428static int atmel_sha_init(struct ahash_request *req)
429{
430 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
431 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
432 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
433 struct atmel_sha_dev *dd = atmel_sha_find_dev(tctx);
434
284 ctx->dd = dd; 435 ctx->dd = dd;
285 436
286 ctx->flags = 0; 437 ctx->flags = 0;
@@ -397,6 +548,19 @@ static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma)
397 atmel_sha_write(dd, SHA_MR, valmr); 548 atmel_sha_write(dd, SHA_MR, valmr);
398} 549}
399 550
551static inline int atmel_sha_wait_for_data_ready(struct atmel_sha_dev *dd,
552 atmel_sha_fn_t resume)
553{
554 u32 isr = atmel_sha_read(dd, SHA_ISR);
555
556 if (unlikely(isr & SHA_INT_DATARDY))
557 return resume(dd);
558
559 dd->resume = resume;
560 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
561 return -EINPROGRESS;
562}
563
400static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, 564static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
401 size_t length, int final) 565 size_t length, int final)
402{ 566{
@@ -404,7 +568,7 @@ static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
404 int count, len32; 568 int count, len32;
405 const u32 *buffer = (const u32 *)buf; 569 const u32 *buffer = (const u32 *)buf;
406 570
407 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 571 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",
408 ctx->digcnt[1], ctx->digcnt[0], length, final); 572 ctx->digcnt[1], ctx->digcnt[0], length, final);
409 573
410 atmel_sha_write_ctrl(dd, 0); 574 atmel_sha_write_ctrl(dd, 0);
@@ -433,7 +597,7 @@ static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
433 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 597 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
434 int len32; 598 int len32;
435 599
436 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 600 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",
437 ctx->digcnt[1], ctx->digcnt[0], length1, final); 601 ctx->digcnt[1], ctx->digcnt[0], length1, final);
438 602
439 len32 = DIV_ROUND_UP(length1, sizeof(u32)); 603 len32 = DIV_ROUND_UP(length1, sizeof(u32));
@@ -467,6 +631,8 @@ static void atmel_sha_dma_callback(void *data)
467{ 631{
468 struct atmel_sha_dev *dd = data; 632 struct atmel_sha_dev *dd = data;
469 633
634 dd->is_async = true;
635
470 /* dma_lch_in - completed - wait DATRDY */ 636 /* dma_lch_in - completed - wait DATRDY */
471 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 637 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
472} 638}
@@ -478,7 +644,7 @@ static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
478 struct dma_async_tx_descriptor *in_desc; 644 struct dma_async_tx_descriptor *in_desc;
479 struct scatterlist sg[2]; 645 struct scatterlist sg[2];
480 646
481 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 647 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",
482 ctx->digcnt[1], ctx->digcnt[0], length1, final); 648 ctx->digcnt[1], ctx->digcnt[0], length1, final);
483 649
484 dd->dma_lch_in.dma_conf.src_maxburst = 16; 650 dd->dma_lch_in.dma_conf.src_maxburst = 16;
@@ -502,7 +668,7 @@ static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
502 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 668 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
503 } 669 }
504 if (!in_desc) 670 if (!in_desc)
505 return -EINVAL; 671 return atmel_sha_complete(dd, -EINVAL);
506 672
507 in_desc->callback = atmel_sha_dma_callback; 673 in_desc->callback = atmel_sha_dma_callback;
508 in_desc->callback_param = dd; 674 in_desc->callback_param = dd;
@@ -557,9 +723,9 @@ static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,
557 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 723 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
558 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 724 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
559 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 725 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
560 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen + 726 dev_err(dd->dev, "dma %zu bytes error\n", ctx->buflen +
561 ctx->block_size); 727 ctx->block_size);
562 return -EINVAL; 728 return atmel_sha_complete(dd, -EINVAL);
563 } 729 }
564 730
565 ctx->flags &= ~SHA_FLAGS_SG; 731 ctx->flags &= ~SHA_FLAGS_SG;
@@ -578,7 +744,7 @@ static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)
578 744
579 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 745 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
580 746
581 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n", 747 dev_dbg(dd->dev, "slow: bufcnt: %zu, digcnt: 0x%llx 0x%llx, final: %d\n",
582 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); 748 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final);
583 749
584 if (final) 750 if (final)
@@ -606,7 +772,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
606 if (ctx->bufcnt || ctx->offset) 772 if (ctx->bufcnt || ctx->offset)
607 return atmel_sha_update_dma_slow(dd); 773 return atmel_sha_update_dma_slow(dd);
608 774
609 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n", 775 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %zd, total: %u\n",
610 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); 776 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total);
611 777
612 sg = ctx->sg; 778 sg = ctx->sg;
@@ -648,9 +814,9 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
648 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 814 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
649 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 815 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
650 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 816 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
651 dev_err(dd->dev, "dma %u bytes error\n", 817 dev_err(dd->dev, "dma %zu bytes error\n",
652 ctx->buflen + ctx->block_size); 818 ctx->buflen + ctx->block_size);
653 return -EINVAL; 819 return atmel_sha_complete(dd, -EINVAL);
654 } 820 }
655 821
656 if (length == 0) { 822 if (length == 0) {
@@ -664,7 +830,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
664 if (!dma_map_sg(dd->dev, ctx->sg, 1, 830 if (!dma_map_sg(dd->dev, ctx->sg, 1,
665 DMA_TO_DEVICE)) { 831 DMA_TO_DEVICE)) {
666 dev_err(dd->dev, "dma_map_sg error\n"); 832 dev_err(dd->dev, "dma_map_sg error\n");
667 return -EINVAL; 833 return atmel_sha_complete(dd, -EINVAL);
668 } 834 }
669 835
670 ctx->flags |= SHA_FLAGS_SG; 836 ctx->flags |= SHA_FLAGS_SG;
@@ -678,7 +844,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
678 844
679 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 845 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
680 dev_err(dd->dev, "dma_map_sg error\n"); 846 dev_err(dd->dev, "dma_map_sg error\n");
681 return -EINVAL; 847 return atmel_sha_complete(dd, -EINVAL);
682 } 848 }
683 849
684 ctx->flags |= SHA_FLAGS_SG; 850 ctx->flags |= SHA_FLAGS_SG;
@@ -796,16 +962,28 @@ static void atmel_sha_copy_ready_hash(struct ahash_request *req)
796 if (!req->result) 962 if (!req->result)
797 return; 963 return;
798 964
799 if (ctx->flags & SHA_FLAGS_SHA1) 965 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
966 default:
967 case SHA_FLAGS_SHA1:
800 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); 968 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
801 else if (ctx->flags & SHA_FLAGS_SHA224) 969 break;
970
971 case SHA_FLAGS_SHA224:
802 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); 972 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
803 else if (ctx->flags & SHA_FLAGS_SHA256) 973 break;
974
975 case SHA_FLAGS_SHA256:
804 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); 976 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
805 else if (ctx->flags & SHA_FLAGS_SHA384) 977 break;
978
979 case SHA_FLAGS_SHA384:
806 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); 980 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE);
807 else 981 break;
982
983 case SHA_FLAGS_SHA512:
808 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); 984 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE);
985 break;
986 }
809} 987}
810 988
811static int atmel_sha_finish(struct ahash_request *req) 989static int atmel_sha_finish(struct ahash_request *req)
@@ -816,7 +994,7 @@ static int atmel_sha_finish(struct ahash_request *req)
816 if (ctx->digcnt[0] || ctx->digcnt[1]) 994 if (ctx->digcnt[0] || ctx->digcnt[1])
817 atmel_sha_copy_ready_hash(req); 995 atmel_sha_copy_ready_hash(req);
818 996
819 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1], 997 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %zd\n", ctx->digcnt[1],
820 ctx->digcnt[0], ctx->bufcnt); 998 ctx->digcnt[0], ctx->bufcnt);
821 999
822 return 0; 1000 return 0;
@@ -836,16 +1014,7 @@ static void atmel_sha_finish_req(struct ahash_request *req, int err)
836 } 1014 }
837 1015
838 /* atomic operation is not needed here */ 1016 /* atomic operation is not needed here */
839 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | 1017 (void)atmel_sha_complete(dd, err);
840 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
841
842 clk_disable(dd->iclk);
843
844 if (req->base.complete)
845 req->base.complete(&req->base, err);
846
847 /* handle new request */
848 tasklet_schedule(&dd->queue_task);
849} 1018}
850 1019
851static int atmel_sha_hw_init(struct atmel_sha_dev *dd) 1020static int atmel_sha_hw_init(struct atmel_sha_dev *dd)
@@ -886,8 +1055,9 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
886 struct ahash_request *req) 1055 struct ahash_request *req)
887{ 1056{
888 struct crypto_async_request *async_req, *backlog; 1057 struct crypto_async_request *async_req, *backlog;
889 struct atmel_sha_reqctx *ctx; 1058 struct atmel_sha_ctx *ctx;
890 unsigned long flags; 1059 unsigned long flags;
1060 bool start_async;
891 int err = 0, ret = 0; 1061 int err = 0, ret = 0;
892 1062
893 spin_lock_irqsave(&dd->lock, flags); 1063 spin_lock_irqsave(&dd->lock, flags);
@@ -912,35 +1082,69 @@ static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
912 if (backlog) 1082 if (backlog)
913 backlog->complete(backlog, -EINPROGRESS); 1083 backlog->complete(backlog, -EINPROGRESS);
914 1084
915 req = ahash_request_cast(async_req); 1085 ctx = crypto_tfm_ctx(async_req->tfm);
916 dd->req = req; 1086
917 ctx = ahash_request_ctx(req); 1087 dd->req = ahash_request_cast(async_req);
1088 start_async = (dd->req != req);
1089 dd->is_async = start_async;
1090 dd->force_complete = false;
1091
1092 /* WARNING: ctx->start() MAY change dd->is_async. */
1093 err = ctx->start(dd);
1094 return (start_async) ? ret : err;
1095}
1096
1097static int atmel_sha_done(struct atmel_sha_dev *dd);
1098
1099static int atmel_sha_start(struct atmel_sha_dev *dd)
1100{
1101 struct ahash_request *req = dd->req;
1102 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1103 int err;
918 1104
919 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 1105 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
920 ctx->op, req->nbytes); 1106 ctx->op, req->nbytes);
921 1107
922 err = atmel_sha_hw_init(dd); 1108 err = atmel_sha_hw_init(dd);
923
924 if (err) 1109 if (err)
925 goto err1; 1110 return atmel_sha_complete(dd, err);
926 1111
1112 /*
1113 * atmel_sha_update_req() and atmel_sha_final_req() can return either:
1114 * -EINPROGRESS: the hardware is busy and the SHA driver will resume
1115 * its job later in the done_task.
1116 * This is the main path.
1117 *
1118 * 0: the SHA driver can continue its job then release the hardware
1119 * later, if needed, with atmel_sha_finish_req().
1120 * This is the alternate path.
1121 *
1122 * < 0: an error has occurred so atmel_sha_complete(dd, err) has already
1123 * been called, hence the hardware has been released.
1124 * The SHA driver must stop its job without calling
1125 * atmel_sha_finish_req(), otherwise atmel_sha_complete() would be
1126 * called a second time.
1127 *
1128 * Please note that currently, atmel_sha_final_req() never returns 0.
1129 */
1130
1131 dd->resume = atmel_sha_done;
927 if (ctx->op == SHA_OP_UPDATE) { 1132 if (ctx->op == SHA_OP_UPDATE) {
928 err = atmel_sha_update_req(dd); 1133 err = atmel_sha_update_req(dd);
929 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) 1134 if (!err && (ctx->flags & SHA_FLAGS_FINUP))
930 /* no final() after finup() */ 1135 /* no final() after finup() */
931 err = atmel_sha_final_req(dd); 1136 err = atmel_sha_final_req(dd);
932 } else if (ctx->op == SHA_OP_FINAL) { 1137 } else if (ctx->op == SHA_OP_FINAL) {
933 err = atmel_sha_final_req(dd); 1138 err = atmel_sha_final_req(dd);
934 } 1139 }
935 1140
936err1: 1141 if (!err)
937 if (err != -EINPROGRESS)
938 /* done_task will not finish it, so do it here */ 1142 /* done_task will not finish it, so do it here */
939 atmel_sha_finish_req(req, err); 1143 atmel_sha_finish_req(req, err);
940 1144
941 dev_dbg(dd->dev, "exit, err: %d\n", err); 1145 dev_dbg(dd->dev, "exit, err: %d\n", err);
942 1146
943 return ret; 1147 return err;
944} 1148}
945 1149
946static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op) 1150static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op)
@@ -1036,8 +1240,11 @@ static int atmel_sha_import(struct ahash_request *req, const void *in)
1036 1240
1037static int atmel_sha_cra_init(struct crypto_tfm *tfm) 1241static int atmel_sha_cra_init(struct crypto_tfm *tfm)
1038{ 1242{
1243 struct atmel_sha_ctx *ctx = crypto_tfm_ctx(tfm);
1244
1039 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1245 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1040 sizeof(struct atmel_sha_reqctx)); 1246 sizeof(struct atmel_sha_reqctx));
1247 ctx->start = atmel_sha_start;
1041 1248
1042 return 0; 1249 return 0;
1043} 1250}
@@ -1176,9 +1383,8 @@ static void atmel_sha_queue_task(unsigned long data)
1176 atmel_sha_handle_queue(dd, NULL); 1383 atmel_sha_handle_queue(dd, NULL);
1177} 1384}
1178 1385
1179static void atmel_sha_done_task(unsigned long data) 1386static int atmel_sha_done(struct atmel_sha_dev *dd)
1180{ 1387{
1181 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1182 int err = 0; 1388 int err = 0;
1183 1389
1184 if (SHA_FLAGS_CPU & dd->flags) { 1390 if (SHA_FLAGS_CPU & dd->flags) {
@@ -1204,11 +1410,21 @@ static void atmel_sha_done_task(unsigned long data)
1204 goto finish; 1410 goto finish;
1205 } 1411 }
1206 } 1412 }
1207 return; 1413 return err;
1208 1414
1209finish: 1415finish:
1210 /* finish curent request */ 1416 /* finish curent request */
1211 atmel_sha_finish_req(dd->req, err); 1417 atmel_sha_finish_req(dd->req, err);
1418
1419 return err;
1420}
1421
1422static void atmel_sha_done_task(unsigned long data)
1423{
1424 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1425
1426 dd->is_async = true;
1427 (void)dd->resume(dd);
1212} 1428}
1213 1429
1214static irqreturn_t atmel_sha_irq(int irq, void *dev_id) 1430static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
@@ -1233,10 +1449,1104 @@ static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
1233 return IRQ_NONE; 1449 return IRQ_NONE;
1234} 1450}
1235 1451
1452
1453/* DMA transfer functions */
1454
1455static bool atmel_sha_dma_check_aligned(struct atmel_sha_dev *dd,
1456 struct scatterlist *sg,
1457 size_t len)
1458{
1459 struct atmel_sha_dma *dma = &dd->dma_lch_in;
1460 struct ahash_request *req = dd->req;
1461 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1462 size_t bs = ctx->block_size;
1463 int nents;
1464
1465 for (nents = 0; sg; sg = sg_next(sg), ++nents) {
1466 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
1467 return false;
1468
1469 /*
1470 * This is the last sg, the only one that is allowed to
1471 * have an unaligned length.
1472 */
1473 if (len <= sg->length) {
1474 dma->nents = nents + 1;
1475 dma->last_sg_length = sg->length;
1476 sg->length = ALIGN(len, sizeof(u32));
1477 return true;
1478 }
1479
1480 /* All other sg lengths MUST be aligned to the block size. */
1481 if (!IS_ALIGNED(sg->length, bs))
1482 return false;
1483
1484 len -= sg->length;
1485 }
1486
1487 return false;
1488}
1489
1490static void atmel_sha_dma_callback2(void *data)
1491{
1492 struct atmel_sha_dev *dd = data;
1493 struct atmel_sha_dma *dma = &dd->dma_lch_in;
1494 struct scatterlist *sg;
1495 int nents;
1496
1497 dmaengine_terminate_all(dma->chan);
1498 dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);
1499
1500 sg = dma->sg;
1501 for (nents = 0; nents < dma->nents - 1; ++nents)
1502 sg = sg_next(sg);
1503 sg->length = dma->last_sg_length;
1504
1505 dd->is_async = true;
1506 (void)atmel_sha_wait_for_data_ready(dd, dd->resume);
1507}
1508
1509static int atmel_sha_dma_start(struct atmel_sha_dev *dd,
1510 struct scatterlist *src,
1511 size_t len,
1512 atmel_sha_fn_t resume)
1513{
1514 struct atmel_sha_dma *dma = &dd->dma_lch_in;
1515 struct dma_slave_config *config = &dma->dma_conf;
1516 struct dma_chan *chan = dma->chan;
1517 struct dma_async_tx_descriptor *desc;
1518 dma_cookie_t cookie;
1519 unsigned int sg_len;
1520 int err;
1521
1522 dd->resume = resume;
1523
1524 /*
1525 * dma->nents has already been initialized by
1526 * atmel_sha_dma_check_aligned().
1527 */
1528 dma->sg = src;
1529 sg_len = dma_map_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);
1530 if (!sg_len) {
1531 err = -ENOMEM;
1532 goto exit;
1533 }
1534
1535 config->src_maxburst = 16;
1536 config->dst_maxburst = 16;
1537 err = dmaengine_slave_config(chan, config);
1538 if (err)
1539 goto unmap_sg;
1540
1541 desc = dmaengine_prep_slave_sg(chan, dma->sg, sg_len, DMA_MEM_TO_DEV,
1542 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1543 if (!desc) {
1544 err = -ENOMEM;
1545 goto unmap_sg;
1546 }
1547
1548 desc->callback = atmel_sha_dma_callback2;
1549 desc->callback_param = dd;
1550 cookie = dmaengine_submit(desc);
1551 err = dma_submit_error(cookie);
1552 if (err)
1553 goto unmap_sg;
1554
1555 dma_async_issue_pending(chan);
1556
1557 return -EINPROGRESS;
1558
1559unmap_sg:
1560 dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);
1561exit:
1562 return atmel_sha_complete(dd, err);
1563}
1564
1565
1566/* CPU transfer functions */
1567
1568static int atmel_sha_cpu_transfer(struct atmel_sha_dev *dd)
1569{
1570 struct ahash_request *req = dd->req;
1571 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1572 const u32 *words = (const u32 *)ctx->buffer;
1573 size_t i, num_words;
1574 u32 isr, din, din_inc;
1575
1576 din_inc = (ctx->flags & SHA_FLAGS_IDATAR0) ? 0 : 1;
1577 for (;;) {
1578 /* Write data into the Input Data Registers. */
1579 num_words = DIV_ROUND_UP(ctx->bufcnt, sizeof(u32));
1580 for (i = 0, din = 0; i < num_words; ++i, din += din_inc)
1581 atmel_sha_write(dd, SHA_REG_DIN(din), words[i]);
1582
1583 ctx->offset += ctx->bufcnt;
1584 ctx->total -= ctx->bufcnt;
1585
1586 if (!ctx->total)
1587 break;
1588
1589 /*
1590 * Prepare next block:
1591 * Fill ctx->buffer now with the next data to be written into
1592 * IDATARx: it gives time for the SHA hardware to process
1593 * the current data so the SHA_INT_DATARDY flag might be set
1594 * in SHA_ISR when polling this register at the beginning of
1595 * the next loop.
1596 */
1597 ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total);
1598 scatterwalk_map_and_copy(ctx->buffer, ctx->sg,
1599 ctx->offset, ctx->bufcnt, 0);
1600
1601 /* Wait for hardware to be ready again. */
1602 isr = atmel_sha_read(dd, SHA_ISR);
1603 if (!(isr & SHA_INT_DATARDY)) {
1604 /* Not ready yet. */
1605 dd->resume = atmel_sha_cpu_transfer;
1606 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
1607 return -EINPROGRESS;
1608 }
1609 }
1610
1611 if (unlikely(!(ctx->flags & SHA_FLAGS_WAIT_DATARDY)))
1612 return dd->cpu_transfer_complete(dd);
1613
1614 return atmel_sha_wait_for_data_ready(dd, dd->cpu_transfer_complete);
1615}
1616
1617static int atmel_sha_cpu_start(struct atmel_sha_dev *dd,
1618 struct scatterlist *sg,
1619 unsigned int len,
1620 bool idatar0_only,
1621 bool wait_data_ready,
1622 atmel_sha_fn_t resume)
1623{
1624 struct ahash_request *req = dd->req;
1625 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1626
1627 if (!len)
1628 return resume(dd);
1629
1630 ctx->flags &= ~(SHA_FLAGS_IDATAR0 | SHA_FLAGS_WAIT_DATARDY);
1631
1632 if (idatar0_only)
1633 ctx->flags |= SHA_FLAGS_IDATAR0;
1634
1635 if (wait_data_ready)
1636 ctx->flags |= SHA_FLAGS_WAIT_DATARDY;
1637
1638 ctx->sg = sg;
1639 ctx->total = len;
1640 ctx->offset = 0;
1641
1642 /* Prepare the first block to be written. */
1643 ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total);
1644 scatterwalk_map_and_copy(ctx->buffer, ctx->sg,
1645 ctx->offset, ctx->bufcnt, 0);
1646
1647 dd->cpu_transfer_complete = resume;
1648 return atmel_sha_cpu_transfer(dd);
1649}
1650
1651static int atmel_sha_cpu_hash(struct atmel_sha_dev *dd,
1652 const void *data, unsigned int datalen,
1653 bool auto_padding,
1654 atmel_sha_fn_t resume)
1655{
1656 struct ahash_request *req = dd->req;
1657 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1658 u32 msglen = (auto_padding) ? datalen : 0;
1659 u32 mr = SHA_MR_MODE_AUTO;
1660
1661 if (!(IS_ALIGNED(datalen, ctx->block_size) || auto_padding))
1662 return atmel_sha_complete(dd, -EINVAL);
1663
1664 mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK);
1665 atmel_sha_write(dd, SHA_MR, mr);
1666 atmel_sha_write(dd, SHA_MSR, msglen);
1667 atmel_sha_write(dd, SHA_BCR, msglen);
1668 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
1669
1670 sg_init_one(&dd->tmp, data, datalen);
1671 return atmel_sha_cpu_start(dd, &dd->tmp, datalen, false, true, resume);
1672}
1673
1674
1675/* hmac functions */
1676
1677struct atmel_sha_hmac_key {
1678 bool valid;
1679 unsigned int keylen;
1680 u8 buffer[SHA512_BLOCK_SIZE];
1681 u8 *keydup;
1682};
1683
1684static inline void atmel_sha_hmac_key_init(struct atmel_sha_hmac_key *hkey)
1685{
1686 memset(hkey, 0, sizeof(*hkey));
1687}
1688
1689static inline void atmel_sha_hmac_key_release(struct atmel_sha_hmac_key *hkey)
1690{
1691 kfree(hkey->keydup);
1692 memset(hkey, 0, sizeof(*hkey));
1693}
1694
1695static inline int atmel_sha_hmac_key_set(struct atmel_sha_hmac_key *hkey,
1696 const u8 *key,
1697 unsigned int keylen)
1698{
1699 atmel_sha_hmac_key_release(hkey);
1700
1701 if (keylen > sizeof(hkey->buffer)) {
1702 hkey->keydup = kmemdup(key, keylen, GFP_KERNEL);
1703 if (!hkey->keydup)
1704 return -ENOMEM;
1705
1706 } else {
1707 memcpy(hkey->buffer, key, keylen);
1708 }
1709
1710 hkey->valid = true;
1711 hkey->keylen = keylen;
1712 return 0;
1713}
1714
1715static inline bool atmel_sha_hmac_key_get(const struct atmel_sha_hmac_key *hkey,
1716 const u8 **key,
1717 unsigned int *keylen)
1718{
1719 if (!hkey->valid)
1720 return false;
1721
1722 *keylen = hkey->keylen;
1723 *key = (hkey->keydup) ? hkey->keydup : hkey->buffer;
1724 return true;
1725}
1726
1727
1728struct atmel_sha_hmac_ctx {
1729 struct atmel_sha_ctx base;
1730
1731 struct atmel_sha_hmac_key hkey;
1732 u32 ipad[SHA512_BLOCK_SIZE / sizeof(u32)];
1733 u32 opad[SHA512_BLOCK_SIZE / sizeof(u32)];
1734 atmel_sha_fn_t resume;
1735};
1736
1737static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd,
1738 atmel_sha_fn_t resume);
1739static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd,
1740 const u8 *key, unsigned int keylen);
1741static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd);
1742static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd);
1743static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd);
1744static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd);
1745
1746static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd);
1747static int atmel_sha_hmac_final(struct atmel_sha_dev *dd);
1748static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd);
1749static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd);
1750
1751static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd,
1752 atmel_sha_fn_t resume)
1753{
1754 struct ahash_request *req = dd->req;
1755 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1756 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1757 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1758 unsigned int keylen;
1759 const u8 *key;
1760 size_t bs;
1761
1762 hmac->resume = resume;
1763 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
1764 case SHA_FLAGS_SHA1:
1765 ctx->block_size = SHA1_BLOCK_SIZE;
1766 ctx->hash_size = SHA1_DIGEST_SIZE;
1767 break;
1768
1769 case SHA_FLAGS_SHA224:
1770 ctx->block_size = SHA224_BLOCK_SIZE;
1771 ctx->hash_size = SHA256_DIGEST_SIZE;
1772 break;
1773
1774 case SHA_FLAGS_SHA256:
1775 ctx->block_size = SHA256_BLOCK_SIZE;
1776 ctx->hash_size = SHA256_DIGEST_SIZE;
1777 break;
1778
1779 case SHA_FLAGS_SHA384:
1780 ctx->block_size = SHA384_BLOCK_SIZE;
1781 ctx->hash_size = SHA512_DIGEST_SIZE;
1782 break;
1783
1784 case SHA_FLAGS_SHA512:
1785 ctx->block_size = SHA512_BLOCK_SIZE;
1786 ctx->hash_size = SHA512_DIGEST_SIZE;
1787 break;
1788
1789 default:
1790 return atmel_sha_complete(dd, -EINVAL);
1791 }
1792 bs = ctx->block_size;
1793
1794 if (likely(!atmel_sha_hmac_key_get(&hmac->hkey, &key, &keylen)))
1795 return resume(dd);
1796
1797 /* Compute K' from K. */
1798 if (unlikely(keylen > bs))
1799 return atmel_sha_hmac_prehash_key(dd, key, keylen);
1800
1801 /* Prepare ipad. */
1802 memcpy((u8 *)hmac->ipad, key, keylen);
1803 memset((u8 *)hmac->ipad + keylen, 0, bs - keylen);
1804 return atmel_sha_hmac_compute_ipad_hash(dd);
1805}
1806
1807static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd,
1808 const u8 *key, unsigned int keylen)
1809{
1810 return atmel_sha_cpu_hash(dd, key, keylen, true,
1811 atmel_sha_hmac_prehash_key_done);
1812}
1813
1814static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd)
1815{
1816 struct ahash_request *req = dd->req;
1817 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1818 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1819 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1820 size_t ds = crypto_ahash_digestsize(tfm);
1821 size_t bs = ctx->block_size;
1822 size_t i, num_words = ds / sizeof(u32);
1823
1824 /* Prepare ipad. */
1825 for (i = 0; i < num_words; ++i)
1826 hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
1827 memset((u8 *)hmac->ipad + ds, 0, bs - ds);
1828 return atmel_sha_hmac_compute_ipad_hash(dd);
1829}
1830
1831static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd)
1832{
1833 struct ahash_request *req = dd->req;
1834 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1835 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1836 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1837 size_t bs = ctx->block_size;
1838 size_t i, num_words = bs / sizeof(u32);
1839
1840 memcpy(hmac->opad, hmac->ipad, bs);
1841 for (i = 0; i < num_words; ++i) {
1842 hmac->ipad[i] ^= 0x36363636;
1843 hmac->opad[i] ^= 0x5c5c5c5c;
1844 }
1845
1846 return atmel_sha_cpu_hash(dd, hmac->ipad, bs, false,
1847 atmel_sha_hmac_compute_opad_hash);
1848}
1849
1850static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd)
1851{
1852 struct ahash_request *req = dd->req;
1853 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1854 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1855 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1856 size_t bs = ctx->block_size;
1857 size_t hs = ctx->hash_size;
1858 size_t i, num_words = hs / sizeof(u32);
1859
1860 for (i = 0; i < num_words; ++i)
1861 hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
1862 return atmel_sha_cpu_hash(dd, hmac->opad, bs, false,
1863 atmel_sha_hmac_setup_done);
1864}
1865
1866static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd)
1867{
1868 struct ahash_request *req = dd->req;
1869 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1870 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1871 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1872 size_t hs = ctx->hash_size;
1873 size_t i, num_words = hs / sizeof(u32);
1874
1875 for (i = 0; i < num_words; ++i)
1876 hmac->opad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
1877 atmel_sha_hmac_key_release(&hmac->hkey);
1878 return hmac->resume(dd);
1879}
1880
1881static int atmel_sha_hmac_start(struct atmel_sha_dev *dd)
1882{
1883 struct ahash_request *req = dd->req;
1884 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1885 int err;
1886
1887 err = atmel_sha_hw_init(dd);
1888 if (err)
1889 return atmel_sha_complete(dd, err);
1890
1891 switch (ctx->op) {
1892 case SHA_OP_INIT:
1893 err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_init_done);
1894 break;
1895
1896 case SHA_OP_UPDATE:
1897 dd->resume = atmel_sha_done;
1898 err = atmel_sha_update_req(dd);
1899 break;
1900
1901 case SHA_OP_FINAL:
1902 dd->resume = atmel_sha_hmac_final;
1903 err = atmel_sha_final_req(dd);
1904 break;
1905
1906 case SHA_OP_DIGEST:
1907 err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_digest2);
1908 break;
1909
1910 default:
1911 return atmel_sha_complete(dd, -EINVAL);
1912 }
1913
1914 return err;
1915}
1916
1917static int atmel_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
1918 unsigned int keylen)
1919{
1920 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1921
1922 if (atmel_sha_hmac_key_set(&hmac->hkey, key, keylen)) {
1923 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1924 return -EINVAL;
1925 }
1926
1927 return 0;
1928}
1929
1930static int atmel_sha_hmac_init(struct ahash_request *req)
1931{
1932 int err;
1933
1934 err = atmel_sha_init(req);
1935 if (err)
1936 return err;
1937
1938 return atmel_sha_enqueue(req, SHA_OP_INIT);
1939}
1940
1941static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd)
1942{
1943 struct ahash_request *req = dd->req;
1944 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1945 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1946 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1947 size_t bs = ctx->block_size;
1948 size_t hs = ctx->hash_size;
1949
1950 ctx->bufcnt = 0;
1951 ctx->digcnt[0] = bs;
1952 ctx->digcnt[1] = 0;
1953 ctx->flags |= SHA_FLAGS_RESTORE;
1954 memcpy(ctx->digest, hmac->ipad, hs);
1955 return atmel_sha_complete(dd, 0);
1956}
1957
1958static int atmel_sha_hmac_final(struct atmel_sha_dev *dd)
1959{
1960 struct ahash_request *req = dd->req;
1961 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1962 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1963 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
1964 u32 *digest = (u32 *)ctx->digest;
1965 size_t ds = crypto_ahash_digestsize(tfm);
1966 size_t bs = ctx->block_size;
1967 size_t hs = ctx->hash_size;
1968 size_t i, num_words;
1969 u32 mr;
1970
1971 /* Save d = SHA((K' + ipad) | msg). */
1972 num_words = ds / sizeof(u32);
1973 for (i = 0; i < num_words; ++i)
1974 digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
1975
1976 /* Restore context to finish computing SHA((K' + opad) | d). */
1977 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
1978 num_words = hs / sizeof(u32);
1979 for (i = 0; i < num_words; ++i)
1980 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);
1981
1982 mr = SHA_MR_MODE_AUTO | SHA_MR_UIHV;
1983 mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK);
1984 atmel_sha_write(dd, SHA_MR, mr);
1985 atmel_sha_write(dd, SHA_MSR, bs + ds);
1986 atmel_sha_write(dd, SHA_BCR, ds);
1987 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
1988
1989 sg_init_one(&dd->tmp, digest, ds);
1990 return atmel_sha_cpu_start(dd, &dd->tmp, ds, false, true,
1991 atmel_sha_hmac_final_done);
1992}
1993
1994static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd)
1995{
1996 /*
1997 * req->result might not be sizeof(u32) aligned, so copy the
1998 * digest into ctx->digest[] before memcpy() the data into
1999 * req->result.
2000 */
2001 atmel_sha_copy_hash(dd->req);
2002 atmel_sha_copy_ready_hash(dd->req);
2003 return atmel_sha_complete(dd, 0);
2004}
2005
2006static int atmel_sha_hmac_digest(struct ahash_request *req)
2007{
2008 int err;
2009
2010 err = atmel_sha_init(req);
2011 if (err)
2012 return err;
2013
2014 return atmel_sha_enqueue(req, SHA_OP_DIGEST);
2015}
2016
2017static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd)
2018{
2019 struct ahash_request *req = dd->req;
2020 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
2021 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2022 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
2023 size_t hs = ctx->hash_size;
2024 size_t i, num_words = hs / sizeof(u32);
2025 bool use_dma = false;
2026 u32 mr;
2027
2028 /* Special case for empty message. */
2029 if (!req->nbytes)
2030 return atmel_sha_complete(dd, -EINVAL); // TODO:
2031
2032 /* Check DMA threshold and alignment. */
2033 if (req->nbytes > ATMEL_SHA_DMA_THRESHOLD &&
2034 atmel_sha_dma_check_aligned(dd, req->src, req->nbytes))
2035 use_dma = true;
2036
2037 /* Write both initial hash values to compute a HMAC. */
2038 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
2039 for (i = 0; i < num_words; ++i)
2040 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]);
2041
2042 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV);
2043 for (i = 0; i < num_words; ++i)
2044 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);
2045
2046 /* Write the Mode, Message Size, Bytes Count then Control Registers. */
2047 mr = (SHA_MR_HMAC | SHA_MR_DUALBUFF);
2048 mr |= ctx->flags & SHA_FLAGS_ALGO_MASK;
2049 if (use_dma)
2050 mr |= SHA_MR_MODE_IDATAR0;
2051 else
2052 mr |= SHA_MR_MODE_AUTO;
2053 atmel_sha_write(dd, SHA_MR, mr);
2054
2055 atmel_sha_write(dd, SHA_MSR, req->nbytes);
2056 atmel_sha_write(dd, SHA_BCR, req->nbytes);
2057
2058 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
2059
2060 /* Process data. */
2061 if (use_dma)
2062 return atmel_sha_dma_start(dd, req->src, req->nbytes,
2063 atmel_sha_hmac_final_done);
2064
2065 return atmel_sha_cpu_start(dd, req->src, req->nbytes, false, true,
2066 atmel_sha_hmac_final_done);
2067}
2068
2069static int atmel_sha_hmac_cra_init(struct crypto_tfm *tfm)
2070{
2071 struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm);
2072
2073 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2074 sizeof(struct atmel_sha_reqctx));
2075 hmac->base.start = atmel_sha_hmac_start;
2076 atmel_sha_hmac_key_init(&hmac->hkey);
2077
2078 return 0;
2079}
2080
2081static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm)
2082{
2083 struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm);
2084
2085 atmel_sha_hmac_key_release(&hmac->hkey);
2086}
2087
2088static struct ahash_alg sha_hmac_algs[] = {
2089{
2090 .init = atmel_sha_hmac_init,
2091 .update = atmel_sha_update,
2092 .final = atmel_sha_final,
2093 .digest = atmel_sha_hmac_digest,
2094 .setkey = atmel_sha_hmac_setkey,
2095 .export = atmel_sha_export,
2096 .import = atmel_sha_import,
2097 .halg = {
2098 .digestsize = SHA1_DIGEST_SIZE,
2099 .statesize = sizeof(struct atmel_sha_reqctx),
2100 .base = {
2101 .cra_name = "hmac(sha1)",
2102 .cra_driver_name = "atmel-hmac-sha1",
2103 .cra_priority = 100,
2104 .cra_flags = CRYPTO_ALG_ASYNC,
2105 .cra_blocksize = SHA1_BLOCK_SIZE,
2106 .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
2107 .cra_alignmask = 0,
2108 .cra_module = THIS_MODULE,
2109 .cra_init = atmel_sha_hmac_cra_init,
2110 .cra_exit = atmel_sha_hmac_cra_exit,
2111 }
2112 }
2113},
2114{
2115 .init = atmel_sha_hmac_init,
2116 .update = atmel_sha_update,
2117 .final = atmel_sha_final,
2118 .digest = atmel_sha_hmac_digest,
2119 .setkey = atmel_sha_hmac_setkey,
2120 .export = atmel_sha_export,
2121 .import = atmel_sha_import,
2122 .halg = {
2123 .digestsize = SHA224_DIGEST_SIZE,
2124 .statesize = sizeof(struct atmel_sha_reqctx),
2125 .base = {
2126 .cra_name = "hmac(sha224)",
2127 .cra_driver_name = "atmel-hmac-sha224",
2128 .cra_priority = 100,
2129 .cra_flags = CRYPTO_ALG_ASYNC,
2130 .cra_blocksize = SHA224_BLOCK_SIZE,
2131 .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
2132 .cra_alignmask = 0,
2133 .cra_module = THIS_MODULE,
2134 .cra_init = atmel_sha_hmac_cra_init,
2135 .cra_exit = atmel_sha_hmac_cra_exit,
2136 }
2137 }
2138},
2139{
2140 .init = atmel_sha_hmac_init,
2141 .update = atmel_sha_update,
2142 .final = atmel_sha_final,
2143 .digest = atmel_sha_hmac_digest,
2144 .setkey = atmel_sha_hmac_setkey,
2145 .export = atmel_sha_export,
2146 .import = atmel_sha_import,
2147 .halg = {
2148 .digestsize = SHA256_DIGEST_SIZE,
2149 .statesize = sizeof(struct atmel_sha_reqctx),
2150 .base = {
2151 .cra_name = "hmac(sha256)",
2152 .cra_driver_name = "atmel-hmac-sha256",
2153 .cra_priority = 100,
2154 .cra_flags = CRYPTO_ALG_ASYNC,
2155 .cra_blocksize = SHA256_BLOCK_SIZE,
2156 .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
2157 .cra_alignmask = 0,
2158 .cra_module = THIS_MODULE,
2159 .cra_init = atmel_sha_hmac_cra_init,
2160 .cra_exit = atmel_sha_hmac_cra_exit,
2161 }
2162 }
2163},
2164{
2165 .init = atmel_sha_hmac_init,
2166 .update = atmel_sha_update,
2167 .final = atmel_sha_final,
2168 .digest = atmel_sha_hmac_digest,
2169 .setkey = atmel_sha_hmac_setkey,
2170 .export = atmel_sha_export,
2171 .import = atmel_sha_import,
2172 .halg = {
2173 .digestsize = SHA384_DIGEST_SIZE,
2174 .statesize = sizeof(struct atmel_sha_reqctx),
2175 .base = {
2176 .cra_name = "hmac(sha384)",
2177 .cra_driver_name = "atmel-hmac-sha384",
2178 .cra_priority = 100,
2179 .cra_flags = CRYPTO_ALG_ASYNC,
2180 .cra_blocksize = SHA384_BLOCK_SIZE,
2181 .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
2182 .cra_alignmask = 0,
2183 .cra_module = THIS_MODULE,
2184 .cra_init = atmel_sha_hmac_cra_init,
2185 .cra_exit = atmel_sha_hmac_cra_exit,
2186 }
2187 }
2188},
2189{
2190 .init = atmel_sha_hmac_init,
2191 .update = atmel_sha_update,
2192 .final = atmel_sha_final,
2193 .digest = atmel_sha_hmac_digest,
2194 .setkey = atmel_sha_hmac_setkey,
2195 .export = atmel_sha_export,
2196 .import = atmel_sha_import,
2197 .halg = {
2198 .digestsize = SHA512_DIGEST_SIZE,
2199 .statesize = sizeof(struct atmel_sha_reqctx),
2200 .base = {
2201 .cra_name = "hmac(sha512)",
2202 .cra_driver_name = "atmel-hmac-sha512",
2203 .cra_priority = 100,
2204 .cra_flags = CRYPTO_ALG_ASYNC,
2205 .cra_blocksize = SHA512_BLOCK_SIZE,
2206 .cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx),
2207 .cra_alignmask = 0,
2208 .cra_module = THIS_MODULE,
2209 .cra_init = atmel_sha_hmac_cra_init,
2210 .cra_exit = atmel_sha_hmac_cra_exit,
2211 }
2212 }
2213},
2214};
2215
2216#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2217/* authenc functions */
2218
2219static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd);
2220static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd);
2221static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd);
2222
2223
2224struct atmel_sha_authenc_ctx {
2225 struct crypto_ahash *tfm;
2226};
2227
2228struct atmel_sha_authenc_reqctx {
2229 struct atmel_sha_reqctx base;
2230
2231 atmel_aes_authenc_fn_t cb;
2232 struct atmel_aes_dev *aes_dev;
2233
2234 /* _init() parameters. */
2235 struct scatterlist *assoc;
2236 u32 assoclen;
2237 u32 textlen;
2238
2239 /* _final() parameters. */
2240 u32 *digest;
2241 unsigned int digestlen;
2242};
2243
2244static void atmel_sha_authenc_complete(struct crypto_async_request *areq,
2245 int err)
2246{
2247 struct ahash_request *req = areq->data;
2248 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2249
2250 authctx->cb(authctx->aes_dev, err, authctx->base.dd->is_async);
2251}
2252
2253static int atmel_sha_authenc_start(struct atmel_sha_dev *dd)
2254{
2255 struct ahash_request *req = dd->req;
2256 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2257 int err;
2258
2259 /*
2260 * Force atmel_sha_complete() to call req->base.complete(), ie
2261 * atmel_sha_authenc_complete(), which in turn calls authctx->cb().
2262 */
2263 dd->force_complete = true;
2264
2265 err = atmel_sha_hw_init(dd);
2266 return authctx->cb(authctx->aes_dev, err, dd->is_async);
2267}
2268
2269bool atmel_sha_authenc_is_ready(void)
2270{
2271 struct atmel_sha_ctx dummy;
2272
2273 dummy.dd = NULL;
2274 return (atmel_sha_find_dev(&dummy) != NULL);
2275}
2276EXPORT_SYMBOL_GPL(atmel_sha_authenc_is_ready);
2277
2278unsigned int atmel_sha_authenc_get_reqsize(void)
2279{
2280 return sizeof(struct atmel_sha_authenc_reqctx);
2281}
2282EXPORT_SYMBOL_GPL(atmel_sha_authenc_get_reqsize);
2283
2284struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode)
2285{
2286 struct atmel_sha_authenc_ctx *auth;
2287 struct crypto_ahash *tfm;
2288 struct atmel_sha_ctx *tctx;
2289 const char *name;
2290 int err = -EINVAL;
2291
2292 switch (mode & SHA_FLAGS_MODE_MASK) {
2293 case SHA_FLAGS_HMAC_SHA1:
2294 name = "atmel-hmac-sha1";
2295 break;
2296
2297 case SHA_FLAGS_HMAC_SHA224:
2298 name = "atmel-hmac-sha224";
2299 break;
2300
2301 case SHA_FLAGS_HMAC_SHA256:
2302 name = "atmel-hmac-sha256";
2303 break;
2304
2305 case SHA_FLAGS_HMAC_SHA384:
2306 name = "atmel-hmac-sha384";
2307 break;
2308
2309 case SHA_FLAGS_HMAC_SHA512:
2310 name = "atmel-hmac-sha512";
2311 break;
2312
2313 default:
2314 goto error;
2315 }
2316
2317 tfm = crypto_alloc_ahash(name,
2318 CRYPTO_ALG_TYPE_AHASH,
2319 CRYPTO_ALG_TYPE_AHASH_MASK);
2320 if (IS_ERR(tfm)) {
2321 err = PTR_ERR(tfm);
2322 goto error;
2323 }
2324 tctx = crypto_ahash_ctx(tfm);
2325 tctx->start = atmel_sha_authenc_start;
2326 tctx->flags = mode;
2327
2328 auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2329 if (!auth) {
2330 err = -ENOMEM;
2331 goto err_free_ahash;
2332 }
2333 auth->tfm = tfm;
2334
2335 return auth;
2336
2337err_free_ahash:
2338 crypto_free_ahash(tfm);
2339error:
2340 return ERR_PTR(err);
2341}
2342EXPORT_SYMBOL_GPL(atmel_sha_authenc_spawn);
2343
2344void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth)
2345{
2346 if (auth)
2347 crypto_free_ahash(auth->tfm);
2348 kfree(auth);
2349}
2350EXPORT_SYMBOL_GPL(atmel_sha_authenc_free);
2351
2352int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth,
2353 const u8 *key, unsigned int keylen,
2354 u32 *flags)
2355{
2356 struct crypto_ahash *tfm = auth->tfm;
2357 int err;
2358
2359 crypto_ahash_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
2360 crypto_ahash_set_flags(tfm, *flags & CRYPTO_TFM_REQ_MASK);
2361 err = crypto_ahash_setkey(tfm, key, keylen);
2362 *flags = crypto_ahash_get_flags(tfm);
2363
2364 return err;
2365}
2366EXPORT_SYMBOL_GPL(atmel_sha_authenc_setkey);
2367
2368int atmel_sha_authenc_schedule(struct ahash_request *req,
2369 struct atmel_sha_authenc_ctx *auth,
2370 atmel_aes_authenc_fn_t cb,
2371 struct atmel_aes_dev *aes_dev)
2372{
2373 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2374 struct atmel_sha_reqctx *ctx = &authctx->base;
2375 struct crypto_ahash *tfm = auth->tfm;
2376 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
2377 struct atmel_sha_dev *dd;
2378
2379 /* Reset request context (MUST be done first). */
2380 memset(authctx, 0, sizeof(*authctx));
2381
2382 /* Get SHA device. */
2383 dd = atmel_sha_find_dev(tctx);
2384 if (!dd)
2385 return cb(aes_dev, -ENODEV, false);
2386
2387 /* Init request context. */
2388 ctx->dd = dd;
2389 ctx->buflen = SHA_BUFFER_LEN;
2390 authctx->cb = cb;
2391 authctx->aes_dev = aes_dev;
2392 ahash_request_set_tfm(req, tfm);
2393 ahash_request_set_callback(req, 0, atmel_sha_authenc_complete, req);
2394
2395 return atmel_sha_handle_queue(dd, req);
2396}
2397EXPORT_SYMBOL_GPL(atmel_sha_authenc_schedule);
2398
2399int atmel_sha_authenc_init(struct ahash_request *req,
2400 struct scatterlist *assoc, unsigned int assoclen,
2401 unsigned int textlen,
2402 atmel_aes_authenc_fn_t cb,
2403 struct atmel_aes_dev *aes_dev)
2404{
2405 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2406 struct atmel_sha_reqctx *ctx = &authctx->base;
2407 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2408 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
2409 struct atmel_sha_dev *dd = ctx->dd;
2410
2411 if (unlikely(!IS_ALIGNED(assoclen, sizeof(u32))))
2412 return atmel_sha_complete(dd, -EINVAL);
2413
2414 authctx->cb = cb;
2415 authctx->aes_dev = aes_dev;
2416 authctx->assoc = assoc;
2417 authctx->assoclen = assoclen;
2418 authctx->textlen = textlen;
2419
2420 ctx->flags = hmac->base.flags;
2421 return atmel_sha_hmac_setup(dd, atmel_sha_authenc_init2);
2422}
2423EXPORT_SYMBOL_GPL(atmel_sha_authenc_init);
2424
2425static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd)
2426{
2427 struct ahash_request *req = dd->req;
2428 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2429 struct atmel_sha_reqctx *ctx = &authctx->base;
2430 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2431 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
2432 size_t hs = ctx->hash_size;
2433 size_t i, num_words = hs / sizeof(u32);
2434 u32 mr, msg_size;
2435
2436 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
2437 for (i = 0; i < num_words; ++i)
2438 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]);
2439
2440 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV);
2441 for (i = 0; i < num_words; ++i)
2442 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);
2443
2444 mr = (SHA_MR_MODE_IDATAR0 |
2445 SHA_MR_HMAC |
2446 SHA_MR_DUALBUFF);
2447 mr |= ctx->flags & SHA_FLAGS_ALGO_MASK;
2448 atmel_sha_write(dd, SHA_MR, mr);
2449
2450 msg_size = authctx->assoclen + authctx->textlen;
2451 atmel_sha_write(dd, SHA_MSR, msg_size);
2452 atmel_sha_write(dd, SHA_BCR, msg_size);
2453
2454 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
2455
2456 /* Process assoc data. */
2457 return atmel_sha_cpu_start(dd, authctx->assoc, authctx->assoclen,
2458 true, false,
2459 atmel_sha_authenc_init_done);
2460}
2461
2462static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd)
2463{
2464 struct ahash_request *req = dd->req;
2465 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2466
2467 return authctx->cb(authctx->aes_dev, 0, dd->is_async);
2468}
2469
2470int atmel_sha_authenc_final(struct ahash_request *req,
2471 u32 *digest, unsigned int digestlen,
2472 atmel_aes_authenc_fn_t cb,
2473 struct atmel_aes_dev *aes_dev)
2474{
2475 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2476 struct atmel_sha_reqctx *ctx = &authctx->base;
2477 struct atmel_sha_dev *dd = ctx->dd;
2478
2479 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
2480 case SHA_FLAGS_SHA1:
2481 authctx->digestlen = SHA1_DIGEST_SIZE;
2482 break;
2483
2484 case SHA_FLAGS_SHA224:
2485 authctx->digestlen = SHA224_DIGEST_SIZE;
2486 break;
2487
2488 case SHA_FLAGS_SHA256:
2489 authctx->digestlen = SHA256_DIGEST_SIZE;
2490 break;
2491
2492 case SHA_FLAGS_SHA384:
2493 authctx->digestlen = SHA384_DIGEST_SIZE;
2494 break;
2495
2496 case SHA_FLAGS_SHA512:
2497 authctx->digestlen = SHA512_DIGEST_SIZE;
2498 break;
2499
2500 default:
2501 return atmel_sha_complete(dd, -EINVAL);
2502 }
2503 if (authctx->digestlen > digestlen)
2504 authctx->digestlen = digestlen;
2505
2506 authctx->cb = cb;
2507 authctx->aes_dev = aes_dev;
2508 authctx->digest = digest;
2509 return atmel_sha_wait_for_data_ready(dd,
2510 atmel_sha_authenc_final_done);
2511}
2512EXPORT_SYMBOL_GPL(atmel_sha_authenc_final);
2513
2514static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd)
2515{
2516 struct ahash_request *req = dd->req;
2517 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2518 size_t i, num_words = authctx->digestlen / sizeof(u32);
2519
2520 for (i = 0; i < num_words; ++i)
2521 authctx->digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));
2522
2523 return atmel_sha_complete(dd, 0);
2524}
2525
2526void atmel_sha_authenc_abort(struct ahash_request *req)
2527{
2528 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);
2529 struct atmel_sha_reqctx *ctx = &authctx->base;
2530 struct atmel_sha_dev *dd = ctx->dd;
2531
2532 /* Prevent atmel_sha_complete() from calling req->base.complete(). */
2533 dd->is_async = false;
2534 dd->force_complete = false;
2535 (void)atmel_sha_complete(dd, 0);
2536}
2537EXPORT_SYMBOL_GPL(atmel_sha_authenc_abort);
2538
2539#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
2540
2541
1236static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) 2542static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
1237{ 2543{
1238 int i; 2544 int i;
1239 2545
2546 if (dd->caps.has_hmac)
2547 for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++)
2548 crypto_unregister_ahash(&sha_hmac_algs[i]);
2549
1240 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) 2550 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++)
1241 crypto_unregister_ahash(&sha_1_256_algs[i]); 2551 crypto_unregister_ahash(&sha_1_256_algs[i]);
1242 2552
@@ -1273,8 +2583,21 @@ static int atmel_sha_register_algs(struct atmel_sha_dev *dd)
1273 } 2583 }
1274 } 2584 }
1275 2585
2586 if (dd->caps.has_hmac) {
2587 for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) {
2588 err = crypto_register_ahash(&sha_hmac_algs[i]);
2589 if (err)
2590 goto err_sha_hmac_algs;
2591 }
2592 }
2593
1276 return 0; 2594 return 0;
1277 2595
2596 /*i = ARRAY_SIZE(sha_hmac_algs);*/
2597err_sha_hmac_algs:
2598 for (j = 0; j < i; j++)
2599 crypto_unregister_ahash(&sha_hmac_algs[j]);
2600 i = ARRAY_SIZE(sha_384_512_algs);
1278err_sha_384_512_algs: 2601err_sha_384_512_algs:
1279 for (j = 0; j < i; j++) 2602 for (j = 0; j < i; j++)
1280 crypto_unregister_ahash(&sha_384_512_algs[j]); 2603 crypto_unregister_ahash(&sha_384_512_algs[j]);
@@ -1344,6 +2667,7 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
1344 dd->caps.has_sha224 = 0; 2667 dd->caps.has_sha224 = 0;
1345 dd->caps.has_sha_384_512 = 0; 2668 dd->caps.has_sha_384_512 = 0;
1346 dd->caps.has_uihv = 0; 2669 dd->caps.has_uihv = 0;
2670 dd->caps.has_hmac = 0;
1347 2671
1348 /* keep only major version number */ 2672 /* keep only major version number */
1349 switch (dd->hw_version & 0xff0) { 2673 switch (dd->hw_version & 0xff0) {
@@ -1353,6 +2677,7 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
1353 dd->caps.has_sha224 = 1; 2677 dd->caps.has_sha224 = 1;
1354 dd->caps.has_sha_384_512 = 1; 2678 dd->caps.has_sha_384_512 = 1;
1355 dd->caps.has_uihv = 1; 2679 dd->caps.has_uihv = 1;
2680 dd->caps.has_hmac = 1;
1356 break; 2681 break;
1357 case 0x420: 2682 case 0x420:
1358 dd->caps.has_dma = 1; 2683 dd->caps.has_dma = 1;
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index bf467d7be35c..b25f1b3c981f 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -150,7 +150,7 @@ static struct atmel_tdes_drv atmel_tdes = {
150static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset, 150static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset,
151 void *buf, size_t buflen, size_t total, int out) 151 void *buf, size_t buflen, size_t total, int out)
152{ 152{
153 unsigned int count, off = 0; 153 size_t count, off = 0;
154 154
155 while (buflen && total) { 155 while (buflen && total) {
156 count = min((*sg)->length - *offset, total); 156 count = min((*sg)->length - *offset, total);
@@ -336,7 +336,7 @@ static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
336 dd->buf_out, dd->buflen, dd->dma_size, 1); 336 dd->buf_out, dd->buflen, dd->dma_size, 1);
337 if (count != dd->dma_size) { 337 if (count != dd->dma_size) {
338 err = -EINVAL; 338 err = -EINVAL;
339 pr_err("not all data converted: %u\n", count); 339 pr_err("not all data converted: %zu\n", count);
340 } 340 }
341 } 341 }
342 342
@@ -361,7 +361,7 @@ static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
361 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, 361 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,
362 dd->buflen, DMA_TO_DEVICE); 362 dd->buflen, DMA_TO_DEVICE);
363 if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { 363 if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
364 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); 364 dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen);
365 err = -EINVAL; 365 err = -EINVAL;
366 goto err_map_in; 366 goto err_map_in;
367 } 367 }
@@ -369,7 +369,7 @@ static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
369 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, 369 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,
370 dd->buflen, DMA_FROM_DEVICE); 370 dd->buflen, DMA_FROM_DEVICE);
371 if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { 371 if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
372 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); 372 dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen);
373 err = -EINVAL; 373 err = -EINVAL;
374 goto err_map_out; 374 goto err_map_out;
375 } 375 }
@@ -525,8 +525,8 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)
525 525
526 526
527 if (fast) { 527 if (fast) {
528 count = min(dd->total, sg_dma_len(dd->in_sg)); 528 count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg));
529 count = min(count, sg_dma_len(dd->out_sg)); 529 count = min_t(size_t, count, sg_dma_len(dd->out_sg));
530 530
531 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 531 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
532 if (!err) { 532 if (!err) {
@@ -661,7 +661,7 @@ static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)
661 dd->buf_out, dd->buflen, dd->dma_size, 1); 661 dd->buf_out, dd->buflen, dd->dma_size, 1);
662 if (count != dd->dma_size) { 662 if (count != dd->dma_size) {
663 err = -EINVAL; 663 err = -EINVAL;
664 pr_err("not all data converted: %u\n", count); 664 pr_err("not all data converted: %zu\n", count);
665 } 665 }
666 } 666 }
667 } 667 }
diff --git a/drivers/crypto/bcm/Makefile b/drivers/crypto/bcm/Makefile
new file mode 100644
index 000000000000..13cb80eb2665
--- /dev/null
+++ b/drivers/crypto/bcm/Makefile
@@ -0,0 +1,15 @@
1# File: drivers/crypto/bcm/Makefile
2#
3# Makefile for crypto acceleration files for Broadcom SPU driver
4#
5# Uncomment to enable debug tracing in the SPU driver.
6# CFLAGS_util.o := -DDEBUG
7# CFLAGS_cipher.o := -DDEBUG
8# CFLAGS_spu.o := -DDEBUG
9# CFLAGS_spu2.o := -DDEBUG
10
11obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) := bcm_crypto_spu.o
12
13bcm_crypto_spu-objs := util.o spu.o spu2.o cipher.o
14
15ccflags-y += -I. -DBCMDRIVER
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
new file mode 100644
index 000000000000..cc0d5b98006e
--- /dev/null
+++ b/drivers/crypto/bcm/cipher.c
@@ -0,0 +1,4963 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#include <linux/err.h>
18#include <linux/module.h>
19#include <linux/init.h>
20#include <linux/errno.h>
21#include <linux/kernel.h>
22#include <linux/interrupt.h>
23#include <linux/platform_device.h>
24#include <linux/scatterlist.h>
25#include <linux/crypto.h>
26#include <linux/kthread.h>
27#include <linux/rtnetlink.h>
28#include <linux/sched.h>
29#include <linux/of_address.h>
30#include <linux/of_device.h>
31#include <linux/io.h>
32#include <linux/bitops.h>
33
34#include <crypto/algapi.h>
35#include <crypto/aead.h>
36#include <crypto/internal/aead.h>
37#include <crypto/aes.h>
38#include <crypto/des.h>
39#include <crypto/sha.h>
40#include <crypto/md5.h>
41#include <crypto/authenc.h>
42#include <crypto/skcipher.h>
43#include <crypto/hash.h>
44#include <crypto/aes.h>
45#include <crypto/sha3.h>
46
47#include "util.h"
48#include "cipher.h"
49#include "spu.h"
50#include "spum.h"
51#include "spu2.h"
52
53/* ================= Device Structure ================== */
54
55struct device_private iproc_priv;
56
57/* ==================== Parameters ===================== */
58
59int flow_debug_logging;
60module_param(flow_debug_logging, int, 0644);
61MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging");
62
63int packet_debug_logging;
64module_param(packet_debug_logging, int, 0644);
65MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging");
66
67int debug_logging_sleep;
68module_param(debug_logging_sleep, int, 0644);
69MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep");
70
71/*
72 * The value of these module parameters is used to set the priority for each
73 * algo type when this driver registers algos with the kernel crypto API.
74 * To use a priority other than the default, set the priority in the insmod or
75 * modprobe. Changing the module priority after init time has no effect.
76 *
77 * The default priorities are chosen to be lower (less preferred) than ARMv8 CE
78 * algos, but more preferred than generic software algos.
79 */
80static int cipher_pri = 150;
81module_param(cipher_pri, int, 0644);
82MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos");
83
84static int hash_pri = 100;
85module_param(hash_pri, int, 0644);
86MODULE_PARM_DESC(hash_pri, "Priority for hash algos");
87
88static int aead_pri = 150;
89module_param(aead_pri, int, 0644);
90MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
91
92#define MAX_SPUS 16
93
94/* A type 3 BCM header, expected to precede the SPU header for SPU-M.
95 * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
96 * 0x60 - ring 0
97 * 0x68 - ring 1
98 * 0x70 - ring 2
99 * 0x78 - ring 3
100 */
101char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
102/*
103 * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
104 * is set dynamically after reading SPU type from device tree.
105 */
106#define BCM_HDR_LEN iproc_priv.bcm_hdr_len
107
108/* min and max time to sleep before retrying when mbox queue is full. usec */
109#define MBOX_SLEEP_MIN 800
110#define MBOX_SLEEP_MAX 1000
111
112/**
113 * select_channel() - Select a SPU channel to handle a crypto request. Selects
114 * channel in round robin order.
115 *
116 * Return: channel index
117 */
118static u8 select_channel(void)
119{
120 u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
121
122 return chan_idx % iproc_priv.spu.num_spu;
123}
124
125/**
126 * spu_ablkcipher_rx_sg_create() - Build up the scatterlist of buffers used to
127 * receive a SPU response message for an ablkcipher request. Includes buffers to
128 * catch SPU message headers and the response data.
129 * @mssg: mailbox message containing the receive sg
130 * @rctx: crypto request context
131 * @rx_frag_num: number of scatterlist elements required to hold the
132 * SPU response message
133 * @chunksize: Number of bytes of response data expected
134 * @stat_pad_len: Number of bytes required to pad the STAT field to
135 * a 4-byte boundary
136 *
137 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
138 * when the request completes, whether the request is handled successfully or
139 * there is an error.
140 *
141 * Returns:
142 * 0 if successful
143 * < 0 if an error
144 */
145static int
146spu_ablkcipher_rx_sg_create(struct brcm_message *mssg,
147 struct iproc_reqctx_s *rctx,
148 u8 rx_frag_num,
149 unsigned int chunksize, u32 stat_pad_len)
150{
151 struct spu_hw *spu = &iproc_priv.spu;
152 struct scatterlist *sg; /* used to build sgs in mbox message */
153 struct iproc_ctx_s *ctx = rctx->ctx;
154 u32 datalen; /* Number of bytes of response data expected */
155
156 mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
157 rctx->gfp);
158 if (!mssg->spu.dst)
159 return -ENOMEM;
160
161 sg = mssg->spu.dst;
162 sg_init_table(sg, rx_frag_num);
163 /* Space for SPU message header */
164 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
165
166 /* If XTS tweak in payload, add buffer to receive encrypted tweak */
167 if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
168 spu->spu_xts_tweak_in_payload())
169 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak,
170 SPU_XTS_TWEAK_SIZE);
171
172 /* Copy in each dst sg entry from request, up to chunksize */
173 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
174 rctx->dst_nents, chunksize);
175 if (datalen < chunksize) {
176 pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u",
177 __func__, chunksize, datalen);
178 return -EFAULT;
179 }
180
181 if (ctx->cipher.alg == CIPHER_ALG_RC4)
182 /* Add buffer to catch 260-byte SUPDT field for RC4 */
183 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_SUPDT_LEN);
184
185 if (stat_pad_len)
186 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
187
188 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
189 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
190
191 return 0;
192}
193
194/**
195 * spu_ablkcipher_tx_sg_create() - Build up the scatterlist of buffers used to
196 * send a SPU request message for an ablkcipher request. Includes SPU message
197 * headers and the request data.
198 * @mssg: mailbox message containing the transmit sg
199 * @rctx: crypto request context
200 * @tx_frag_num: number of scatterlist elements required to construct the
201 * SPU request message
202 * @chunksize: Number of bytes of request data
203 * @pad_len: Number of pad bytes
204 *
205 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
206 * when the request completes, whether the request is handled successfully or
207 * there is an error.
208 *
209 * Returns:
210 * 0 if successful
211 * < 0 if an error
212 */
213static int
214spu_ablkcipher_tx_sg_create(struct brcm_message *mssg,
215 struct iproc_reqctx_s *rctx,
216 u8 tx_frag_num, unsigned int chunksize, u32 pad_len)
217{
218 struct spu_hw *spu = &iproc_priv.spu;
219 struct scatterlist *sg; /* used to build sgs in mbox message */
220 struct iproc_ctx_s *ctx = rctx->ctx;
221 u32 datalen; /* Number of bytes of response data expected */
222 u32 stat_len;
223
224 mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
225 rctx->gfp);
226 if (unlikely(!mssg->spu.src))
227 return -ENOMEM;
228
229 sg = mssg->spu.src;
230 sg_init_table(sg, tx_frag_num);
231
232 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
233 BCM_HDR_LEN + ctx->spu_req_hdr_len);
234
235 /* if XTS tweak in payload, copy from IV (where crypto API puts it) */
236 if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
237 spu->spu_xts_tweak_in_payload())
238 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE);
239
240 /* Copy in each src sg entry from request, up to chunksize */
241 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
242 rctx->src_nents, chunksize);
243 if (unlikely(datalen < chunksize)) {
244 pr_err("%s(): failed to copy src sg to mbox msg",
245 __func__);
246 return -EFAULT;
247 }
248
249 if (pad_len)
250 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
251
252 stat_len = spu->spu_tx_status_len();
253 if (stat_len) {
254 memset(rctx->msg_buf.tx_stat, 0, stat_len);
255 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
256 }
257 return 0;
258}
259
260/**
261 * handle_ablkcipher_req() - Submit as much of a block cipher request as fits in
262 * a single SPU request message, starting at the current position in the request
263 * data.
264 * @rctx: Crypto request context
265 *
266 * This may be called on the crypto API thread, or, when a request is so large
267 * it must be broken into multiple SPU messages, on the thread used to invoke
268 * the response callback. When requests are broken into multiple SPU
269 * messages, we assume subsequent messages depend on previous results, and
270 * thus always wait for previous results before submitting the next message.
271 * Because requests are submitted in lock step like this, there is no need
272 * to synchronize access to request data structures.
273 *
274 * Return: -EINPROGRESS: request has been accepted and result will be returned
275 * asynchronously
276 * Any other value indicates an error
277 */
278static int handle_ablkcipher_req(struct iproc_reqctx_s *rctx)
279{
280 struct spu_hw *spu = &iproc_priv.spu;
281 struct crypto_async_request *areq = rctx->parent;
282 struct ablkcipher_request *req =
283 container_of(areq, struct ablkcipher_request, base);
284 struct iproc_ctx_s *ctx = rctx->ctx;
285 struct spu_cipher_parms cipher_parms;
286 int err = 0;
287 unsigned int chunksize = 0; /* Num bytes of request to submit */
288 int remaining = 0; /* Bytes of request still to process */
289 int chunk_start; /* Beginning of data for current SPU msg */
290
291 /* IV or ctr value to use in this SPU msg */
292 u8 local_iv_ctr[MAX_IV_SIZE];
293 u32 stat_pad_len; /* num bytes to align status field */
294 u32 pad_len; /* total length of all padding */
295 bool update_key = false;
296 struct brcm_message *mssg; /* mailbox message */
297 int retry_cnt = 0;
298
299 /* number of entries in src and dst sg in mailbox message. */
300 u8 rx_frag_num = 2; /* response header and STATUS */
301 u8 tx_frag_num = 1; /* request header */
302
303 flow_log("%s\n", __func__);
304
305 cipher_parms.alg = ctx->cipher.alg;
306 cipher_parms.mode = ctx->cipher.mode;
307 cipher_parms.type = ctx->cipher_type;
308 cipher_parms.key_len = ctx->enckeylen;
309 cipher_parms.key_buf = ctx->enckey;
310 cipher_parms.iv_buf = local_iv_ctr;
311 cipher_parms.iv_len = rctx->iv_ctr_len;
312
313 mssg = &rctx->mb_mssg;
314 chunk_start = rctx->src_sent;
315 remaining = rctx->total_todo - chunk_start;
316
317 /* determine the chunk we are breaking off and update the indexes */
318 if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
319 (remaining > ctx->max_payload))
320 chunksize = ctx->max_payload;
321 else
322 chunksize = remaining;
323
324 rctx->src_sent += chunksize;
325 rctx->total_sent = rctx->src_sent;
326
327 /* Count number of sg entries to be included in this request */
328 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
329 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
330
331 if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
332 rctx->is_encrypt && chunk_start)
333 /*
334 * Encrypting non-first first chunk. Copy last block of
335 * previous result to IV for this chunk.
336 */
337 sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr,
338 rctx->iv_ctr_len,
339 chunk_start - rctx->iv_ctr_len);
340
341 if (rctx->iv_ctr_len) {
342 /* get our local copy of the iv */
343 __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr,
344 rctx->iv_ctr_len);
345
346 /* generate the next IV if possible */
347 if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
348 !rctx->is_encrypt) {
349 /*
350 * CBC Decrypt: next IV is the last ciphertext block in
351 * this chunk
352 */
353 sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr,
354 rctx->iv_ctr_len,
355 rctx->src_sent - rctx->iv_ctr_len);
356 } else if (ctx->cipher.mode == CIPHER_MODE_CTR) {
357 /*
358 * The SPU hardware increments the counter once for
359 * each AES block of 16 bytes. So update the counter
360 * for the next chunk, if there is one. Note that for
361 * this chunk, the counter has already been copied to
362 * local_iv_ctr. We can assume a block size of 16,
363 * because we only support CTR mode for AES, not for
364 * any other cipher alg.
365 */
366 add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4);
367 }
368 }
369
370 if (ctx->cipher.alg == CIPHER_ALG_RC4) {
371 rx_frag_num++;
372 if (chunk_start) {
373 /*
374 * for non-first RC4 chunks, use SUPDT from previous
375 * response as key for this chunk.
376 */
377 cipher_parms.key_buf = rctx->msg_buf.c.supdt_tweak;
378 update_key = true;
379 cipher_parms.type = CIPHER_TYPE_UPDT;
380 } else if (!rctx->is_encrypt) {
381 /*
382 * First RC4 chunk. For decrypt, key in pre-built msg
383 * header may have been changed if encrypt required
384 * multiple chunks. So revert the key to the
385 * ctx->enckey value.
386 */
387 update_key = true;
388 cipher_parms.type = CIPHER_TYPE_INIT;
389 }
390 }
391
392 if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
393 flow_log("max_payload infinite\n");
394 else
395 flow_log("max_payload %u\n", ctx->max_payload);
396
397 flow_log("sent:%u start:%u remains:%u size:%u\n",
398 rctx->src_sent, chunk_start, remaining, chunksize);
399
400 /* Copy SPU header template created at setkey time */
401 memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
402 sizeof(rctx->msg_buf.bcm_spu_req_hdr));
403
404 /*
405 * Pass SUPDT field as key. Key field in finish() call is only used
406 * when update_key has been set above for RC4. Will be ignored in
407 * all other cases.
408 */
409 spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
410 ctx->spu_req_hdr_len, !(rctx->is_encrypt),
411 &cipher_parms, update_key, chunksize);
412
413 atomic64_add(chunksize, &iproc_priv.bytes_out);
414
415 stat_pad_len = spu->spu_wordalign_padlen(chunksize);
416 if (stat_pad_len)
417 rx_frag_num++;
418 pad_len = stat_pad_len;
419 if (pad_len) {
420 tx_frag_num++;
421 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0,
422 0, ctx->auth.alg, ctx->auth.mode,
423 rctx->total_sent, stat_pad_len);
424 }
425
426 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
427 ctx->spu_req_hdr_len);
428 packet_log("payload:\n");
429 dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
430 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
431
432 /*
433 * Build mailbox message containing SPU request msg and rx buffers
434 * to catch response message
435 */
436 memset(mssg, 0, sizeof(*mssg));
437 mssg->type = BRCM_MESSAGE_SPU;
438 mssg->ctx = rctx; /* Will be returned in response */
439
440 /* Create rx scatterlist to catch result */
441 rx_frag_num += rctx->dst_nents;
442
443 if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
444 spu->spu_xts_tweak_in_payload())
445 rx_frag_num++; /* extra sg to insert tweak */
446
447 err = spu_ablkcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize,
448 stat_pad_len);
449 if (err)
450 return err;
451
452 /* Create tx scatterlist containing SPU request message */
453 tx_frag_num += rctx->src_nents;
454 if (spu->spu_tx_status_len())
455 tx_frag_num++;
456
457 if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
458 spu->spu_xts_tweak_in_payload())
459 tx_frag_num++; /* extra sg to insert tweak */
460
461 err = spu_ablkcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize,
462 pad_len);
463 if (err)
464 return err;
465
466 err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg);
467 if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
468 while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
469 /*
470 * Mailbox queue is full. Since MAY_SLEEP is set, assume
471 * not in atomic context and we can wait and try again.
472 */
473 retry_cnt++;
474 usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
475 err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx],
476 mssg);
477 atomic_inc(&iproc_priv.mb_no_spc);
478 }
479 }
480 if (unlikely(err < 0)) {
481 atomic_inc(&iproc_priv.mb_send_fail);
482 return err;
483 }
484
485 return -EINPROGRESS;
486}
487
488/**
489 * handle_ablkcipher_resp() - Process a block cipher SPU response. Updates the
490 * total received count for the request and updates global stats.
491 * @rctx: Crypto request context
492 */
493static void handle_ablkcipher_resp(struct iproc_reqctx_s *rctx)
494{
495 struct spu_hw *spu = &iproc_priv.spu;
496#ifdef DEBUG
497 struct crypto_async_request *areq = rctx->parent;
498 struct ablkcipher_request *req = ablkcipher_request_cast(areq);
499#endif
500 struct iproc_ctx_s *ctx = rctx->ctx;
501 u32 payload_len;
502
503 /* See how much data was returned */
504 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
505
506 /*
507 * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the
508 * encrypted tweak ("i") value; we don't count those.
509 */
510 if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
511 spu->spu_xts_tweak_in_payload() &&
512 (payload_len >= SPU_XTS_TWEAK_SIZE))
513 payload_len -= SPU_XTS_TWEAK_SIZE;
514
515 atomic64_add(payload_len, &iproc_priv.bytes_in);
516
517 flow_log("%s() offset: %u, bd_len: %u BD:\n",
518 __func__, rctx->total_received, payload_len);
519
520 dump_sg(req->dst, rctx->total_received, payload_len);
521 if (ctx->cipher.alg == CIPHER_ALG_RC4)
522 packet_dump(" supdt ", rctx->msg_buf.c.supdt_tweak,
523 SPU_SUPDT_LEN);
524
525 rctx->total_received += payload_len;
526 if (rctx->total_received == rctx->total_todo) {
527 atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]);
528 atomic_inc(
529 &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);
530 }
531}
532
533/**
534 * spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to
535 * receive a SPU response message for an ahash request.
536 * @mssg: mailbox message containing the receive sg
537 * @rctx: crypto request context
538 * @rx_frag_num: number of scatterlist elements required to hold the
539 * SPU response message
540 * @digestsize: length of hash digest, in bytes
541 * @stat_pad_len: Number of bytes required to pad the STAT field to
542 * a 4-byte boundary
543 *
544 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
545 * when the request completes, whether the request is handled successfully or
546 * there is an error.
547 *
548 * Return:
549 * 0 if successful
550 * < 0 if an error
551 */
552static int
553spu_ahash_rx_sg_create(struct brcm_message *mssg,
554 struct iproc_reqctx_s *rctx,
555 u8 rx_frag_num, unsigned int digestsize,
556 u32 stat_pad_len)
557{
558 struct spu_hw *spu = &iproc_priv.spu;
559 struct scatterlist *sg; /* used to build sgs in mbox message */
560 struct iproc_ctx_s *ctx = rctx->ctx;
561
562 mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
563 rctx->gfp);
564 if (!mssg->spu.dst)
565 return -ENOMEM;
566
567 sg = mssg->spu.dst;
568 sg_init_table(sg, rx_frag_num);
569 /* Space for SPU message header */
570 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
571
572 /* Space for digest */
573 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
574
575 if (stat_pad_len)
576 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
577
578 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
579 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
580 return 0;
581}
582
583/**
584 * spu_ahash_tx_sg_create() - Build up the scatterlist of buffers used to send
585 * a SPU request message for an ahash request. Includes SPU message headers and
586 * the request data.
587 * @mssg: mailbox message containing the transmit sg
588 * @rctx: crypto request context
589 * @tx_frag_num: number of scatterlist elements required to construct the
590 * SPU request message
591 * @spu_hdr_len: length in bytes of SPU message header
592 * @hash_carry_len: Number of bytes of data carried over from previous req
593 * @new_data_len: Number of bytes of new request data
594 * @pad_len: Number of pad bytes
595 *
596 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
597 * when the request completes, whether the request is handled successfully or
598 * there is an error.
599 *
600 * Return:
601 * 0 if successful
602 * < 0 if an error
603 */
604static int
605spu_ahash_tx_sg_create(struct brcm_message *mssg,
606 struct iproc_reqctx_s *rctx,
607 u8 tx_frag_num,
608 u32 spu_hdr_len,
609 unsigned int hash_carry_len,
610 unsigned int new_data_len, u32 pad_len)
611{
612 struct spu_hw *spu = &iproc_priv.spu;
613 struct scatterlist *sg; /* used to build sgs in mbox message */
614 u32 datalen; /* Number of bytes of response data expected */
615 u32 stat_len;
616
617 mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
618 rctx->gfp);
619 if (!mssg->spu.src)
620 return -ENOMEM;
621
622 sg = mssg->spu.src;
623 sg_init_table(sg, tx_frag_num);
624
625 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
626 BCM_HDR_LEN + spu_hdr_len);
627
628 if (hash_carry_len)
629 sg_set_buf(sg++, rctx->hash_carry, hash_carry_len);
630
631 if (new_data_len) {
632 /* Copy in each src sg entry from request, up to chunksize */
633 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
634 rctx->src_nents, new_data_len);
635 if (datalen < new_data_len) {
636 pr_err("%s(): failed to copy src sg to mbox msg",
637 __func__);
638 return -EFAULT;
639 }
640 }
641
642 if (pad_len)
643 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
644
645 stat_len = spu->spu_tx_status_len();
646 if (stat_len) {
647 memset(rctx->msg_buf.tx_stat, 0, stat_len);
648 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
649 }
650
651 return 0;
652}
653
654/**
655 * handle_ahash_req() - Process an asynchronous hash request from the crypto
656 * API.
657 * @rctx: Crypto request context
658 *
659 * Builds a SPU request message embedded in a mailbox message and submits the
660 * mailbox message on a selected mailbox channel. The SPU request message is
661 * constructed as a scatterlist, including entries from the crypto API's
662 * src scatterlist to avoid copying the data to be hashed. This function is
663 * called either on the thread from the crypto API, or, in the case that the
664 * crypto API request is too large to fit in a single SPU request message,
665 * on the thread that invokes the receive callback with a response message.
666 * Because some operations require the response from one chunk before the next
667 * chunk can be submitted, we always wait for the response for the previous
668 * chunk before submitting the next chunk. Because requests are submitted in
669 * lock step like this, there is no need to synchronize access to request data
670 * structures.
671 *
672 * Return:
673 * -EINPROGRESS: request has been submitted to SPU and response will be
674 * returned asynchronously
675 * -EAGAIN: non-final request included a small amount of data, which for
676 * efficiency we did not submit to the SPU, but instead stored
677 * to be submitted to the SPU with the next part of the request
678 * other: an error code
679 */
680static int handle_ahash_req(struct iproc_reqctx_s *rctx)
681{
682 struct spu_hw *spu = &iproc_priv.spu;
683 struct crypto_async_request *areq = rctx->parent;
684 struct ahash_request *req = ahash_request_cast(areq);
685 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
686 struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
687 unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
688 struct iproc_ctx_s *ctx = rctx->ctx;
689
690 /* number of bytes still to be hashed in this req */
691 unsigned int nbytes_to_hash = 0;
692 int err = 0;
693 unsigned int chunksize = 0; /* length of hash carry + new data */
694 /*
695 * length of new data, not from hash carry, to be submitted in
696 * this hw request
697 */
698 unsigned int new_data_len;
699
700 unsigned int chunk_start = 0;
701 u32 db_size; /* Length of data field, incl gcm and hash padding */
702 int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
703 u32 data_pad_len = 0; /* length of GCM/CCM padding */
704 u32 stat_pad_len = 0; /* length of padding to align STATUS word */
705 struct brcm_message *mssg; /* mailbox message */
706 struct spu_request_opts req_opts;
707 struct spu_cipher_parms cipher_parms;
708 struct spu_hash_parms hash_parms;
709 struct spu_aead_parms aead_parms;
710 unsigned int local_nbuf;
711 u32 spu_hdr_len;
712 unsigned int digestsize;
713 u16 rem = 0;
714 int retry_cnt = 0;
715
716 /*
717 * number of entries in src and dst sg. Always includes SPU msg header.
718 * rx always includes a buffer to catch digest and STATUS.
719 */
720 u8 rx_frag_num = 3;
721 u8 tx_frag_num = 1;
722
723 flow_log("total_todo %u, total_sent %u\n",
724 rctx->total_todo, rctx->total_sent);
725
726 memset(&req_opts, 0, sizeof(req_opts));
727 memset(&cipher_parms, 0, sizeof(cipher_parms));
728 memset(&hash_parms, 0, sizeof(hash_parms));
729 memset(&aead_parms, 0, sizeof(aead_parms));
730
731 req_opts.bd_suppress = true;
732 hash_parms.alg = ctx->auth.alg;
733 hash_parms.mode = ctx->auth.mode;
734 hash_parms.type = HASH_TYPE_NONE;
735 hash_parms.key_buf = (u8 *)ctx->authkey;
736 hash_parms.key_len = ctx->authkeylen;
737
738 /*
739 * For hash algorithms below assignment looks bit odd but
740 * it's needed for AES-XCBC and AES-CMAC hash algorithms
741 * to differentiate between 128, 192, 256 bit key values.
742 * Based on the key values, hash algorithm is selected.
743 * For example for 128 bit key, hash algorithm is AES-128.
744 */
745 cipher_parms.type = ctx->cipher_type;
746
747 mssg = &rctx->mb_mssg;
748 chunk_start = rctx->src_sent;
749
750 /*
751 * Compute the amount remaining to hash. This may include data
752 * carried over from previous requests.
753 */
754 nbytes_to_hash = rctx->total_todo - rctx->total_sent;
755 chunksize = nbytes_to_hash;
756 if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
757 (chunksize > ctx->max_payload))
758 chunksize = ctx->max_payload;
759
760 /*
761 * If this is not a final request and the request data is not a multiple
762 * of a full block, then simply park the extra data and prefix it to the
763 * data for the next request.
764 */
765 if (!rctx->is_final) {
766 u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
767 u16 new_len; /* len of data to add to hash carry */
768
769 rem = chunksize % blocksize; /* remainder */
770 if (rem) {
771 /* chunksize not a multiple of blocksize */
772 chunksize -= rem;
773 if (chunksize == 0) {
774 /* Don't have a full block to submit to hw */
775 new_len = rem - rctx->hash_carry_len;
776 sg_copy_part_to_buf(req->src, dest, new_len,
777 rctx->src_sent);
778 rctx->hash_carry_len = rem;
779 flow_log("Exiting with hash carry len: %u\n",
780 rctx->hash_carry_len);
781 packet_dump(" buf: ",
782 rctx->hash_carry,
783 rctx->hash_carry_len);
784 return -EAGAIN;
785 }
786 }
787 }
788
789 /* if we have hash carry, then prefix it to the data in this request */
790 local_nbuf = rctx->hash_carry_len;
791 rctx->hash_carry_len = 0;
792 if (local_nbuf)
793 tx_frag_num++;
794 new_data_len = chunksize - local_nbuf;
795
796 /* Count number of sg entries to be used in this request */
797 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
798 new_data_len);
799
800 /* AES hashing keeps key size in type field, so need to copy it here */
801 if (hash_parms.alg == HASH_ALG_AES)
802 hash_parms.type = cipher_parms.type;
803 else
804 hash_parms.type = spu->spu_hash_type(rctx->total_sent);
805
806 digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
807 hash_parms.type);
808 hash_parms.digestsize = digestsize;
809
810 /* update the indexes */
811 rctx->total_sent += chunksize;
812 /* if you sent a prebuf then that wasn't from this req->src */
813 rctx->src_sent += new_data_len;
814
815 if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
816 hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
817 hash_parms.mode,
818 chunksize,
819 blocksize);
820
821 /*
822 * If a non-first chunk, then include the digest returned from the
823 * previous chunk so that hw can add to it (except for AES types).
824 */
825 if ((hash_parms.type == HASH_TYPE_UPDT) &&
826 (hash_parms.alg != HASH_ALG_AES)) {
827 hash_parms.key_buf = rctx->incr_hash;
828 hash_parms.key_len = digestsize;
829 }
830
831 atomic64_add(chunksize, &iproc_priv.bytes_out);
832
833 flow_log("%s() final: %u nbuf: %u ",
834 __func__, rctx->is_final, local_nbuf);
835
836 if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
837 flow_log("max_payload infinite\n");
838 else
839 flow_log("max_payload %u\n", ctx->max_payload);
840
841 flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
842
843 /* Prepend SPU header with type 3 BCM header */
844 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
845
846 hash_parms.prebuf_len = local_nbuf;
847 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
848 BCM_HDR_LEN,
849 &req_opts, &cipher_parms,
850 &hash_parms, &aead_parms,
851 new_data_len);
852
853 if (spu_hdr_len == 0) {
854 pr_err("Failed to create SPU request header\n");
855 return -EFAULT;
856 }
857
858 /*
859 * Determine total length of padding required. Put all padding in one
860 * buffer.
861 */
862 data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
863 db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
864 0, 0, hash_parms.pad_len);
865 if (spu->spu_tx_status_len())
866 stat_pad_len = spu->spu_wordalign_padlen(db_size);
867 if (stat_pad_len)
868 rx_frag_num++;
869 pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
870 if (pad_len) {
871 tx_frag_num++;
872 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
873 hash_parms.pad_len, ctx->auth.alg,
874 ctx->auth.mode, rctx->total_sent,
875 stat_pad_len);
876 }
877
878 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
879 spu_hdr_len);
880 packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf);
881 flow_log("Data:\n");
882 dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
883 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
884
885 /*
886 * Build mailbox message containing SPU request msg and rx buffers
887 * to catch response message
888 */
889 memset(mssg, 0, sizeof(*mssg));
890 mssg->type = BRCM_MESSAGE_SPU;
891 mssg->ctx = rctx; /* Will be returned in response */
892
893 /* Create rx scatterlist to catch result */
894 err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
895 stat_pad_len);
896 if (err)
897 return err;
898
899 /* Create tx scatterlist containing SPU request message */
900 tx_frag_num += rctx->src_nents;
901 if (spu->spu_tx_status_len())
902 tx_frag_num++;
903 err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
904 local_nbuf, new_data_len, pad_len);
905 if (err)
906 return err;
907
908 err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg);
909 if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
910 while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
911 /*
912 * Mailbox queue is full. Since MAY_SLEEP is set, assume
913 * not in atomic context and we can wait and try again.
914 */
915 retry_cnt++;
916 usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
917 err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx],
918 mssg);
919 atomic_inc(&iproc_priv.mb_no_spc);
920 }
921 }
922 if (err < 0) {
923 atomic_inc(&iproc_priv.mb_send_fail);
924 return err;
925 }
926 return -EINPROGRESS;
927}
928
929/**
930 * spu_hmac_outer_hash() - Request synchonous software compute of the outer hash
931 * for an HMAC request.
932 * @req: The HMAC request from the crypto API
933 * @ctx: The session context
934 *
935 * Return: 0 if synchronous hash operation successful
936 * -EINVAL if the hash algo is unrecognized
937 * any other value indicates an error
938 */
939static int spu_hmac_outer_hash(struct ahash_request *req,
940 struct iproc_ctx_s *ctx)
941{
942 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
943 unsigned int blocksize =
944 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
945 int rc;
946
947 switch (ctx->auth.alg) {
948 case HASH_ALG_MD5:
949 rc = do_shash("md5", req->result, ctx->opad, blocksize,
950 req->result, ctx->digestsize, NULL, 0);
951 break;
952 case HASH_ALG_SHA1:
953 rc = do_shash("sha1", req->result, ctx->opad, blocksize,
954 req->result, ctx->digestsize, NULL, 0);
955 break;
956 case HASH_ALG_SHA224:
957 rc = do_shash("sha224", req->result, ctx->opad, blocksize,
958 req->result, ctx->digestsize, NULL, 0);
959 break;
960 case HASH_ALG_SHA256:
961 rc = do_shash("sha256", req->result, ctx->opad, blocksize,
962 req->result, ctx->digestsize, NULL, 0);
963 break;
964 case HASH_ALG_SHA384:
965 rc = do_shash("sha384", req->result, ctx->opad, blocksize,
966 req->result, ctx->digestsize, NULL, 0);
967 break;
968 case HASH_ALG_SHA512:
969 rc = do_shash("sha512", req->result, ctx->opad, blocksize,
970 req->result, ctx->digestsize, NULL, 0);
971 break;
972 default:
973 pr_err("%s() Error : unknown hmac type\n", __func__);
974 rc = -EINVAL;
975 }
976 return rc;
977}
978
979/**
980 * ahash_req_done() - Process a hash result from the SPU hardware.
981 * @rctx: Crypto request context
982 *
983 * Return: 0 if successful
984 * < 0 if an error
985 */
986static int ahash_req_done(struct iproc_reqctx_s *rctx)
987{
988 struct spu_hw *spu = &iproc_priv.spu;
989 struct crypto_async_request *areq = rctx->parent;
990 struct ahash_request *req = ahash_request_cast(areq);
991 struct iproc_ctx_s *ctx = rctx->ctx;
992 int err;
993
994 memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize);
995
996 if (spu->spu_type == SPU_TYPE_SPUM) {
997 /* byte swap the output from the UPDT function to network byte
998 * order
999 */
1000 if (ctx->auth.alg == HASH_ALG_MD5) {
1001 __swab32s((u32 *)req->result);
1002 __swab32s(((u32 *)req->result) + 1);
1003 __swab32s(((u32 *)req->result) + 2);
1004 __swab32s(((u32 *)req->result) + 3);
1005 __swab32s(((u32 *)req->result) + 4);
1006 }
1007 }
1008
1009 flow_dump(" digest ", req->result, ctx->digestsize);
1010
1011 /* if this an HMAC then do the outer hash */
1012 if (rctx->is_sw_hmac) {
1013 err = spu_hmac_outer_hash(req, ctx);
1014 if (err < 0)
1015 return err;
1016 flow_dump(" hmac: ", req->result, ctx->digestsize);
1017 }
1018
1019 if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) {
1020 atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]);
1021 atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]);
1022 } else {
1023 atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]);
1024 atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]);
1025 }
1026
1027 return 0;
1028}
1029
1030/**
1031 * handle_ahash_resp() - Process a SPU response message for a hash request.
1032 * Checks if the entire crypto API request has been processed, and if so,
1033 * invokes post processing on the result.
1034 * @rctx: Crypto request context
1035 */
1036static void handle_ahash_resp(struct iproc_reqctx_s *rctx)
1037{
1038 struct iproc_ctx_s *ctx = rctx->ctx;
1039#ifdef DEBUG
1040 struct crypto_async_request *areq = rctx->parent;
1041 struct ahash_request *req = ahash_request_cast(areq);
1042 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1043 unsigned int blocksize =
1044 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
1045#endif
1046 /*
1047 * Save hash to use as input to next op if incremental. Might be copying
1048 * too much, but that's easier than figuring out actual digest size here
1049 */
1050 memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE);
1051
1052 flow_log("%s() blocksize:%u digestsize:%u\n",
1053 __func__, blocksize, ctx->digestsize);
1054
1055 atomic64_add(ctx->digestsize, &iproc_priv.bytes_in);
1056
1057 if (rctx->is_final && (rctx->total_sent == rctx->total_todo))
1058 ahash_req_done(rctx);
1059}
1060
1061/**
1062 * spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive
1063 * a SPU response message for an AEAD request. Includes buffers to catch SPU
1064 * message headers and the response data.
1065 * @mssg: mailbox message containing the receive sg
1066 * @rctx: crypto request context
1067 * @rx_frag_num: number of scatterlist elements required to hold the
1068 * SPU response message
1069 * @assoc_len: Length of associated data included in the crypto request
1070 * @ret_iv_len: Length of IV returned in response
1071 * @resp_len: Number of bytes of response data expected to be written to
1072 * dst buffer from crypto API
1073 * @digestsize: Length of hash digest, in bytes
1074 * @stat_pad_len: Number of bytes required to pad the STAT field to
1075 * a 4-byte boundary
1076 *
1077 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1078 * when the request completes, whether the request is handled successfully or
1079 * there is an error.
1080 *
1081 * Returns:
1082 * 0 if successful
1083 * < 0 if an error
1084 */
1085static int spu_aead_rx_sg_create(struct brcm_message *mssg,
1086 struct aead_request *req,
1087 struct iproc_reqctx_s *rctx,
1088 u8 rx_frag_num,
1089 unsigned int assoc_len,
1090 u32 ret_iv_len, unsigned int resp_len,
1091 unsigned int digestsize, u32 stat_pad_len)
1092{
1093 struct spu_hw *spu = &iproc_priv.spu;
1094 struct scatterlist *sg; /* used to build sgs in mbox message */
1095 struct iproc_ctx_s *ctx = rctx->ctx;
1096 u32 datalen; /* Number of bytes of response data expected */
1097 u32 assoc_buf_len;
1098 u8 data_padlen = 0;
1099
1100 if (ctx->is_rfc4543) {
1101 /* RFC4543: only pad after data, not after AAD */
1102 data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1103 assoc_len + resp_len);
1104 assoc_buf_len = assoc_len;
1105 } else {
1106 data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1107 resp_len);
1108 assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode,
1109 assoc_len, ret_iv_len,
1110 rctx->is_encrypt);
1111 }
1112
1113 if (ctx->cipher.mode == CIPHER_MODE_CCM)
1114 /* ICV (after data) must be in the next 32-bit word for CCM */
1115 data_padlen += spu->spu_wordalign_padlen(assoc_buf_len +
1116 resp_len +
1117 data_padlen);
1118
1119 if (data_padlen)
1120 /* have to catch gcm pad in separate buffer */
1121 rx_frag_num++;
1122
1123 mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
1124 rctx->gfp);
1125 if (!mssg->spu.dst)
1126 return -ENOMEM;
1127
1128 sg = mssg->spu.dst;
1129 sg_init_table(sg, rx_frag_num);
1130
1131 /* Space for SPU message header */
1132 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
1133
1134 if (assoc_buf_len) {
1135 /*
1136 * Don't write directly to req->dst, because SPU may pad the
1137 * assoc data in the response
1138 */
1139 memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len);
1140 sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len);
1141 }
1142
1143 if (resp_len) {
1144 /*
1145 * Copy in each dst sg entry from request, up to chunksize.
1146 * dst sg catches just the data. digest caught in separate buf.
1147 */
1148 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
1149 rctx->dst_nents, resp_len);
1150 if (datalen < (resp_len)) {
1151 pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u",
1152 __func__, resp_len, datalen);
1153 return -EFAULT;
1154 }
1155 }
1156
1157 /* If GCM/CCM data is padded, catch padding in separate buffer */
1158 if (data_padlen) {
1159 memset(rctx->msg_buf.a.gcmpad, 0, data_padlen);
1160 sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen);
1161 }
1162
1163 /* Always catch ICV in separate buffer */
1164 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
1165
1166 flow_log("stat_pad_len %u\n", stat_pad_len);
1167 if (stat_pad_len) {
1168 memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len);
1169 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
1170 }
1171
1172 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
1173 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
1174
1175 return 0;
1176}
1177
1178/**
1179 * spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a
1180 * SPU request message for an AEAD request. Includes SPU message headers and the
1181 * request data.
1182 * @mssg: mailbox message containing the transmit sg
1183 * @rctx: crypto request context
1184 * @tx_frag_num: number of scatterlist elements required to construct the
1185 * SPU request message
1186 * @spu_hdr_len: length of SPU message header in bytes
1187 * @assoc: crypto API associated data scatterlist
1188 * @assoc_len: length of associated data
1189 * @assoc_nents: number of scatterlist entries containing assoc data
1190 * @aead_iv_len: length of AEAD IV, if included
1191 * @chunksize: Number of bytes of request data
1192 * @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM.
1193 * @pad_len: Number of pad bytes
1194 * @incl_icv: If true, write separate ICV buffer after data and
1195 * any padding
1196 *
1197 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1198 * when the request completes, whether the request is handled successfully or
1199 * there is an error.
1200 *
1201 * Return:
1202 * 0 if successful
1203 * < 0 if an error
1204 */
1205static int spu_aead_tx_sg_create(struct brcm_message *mssg,
1206 struct iproc_reqctx_s *rctx,
1207 u8 tx_frag_num,
1208 u32 spu_hdr_len,
1209 struct scatterlist *assoc,
1210 unsigned int assoc_len,
1211 int assoc_nents,
1212 unsigned int aead_iv_len,
1213 unsigned int chunksize,
1214 u32 aad_pad_len, u32 pad_len, bool incl_icv)
1215{
1216 struct spu_hw *spu = &iproc_priv.spu;
1217 struct scatterlist *sg; /* used to build sgs in mbox message */
1218 struct scatterlist *assoc_sg = assoc;
1219 struct iproc_ctx_s *ctx = rctx->ctx;
1220 u32 datalen; /* Number of bytes of data to write */
1221 u32 written; /* Number of bytes of data written */
1222 u32 assoc_offset = 0;
1223 u32 stat_len;
1224
1225 mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
1226 rctx->gfp);
1227 if (!mssg->spu.src)
1228 return -ENOMEM;
1229
1230 sg = mssg->spu.src;
1231 sg_init_table(sg, tx_frag_num);
1232
1233 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
1234 BCM_HDR_LEN + spu_hdr_len);
1235
1236 if (assoc_len) {
1237 /* Copy in each associated data sg entry from request */
1238 written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset,
1239 assoc_nents, assoc_len);
1240 if (written < assoc_len) {
1241 pr_err("%s(): failed to copy assoc sg to mbox msg",
1242 __func__);
1243 return -EFAULT;
1244 }
1245 }
1246
1247 if (aead_iv_len)
1248 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len);
1249
1250 if (aad_pad_len) {
1251 memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len);
1252 sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len);
1253 }
1254
1255 datalen = chunksize;
1256 if ((chunksize > ctx->digestsize) && incl_icv)
1257 datalen -= ctx->digestsize;
1258 if (datalen) {
1259 /* For aead, a single msg should consume the entire src sg */
1260 written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
1261 rctx->src_nents, datalen);
1262 if (written < datalen) {
1263 pr_err("%s(): failed to copy src sg to mbox msg",
1264 __func__);
1265 return -EFAULT;
1266 }
1267 }
1268
1269 if (pad_len) {
1270 memset(rctx->msg_buf.spu_req_pad, 0, pad_len);
1271 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
1272 }
1273
1274 if (incl_icv)
1275 sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize);
1276
1277 stat_len = spu->spu_tx_status_len();
1278 if (stat_len) {
1279 memset(rctx->msg_buf.tx_stat, 0, stat_len);
1280 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
1281 }
1282 return 0;
1283}
1284
1285/**
1286 * handle_aead_req() - Submit a SPU request message for the next chunk of the
1287 * current AEAD request.
1288 * @rctx: Crypto request context
1289 *
1290 * Unlike other operation types, we assume the length of the request fits in
1291 * a single SPU request message. aead_enqueue() makes sure this is true.
1292 * Comments for other op types regarding threads applies here as well.
1293 *
1294 * Unlike incremental hash ops, where the spu returns the entire hash for
1295 * truncated algs like sha-224, the SPU returns just the truncated hash in
1296 * response to aead requests. So digestsize is always ctx->digestsize here.
1297 *
1298 * Return: -EINPROGRESS: crypto request has been accepted and result will be
1299 * returned asynchronously
1300 * Any other value indicates an error
1301 */
1302static int handle_aead_req(struct iproc_reqctx_s *rctx)
1303{
1304 struct spu_hw *spu = &iproc_priv.spu;
1305 struct crypto_async_request *areq = rctx->parent;
1306 struct aead_request *req = container_of(areq,
1307 struct aead_request, base);
1308 struct iproc_ctx_s *ctx = rctx->ctx;
1309 int err;
1310 unsigned int chunksize;
1311 unsigned int resp_len;
1312 u32 spu_hdr_len;
1313 u32 db_size;
1314 u32 stat_pad_len;
1315 u32 pad_len;
1316 struct brcm_message *mssg; /* mailbox message */
1317 struct spu_request_opts req_opts;
1318 struct spu_cipher_parms cipher_parms;
1319 struct spu_hash_parms hash_parms;
1320 struct spu_aead_parms aead_parms;
1321 int assoc_nents = 0;
1322 bool incl_icv = false;
1323 unsigned int digestsize = ctx->digestsize;
1324 int retry_cnt = 0;
1325
1326 /* number of entries in src and dst sg. Always includes SPU msg header.
1327 */
1328 u8 rx_frag_num = 2; /* and STATUS */
1329 u8 tx_frag_num = 1;
1330
1331 /* doing the whole thing at once */
1332 chunksize = rctx->total_todo;
1333
1334 flow_log("%s: chunksize %u\n", __func__, chunksize);
1335
1336 memset(&req_opts, 0, sizeof(req_opts));
1337 memset(&hash_parms, 0, sizeof(hash_parms));
1338 memset(&aead_parms, 0, sizeof(aead_parms));
1339
1340 req_opts.is_inbound = !(rctx->is_encrypt);
1341 req_opts.auth_first = ctx->auth_first;
1342 req_opts.is_aead = true;
1343 req_opts.is_esp = ctx->is_esp;
1344
1345 cipher_parms.alg = ctx->cipher.alg;
1346 cipher_parms.mode = ctx->cipher.mode;
1347 cipher_parms.type = ctx->cipher_type;
1348 cipher_parms.key_buf = ctx->enckey;
1349 cipher_parms.key_len = ctx->enckeylen;
1350 cipher_parms.iv_buf = rctx->msg_buf.iv_ctr;
1351 cipher_parms.iv_len = rctx->iv_ctr_len;
1352
1353 hash_parms.alg = ctx->auth.alg;
1354 hash_parms.mode = ctx->auth.mode;
1355 hash_parms.type = HASH_TYPE_NONE;
1356 hash_parms.key_buf = (u8 *)ctx->authkey;
1357 hash_parms.key_len = ctx->authkeylen;
1358 hash_parms.digestsize = digestsize;
1359
1360 if ((ctx->auth.alg == HASH_ALG_SHA224) &&
1361 (ctx->authkeylen < SHA224_DIGEST_SIZE))
1362 hash_parms.key_len = SHA224_DIGEST_SIZE;
1363
1364 aead_parms.assoc_size = req->assoclen;
1365 if (ctx->is_esp && !ctx->is_rfc4543) {
1366 /*
1367 * 8-byte IV is included assoc data in request. SPU2
1368 * expects AAD to include just SPI and seqno. So
1369 * subtract off the IV len.
1370 */
1371 aead_parms.assoc_size -= GCM_ESP_IV_SIZE;
1372
1373 if (rctx->is_encrypt) {
1374 aead_parms.return_iv = true;
1375 aead_parms.ret_iv_len = GCM_ESP_IV_SIZE;
1376 aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE;
1377 }
1378 } else {
1379 aead_parms.ret_iv_len = 0;
1380 }
1381
1382 /*
1383 * Count number of sg entries from the crypto API request that are to
1384 * be included in this mailbox message. For dst sg, don't count space
1385 * for digest. Digest gets caught in a separate buffer and copied back
1386 * to dst sg when processing response.
1387 */
1388 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
1389 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
1390 if (aead_parms.assoc_size)
1391 assoc_nents = spu_sg_count(rctx->assoc, 0,
1392 aead_parms.assoc_size);
1393
1394 mssg = &rctx->mb_mssg;
1395
1396 rctx->total_sent = chunksize;
1397 rctx->src_sent = chunksize;
1398 if (spu->spu_assoc_resp_len(ctx->cipher.mode,
1399 aead_parms.assoc_size,
1400 aead_parms.ret_iv_len,
1401 rctx->is_encrypt))
1402 rx_frag_num++;
1403
1404 aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode,
1405 rctx->iv_ctr_len);
1406
1407 if (ctx->auth.alg == HASH_ALG_AES)
1408 hash_parms.type = ctx->cipher_type;
1409
1410 /* General case AAD padding (CCM and RFC4543 special cases below) */
1411 aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1412 aead_parms.assoc_size);
1413
1414 /* General case data padding (CCM decrypt special case below) */
1415 aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1416 chunksize);
1417
1418 if (ctx->cipher.mode == CIPHER_MODE_CCM) {
1419 /*
1420 * for CCM, AAD len + 2 (rather than AAD len) needs to be
1421 * 128-bit aligned
1422 */
1423 aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(
1424 ctx->cipher.mode,
1425 aead_parms.assoc_size + 2);
1426
1427 /*
1428 * And when decrypting CCM, need to pad without including
1429 * size of ICV which is tacked on to end of chunk
1430 */
1431 if (!rctx->is_encrypt)
1432 aead_parms.data_pad_len =
1433 spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1434 chunksize - digestsize);
1435
1436 /* CCM also requires software to rewrite portions of IV: */
1437 spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen,
1438 chunksize, rctx->is_encrypt,
1439 ctx->is_esp);
1440 }
1441
1442 if (ctx->is_rfc4543) {
1443 /*
1444 * RFC4543: data is included in AAD, so don't pad after AAD
1445 * and pad data based on both AAD + data size
1446 */
1447 aead_parms.aad_pad_len = 0;
1448 if (!rctx->is_encrypt)
1449 aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1450 ctx->cipher.mode,
1451 aead_parms.assoc_size + chunksize -
1452 digestsize);
1453 else
1454 aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1455 ctx->cipher.mode,
1456 aead_parms.assoc_size + chunksize);
1457
1458 req_opts.is_rfc4543 = true;
1459 }
1460
1461 if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) {
1462 incl_icv = true;
1463 tx_frag_num++;
1464 /* Copy ICV from end of src scatterlist to digest buf */
1465 sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize,
1466 req->assoclen + rctx->total_sent -
1467 digestsize);
1468 }
1469
1470 atomic64_add(chunksize, &iproc_priv.bytes_out);
1471
1472 flow_log("%s()-sent chunksize:%u\n", __func__, chunksize);
1473
1474 /* Prepend SPU header with type 3 BCM header */
1475 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1476
1477 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
1478 BCM_HDR_LEN, &req_opts,
1479 &cipher_parms, &hash_parms,
1480 &aead_parms, chunksize);
1481
1482 /* Determine total length of padding. Put all padding in one buffer. */
1483 db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0,
1484 chunksize, aead_parms.aad_pad_len,
1485 aead_parms.data_pad_len, 0);
1486
1487 stat_pad_len = spu->spu_wordalign_padlen(db_size);
1488
1489 if (stat_pad_len)
1490 rx_frag_num++;
1491 pad_len = aead_parms.data_pad_len + stat_pad_len;
1492 if (pad_len) {
1493 tx_frag_num++;
1494 spu->spu_request_pad(rctx->msg_buf.spu_req_pad,
1495 aead_parms.data_pad_len, 0,
1496 ctx->auth.alg, ctx->auth.mode,
1497 rctx->total_sent, stat_pad_len);
1498 }
1499
1500 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
1501 spu_hdr_len);
1502 dump_sg(rctx->assoc, 0, aead_parms.assoc_size);
1503 packet_dump(" aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len);
1504 packet_log("BD:\n");
1505 dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
1506 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
1507
1508 /*
1509 * Build mailbox message containing SPU request msg and rx buffers
1510 * to catch response message
1511 */
1512 memset(mssg, 0, sizeof(*mssg));
1513 mssg->type = BRCM_MESSAGE_SPU;
1514 mssg->ctx = rctx; /* Will be returned in response */
1515
1516 /* Create rx scatterlist to catch result */
1517 rx_frag_num += rctx->dst_nents;
1518 resp_len = chunksize;
1519
1520 /*
1521 * Always catch ICV in separate buffer. Have to for GCM/CCM because of
1522 * padding. Have to for SHA-224 and other truncated SHAs because SPU
1523 * sends entire digest back.
1524 */
1525 rx_frag_num++;
1526
1527 if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
1528 (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) {
1529 /*
1530 * Input is ciphertxt plus ICV, but ICV not incl
1531 * in output.
1532 */
1533 resp_len -= ctx->digestsize;
1534 if (resp_len == 0)
1535 /* no rx frags to catch output data */
1536 rx_frag_num -= rctx->dst_nents;
1537 }
1538
1539 err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num,
1540 aead_parms.assoc_size,
1541 aead_parms.ret_iv_len, resp_len, digestsize,
1542 stat_pad_len);
1543 if (err)
1544 return err;
1545
1546 /* Create tx scatterlist containing SPU request message */
1547 tx_frag_num += rctx->src_nents;
1548 tx_frag_num += assoc_nents;
1549 if (aead_parms.aad_pad_len)
1550 tx_frag_num++;
1551 if (aead_parms.iv_len)
1552 tx_frag_num++;
1553 if (spu->spu_tx_status_len())
1554 tx_frag_num++;
1555 err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
1556 rctx->assoc, aead_parms.assoc_size,
1557 assoc_nents, aead_parms.iv_len, chunksize,
1558 aead_parms.aad_pad_len, pad_len, incl_icv);
1559 if (err)
1560 return err;
1561
1562 err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg);
1563 if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
1564 while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
1565 /*
1566 * Mailbox queue is full. Since MAY_SLEEP is set, assume
1567 * not in atomic context and we can wait and try again.
1568 */
1569 retry_cnt++;
1570 usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
1571 err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx],
1572 mssg);
1573 atomic_inc(&iproc_priv.mb_no_spc);
1574 }
1575 }
1576 if (err < 0) {
1577 atomic_inc(&iproc_priv.mb_send_fail);
1578 return err;
1579 }
1580
1581 return -EINPROGRESS;
1582}
1583
1584/**
1585 * handle_aead_resp() - Process a SPU response message for an AEAD request.
1586 * @rctx: Crypto request context
1587 */
1588static void handle_aead_resp(struct iproc_reqctx_s *rctx)
1589{
1590 struct spu_hw *spu = &iproc_priv.spu;
1591 struct crypto_async_request *areq = rctx->parent;
1592 struct aead_request *req = container_of(areq,
1593 struct aead_request, base);
1594 struct iproc_ctx_s *ctx = rctx->ctx;
1595 u32 payload_len;
1596 unsigned int icv_offset;
1597 u32 result_len;
1598
1599 /* See how much data was returned */
1600 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
1601 flow_log("payload_len %u\n", payload_len);
1602
1603 /* only count payload */
1604 atomic64_add(payload_len, &iproc_priv.bytes_in);
1605
1606 if (req->assoclen)
1607 packet_dump(" assoc_data ", rctx->msg_buf.a.resp_aad,
1608 req->assoclen);
1609
1610 /*
1611 * Copy the ICV back to the destination
1612 * buffer. In decrypt case, SPU gives us back the digest, but crypto
1613 * API doesn't expect ICV in dst buffer.
1614 */
1615 result_len = req->cryptlen;
1616 if (rctx->is_encrypt) {
1617 icv_offset = req->assoclen + rctx->total_sent;
1618 packet_dump(" ICV: ", rctx->msg_buf.digest, ctx->digestsize);
1619 flow_log("copying ICV to dst sg at offset %u\n", icv_offset);
1620 sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest,
1621 ctx->digestsize, icv_offset);
1622 result_len += ctx->digestsize;
1623 }
1624
1625 packet_log("response data: ");
1626 dump_sg(req->dst, req->assoclen, result_len);
1627
1628 atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]);
1629 if (ctx->cipher.alg == CIPHER_ALG_AES) {
1630 if (ctx->cipher.mode == CIPHER_MODE_CCM)
1631 atomic_inc(&iproc_priv.aead_cnt[AES_CCM]);
1632 else if (ctx->cipher.mode == CIPHER_MODE_GCM)
1633 atomic_inc(&iproc_priv.aead_cnt[AES_GCM]);
1634 else
1635 atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1636 } else {
1637 atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1638 }
1639}
1640
1641/**
1642 * spu_chunk_cleanup() - Do cleanup after processing one chunk of a request
1643 * @rctx: request context
1644 *
1645 * Mailbox scatterlists are allocated for each chunk. So free them after
1646 * processing each chunk.
1647 */
1648static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx)
1649{
1650 /* mailbox message used to tx request */
1651 struct brcm_message *mssg = &rctx->mb_mssg;
1652
1653 kfree(mssg->spu.src);
1654 kfree(mssg->spu.dst);
1655 memset(mssg, 0, sizeof(struct brcm_message));
1656}
1657
1658/**
1659 * finish_req() - Used to invoke the complete callback from the requester when
1660 * a request has been handled asynchronously.
1661 * @rctx: Request context
1662 * @err: Indicates whether the request was successful or not
1663 *
1664 * Ensures that cleanup has been done for request
1665 */
1666static void finish_req(struct iproc_reqctx_s *rctx, int err)
1667{
1668 struct crypto_async_request *areq = rctx->parent;
1669
1670 flow_log("%s() err:%d\n\n", __func__, err);
1671
1672 /* No harm done if already called */
1673 spu_chunk_cleanup(rctx);
1674
1675 if (areq)
1676 areq->complete(areq, err);
1677}
1678
1679/**
1680 * spu_rx_callback() - Callback from mailbox framework with a SPU response.
1681 * @cl: mailbox client structure for SPU driver
1682 * @msg: mailbox message containing SPU response
1683 */
1684static void spu_rx_callback(struct mbox_client *cl, void *msg)
1685{
1686 struct spu_hw *spu = &iproc_priv.spu;
1687 struct brcm_message *mssg = msg;
1688 struct iproc_reqctx_s *rctx;
1689 struct iproc_ctx_s *ctx;
1690 struct crypto_async_request *areq;
1691 int err = 0;
1692
1693 rctx = mssg->ctx;
1694 if (unlikely(!rctx)) {
1695 /* This is fatal */
1696 pr_err("%s(): no request context", __func__);
1697 err = -EFAULT;
1698 goto cb_finish;
1699 }
1700 areq = rctx->parent;
1701 ctx = rctx->ctx;
1702
1703 /* process the SPU status */
1704 err = spu->spu_status_process(rctx->msg_buf.rx_stat);
1705 if (err != 0) {
1706 if (err == SPU_INVALID_ICV)
1707 atomic_inc(&iproc_priv.bad_icv);
1708 err = -EBADMSG;
1709 goto cb_finish;
1710 }
1711
1712 /* Process the SPU response message */
1713 switch (rctx->ctx->alg->type) {
1714 case CRYPTO_ALG_TYPE_ABLKCIPHER:
1715 handle_ablkcipher_resp(rctx);
1716 break;
1717 case CRYPTO_ALG_TYPE_AHASH:
1718 handle_ahash_resp(rctx);
1719 break;
1720 case CRYPTO_ALG_TYPE_AEAD:
1721 handle_aead_resp(rctx);
1722 break;
1723 default:
1724 err = -EINVAL;
1725 goto cb_finish;
1726 }
1727
1728 /*
1729 * If this response does not complete the request, then send the next
1730 * request chunk.
1731 */
1732 if (rctx->total_sent < rctx->total_todo) {
1733 /* Deallocate anything specific to previous chunk */
1734 spu_chunk_cleanup(rctx);
1735
1736 switch (rctx->ctx->alg->type) {
1737 case CRYPTO_ALG_TYPE_ABLKCIPHER:
1738 err = handle_ablkcipher_req(rctx);
1739 break;
1740 case CRYPTO_ALG_TYPE_AHASH:
1741 err = handle_ahash_req(rctx);
1742 if (err == -EAGAIN)
1743 /*
1744 * we saved data in hash carry, but tell crypto
1745 * API we successfully completed request.
1746 */
1747 err = 0;
1748 break;
1749 case CRYPTO_ALG_TYPE_AEAD:
1750 err = handle_aead_req(rctx);
1751 break;
1752 default:
1753 err = -EINVAL;
1754 }
1755
1756 if (err == -EINPROGRESS)
1757 /* Successfully submitted request for next chunk */
1758 return;
1759 }
1760
1761cb_finish:
1762 finish_req(rctx, err);
1763}
1764
1765/* ==================== Kernel Cryptographic API ==================== */
1766
1767/**
1768 * ablkcipher_enqueue() - Handle ablkcipher encrypt or decrypt request.
1769 * @req: Crypto API request
1770 * @encrypt: true if encrypting; false if decrypting
1771 *
1772 * Return: -EINPROGRESS if request accepted and result will be returned
1773 * asynchronously
1774 * < 0 if an error
1775 */
1776static int ablkcipher_enqueue(struct ablkcipher_request *req, bool encrypt)
1777{
1778 struct iproc_reqctx_s *rctx = ablkcipher_request_ctx(req);
1779 struct iproc_ctx_s *ctx =
1780 crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
1781 int err;
1782
1783 flow_log("%s() enc:%u\n", __func__, encrypt);
1784
1785 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1786 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1787 rctx->parent = &req->base;
1788 rctx->is_encrypt = encrypt;
1789 rctx->bd_suppress = false;
1790 rctx->total_todo = req->nbytes;
1791 rctx->src_sent = 0;
1792 rctx->total_sent = 0;
1793 rctx->total_received = 0;
1794 rctx->ctx = ctx;
1795
1796 /* Initialize current position in src and dst scatterlists */
1797 rctx->src_sg = req->src;
1798 rctx->src_nents = 0;
1799 rctx->src_skip = 0;
1800 rctx->dst_sg = req->dst;
1801 rctx->dst_nents = 0;
1802 rctx->dst_skip = 0;
1803
1804 if (ctx->cipher.mode == CIPHER_MODE_CBC ||
1805 ctx->cipher.mode == CIPHER_MODE_CTR ||
1806 ctx->cipher.mode == CIPHER_MODE_OFB ||
1807 ctx->cipher.mode == CIPHER_MODE_XTS ||
1808 ctx->cipher.mode == CIPHER_MODE_GCM ||
1809 ctx->cipher.mode == CIPHER_MODE_CCM) {
1810 rctx->iv_ctr_len =
1811 crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
1812 memcpy(rctx->msg_buf.iv_ctr, req->info, rctx->iv_ctr_len);
1813 } else {
1814 rctx->iv_ctr_len = 0;
1815 }
1816
1817 /* Choose a SPU to process this request */
1818 rctx->chan_idx = select_channel();
1819 err = handle_ablkcipher_req(rctx);
1820 if (err != -EINPROGRESS)
1821 /* synchronous result */
1822 spu_chunk_cleanup(rctx);
1823
1824 return err;
1825}
1826
1827static int des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1828 unsigned int keylen)
1829{
1830 struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
1831 u32 tmp[DES_EXPKEY_WORDS];
1832
1833 if (keylen == DES_KEY_SIZE) {
1834 if (des_ekey(tmp, key) == 0) {
1835 if (crypto_ablkcipher_get_flags(cipher) &
1836 CRYPTO_TFM_REQ_WEAK_KEY) {
1837 u32 flags = CRYPTO_TFM_RES_WEAK_KEY;
1838
1839 crypto_ablkcipher_set_flags(cipher, flags);
1840 return -EINVAL;
1841 }
1842 }
1843
1844 ctx->cipher_type = CIPHER_TYPE_DES;
1845 } else {
1846 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1847 return -EINVAL;
1848 }
1849 return 0;
1850}
1851
1852static int threedes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1853 unsigned int keylen)
1854{
1855 struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
1856
1857 if (keylen == (DES_KEY_SIZE * 3)) {
1858 const u32 *K = (const u32 *)key;
1859 u32 flags = CRYPTO_TFM_RES_BAD_KEY_SCHED;
1860
1861 if (!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
1862 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) {
1863 crypto_ablkcipher_set_flags(cipher, flags);
1864 return -EINVAL;
1865 }
1866
1867 ctx->cipher_type = CIPHER_TYPE_3DES;
1868 } else {
1869 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1870 return -EINVAL;
1871 }
1872 return 0;
1873}
1874
1875static int aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1876 unsigned int keylen)
1877{
1878 struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
1879
1880 if (ctx->cipher.mode == CIPHER_MODE_XTS)
1881 /* XTS includes two keys of equal length */
1882 keylen = keylen / 2;
1883
1884 switch (keylen) {
1885 case AES_KEYSIZE_128:
1886 ctx->cipher_type = CIPHER_TYPE_AES128;
1887 break;
1888 case AES_KEYSIZE_192:
1889 ctx->cipher_type = CIPHER_TYPE_AES192;
1890 break;
1891 case AES_KEYSIZE_256:
1892 ctx->cipher_type = CIPHER_TYPE_AES256;
1893 break;
1894 default:
1895 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1896 return -EINVAL;
1897 }
1898 WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
1899 ((ctx->max_payload % AES_BLOCK_SIZE) != 0));
1900 return 0;
1901}
1902
1903static int rc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1904 unsigned int keylen)
1905{
1906 struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
1907 int i;
1908
1909 ctx->enckeylen = ARC4_MAX_KEY_SIZE + ARC4_STATE_SIZE;
1910
1911 ctx->enckey[0] = 0x00; /* 0x00 */
1912 ctx->enckey[1] = 0x00; /* i */
1913 ctx->enckey[2] = 0x00; /* 0x00 */
1914 ctx->enckey[3] = 0x00; /* j */
1915 for (i = 0; i < ARC4_MAX_KEY_SIZE; i++)
1916 ctx->enckey[i + ARC4_STATE_SIZE] = key[i % keylen];
1917
1918 ctx->cipher_type = CIPHER_TYPE_INIT;
1919
1920 return 0;
1921}
1922
1923static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1924 unsigned int keylen)
1925{
1926 struct spu_hw *spu = &iproc_priv.spu;
1927 struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
1928 struct spu_cipher_parms cipher_parms;
1929 u32 alloc_len = 0;
1930 int err;
1931
1932 flow_log("ablkcipher_setkey() keylen: %d\n", keylen);
1933 flow_dump(" key: ", key, keylen);
1934
1935 switch (ctx->cipher.alg) {
1936 case CIPHER_ALG_DES:
1937 err = des_setkey(cipher, key, keylen);
1938 break;
1939 case CIPHER_ALG_3DES:
1940 err = threedes_setkey(cipher, key, keylen);
1941 break;
1942 case CIPHER_ALG_AES:
1943 err = aes_setkey(cipher, key, keylen);
1944 break;
1945 case CIPHER_ALG_RC4:
1946 err = rc4_setkey(cipher, key, keylen);
1947 break;
1948 default:
1949 pr_err("%s() Error: unknown cipher alg\n", __func__);
1950 err = -EINVAL;
1951 }
1952 if (err)
1953 return err;
1954
1955 /* RC4 already populated ctx->enkey */
1956 if (ctx->cipher.alg != CIPHER_ALG_RC4) {
1957 memcpy(ctx->enckey, key, keylen);
1958 ctx->enckeylen = keylen;
1959 }
1960 /* SPU needs XTS keys in the reverse order the crypto API presents */
1961 if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
1962 (ctx->cipher.mode == CIPHER_MODE_XTS)) {
1963 unsigned int xts_keylen = keylen / 2;
1964
1965 memcpy(ctx->enckey, key + xts_keylen, xts_keylen);
1966 memcpy(ctx->enckey + xts_keylen, key, xts_keylen);
1967 }
1968
1969 if (spu->spu_type == SPU_TYPE_SPUM)
1970 alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN;
1971 else if (spu->spu_type == SPU_TYPE_SPU2)
1972 alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN;
1973 memset(ctx->bcm_spu_req_hdr, 0, alloc_len);
1974 cipher_parms.iv_buf = NULL;
1975 cipher_parms.iv_len = crypto_ablkcipher_ivsize(cipher);
1976 flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len);
1977
1978 cipher_parms.alg = ctx->cipher.alg;
1979 cipher_parms.mode = ctx->cipher.mode;
1980 cipher_parms.type = ctx->cipher_type;
1981 cipher_parms.key_buf = ctx->enckey;
1982 cipher_parms.key_len = ctx->enckeylen;
1983
1984 /* Prepend SPU request message with BCM header */
1985 memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1986 ctx->spu_req_hdr_len =
1987 spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN,
1988 &cipher_parms);
1989
1990 ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
1991 ctx->enckeylen,
1992 false);
1993
1994 atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]);
1995
1996 return 0;
1997}
1998
1999static int ablkcipher_encrypt(struct ablkcipher_request *req)
2000{
2001 flow_log("ablkcipher_encrypt() nbytes:%u\n", req->nbytes);
2002
2003 return ablkcipher_enqueue(req, true);
2004}
2005
2006static int ablkcipher_decrypt(struct ablkcipher_request *req)
2007{
2008 flow_log("ablkcipher_decrypt() nbytes:%u\n", req->nbytes);
2009 return ablkcipher_enqueue(req, false);
2010}
2011
2012static int ahash_enqueue(struct ahash_request *req)
2013{
2014 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2015 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2016 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2017 int err = 0;
2018 const char *alg_name;
2019
2020 flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
2021
2022 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2023 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2024 rctx->parent = &req->base;
2025 rctx->ctx = ctx;
2026 rctx->bd_suppress = true;
2027 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2028
2029 /* Initialize position in src scatterlist */
2030 rctx->src_sg = req->src;
2031 rctx->src_skip = 0;
2032 rctx->src_nents = 0;
2033 rctx->dst_sg = NULL;
2034 rctx->dst_skip = 0;
2035 rctx->dst_nents = 0;
2036
2037 /* SPU2 hardware does not compute hash of zero length data */
2038 if ((rctx->is_final == 1) && (rctx->total_todo == 0) &&
2039 (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) {
2040 alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
2041 flow_log("Doing %sfinal %s zero-len hash request in software\n",
2042 rctx->is_final ? "" : "non-", alg_name);
2043 err = do_shash((unsigned char *)alg_name, req->result,
2044 NULL, 0, NULL, 0, ctx->authkey,
2045 ctx->authkeylen);
2046 if (err < 0)
2047 flow_log("Hash request failed with error %d\n", err);
2048 return err;
2049 }
2050 /* Choose a SPU to process this request */
2051 rctx->chan_idx = select_channel();
2052
2053 err = handle_ahash_req(rctx);
2054 if (err != -EINPROGRESS)
2055 /* synchronous result */
2056 spu_chunk_cleanup(rctx);
2057
2058 if (err == -EAGAIN)
2059 /*
2060 * we saved data in hash carry, but tell crypto API
2061 * we successfully completed request.
2062 */
2063 err = 0;
2064
2065 return err;
2066}
2067
2068static int __ahash_init(struct ahash_request *req)
2069{
2070 struct spu_hw *spu = &iproc_priv.spu;
2071 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2072 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2073 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2074
2075 flow_log("%s()\n", __func__);
2076
2077 /* Initialize the context */
2078 rctx->hash_carry_len = 0;
2079 rctx->is_final = 0;
2080
2081 rctx->total_todo = 0;
2082 rctx->src_sent = 0;
2083 rctx->total_sent = 0;
2084 rctx->total_received = 0;
2085
2086 ctx->digestsize = crypto_ahash_digestsize(tfm);
2087 /* If we add a hash whose digest is larger, catch it here. */
2088 WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE);
2089
2090 rctx->is_sw_hmac = false;
2091
2092 ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0,
2093 true);
2094
2095 return 0;
2096}
2097
2098/**
2099 * spu_no_incr_hash() - Determine whether incremental hashing is supported.
2100 * @ctx: Crypto session context
2101 *
2102 * SPU-2 does not support incremental hashing (we'll have to revisit and
2103 * condition based on chip revision or device tree entry if future versions do
2104 * support incremental hash)
2105 *
2106 * SPU-M also doesn't support incremental hashing of AES-XCBC
2107 *
2108 * Return: true if incremental hashing is not supported
2109 * false otherwise
2110 */
2111bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
2112{
2113 struct spu_hw *spu = &iproc_priv.spu;
2114
2115 if (spu->spu_type == SPU_TYPE_SPU2)
2116 return true;
2117
2118 if ((ctx->auth.alg == HASH_ALG_AES) &&
2119 (ctx->auth.mode == HASH_MODE_XCBC))
2120 return true;
2121
2122 /* Otherwise, incremental hashing is supported */
2123 return false;
2124}
2125
2126static int ahash_init(struct ahash_request *req)
2127{
2128 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2129 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2130 const char *alg_name;
2131 struct crypto_shash *hash;
2132 int ret;
2133 gfp_t gfp;
2134
2135 if (spu_no_incr_hash(ctx)) {
2136 /*
2137 * If we get an incremental hashing request and it's not
2138 * supported by the hardware, we need to handle it in software
2139 * by calling synchronous hash functions.
2140 */
2141 alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
2142 hash = crypto_alloc_shash(alg_name, 0, 0);
2143 if (IS_ERR(hash)) {
2144 ret = PTR_ERR(hash);
2145 goto err;
2146 }
2147
2148 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2149 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2150 ctx->shash = kmalloc(sizeof(*ctx->shash) +
2151 crypto_shash_descsize(hash), gfp);
2152 if (!ctx->shash) {
2153 ret = -ENOMEM;
2154 goto err_hash;
2155 }
2156 ctx->shash->tfm = hash;
2157 ctx->shash->flags = 0;
2158
2159 /* Set the key using data we already have from setkey */
2160 if (ctx->authkeylen > 0) {
2161 ret = crypto_shash_setkey(hash, ctx->authkey,
2162 ctx->authkeylen);
2163 if (ret)
2164 goto err_shash;
2165 }
2166
2167 /* Initialize hash w/ this key and other params */
2168 ret = crypto_shash_init(ctx->shash);
2169 if (ret)
2170 goto err_shash;
2171 } else {
2172 /* Otherwise call the internal function which uses SPU hw */
2173 ret = __ahash_init(req);
2174 }
2175
2176 return ret;
2177
2178err_shash:
2179 kfree(ctx->shash);
2180err_hash:
2181 crypto_free_shash(hash);
2182err:
2183 return ret;
2184}
2185
2186static int __ahash_update(struct ahash_request *req)
2187{
2188 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2189
2190 flow_log("ahash_update() nbytes:%u\n", req->nbytes);
2191
2192 if (!req->nbytes)
2193 return 0;
2194 rctx->total_todo += req->nbytes;
2195 rctx->src_sent = 0;
2196
2197 return ahash_enqueue(req);
2198}
2199
2200static int ahash_update(struct ahash_request *req)
2201{
2202 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2203 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2204 u8 *tmpbuf;
2205 int ret;
2206 int nents;
2207 gfp_t gfp;
2208
2209 if (spu_no_incr_hash(ctx)) {
2210 /*
2211 * If we get an incremental hashing request and it's not
2212 * supported by the hardware, we need to handle it in software
2213 * by calling synchronous hash functions.
2214 */
2215 if (req->src)
2216 nents = sg_nents(req->src);
2217 else
2218 return -EINVAL;
2219
2220 /* Copy data from req scatterlist to tmp buffer */
2221 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2222 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2223 tmpbuf = kmalloc(req->nbytes, gfp);
2224 if (!tmpbuf)
2225 return -ENOMEM;
2226
2227 if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2228 req->nbytes) {
2229 kfree(tmpbuf);
2230 return -EINVAL;
2231 }
2232
2233 /* Call synchronous update */
2234 ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes);
2235 kfree(tmpbuf);
2236 } else {
2237 /* Otherwise call the internal function which uses SPU hw */
2238 ret = __ahash_update(req);
2239 }
2240
2241 return ret;
2242}
2243
2244static int __ahash_final(struct ahash_request *req)
2245{
2246 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2247
2248 flow_log("ahash_final() nbytes:%u\n", req->nbytes);
2249
2250 rctx->is_final = 1;
2251
2252 return ahash_enqueue(req);
2253}
2254
2255static int ahash_final(struct ahash_request *req)
2256{
2257 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2258 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2259 int ret;
2260
2261 if (spu_no_incr_hash(ctx)) {
2262 /*
2263 * If we get an incremental hashing request and it's not
2264 * supported by the hardware, we need to handle it in software
2265 * by calling synchronous hash functions.
2266 */
2267 ret = crypto_shash_final(ctx->shash, req->result);
2268
2269 /* Done with hash, can deallocate it now */
2270 crypto_free_shash(ctx->shash->tfm);
2271 kfree(ctx->shash);
2272
2273 } else {
2274 /* Otherwise call the internal function which uses SPU hw */
2275 ret = __ahash_final(req);
2276 }
2277
2278 return ret;
2279}
2280
2281static int __ahash_finup(struct ahash_request *req)
2282{
2283 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2284
2285 flow_log("ahash_finup() nbytes:%u\n", req->nbytes);
2286
2287 rctx->total_todo += req->nbytes;
2288 rctx->src_sent = 0;
2289 rctx->is_final = 1;
2290
2291 return ahash_enqueue(req);
2292}
2293
2294static int ahash_finup(struct ahash_request *req)
2295{
2296 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2297 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2298 u8 *tmpbuf;
2299 int ret;
2300 int nents;
2301 gfp_t gfp;
2302
2303 if (spu_no_incr_hash(ctx)) {
2304 /*
2305 * If we get an incremental hashing request and it's not
2306 * supported by the hardware, we need to handle it in software
2307 * by calling synchronous hash functions.
2308 */
2309 if (req->src) {
2310 nents = sg_nents(req->src);
2311 } else {
2312 ret = -EINVAL;
2313 goto ahash_finup_exit;
2314 }
2315
2316 /* Copy data from req scatterlist to tmp buffer */
2317 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2318 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2319 tmpbuf = kmalloc(req->nbytes, gfp);
2320 if (!tmpbuf) {
2321 ret = -ENOMEM;
2322 goto ahash_finup_exit;
2323 }
2324
2325 if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2326 req->nbytes) {
2327 ret = -EINVAL;
2328 goto ahash_finup_free;
2329 }
2330
2331 /* Call synchronous update */
2332 ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes,
2333 req->result);
2334 } else {
2335 /* Otherwise call the internal function which uses SPU hw */
2336 return __ahash_finup(req);
2337 }
2338ahash_finup_free:
2339 kfree(tmpbuf);
2340
2341ahash_finup_exit:
2342 /* Done with hash, can deallocate it now */
2343 crypto_free_shash(ctx->shash->tfm);
2344 kfree(ctx->shash);
2345 return ret;
2346}
2347
2348static int ahash_digest(struct ahash_request *req)
2349{
2350 int err = 0;
2351
2352 flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
2353
2354 /* whole thing at once */
2355 err = __ahash_init(req);
2356 if (!err)
2357 err = __ahash_finup(req);
2358
2359 return err;
2360}
2361
2362static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
2363 unsigned int keylen)
2364{
2365 struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2366
2367 flow_log("%s() ahash:%p key:%p keylen:%u\n",
2368 __func__, ahash, key, keylen);
2369 flow_dump(" key: ", key, keylen);
2370
2371 if (ctx->auth.alg == HASH_ALG_AES) {
2372 switch (keylen) {
2373 case AES_KEYSIZE_128:
2374 ctx->cipher_type = CIPHER_TYPE_AES128;
2375 break;
2376 case AES_KEYSIZE_192:
2377 ctx->cipher_type = CIPHER_TYPE_AES192;
2378 break;
2379 case AES_KEYSIZE_256:
2380 ctx->cipher_type = CIPHER_TYPE_AES256;
2381 break;
2382 default:
2383 pr_err("%s() Error: Invalid key length\n", __func__);
2384 return -EINVAL;
2385 }
2386 } else {
2387 pr_err("%s() Error: unknown hash alg\n", __func__);
2388 return -EINVAL;
2389 }
2390 memcpy(ctx->authkey, key, keylen);
2391 ctx->authkeylen = keylen;
2392
2393 return 0;
2394}
2395
2396static int ahash_export(struct ahash_request *req, void *out)
2397{
2398 const struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2399 struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)out;
2400
2401 spu_exp->total_todo = rctx->total_todo;
2402 spu_exp->total_sent = rctx->total_sent;
2403 spu_exp->is_sw_hmac = rctx->is_sw_hmac;
2404 memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry));
2405 spu_exp->hash_carry_len = rctx->hash_carry_len;
2406 memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash));
2407
2408 return 0;
2409}
2410
2411static int ahash_import(struct ahash_request *req, const void *in)
2412{
2413 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2414 struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)in;
2415
2416 rctx->total_todo = spu_exp->total_todo;
2417 rctx->total_sent = spu_exp->total_sent;
2418 rctx->is_sw_hmac = spu_exp->is_sw_hmac;
2419 memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry));
2420 rctx->hash_carry_len = spu_exp->hash_carry_len;
2421 memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash));
2422
2423 return 0;
2424}
2425
2426static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
2427 unsigned int keylen)
2428{
2429 struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2430 unsigned int blocksize =
2431 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
2432 unsigned int digestsize = crypto_ahash_digestsize(ahash);
2433 unsigned int index;
2434 int rc;
2435
2436 flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n",
2437 __func__, ahash, key, keylen, blocksize, digestsize);
2438 flow_dump(" key: ", key, keylen);
2439
2440 if (keylen > blocksize) {
2441 switch (ctx->auth.alg) {
2442 case HASH_ALG_MD5:
2443 rc = do_shash("md5", ctx->authkey, key, keylen, NULL,
2444 0, NULL, 0);
2445 break;
2446 case HASH_ALG_SHA1:
2447 rc = do_shash("sha1", ctx->authkey, key, keylen, NULL,
2448 0, NULL, 0);
2449 break;
2450 case HASH_ALG_SHA224:
2451 rc = do_shash("sha224", ctx->authkey, key, keylen, NULL,
2452 0, NULL, 0);
2453 break;
2454 case HASH_ALG_SHA256:
2455 rc = do_shash("sha256", ctx->authkey, key, keylen, NULL,
2456 0, NULL, 0);
2457 break;
2458 case HASH_ALG_SHA384:
2459 rc = do_shash("sha384", ctx->authkey, key, keylen, NULL,
2460 0, NULL, 0);
2461 break;
2462 case HASH_ALG_SHA512:
2463 rc = do_shash("sha512", ctx->authkey, key, keylen, NULL,
2464 0, NULL, 0);
2465 break;
2466 case HASH_ALG_SHA3_224:
2467 rc = do_shash("sha3-224", ctx->authkey, key, keylen,
2468 NULL, 0, NULL, 0);
2469 break;
2470 case HASH_ALG_SHA3_256:
2471 rc = do_shash("sha3-256", ctx->authkey, key, keylen,
2472 NULL, 0, NULL, 0);
2473 break;
2474 case HASH_ALG_SHA3_384:
2475 rc = do_shash("sha3-384", ctx->authkey, key, keylen,
2476 NULL, 0, NULL, 0);
2477 break;
2478 case HASH_ALG_SHA3_512:
2479 rc = do_shash("sha3-512", ctx->authkey, key, keylen,
2480 NULL, 0, NULL, 0);
2481 break;
2482 default:
2483 pr_err("%s() Error: unknown hash alg\n", __func__);
2484 return -EINVAL;
2485 }
2486 if (rc < 0) {
2487 pr_err("%s() Error %d computing shash for %s\n",
2488 __func__, rc, hash_alg_name[ctx->auth.alg]);
2489 return rc;
2490 }
2491 ctx->authkeylen = digestsize;
2492
2493 flow_log(" keylen > digestsize... hashed\n");
2494 flow_dump(" newkey: ", ctx->authkey, ctx->authkeylen);
2495 } else {
2496 memcpy(ctx->authkey, key, keylen);
2497 ctx->authkeylen = keylen;
2498 }
2499
2500 /*
2501 * Full HMAC operation in SPUM is not verified,
2502 * So keeping the generation of IPAD, OPAD and
2503 * outer hashing in software.
2504 */
2505 if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) {
2506 memcpy(ctx->ipad, ctx->authkey, ctx->authkeylen);
2507 memset(ctx->ipad + ctx->authkeylen, 0,
2508 blocksize - ctx->authkeylen);
2509 ctx->authkeylen = 0;
2510 memcpy(ctx->opad, ctx->ipad, blocksize);
2511
2512 for (index = 0; index < blocksize; index++) {
2513 ctx->ipad[index] ^= 0x36;
2514 ctx->opad[index] ^= 0x5c;
2515 }
2516
2517 flow_dump(" ipad: ", ctx->ipad, blocksize);
2518 flow_dump(" opad: ", ctx->opad, blocksize);
2519 }
2520 ctx->digestsize = digestsize;
2521 atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]);
2522
2523 return 0;
2524}
2525
2526static int ahash_hmac_init(struct ahash_request *req)
2527{
2528 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2529 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2530 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2531 unsigned int blocksize =
2532 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2533
2534 flow_log("ahash_hmac_init()\n");
2535
2536 /* init the context as a hash */
2537 ahash_init(req);
2538
2539 if (!spu_no_incr_hash(ctx)) {
2540 /* SPU-M can do incr hashing but needs sw for outer HMAC */
2541 rctx->is_sw_hmac = true;
2542 ctx->auth.mode = HASH_MODE_HASH;
2543 /* start with a prepended ipad */
2544 memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2545 rctx->hash_carry_len = blocksize;
2546 rctx->total_todo += blocksize;
2547 }
2548
2549 return 0;
2550}
2551
2552static int ahash_hmac_update(struct ahash_request *req)
2553{
2554 flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes);
2555
2556 if (!req->nbytes)
2557 return 0;
2558
2559 return ahash_update(req);
2560}
2561
2562static int ahash_hmac_final(struct ahash_request *req)
2563{
2564 flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes);
2565
2566 return ahash_final(req);
2567}
2568
2569static int ahash_hmac_finup(struct ahash_request *req)
2570{
2571 flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes);
2572
2573 return ahash_finup(req);
2574}
2575
2576static int ahash_hmac_digest(struct ahash_request *req)
2577{
2578 struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2579 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2580 struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2581 unsigned int blocksize =
2582 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2583
2584 flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes);
2585
2586 /* Perform initialization and then call finup */
2587 __ahash_init(req);
2588
2589 if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) {
2590 /*
2591 * SPU2 supports full HMAC implementation in the
2592 * hardware, need not to generate IPAD, OPAD and
2593 * outer hash in software.
2594 * Only for hash key len > hash block size, SPU2
2595 * expects to perform hashing on the key, shorten
2596 * it to digest size and feed it as hash key.
2597 */
2598 rctx->is_sw_hmac = false;
2599 ctx->auth.mode = HASH_MODE_HMAC;
2600 } else {
2601 rctx->is_sw_hmac = true;
2602 ctx->auth.mode = HASH_MODE_HASH;
2603 /* start with a prepended ipad */
2604 memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2605 rctx->hash_carry_len = blocksize;
2606 rctx->total_todo += blocksize;
2607 }
2608
2609 return __ahash_finup(req);
2610}
2611
2612/* aead helpers */
2613
2614static int aead_need_fallback(struct aead_request *req)
2615{
2616 struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2617 struct spu_hw *spu = &iproc_priv.spu;
2618 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2619 struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2620 u32 payload_len;
2621
2622 /*
2623 * SPU hardware cannot handle the AES-GCM/CCM case where plaintext
2624 * and AAD are both 0 bytes long. So use fallback in this case.
2625 */
2626 if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
2627 (ctx->cipher.mode == CIPHER_MODE_CCM)) &&
2628 (req->assoclen == 0)) {
2629 if ((rctx->is_encrypt && (req->cryptlen == 0)) ||
2630 (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) {
2631 flow_log("AES GCM/CCM needs fallback for 0 len req\n");
2632 return 1;
2633 }
2634 }
2635
2636 /* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */
2637 if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2638 (spu->spu_type == SPU_TYPE_SPUM) &&
2639 (ctx->digestsize != 8) && (ctx->digestsize != 12) &&
2640 (ctx->digestsize != 16)) {
2641 flow_log("%s() AES CCM needs fallbck for digest size %d\n",
2642 __func__, ctx->digestsize);
2643 return 1;
2644 }
2645
2646 /*
2647 * SPU-M on NSP has an issue where AES-CCM hash is not correct
2648 * when AAD size is 0
2649 */
2650 if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2651 (spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) &&
2652 (req->assoclen == 0)) {
2653 flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n",
2654 __func__);
2655 return 1;
2656 }
2657
2658 payload_len = req->cryptlen;
2659 if (spu->spu_type == SPU_TYPE_SPUM)
2660 payload_len += req->assoclen;
2661
2662 flow_log("%s() payload len: %u\n", __func__, payload_len);
2663
2664 if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2665 return 0;
2666 else
2667 return payload_len > ctx->max_payload;
2668}
2669
2670static void aead_complete(struct crypto_async_request *areq, int err)
2671{
2672 struct aead_request *req =
2673 container_of(areq, struct aead_request, base);
2674 struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2675 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2676
2677 flow_log("%s() err:%d\n", __func__, err);
2678
2679 areq->tfm = crypto_aead_tfm(aead);
2680
2681 areq->complete = rctx->old_complete;
2682 areq->data = rctx->old_data;
2683
2684 areq->complete(areq, err);
2685}
2686
2687static int aead_do_fallback(struct aead_request *req, bool is_encrypt)
2688{
2689 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2690 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2691 struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2692 struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
2693 int err;
2694 u32 req_flags;
2695
2696 flow_log("%s() enc:%u\n", __func__, is_encrypt);
2697
2698 if (ctx->fallback_cipher) {
2699 /* Store the cipher tfm and then use the fallback tfm */
2700 rctx->old_tfm = tfm;
2701 aead_request_set_tfm(req, ctx->fallback_cipher);
2702 /*
2703 * Save the callback and chain ourselves in, so we can restore
2704 * the tfm
2705 */
2706 rctx->old_complete = req->base.complete;
2707 rctx->old_data = req->base.data;
2708 req_flags = aead_request_flags(req);
2709 aead_request_set_callback(req, req_flags, aead_complete, req);
2710 err = is_encrypt ? crypto_aead_encrypt(req) :
2711 crypto_aead_decrypt(req);
2712
2713 if (err == 0) {
2714 /*
2715 * fallback was synchronous (did not return
2716 * -EINPROGRESS). So restore request state here.
2717 */
2718 aead_request_set_callback(req, req_flags,
2719 rctx->old_complete, req);
2720 req->base.data = rctx->old_data;
2721 aead_request_set_tfm(req, aead);
2722 flow_log("%s() fallback completed successfully\n\n",
2723 __func__);
2724 }
2725 } else {
2726 err = -EINVAL;
2727 }
2728
2729 return err;
2730}
2731
2732static int aead_enqueue(struct aead_request *req, bool is_encrypt)
2733{
2734 struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2735 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2736 struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2737 int err;
2738
2739 flow_log("%s() enc:%u\n", __func__, is_encrypt);
2740
2741 if (req->assoclen > MAX_ASSOC_SIZE) {
2742 pr_err
2743 ("%s() Error: associated data too long. (%u > %u bytes)\n",
2744 __func__, req->assoclen, MAX_ASSOC_SIZE);
2745 return -EINVAL;
2746 }
2747
2748 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2749 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2750 rctx->parent = &req->base;
2751 rctx->is_encrypt = is_encrypt;
2752 rctx->bd_suppress = false;
2753 rctx->total_todo = req->cryptlen;
2754 rctx->src_sent = 0;
2755 rctx->total_sent = 0;
2756 rctx->total_received = 0;
2757 rctx->is_sw_hmac = false;
2758 rctx->ctx = ctx;
2759 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2760
2761 /* assoc data is at start of src sg */
2762 rctx->assoc = req->src;
2763
2764 /*
2765 * Init current position in src scatterlist to be after assoc data.
2766 * src_skip set to buffer offset where data begins. (Assoc data could
2767 * end in the middle of a buffer.)
2768 */
2769 if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg,
2770 &rctx->src_skip) < 0) {
2771 pr_err("%s() Error: Unable to find start of src data\n",
2772 __func__);
2773 return -EINVAL;
2774 }
2775
2776 rctx->src_nents = 0;
2777 rctx->dst_nents = 0;
2778 if (req->dst == req->src) {
2779 rctx->dst_sg = rctx->src_sg;
2780 rctx->dst_skip = rctx->src_skip;
2781 } else {
2782 /*
2783 * Expect req->dst to have room for assoc data followed by
2784 * output data and ICV, if encrypt. So initialize dst_sg
2785 * to point beyond assoc len offset.
2786 */
2787 if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg,
2788 &rctx->dst_skip) < 0) {
2789 pr_err("%s() Error: Unable to find start of dst data\n",
2790 __func__);
2791 return -EINVAL;
2792 }
2793 }
2794
2795 if (ctx->cipher.mode == CIPHER_MODE_CBC ||
2796 ctx->cipher.mode == CIPHER_MODE_CTR ||
2797 ctx->cipher.mode == CIPHER_MODE_OFB ||
2798 ctx->cipher.mode == CIPHER_MODE_XTS ||
2799 ctx->cipher.mode == CIPHER_MODE_GCM) {
2800 rctx->iv_ctr_len =
2801 ctx->salt_len +
2802 crypto_aead_ivsize(crypto_aead_reqtfm(req));
2803 } else if (ctx->cipher.mode == CIPHER_MODE_CCM) {
2804 rctx->iv_ctr_len = CCM_AES_IV_SIZE;
2805 } else {
2806 rctx->iv_ctr_len = 0;
2807 }
2808
2809 rctx->hash_carry_len = 0;
2810
2811 flow_log(" src sg: %p\n", req->src);
2812 flow_log(" rctx->src_sg: %p, src_skip %u\n",
2813 rctx->src_sg, rctx->src_skip);
2814 flow_log(" assoc: %p, assoclen %u\n", rctx->assoc, req->assoclen);
2815 flow_log(" dst sg: %p\n", req->dst);
2816 flow_log(" rctx->dst_sg: %p, dst_skip %u\n",
2817 rctx->dst_sg, rctx->dst_skip);
2818 flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len);
2819 flow_dump(" iv: ", req->iv, rctx->iv_ctr_len);
2820 flow_log(" authkeylen:%u\n", ctx->authkeylen);
2821 flow_log(" is_esp: %s\n", ctx->is_esp ? "yes" : "no");
2822
2823 if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2824 flow_log(" max_payload infinite");
2825 else
2826 flow_log(" max_payload: %u\n", ctx->max_payload);
2827
2828 if (unlikely(aead_need_fallback(req)))
2829 return aead_do_fallback(req, is_encrypt);
2830
2831 /*
2832 * Do memory allocations for request after fallback check, because if we
2833 * do fallback, we won't call finish_req() to dealloc.
2834 */
2835 if (rctx->iv_ctr_len) {
2836 if (ctx->salt_len)
2837 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset,
2838 ctx->salt, ctx->salt_len);
2839 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len,
2840 req->iv,
2841 rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset);
2842 }
2843
2844 rctx->chan_idx = select_channel();
2845 err = handle_aead_req(rctx);
2846 if (err != -EINPROGRESS)
2847 /* synchronous result */
2848 spu_chunk_cleanup(rctx);
2849
2850 return err;
2851}
2852
2853static int aead_authenc_setkey(struct crypto_aead *cipher,
2854 const u8 *key, unsigned int keylen)
2855{
2856 struct spu_hw *spu = &iproc_priv.spu;
2857 struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2858 struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2859 struct rtattr *rta = (void *)key;
2860 struct crypto_authenc_key_param *param;
2861 const u8 *origkey = key;
2862 const unsigned int origkeylen = keylen;
2863
2864 int ret = 0;
2865
2866 flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key,
2867 keylen);
2868 flow_dump(" key: ", key, keylen);
2869
2870 if (!RTA_OK(rta, keylen))
2871 goto badkey;
2872 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
2873 goto badkey;
2874 if (RTA_PAYLOAD(rta) < sizeof(*param))
2875 goto badkey;
2876
2877 param = RTA_DATA(rta);
2878 ctx->enckeylen = be32_to_cpu(param->enckeylen);
2879
2880 key += RTA_ALIGN(rta->rta_len);
2881 keylen -= RTA_ALIGN(rta->rta_len);
2882
2883 if (keylen < ctx->enckeylen)
2884 goto badkey;
2885 if (ctx->enckeylen > MAX_KEY_SIZE)
2886 goto badkey;
2887
2888 ctx->authkeylen = keylen - ctx->enckeylen;
2889
2890 if (ctx->authkeylen > MAX_KEY_SIZE)
2891 goto badkey;
2892
2893 memcpy(ctx->enckey, key + ctx->authkeylen, ctx->enckeylen);
2894 /* May end up padding auth key. So make sure it's zeroed. */
2895 memset(ctx->authkey, 0, sizeof(ctx->authkey));
2896 memcpy(ctx->authkey, key, ctx->authkeylen);
2897
2898 switch (ctx->alg->cipher_info.alg) {
2899 case CIPHER_ALG_DES:
2900 if (ctx->enckeylen == DES_KEY_SIZE) {
2901 u32 tmp[DES_EXPKEY_WORDS];
2902 u32 flags = CRYPTO_TFM_RES_WEAK_KEY;
2903
2904 if (des_ekey(tmp, key) == 0) {
2905 if (crypto_aead_get_flags(cipher) &
2906 CRYPTO_TFM_REQ_WEAK_KEY) {
2907 crypto_aead_set_flags(cipher, flags);
2908 return -EINVAL;
2909 }
2910 }
2911
2912 ctx->cipher_type = CIPHER_TYPE_DES;
2913 } else {
2914 goto badkey;
2915 }
2916 break;
2917 case CIPHER_ALG_3DES:
2918 if (ctx->enckeylen == (DES_KEY_SIZE * 3)) {
2919 const u32 *K = (const u32 *)key;
2920 u32 flags = CRYPTO_TFM_RES_BAD_KEY_SCHED;
2921
2922 if (!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
2923 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) {
2924 crypto_aead_set_flags(cipher, flags);
2925 return -EINVAL;
2926 }
2927
2928 ctx->cipher_type = CIPHER_TYPE_3DES;
2929 } else {
2930 crypto_aead_set_flags(cipher,
2931 CRYPTO_TFM_RES_BAD_KEY_LEN);
2932 return -EINVAL;
2933 }
2934 break;
2935 case CIPHER_ALG_AES:
2936 switch (ctx->enckeylen) {
2937 case AES_KEYSIZE_128:
2938 ctx->cipher_type = CIPHER_TYPE_AES128;
2939 break;
2940 case AES_KEYSIZE_192:
2941 ctx->cipher_type = CIPHER_TYPE_AES192;
2942 break;
2943 case AES_KEYSIZE_256:
2944 ctx->cipher_type = CIPHER_TYPE_AES256;
2945 break;
2946 default:
2947 goto badkey;
2948 }
2949 break;
2950 case CIPHER_ALG_RC4:
2951 ctx->cipher_type = CIPHER_TYPE_INIT;
2952 break;
2953 default:
2954 pr_err("%s() Error: Unknown cipher alg\n", __func__);
2955 return -EINVAL;
2956 }
2957
2958 flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2959 ctx->authkeylen);
2960 flow_dump(" enc: ", ctx->enckey, ctx->enckeylen);
2961 flow_dump(" auth: ", ctx->authkey, ctx->authkeylen);
2962
2963 /* setkey the fallback just in case we needto use it */
2964 if (ctx->fallback_cipher) {
2965 flow_log(" running fallback setkey()\n");
2966
2967 ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2968 ctx->fallback_cipher->base.crt_flags |=
2969 tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2970 ret =
2971 crypto_aead_setkey(ctx->fallback_cipher, origkey,
2972 origkeylen);
2973 if (ret) {
2974 flow_log(" fallback setkey() returned:%d\n", ret);
2975 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
2976 tfm->crt_flags |=
2977 (ctx->fallback_cipher->base.crt_flags &
2978 CRYPTO_TFM_RES_MASK);
2979 }
2980 }
2981
2982 ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2983 ctx->enckeylen,
2984 false);
2985
2986 atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2987
2988 return ret;
2989
2990badkey:
2991 ctx->enckeylen = 0;
2992 ctx->authkeylen = 0;
2993 ctx->digestsize = 0;
2994
2995 crypto_aead_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
2996 return -EINVAL;
2997}
2998
2999static int aead_gcm_ccm_setkey(struct crypto_aead *cipher,
3000 const u8 *key, unsigned int keylen)
3001{
3002 struct spu_hw *spu = &iproc_priv.spu;
3003 struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3004 struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
3005
3006 int ret = 0;
3007
3008 flow_log("%s() keylen:%u\n", __func__, keylen);
3009 flow_dump(" key: ", key, keylen);
3010
3011 if (!ctx->is_esp)
3012 ctx->digestsize = keylen;
3013
3014 ctx->enckeylen = keylen;
3015 ctx->authkeylen = 0;
3016 memcpy(ctx->enckey, key, ctx->enckeylen);
3017
3018 switch (ctx->enckeylen) {
3019 case AES_KEYSIZE_128:
3020 ctx->cipher_type = CIPHER_TYPE_AES128;
3021 break;
3022 case AES_KEYSIZE_192:
3023 ctx->cipher_type = CIPHER_TYPE_AES192;
3024 break;
3025 case AES_KEYSIZE_256:
3026 ctx->cipher_type = CIPHER_TYPE_AES256;
3027 break;
3028 default:
3029 goto badkey;
3030 }
3031
3032 flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
3033 ctx->authkeylen);
3034 flow_dump(" enc: ", ctx->enckey, ctx->enckeylen);
3035 flow_dump(" auth: ", ctx->authkey, ctx->authkeylen);
3036
3037 /* setkey the fallback just in case we need to use it */
3038 if (ctx->fallback_cipher) {
3039 flow_log(" running fallback setkey()\n");
3040
3041 ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
3042 ctx->fallback_cipher->base.crt_flags |=
3043 tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
3044 ret = crypto_aead_setkey(ctx->fallback_cipher, key,
3045 keylen + ctx->salt_len);
3046 if (ret) {
3047 flow_log(" fallback setkey() returned:%d\n", ret);
3048 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
3049 tfm->crt_flags |=
3050 (ctx->fallback_cipher->base.crt_flags &
3051 CRYPTO_TFM_RES_MASK);
3052 }
3053 }
3054
3055 ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
3056 ctx->enckeylen,
3057 false);
3058
3059 atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
3060
3061 flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
3062 ctx->authkeylen);
3063
3064 return ret;
3065
3066badkey:
3067 ctx->enckeylen = 0;
3068 ctx->authkeylen = 0;
3069 ctx->digestsize = 0;
3070
3071 crypto_aead_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
3072 return -EINVAL;
3073}
3074
3075/**
3076 * aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES.
3077 * @cipher: AEAD structure
3078 * @key: Key followed by 4 bytes of salt
3079 * @keylen: Length of key plus salt, in bytes
3080 *
3081 * Extracts salt from key and stores it to be prepended to IV on each request.
3082 * Digest is always 16 bytes
3083 *
3084 * Return: Value from generic gcm setkey.
3085 */
3086static int aead_gcm_esp_setkey(struct crypto_aead *cipher,
3087 const u8 *key, unsigned int keylen)
3088{
3089 struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3090
3091 flow_log("%s\n", __func__);
3092 ctx->salt_len = GCM_ESP_SALT_SIZE;
3093 ctx->salt_offset = GCM_ESP_SALT_OFFSET;
3094 memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
3095 keylen -= GCM_ESP_SALT_SIZE;
3096 ctx->digestsize = GCM_ESP_DIGESTSIZE;
3097 ctx->is_esp = true;
3098 flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
3099
3100 return aead_gcm_ccm_setkey(cipher, key, keylen);
3101}
3102
3103/**
3104 * rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC.
3105 * cipher: AEAD structure
3106 * key: Key followed by 4 bytes of salt
3107 * keylen: Length of key plus salt, in bytes
3108 *
3109 * Extracts salt from key and stores it to be prepended to IV on each request.
3110 * Digest is always 16 bytes
3111 *
3112 * Return: Value from generic gcm setkey.
3113 */
3114static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher,
3115 const u8 *key, unsigned int keylen)
3116{
3117 struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3118
3119 flow_log("%s\n", __func__);
3120 ctx->salt_len = GCM_ESP_SALT_SIZE;
3121 ctx->salt_offset = GCM_ESP_SALT_OFFSET;
3122 memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
3123 keylen -= GCM_ESP_SALT_SIZE;
3124 ctx->digestsize = GCM_ESP_DIGESTSIZE;
3125 ctx->is_esp = true;
3126 ctx->is_rfc4543 = true;
3127 flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
3128
3129 return aead_gcm_ccm_setkey(cipher, key, keylen);
3130}
3131
3132/**
3133 * aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES.
3134 * @cipher: AEAD structure
3135 * @key: Key followed by 4 bytes of salt
3136 * @keylen: Length of key plus salt, in bytes
3137 *
3138 * Extracts salt from key and stores it to be prepended to IV on each request.
3139 * Digest is always 16 bytes
3140 *
3141 * Return: Value from generic ccm setkey.
3142 */
3143static int aead_ccm_esp_setkey(struct crypto_aead *cipher,
3144 const u8 *key, unsigned int keylen)
3145{
3146 struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3147
3148 flow_log("%s\n", __func__);
3149 ctx->salt_len = CCM_ESP_SALT_SIZE;
3150 ctx->salt_offset = CCM_ESP_SALT_OFFSET;
3151 memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
3152 keylen -= CCM_ESP_SALT_SIZE;
3153 ctx->is_esp = true;
3154 flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE);
3155
3156 return aead_gcm_ccm_setkey(cipher, key, keylen);
3157}
3158
3159static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize)
3160{
3161 struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3162 int ret = 0;
3163
3164 flow_log("%s() authkeylen:%u authsize:%u\n",
3165 __func__, ctx->authkeylen, authsize);
3166
3167 ctx->digestsize = authsize;
3168
3169 /* setkey the fallback just in case we needto use it */
3170 if (ctx->fallback_cipher) {
3171 flow_log(" running fallback setauth()\n");
3172
3173 ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize);
3174 if (ret)
3175 flow_log(" fallback setauth() returned:%d\n", ret);
3176 }
3177
3178 return ret;
3179}
3180
3181static int aead_encrypt(struct aead_request *req)
3182{
3183 flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen,
3184 req->cryptlen);
3185 dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3186 flow_log(" assoc_len:%u\n", req->assoclen);
3187
3188 return aead_enqueue(req, true);
3189}
3190
3191static int aead_decrypt(struct aead_request *req)
3192{
3193 flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen);
3194 dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3195 flow_log(" assoc_len:%u\n", req->assoclen);
3196
3197 return aead_enqueue(req, false);
3198}
3199
3200/* ==================== Supported Cipher Algorithms ==================== */
3201
3202static struct iproc_alg_s driver_algs[] = {
3203 {
3204 .type = CRYPTO_ALG_TYPE_AEAD,
3205 .alg.aead = {
3206 .base = {
3207 .cra_name = "gcm(aes)",
3208 .cra_driver_name = "gcm-aes-iproc",
3209 .cra_blocksize = AES_BLOCK_SIZE,
3210 .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3211 },
3212 .setkey = aead_gcm_ccm_setkey,
3213 .ivsize = GCM_AES_IV_SIZE,
3214 .maxauthsize = AES_BLOCK_SIZE,
3215 },
3216 .cipher_info = {
3217 .alg = CIPHER_ALG_AES,
3218 .mode = CIPHER_MODE_GCM,
3219 },
3220 .auth_info = {
3221 .alg = HASH_ALG_AES,
3222 .mode = HASH_MODE_GCM,
3223 },
3224 .auth_first = 0,
3225 },
3226 {
3227 .type = CRYPTO_ALG_TYPE_AEAD,
3228 .alg.aead = {
3229 .base = {
3230 .cra_name = "ccm(aes)",
3231 .cra_driver_name = "ccm-aes-iproc",
3232 .cra_blocksize = AES_BLOCK_SIZE,
3233 .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3234 },
3235 .setkey = aead_gcm_ccm_setkey,
3236 .ivsize = CCM_AES_IV_SIZE,
3237 .maxauthsize = AES_BLOCK_SIZE,
3238 },
3239 .cipher_info = {
3240 .alg = CIPHER_ALG_AES,
3241 .mode = CIPHER_MODE_CCM,
3242 },
3243 .auth_info = {
3244 .alg = HASH_ALG_AES,
3245 .mode = HASH_MODE_CCM,
3246 },
3247 .auth_first = 0,
3248 },
3249 {
3250 .type = CRYPTO_ALG_TYPE_AEAD,
3251 .alg.aead = {
3252 .base = {
3253 .cra_name = "rfc4106(gcm(aes))",
3254 .cra_driver_name = "gcm-aes-esp-iproc",
3255 .cra_blocksize = AES_BLOCK_SIZE,
3256 .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3257 },
3258 .setkey = aead_gcm_esp_setkey,
3259 .ivsize = GCM_ESP_IV_SIZE,
3260 .maxauthsize = AES_BLOCK_SIZE,
3261 },
3262 .cipher_info = {
3263 .alg = CIPHER_ALG_AES,
3264 .mode = CIPHER_MODE_GCM,
3265 },
3266 .auth_info = {
3267 .alg = HASH_ALG_AES,
3268 .mode = HASH_MODE_GCM,
3269 },
3270 .auth_first = 0,
3271 },
3272 {
3273 .type = CRYPTO_ALG_TYPE_AEAD,
3274 .alg.aead = {
3275 .base = {
3276 .cra_name = "rfc4309(ccm(aes))",
3277 .cra_driver_name = "ccm-aes-esp-iproc",
3278 .cra_blocksize = AES_BLOCK_SIZE,
3279 .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3280 },
3281 .setkey = aead_ccm_esp_setkey,
3282 .ivsize = CCM_AES_IV_SIZE,
3283 .maxauthsize = AES_BLOCK_SIZE,
3284 },
3285 .cipher_info = {
3286 .alg = CIPHER_ALG_AES,
3287 .mode = CIPHER_MODE_CCM,
3288 },
3289 .auth_info = {
3290 .alg = HASH_ALG_AES,
3291 .mode = HASH_MODE_CCM,
3292 },
3293 .auth_first = 0,
3294 },
3295 {
3296 .type = CRYPTO_ALG_TYPE_AEAD,
3297 .alg.aead = {
3298 .base = {
3299 .cra_name = "rfc4543(gcm(aes))",
3300 .cra_driver_name = "gmac-aes-esp-iproc",
3301 .cra_blocksize = AES_BLOCK_SIZE,
3302 .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3303 },
3304 .setkey = rfc4543_gcm_esp_setkey,
3305 .ivsize = GCM_ESP_IV_SIZE,
3306 .maxauthsize = AES_BLOCK_SIZE,
3307 },
3308 .cipher_info = {
3309 .alg = CIPHER_ALG_AES,
3310 .mode = CIPHER_MODE_GCM,
3311 },
3312 .auth_info = {
3313 .alg = HASH_ALG_AES,
3314 .mode = HASH_MODE_GCM,
3315 },
3316 .auth_first = 0,
3317 },
3318 {
3319 .type = CRYPTO_ALG_TYPE_AEAD,
3320 .alg.aead = {
3321 .base = {
3322 .cra_name = "authenc(hmac(md5),cbc(aes))",
3323 .cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc",
3324 .cra_blocksize = AES_BLOCK_SIZE,
3325 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3326 },
3327 .setkey = aead_authenc_setkey,
3328 .ivsize = AES_BLOCK_SIZE,
3329 .maxauthsize = MD5_DIGEST_SIZE,
3330 },
3331 .cipher_info = {
3332 .alg = CIPHER_ALG_AES,
3333 .mode = CIPHER_MODE_CBC,
3334 },
3335 .auth_info = {
3336 .alg = HASH_ALG_MD5,
3337 .mode = HASH_MODE_HMAC,
3338 },
3339 .auth_first = 0,
3340 },
3341 {
3342 .type = CRYPTO_ALG_TYPE_AEAD,
3343 .alg.aead = {
3344 .base = {
3345 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3346 .cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc",
3347 .cra_blocksize = AES_BLOCK_SIZE,
3348 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3349 },
3350 .setkey = aead_authenc_setkey,
3351 .ivsize = AES_BLOCK_SIZE,
3352 .maxauthsize = SHA1_DIGEST_SIZE,
3353 },
3354 .cipher_info = {
3355 .alg = CIPHER_ALG_AES,
3356 .mode = CIPHER_MODE_CBC,
3357 },
3358 .auth_info = {
3359 .alg = HASH_ALG_SHA1,
3360 .mode = HASH_MODE_HMAC,
3361 },
3362 .auth_first = 0,
3363 },
3364 {
3365 .type = CRYPTO_ALG_TYPE_AEAD,
3366 .alg.aead = {
3367 .base = {
3368 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3369 .cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc",
3370 .cra_blocksize = AES_BLOCK_SIZE,
3371 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3372 },
3373 .setkey = aead_authenc_setkey,
3374 .ivsize = AES_BLOCK_SIZE,
3375 .maxauthsize = SHA256_DIGEST_SIZE,
3376 },
3377 .cipher_info = {
3378 .alg = CIPHER_ALG_AES,
3379 .mode = CIPHER_MODE_CBC,
3380 },
3381 .auth_info = {
3382 .alg = HASH_ALG_SHA256,
3383 .mode = HASH_MODE_HMAC,
3384 },
3385 .auth_first = 0,
3386 },
3387 {
3388 .type = CRYPTO_ALG_TYPE_AEAD,
3389 .alg.aead = {
3390 .base = {
3391 .cra_name = "authenc(hmac(md5),cbc(des))",
3392 .cra_driver_name = "authenc-hmac-md5-cbc-des-iproc",
3393 .cra_blocksize = DES_BLOCK_SIZE,
3394 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3395 },
3396 .setkey = aead_authenc_setkey,
3397 .ivsize = DES_BLOCK_SIZE,
3398 .maxauthsize = MD5_DIGEST_SIZE,
3399 },
3400 .cipher_info = {
3401 .alg = CIPHER_ALG_DES,
3402 .mode = CIPHER_MODE_CBC,
3403 },
3404 .auth_info = {
3405 .alg = HASH_ALG_MD5,
3406 .mode = HASH_MODE_HMAC,
3407 },
3408 .auth_first = 0,
3409 },
3410 {
3411 .type = CRYPTO_ALG_TYPE_AEAD,
3412 .alg.aead = {
3413 .base = {
3414 .cra_name = "authenc(hmac(sha1),cbc(des))",
3415 .cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc",
3416 .cra_blocksize = DES_BLOCK_SIZE,
3417 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3418 },
3419 .setkey = aead_authenc_setkey,
3420 .ivsize = DES_BLOCK_SIZE,
3421 .maxauthsize = SHA1_DIGEST_SIZE,
3422 },
3423 .cipher_info = {
3424 .alg = CIPHER_ALG_DES,
3425 .mode = CIPHER_MODE_CBC,
3426 },
3427 .auth_info = {
3428 .alg = HASH_ALG_SHA1,
3429 .mode = HASH_MODE_HMAC,
3430 },
3431 .auth_first = 0,
3432 },
3433 {
3434 .type = CRYPTO_ALG_TYPE_AEAD,
3435 .alg.aead = {
3436 .base = {
3437 .cra_name = "authenc(hmac(sha224),cbc(des))",
3438 .cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc",
3439 .cra_blocksize = DES_BLOCK_SIZE,
3440 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3441 },
3442 .setkey = aead_authenc_setkey,
3443 .ivsize = DES_BLOCK_SIZE,
3444 .maxauthsize = SHA224_DIGEST_SIZE,
3445 },
3446 .cipher_info = {
3447 .alg = CIPHER_ALG_DES,
3448 .mode = CIPHER_MODE_CBC,
3449 },
3450 .auth_info = {
3451 .alg = HASH_ALG_SHA224,
3452 .mode = HASH_MODE_HMAC,
3453 },
3454 .auth_first = 0,
3455 },
3456 {
3457 .type = CRYPTO_ALG_TYPE_AEAD,
3458 .alg.aead = {
3459 .base = {
3460 .cra_name = "authenc(hmac(sha256),cbc(des))",
3461 .cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc",
3462 .cra_blocksize = DES_BLOCK_SIZE,
3463 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3464 },
3465 .setkey = aead_authenc_setkey,
3466 .ivsize = DES_BLOCK_SIZE,
3467 .maxauthsize = SHA256_DIGEST_SIZE,
3468 },
3469 .cipher_info = {
3470 .alg = CIPHER_ALG_DES,
3471 .mode = CIPHER_MODE_CBC,
3472 },
3473 .auth_info = {
3474 .alg = HASH_ALG_SHA256,
3475 .mode = HASH_MODE_HMAC,
3476 },
3477 .auth_first = 0,
3478 },
3479 {
3480 .type = CRYPTO_ALG_TYPE_AEAD,
3481 .alg.aead = {
3482 .base = {
3483 .cra_name = "authenc(hmac(sha384),cbc(des))",
3484 .cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc",
3485 .cra_blocksize = DES_BLOCK_SIZE,
3486 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3487 },
3488 .setkey = aead_authenc_setkey,
3489 .ivsize = DES_BLOCK_SIZE,
3490 .maxauthsize = SHA384_DIGEST_SIZE,
3491 },
3492 .cipher_info = {
3493 .alg = CIPHER_ALG_DES,
3494 .mode = CIPHER_MODE_CBC,
3495 },
3496 .auth_info = {
3497 .alg = HASH_ALG_SHA384,
3498 .mode = HASH_MODE_HMAC,
3499 },
3500 .auth_first = 0,
3501 },
3502 {
3503 .type = CRYPTO_ALG_TYPE_AEAD,
3504 .alg.aead = {
3505 .base = {
3506 .cra_name = "authenc(hmac(sha512),cbc(des))",
3507 .cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc",
3508 .cra_blocksize = DES_BLOCK_SIZE,
3509 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3510 },
3511 .setkey = aead_authenc_setkey,
3512 .ivsize = DES_BLOCK_SIZE,
3513 .maxauthsize = SHA512_DIGEST_SIZE,
3514 },
3515 .cipher_info = {
3516 .alg = CIPHER_ALG_DES,
3517 .mode = CIPHER_MODE_CBC,
3518 },
3519 .auth_info = {
3520 .alg = HASH_ALG_SHA512,
3521 .mode = HASH_MODE_HMAC,
3522 },
3523 .auth_first = 0,
3524 },
3525 {
3526 .type = CRYPTO_ALG_TYPE_AEAD,
3527 .alg.aead = {
3528 .base = {
3529 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3530 .cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc",
3531 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3532 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3533 },
3534 .setkey = aead_authenc_setkey,
3535 .ivsize = DES3_EDE_BLOCK_SIZE,
3536 .maxauthsize = MD5_DIGEST_SIZE,
3537 },
3538 .cipher_info = {
3539 .alg = CIPHER_ALG_3DES,
3540 .mode = CIPHER_MODE_CBC,
3541 },
3542 .auth_info = {
3543 .alg = HASH_ALG_MD5,
3544 .mode = HASH_MODE_HMAC,
3545 },
3546 .auth_first = 0,
3547 },
3548 {
3549 .type = CRYPTO_ALG_TYPE_AEAD,
3550 .alg.aead = {
3551 .base = {
3552 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
3553 .cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc",
3554 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3555 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3556 },
3557 .setkey = aead_authenc_setkey,
3558 .ivsize = DES3_EDE_BLOCK_SIZE,
3559 .maxauthsize = SHA1_DIGEST_SIZE,
3560 },
3561 .cipher_info = {
3562 .alg = CIPHER_ALG_3DES,
3563 .mode = CIPHER_MODE_CBC,
3564 },
3565 .auth_info = {
3566 .alg = HASH_ALG_SHA1,
3567 .mode = HASH_MODE_HMAC,
3568 },
3569 .auth_first = 0,
3570 },
3571 {
3572 .type = CRYPTO_ALG_TYPE_AEAD,
3573 .alg.aead = {
3574 .base = {
3575 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
3576 .cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc",
3577 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3578 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3579 },
3580 .setkey = aead_authenc_setkey,
3581 .ivsize = DES3_EDE_BLOCK_SIZE,
3582 .maxauthsize = SHA224_DIGEST_SIZE,
3583 },
3584 .cipher_info = {
3585 .alg = CIPHER_ALG_3DES,
3586 .mode = CIPHER_MODE_CBC,
3587 },
3588 .auth_info = {
3589 .alg = HASH_ALG_SHA224,
3590 .mode = HASH_MODE_HMAC,
3591 },
3592 .auth_first = 0,
3593 },
3594 {
3595 .type = CRYPTO_ALG_TYPE_AEAD,
3596 .alg.aead = {
3597 .base = {
3598 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
3599 .cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc",
3600 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3601 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3602 },
3603 .setkey = aead_authenc_setkey,
3604 .ivsize = DES3_EDE_BLOCK_SIZE,
3605 .maxauthsize = SHA256_DIGEST_SIZE,
3606 },
3607 .cipher_info = {
3608 .alg = CIPHER_ALG_3DES,
3609 .mode = CIPHER_MODE_CBC,
3610 },
3611 .auth_info = {
3612 .alg = HASH_ALG_SHA256,
3613 .mode = HASH_MODE_HMAC,
3614 },
3615 .auth_first = 0,
3616 },
3617 {
3618 .type = CRYPTO_ALG_TYPE_AEAD,
3619 .alg.aead = {
3620 .base = {
3621 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
3622 .cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc",
3623 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3624 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3625 },
3626 .setkey = aead_authenc_setkey,
3627 .ivsize = DES3_EDE_BLOCK_SIZE,
3628 .maxauthsize = SHA384_DIGEST_SIZE,
3629 },
3630 .cipher_info = {
3631 .alg = CIPHER_ALG_3DES,
3632 .mode = CIPHER_MODE_CBC,
3633 },
3634 .auth_info = {
3635 .alg = HASH_ALG_SHA384,
3636 .mode = HASH_MODE_HMAC,
3637 },
3638 .auth_first = 0,
3639 },
3640 {
3641 .type = CRYPTO_ALG_TYPE_AEAD,
3642 .alg.aead = {
3643 .base = {
3644 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
3645 .cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc",
3646 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3647 .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC
3648 },
3649 .setkey = aead_authenc_setkey,
3650 .ivsize = DES3_EDE_BLOCK_SIZE,
3651 .maxauthsize = SHA512_DIGEST_SIZE,
3652 },
3653 .cipher_info = {
3654 .alg = CIPHER_ALG_3DES,
3655 .mode = CIPHER_MODE_CBC,
3656 },
3657 .auth_info = {
3658 .alg = HASH_ALG_SHA512,
3659 .mode = HASH_MODE_HMAC,
3660 },
3661 .auth_first = 0,
3662 },
3663
3664/* ABLKCIPHER algorithms. */
3665 {
3666 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3667 .alg.crypto = {
3668 .cra_name = "ecb(arc4)",
3669 .cra_driver_name = "ecb-arc4-iproc",
3670 .cra_blocksize = ARC4_BLOCK_SIZE,
3671 .cra_ablkcipher = {
3672 .min_keysize = ARC4_MIN_KEY_SIZE,
3673 .max_keysize = ARC4_MAX_KEY_SIZE,
3674 .ivsize = 0,
3675 }
3676 },
3677 .cipher_info = {
3678 .alg = CIPHER_ALG_RC4,
3679 .mode = CIPHER_MODE_NONE,
3680 },
3681 .auth_info = {
3682 .alg = HASH_ALG_NONE,
3683 .mode = HASH_MODE_NONE,
3684 },
3685 },
3686 {
3687 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3688 .alg.crypto = {
3689 .cra_name = "ofb(des)",
3690 .cra_driver_name = "ofb-des-iproc",
3691 .cra_blocksize = DES_BLOCK_SIZE,
3692 .cra_ablkcipher = {
3693 .min_keysize = DES_KEY_SIZE,
3694 .max_keysize = DES_KEY_SIZE,
3695 .ivsize = DES_BLOCK_SIZE,
3696 }
3697 },
3698 .cipher_info = {
3699 .alg = CIPHER_ALG_DES,
3700 .mode = CIPHER_MODE_OFB,
3701 },
3702 .auth_info = {
3703 .alg = HASH_ALG_NONE,
3704 .mode = HASH_MODE_NONE,
3705 },
3706 },
3707 {
3708 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3709 .alg.crypto = {
3710 .cra_name = "cbc(des)",
3711 .cra_driver_name = "cbc-des-iproc",
3712 .cra_blocksize = DES_BLOCK_SIZE,
3713 .cra_ablkcipher = {
3714 .min_keysize = DES_KEY_SIZE,
3715 .max_keysize = DES_KEY_SIZE,
3716 .ivsize = DES_BLOCK_SIZE,
3717 }
3718 },
3719 .cipher_info = {
3720 .alg = CIPHER_ALG_DES,
3721 .mode = CIPHER_MODE_CBC,
3722 },
3723 .auth_info = {
3724 .alg = HASH_ALG_NONE,
3725 .mode = HASH_MODE_NONE,
3726 },
3727 },
3728 {
3729 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3730 .alg.crypto = {
3731 .cra_name = "ecb(des)",
3732 .cra_driver_name = "ecb-des-iproc",
3733 .cra_blocksize = DES_BLOCK_SIZE,
3734 .cra_ablkcipher = {
3735 .min_keysize = DES_KEY_SIZE,
3736 .max_keysize = DES_KEY_SIZE,
3737 .ivsize = 0,
3738 }
3739 },
3740 .cipher_info = {
3741 .alg = CIPHER_ALG_DES,
3742 .mode = CIPHER_MODE_ECB,
3743 },
3744 .auth_info = {
3745 .alg = HASH_ALG_NONE,
3746 .mode = HASH_MODE_NONE,
3747 },
3748 },
3749 {
3750 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3751 .alg.crypto = {
3752 .cra_name = "ofb(des3_ede)",
3753 .cra_driver_name = "ofb-des3-iproc",
3754 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3755 .cra_ablkcipher = {
3756 .min_keysize = DES3_EDE_KEY_SIZE,
3757 .max_keysize = DES3_EDE_KEY_SIZE,
3758 .ivsize = DES3_EDE_BLOCK_SIZE,
3759 }
3760 },
3761 .cipher_info = {
3762 .alg = CIPHER_ALG_3DES,
3763 .mode = CIPHER_MODE_OFB,
3764 },
3765 .auth_info = {
3766 .alg = HASH_ALG_NONE,
3767 .mode = HASH_MODE_NONE,
3768 },
3769 },
3770 {
3771 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3772 .alg.crypto = {
3773 .cra_name = "cbc(des3_ede)",
3774 .cra_driver_name = "cbc-des3-iproc",
3775 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3776 .cra_ablkcipher = {
3777 .min_keysize = DES3_EDE_KEY_SIZE,
3778 .max_keysize = DES3_EDE_KEY_SIZE,
3779 .ivsize = DES3_EDE_BLOCK_SIZE,
3780 }
3781 },
3782 .cipher_info = {
3783 .alg = CIPHER_ALG_3DES,
3784 .mode = CIPHER_MODE_CBC,
3785 },
3786 .auth_info = {
3787 .alg = HASH_ALG_NONE,
3788 .mode = HASH_MODE_NONE,
3789 },
3790 },
3791 {
3792 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3793 .alg.crypto = {
3794 .cra_name = "ecb(des3_ede)",
3795 .cra_driver_name = "ecb-des3-iproc",
3796 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3797 .cra_ablkcipher = {
3798 .min_keysize = DES3_EDE_KEY_SIZE,
3799 .max_keysize = DES3_EDE_KEY_SIZE,
3800 .ivsize = 0,
3801 }
3802 },
3803 .cipher_info = {
3804 .alg = CIPHER_ALG_3DES,
3805 .mode = CIPHER_MODE_ECB,
3806 },
3807 .auth_info = {
3808 .alg = HASH_ALG_NONE,
3809 .mode = HASH_MODE_NONE,
3810 },
3811 },
3812 {
3813 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3814 .alg.crypto = {
3815 .cra_name = "ofb(aes)",
3816 .cra_driver_name = "ofb-aes-iproc",
3817 .cra_blocksize = AES_BLOCK_SIZE,
3818 .cra_ablkcipher = {
3819 .min_keysize = AES_MIN_KEY_SIZE,
3820 .max_keysize = AES_MAX_KEY_SIZE,
3821 .ivsize = AES_BLOCK_SIZE,
3822 }
3823 },
3824 .cipher_info = {
3825 .alg = CIPHER_ALG_AES,
3826 .mode = CIPHER_MODE_OFB,
3827 },
3828 .auth_info = {
3829 .alg = HASH_ALG_NONE,
3830 .mode = HASH_MODE_NONE,
3831 },
3832 },
3833 {
3834 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3835 .alg.crypto = {
3836 .cra_name = "cbc(aes)",
3837 .cra_driver_name = "cbc-aes-iproc",
3838 .cra_blocksize = AES_BLOCK_SIZE,
3839 .cra_ablkcipher = {
3840 .min_keysize = AES_MIN_KEY_SIZE,
3841 .max_keysize = AES_MAX_KEY_SIZE,
3842 .ivsize = AES_BLOCK_SIZE,
3843 }
3844 },
3845 .cipher_info = {
3846 .alg = CIPHER_ALG_AES,
3847 .mode = CIPHER_MODE_CBC,
3848 },
3849 .auth_info = {
3850 .alg = HASH_ALG_NONE,
3851 .mode = HASH_MODE_NONE,
3852 },
3853 },
3854 {
3855 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3856 .alg.crypto = {
3857 .cra_name = "ecb(aes)",
3858 .cra_driver_name = "ecb-aes-iproc",
3859 .cra_blocksize = AES_BLOCK_SIZE,
3860 .cra_ablkcipher = {
3861 .min_keysize = AES_MIN_KEY_SIZE,
3862 .max_keysize = AES_MAX_KEY_SIZE,
3863 .ivsize = 0,
3864 }
3865 },
3866 .cipher_info = {
3867 .alg = CIPHER_ALG_AES,
3868 .mode = CIPHER_MODE_ECB,
3869 },
3870 .auth_info = {
3871 .alg = HASH_ALG_NONE,
3872 .mode = HASH_MODE_NONE,
3873 },
3874 },
3875 {
3876 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3877 .alg.crypto = {
3878 .cra_name = "ctr(aes)",
3879 .cra_driver_name = "ctr-aes-iproc",
3880 .cra_blocksize = AES_BLOCK_SIZE,
3881 .cra_ablkcipher = {
3882 /* .geniv = "chainiv", */
3883 .min_keysize = AES_MIN_KEY_SIZE,
3884 .max_keysize = AES_MAX_KEY_SIZE,
3885 .ivsize = AES_BLOCK_SIZE,
3886 }
3887 },
3888 .cipher_info = {
3889 .alg = CIPHER_ALG_AES,
3890 .mode = CIPHER_MODE_CTR,
3891 },
3892 .auth_info = {
3893 .alg = HASH_ALG_NONE,
3894 .mode = HASH_MODE_NONE,
3895 },
3896 },
3897{
3898 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3899 .alg.crypto = {
3900 .cra_name = "xts(aes)",
3901 .cra_driver_name = "xts-aes-iproc",
3902 .cra_blocksize = AES_BLOCK_SIZE,
3903 .cra_ablkcipher = {
3904 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3905 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3906 .ivsize = AES_BLOCK_SIZE,
3907 }
3908 },
3909 .cipher_info = {
3910 .alg = CIPHER_ALG_AES,
3911 .mode = CIPHER_MODE_XTS,
3912 },
3913 .auth_info = {
3914 .alg = HASH_ALG_NONE,
3915 .mode = HASH_MODE_NONE,
3916 },
3917 },
3918
3919/* AHASH algorithms. */
3920 {
3921 .type = CRYPTO_ALG_TYPE_AHASH,
3922 .alg.hash = {
3923 .halg.digestsize = MD5_DIGEST_SIZE,
3924 .halg.base = {
3925 .cra_name = "md5",
3926 .cra_driver_name = "md5-iproc",
3927 .cra_blocksize = MD5_BLOCK_WORDS * 4,
3928 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
3929 CRYPTO_ALG_ASYNC,
3930 }
3931 },
3932 .cipher_info = {
3933 .alg = CIPHER_ALG_NONE,
3934 .mode = CIPHER_MODE_NONE,
3935 },
3936 .auth_info = {
3937 .alg = HASH_ALG_MD5,
3938 .mode = HASH_MODE_HASH,
3939 },
3940 },
3941 {
3942 .type = CRYPTO_ALG_TYPE_AHASH,
3943 .alg.hash = {
3944 .halg.digestsize = MD5_DIGEST_SIZE,
3945 .halg.base = {
3946 .cra_name = "hmac(md5)",
3947 .cra_driver_name = "hmac-md5-iproc",
3948 .cra_blocksize = MD5_BLOCK_WORDS * 4,
3949 }
3950 },
3951 .cipher_info = {
3952 .alg = CIPHER_ALG_NONE,
3953 .mode = CIPHER_MODE_NONE,
3954 },
3955 .auth_info = {
3956 .alg = HASH_ALG_MD5,
3957 .mode = HASH_MODE_HMAC,
3958 },
3959 },
3960 {.type = CRYPTO_ALG_TYPE_AHASH,
3961 .alg.hash = {
3962 .halg.digestsize = SHA1_DIGEST_SIZE,
3963 .halg.base = {
3964 .cra_name = "sha1",
3965 .cra_driver_name = "sha1-iproc",
3966 .cra_blocksize = SHA1_BLOCK_SIZE,
3967 }
3968 },
3969 .cipher_info = {
3970 .alg = CIPHER_ALG_NONE,
3971 .mode = CIPHER_MODE_NONE,
3972 },
3973 .auth_info = {
3974 .alg = HASH_ALG_SHA1,
3975 .mode = HASH_MODE_HASH,
3976 },
3977 },
3978 {.type = CRYPTO_ALG_TYPE_AHASH,
3979 .alg.hash = {
3980 .halg.digestsize = SHA1_DIGEST_SIZE,
3981 .halg.base = {
3982 .cra_name = "hmac(sha1)",
3983 .cra_driver_name = "hmac-sha1-iproc",
3984 .cra_blocksize = SHA1_BLOCK_SIZE,
3985 }
3986 },
3987 .cipher_info = {
3988 .alg = CIPHER_ALG_NONE,
3989 .mode = CIPHER_MODE_NONE,
3990 },
3991 .auth_info = {
3992 .alg = HASH_ALG_SHA1,
3993 .mode = HASH_MODE_HMAC,
3994 },
3995 },
3996 {.type = CRYPTO_ALG_TYPE_AHASH,
3997 .alg.hash = {
3998 .halg.digestsize = SHA224_DIGEST_SIZE,
3999 .halg.base = {
4000 .cra_name = "sha224",
4001 .cra_driver_name = "sha224-iproc",
4002 .cra_blocksize = SHA224_BLOCK_SIZE,
4003 }
4004 },
4005 .cipher_info = {
4006 .alg = CIPHER_ALG_NONE,
4007 .mode = CIPHER_MODE_NONE,
4008 },
4009 .auth_info = {
4010 .alg = HASH_ALG_SHA224,
4011 .mode = HASH_MODE_HASH,
4012 },
4013 },
4014 {.type = CRYPTO_ALG_TYPE_AHASH,
4015 .alg.hash = {
4016 .halg.digestsize = SHA224_DIGEST_SIZE,
4017 .halg.base = {
4018 .cra_name = "hmac(sha224)",
4019 .cra_driver_name = "hmac-sha224-iproc",
4020 .cra_blocksize = SHA224_BLOCK_SIZE,
4021 }
4022 },
4023 .cipher_info = {
4024 .alg = CIPHER_ALG_NONE,
4025 .mode = CIPHER_MODE_NONE,
4026 },
4027 .auth_info = {
4028 .alg = HASH_ALG_SHA224,
4029 .mode = HASH_MODE_HMAC,
4030 },
4031 },
4032 {.type = CRYPTO_ALG_TYPE_AHASH,
4033 .alg.hash = {
4034 .halg.digestsize = SHA256_DIGEST_SIZE,
4035 .halg.base = {
4036 .cra_name = "sha256",
4037 .cra_driver_name = "sha256-iproc",
4038 .cra_blocksize = SHA256_BLOCK_SIZE,
4039 }
4040 },
4041 .cipher_info = {
4042 .alg = CIPHER_ALG_NONE,
4043 .mode = CIPHER_MODE_NONE,
4044 },
4045 .auth_info = {
4046 .alg = HASH_ALG_SHA256,
4047 .mode = HASH_MODE_HASH,
4048 },
4049 },
4050 {.type = CRYPTO_ALG_TYPE_AHASH,
4051 .alg.hash = {
4052 .halg.digestsize = SHA256_DIGEST_SIZE,
4053 .halg.base = {
4054 .cra_name = "hmac(sha256)",
4055 .cra_driver_name = "hmac-sha256-iproc",
4056 .cra_blocksize = SHA256_BLOCK_SIZE,
4057 }
4058 },
4059 .cipher_info = {
4060 .alg = CIPHER_ALG_NONE,
4061 .mode = CIPHER_MODE_NONE,
4062 },
4063 .auth_info = {
4064 .alg = HASH_ALG_SHA256,
4065 .mode = HASH_MODE_HMAC,
4066 },
4067 },
4068 {
4069 .type = CRYPTO_ALG_TYPE_AHASH,
4070 .alg.hash = {
4071 .halg.digestsize = SHA384_DIGEST_SIZE,
4072 .halg.base = {
4073 .cra_name = "sha384",
4074 .cra_driver_name = "sha384-iproc",
4075 .cra_blocksize = SHA384_BLOCK_SIZE,
4076 }
4077 },
4078 .cipher_info = {
4079 .alg = CIPHER_ALG_NONE,
4080 .mode = CIPHER_MODE_NONE,
4081 },
4082 .auth_info = {
4083 .alg = HASH_ALG_SHA384,
4084 .mode = HASH_MODE_HASH,
4085 },
4086 },
4087 {
4088 .type = CRYPTO_ALG_TYPE_AHASH,
4089 .alg.hash = {
4090 .halg.digestsize = SHA384_DIGEST_SIZE,
4091 .halg.base = {
4092 .cra_name = "hmac(sha384)",
4093 .cra_driver_name = "hmac-sha384-iproc",
4094 .cra_blocksize = SHA384_BLOCK_SIZE,
4095 }
4096 },
4097 .cipher_info = {
4098 .alg = CIPHER_ALG_NONE,
4099 .mode = CIPHER_MODE_NONE,
4100 },
4101 .auth_info = {
4102 .alg = HASH_ALG_SHA384,
4103 .mode = HASH_MODE_HMAC,
4104 },
4105 },
4106 {
4107 .type = CRYPTO_ALG_TYPE_AHASH,
4108 .alg.hash = {
4109 .halg.digestsize = SHA512_DIGEST_SIZE,
4110 .halg.base = {
4111 .cra_name = "sha512",
4112 .cra_driver_name = "sha512-iproc",
4113 .cra_blocksize = SHA512_BLOCK_SIZE,
4114 }
4115 },
4116 .cipher_info = {
4117 .alg = CIPHER_ALG_NONE,
4118 .mode = CIPHER_MODE_NONE,
4119 },
4120 .auth_info = {
4121 .alg = HASH_ALG_SHA512,
4122 .mode = HASH_MODE_HASH,
4123 },
4124 },
4125 {
4126 .type = CRYPTO_ALG_TYPE_AHASH,
4127 .alg.hash = {
4128 .halg.digestsize = SHA512_DIGEST_SIZE,
4129 .halg.base = {
4130 .cra_name = "hmac(sha512)",
4131 .cra_driver_name = "hmac-sha512-iproc",
4132 .cra_blocksize = SHA512_BLOCK_SIZE,
4133 }
4134 },
4135 .cipher_info = {
4136 .alg = CIPHER_ALG_NONE,
4137 .mode = CIPHER_MODE_NONE,
4138 },
4139 .auth_info = {
4140 .alg = HASH_ALG_SHA512,
4141 .mode = HASH_MODE_HMAC,
4142 },
4143 },
4144 {
4145 .type = CRYPTO_ALG_TYPE_AHASH,
4146 .alg.hash = {
4147 .halg.digestsize = SHA3_224_DIGEST_SIZE,
4148 .halg.base = {
4149 .cra_name = "sha3-224",
4150 .cra_driver_name = "sha3-224-iproc",
4151 .cra_blocksize = SHA3_224_BLOCK_SIZE,
4152 }
4153 },
4154 .cipher_info = {
4155 .alg = CIPHER_ALG_NONE,
4156 .mode = CIPHER_MODE_NONE,
4157 },
4158 .auth_info = {
4159 .alg = HASH_ALG_SHA3_224,
4160 .mode = HASH_MODE_HASH,
4161 },
4162 },
4163 {
4164 .type = CRYPTO_ALG_TYPE_AHASH,
4165 .alg.hash = {
4166 .halg.digestsize = SHA3_224_DIGEST_SIZE,
4167 .halg.base = {
4168 .cra_name = "hmac(sha3-224)",
4169 .cra_driver_name = "hmac-sha3-224-iproc",
4170 .cra_blocksize = SHA3_224_BLOCK_SIZE,
4171 }
4172 },
4173 .cipher_info = {
4174 .alg = CIPHER_ALG_NONE,
4175 .mode = CIPHER_MODE_NONE,
4176 },
4177 .auth_info = {
4178 .alg = HASH_ALG_SHA3_224,
4179 .mode = HASH_MODE_HMAC
4180 },
4181 },
4182 {
4183 .type = CRYPTO_ALG_TYPE_AHASH,
4184 .alg.hash = {
4185 .halg.digestsize = SHA3_256_DIGEST_SIZE,
4186 .halg.base = {
4187 .cra_name = "sha3-256",
4188 .cra_driver_name = "sha3-256-iproc",
4189 .cra_blocksize = SHA3_256_BLOCK_SIZE,
4190 }
4191 },
4192 .cipher_info = {
4193 .alg = CIPHER_ALG_NONE,
4194 .mode = CIPHER_MODE_NONE,
4195 },
4196 .auth_info = {
4197 .alg = HASH_ALG_SHA3_256,
4198 .mode = HASH_MODE_HASH,
4199 },
4200 },
4201 {
4202 .type = CRYPTO_ALG_TYPE_AHASH,
4203 .alg.hash = {
4204 .halg.digestsize = SHA3_256_DIGEST_SIZE,
4205 .halg.base = {
4206 .cra_name = "hmac(sha3-256)",
4207 .cra_driver_name = "hmac-sha3-256-iproc",
4208 .cra_blocksize = SHA3_256_BLOCK_SIZE,
4209 }
4210 },
4211 .cipher_info = {
4212 .alg = CIPHER_ALG_NONE,
4213 .mode = CIPHER_MODE_NONE,
4214 },
4215 .auth_info = {
4216 .alg = HASH_ALG_SHA3_256,
4217 .mode = HASH_MODE_HMAC,
4218 },
4219 },
4220 {
4221 .type = CRYPTO_ALG_TYPE_AHASH,
4222 .alg.hash = {
4223 .halg.digestsize = SHA3_384_DIGEST_SIZE,
4224 .halg.base = {
4225 .cra_name = "sha3-384",
4226 .cra_driver_name = "sha3-384-iproc",
4227 .cra_blocksize = SHA3_224_BLOCK_SIZE,
4228 }
4229 },
4230 .cipher_info = {
4231 .alg = CIPHER_ALG_NONE,
4232 .mode = CIPHER_MODE_NONE,
4233 },
4234 .auth_info = {
4235 .alg = HASH_ALG_SHA3_384,
4236 .mode = HASH_MODE_HASH,
4237 },
4238 },
4239 {
4240 .type = CRYPTO_ALG_TYPE_AHASH,
4241 .alg.hash = {
4242 .halg.digestsize = SHA3_384_DIGEST_SIZE,
4243 .halg.base = {
4244 .cra_name = "hmac(sha3-384)",
4245 .cra_driver_name = "hmac-sha3-384-iproc",
4246 .cra_blocksize = SHA3_384_BLOCK_SIZE,
4247 }
4248 },
4249 .cipher_info = {
4250 .alg = CIPHER_ALG_NONE,
4251 .mode = CIPHER_MODE_NONE,
4252 },
4253 .auth_info = {
4254 .alg = HASH_ALG_SHA3_384,
4255 .mode = HASH_MODE_HMAC,
4256 },
4257 },
4258 {
4259 .type = CRYPTO_ALG_TYPE_AHASH,
4260 .alg.hash = {
4261 .halg.digestsize = SHA3_512_DIGEST_SIZE,
4262 .halg.base = {
4263 .cra_name = "sha3-512",
4264 .cra_driver_name = "sha3-512-iproc",
4265 .cra_blocksize = SHA3_512_BLOCK_SIZE,
4266 }
4267 },
4268 .cipher_info = {
4269 .alg = CIPHER_ALG_NONE,
4270 .mode = CIPHER_MODE_NONE,
4271 },
4272 .auth_info = {
4273 .alg = HASH_ALG_SHA3_512,
4274 .mode = HASH_MODE_HASH,
4275 },
4276 },
4277 {
4278 .type = CRYPTO_ALG_TYPE_AHASH,
4279 .alg.hash = {
4280 .halg.digestsize = SHA3_512_DIGEST_SIZE,
4281 .halg.base = {
4282 .cra_name = "hmac(sha3-512)",
4283 .cra_driver_name = "hmac-sha3-512-iproc",
4284 .cra_blocksize = SHA3_512_BLOCK_SIZE,
4285 }
4286 },
4287 .cipher_info = {
4288 .alg = CIPHER_ALG_NONE,
4289 .mode = CIPHER_MODE_NONE,
4290 },
4291 .auth_info = {
4292 .alg = HASH_ALG_SHA3_512,
4293 .mode = HASH_MODE_HMAC,
4294 },
4295 },
4296 {
4297 .type = CRYPTO_ALG_TYPE_AHASH,
4298 .alg.hash = {
4299 .halg.digestsize = AES_BLOCK_SIZE,
4300 .halg.base = {
4301 .cra_name = "xcbc(aes)",
4302 .cra_driver_name = "xcbc-aes-iproc",
4303 .cra_blocksize = AES_BLOCK_SIZE,
4304 }
4305 },
4306 .cipher_info = {
4307 .alg = CIPHER_ALG_NONE,
4308 .mode = CIPHER_MODE_NONE,
4309 },
4310 .auth_info = {
4311 .alg = HASH_ALG_AES,
4312 .mode = HASH_MODE_XCBC,
4313 },
4314 },
4315 {
4316 .type = CRYPTO_ALG_TYPE_AHASH,
4317 .alg.hash = {
4318 .halg.digestsize = AES_BLOCK_SIZE,
4319 .halg.base = {
4320 .cra_name = "cmac(aes)",
4321 .cra_driver_name = "cmac-aes-iproc",
4322 .cra_blocksize = AES_BLOCK_SIZE,
4323 }
4324 },
4325 .cipher_info = {
4326 .alg = CIPHER_ALG_NONE,
4327 .mode = CIPHER_MODE_NONE,
4328 },
4329 .auth_info = {
4330 .alg = HASH_ALG_AES,
4331 .mode = HASH_MODE_CMAC,
4332 },
4333 },
4334};
4335
4336static int generic_cra_init(struct crypto_tfm *tfm,
4337 struct iproc_alg_s *cipher_alg)
4338{
4339 struct spu_hw *spu = &iproc_priv.spu;
4340 struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4341 unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
4342
4343 flow_log("%s()\n", __func__);
4344
4345 ctx->alg = cipher_alg;
4346 ctx->cipher = cipher_alg->cipher_info;
4347 ctx->auth = cipher_alg->auth_info;
4348 ctx->auth_first = cipher_alg->auth_first;
4349 ctx->max_payload = spu->spu_ctx_max_payload(ctx->cipher.alg,
4350 ctx->cipher.mode,
4351 blocksize);
4352 ctx->fallback_cipher = NULL;
4353
4354 ctx->enckeylen = 0;
4355 ctx->authkeylen = 0;
4356
4357 atomic_inc(&iproc_priv.stream_count);
4358 atomic_inc(&iproc_priv.session_count);
4359
4360 return 0;
4361}
4362
4363static int ablkcipher_cra_init(struct crypto_tfm *tfm)
4364{
4365 struct crypto_alg *alg = tfm->__crt_alg;
4366 struct iproc_alg_s *cipher_alg;
4367
4368 flow_log("%s()\n", __func__);
4369
4370 tfm->crt_ablkcipher.reqsize = sizeof(struct iproc_reqctx_s);
4371
4372 cipher_alg = container_of(alg, struct iproc_alg_s, alg.crypto);
4373 return generic_cra_init(tfm, cipher_alg);
4374}
4375
4376static int ahash_cra_init(struct crypto_tfm *tfm)
4377{
4378 int err;
4379 struct crypto_alg *alg = tfm->__crt_alg;
4380 struct iproc_alg_s *cipher_alg;
4381
4382 cipher_alg = container_of(__crypto_ahash_alg(alg), struct iproc_alg_s,
4383 alg.hash);
4384
4385 err = generic_cra_init(tfm, cipher_alg);
4386 flow_log("%s()\n", __func__);
4387
4388 /*
4389 * export state size has to be < 512 bytes. So don't include msg bufs
4390 * in state size.
4391 */
4392 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4393 sizeof(struct iproc_reqctx_s));
4394
4395 return err;
4396}
4397
4398static int aead_cra_init(struct crypto_aead *aead)
4399{
4400 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4401 struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4402 struct crypto_alg *alg = tfm->__crt_alg;
4403 struct aead_alg *aalg = container_of(alg, struct aead_alg, base);
4404 struct iproc_alg_s *cipher_alg = container_of(aalg, struct iproc_alg_s,
4405 alg.aead);
4406
4407 int err = generic_cra_init(tfm, cipher_alg);
4408
4409 flow_log("%s()\n", __func__);
4410
4411 crypto_aead_set_reqsize(aead, sizeof(struct iproc_reqctx_s));
4412 ctx->is_esp = false;
4413 ctx->salt_len = 0;
4414 ctx->salt_offset = 0;
4415
4416 /* random first IV */
4417 get_random_bytes(ctx->iv, MAX_IV_SIZE);
4418 flow_dump(" iv: ", ctx->iv, MAX_IV_SIZE);
4419
4420 if (!err) {
4421 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
4422 flow_log("%s() creating fallback cipher\n", __func__);
4423
4424 ctx->fallback_cipher =
4425 crypto_alloc_aead(alg->cra_name, 0,
4426 CRYPTO_ALG_ASYNC |
4427 CRYPTO_ALG_NEED_FALLBACK);
4428 if (IS_ERR(ctx->fallback_cipher)) {
4429 pr_err("%s() Error: failed to allocate fallback for %s\n",
4430 __func__, alg->cra_name);
4431 return PTR_ERR(ctx->fallback_cipher);
4432 }
4433 }
4434 }
4435
4436 return err;
4437}
4438
4439static void generic_cra_exit(struct crypto_tfm *tfm)
4440{
4441 atomic_dec(&iproc_priv.session_count);
4442}
4443
4444static void aead_cra_exit(struct crypto_aead *aead)
4445{
4446 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4447 struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4448
4449 generic_cra_exit(tfm);
4450
4451 if (ctx->fallback_cipher) {
4452 crypto_free_aead(ctx->fallback_cipher);
4453 ctx->fallback_cipher = NULL;
4454 }
4455}
4456
4457/**
4458 * spu_functions_register() - Specify hardware-specific SPU functions based on
4459 * SPU type read from device tree.
4460 * @dev: device structure
4461 * @spu_type: SPU hardware generation
4462 * @spu_subtype: SPU hardware version
4463 */
4464static void spu_functions_register(struct device *dev,
4465 enum spu_spu_type spu_type,
4466 enum spu_spu_subtype spu_subtype)
4467{
4468 struct spu_hw *spu = &iproc_priv.spu;
4469
4470 if (spu_type == SPU_TYPE_SPUM) {
4471 dev_dbg(dev, "Registering SPUM functions");
4472 spu->spu_dump_msg_hdr = spum_dump_msg_hdr;
4473 spu->spu_payload_length = spum_payload_length;
4474 spu->spu_response_hdr_len = spum_response_hdr_len;
4475 spu->spu_hash_pad_len = spum_hash_pad_len;
4476 spu->spu_gcm_ccm_pad_len = spum_gcm_ccm_pad_len;
4477 spu->spu_assoc_resp_len = spum_assoc_resp_len;
4478 spu->spu_aead_ivlen = spum_aead_ivlen;
4479 spu->spu_hash_type = spum_hash_type;
4480 spu->spu_digest_size = spum_digest_size;
4481 spu->spu_create_request = spum_create_request;
4482 spu->spu_cipher_req_init = spum_cipher_req_init;
4483 spu->spu_cipher_req_finish = spum_cipher_req_finish;
4484 spu->spu_request_pad = spum_request_pad;
4485 spu->spu_tx_status_len = spum_tx_status_len;
4486 spu->spu_rx_status_len = spum_rx_status_len;
4487 spu->spu_status_process = spum_status_process;
4488 spu->spu_xts_tweak_in_payload = spum_xts_tweak_in_payload;
4489 spu->spu_ccm_update_iv = spum_ccm_update_iv;
4490 spu->spu_wordalign_padlen = spum_wordalign_padlen;
4491 if (spu_subtype == SPU_SUBTYPE_SPUM_NS2)
4492 spu->spu_ctx_max_payload = spum_ns2_ctx_max_payload;
4493 else
4494 spu->spu_ctx_max_payload = spum_nsp_ctx_max_payload;
4495 } else {
4496 dev_dbg(dev, "Registering SPU2 functions");
4497 spu->spu_dump_msg_hdr = spu2_dump_msg_hdr;
4498 spu->spu_ctx_max_payload = spu2_ctx_max_payload;
4499 spu->spu_payload_length = spu2_payload_length;
4500 spu->spu_response_hdr_len = spu2_response_hdr_len;
4501 spu->spu_hash_pad_len = spu2_hash_pad_len;
4502 spu->spu_gcm_ccm_pad_len = spu2_gcm_ccm_pad_len;
4503 spu->spu_assoc_resp_len = spu2_assoc_resp_len;
4504 spu->spu_aead_ivlen = spu2_aead_ivlen;
4505 spu->spu_hash_type = spu2_hash_type;
4506 spu->spu_digest_size = spu2_digest_size;
4507 spu->spu_create_request = spu2_create_request;
4508 spu->spu_cipher_req_init = spu2_cipher_req_init;
4509 spu->spu_cipher_req_finish = spu2_cipher_req_finish;
4510 spu->spu_request_pad = spu2_request_pad;
4511 spu->spu_tx_status_len = spu2_tx_status_len;
4512 spu->spu_rx_status_len = spu2_rx_status_len;
4513 spu->spu_status_process = spu2_status_process;
4514 spu->spu_xts_tweak_in_payload = spu2_xts_tweak_in_payload;
4515 spu->spu_ccm_update_iv = spu2_ccm_update_iv;
4516 spu->spu_wordalign_padlen = spu2_wordalign_padlen;
4517 }
4518}
4519
4520/**
4521 * spu_mb_init() - Initialize mailbox client. Request ownership of a mailbox
4522 * channel for the SPU being probed.
4523 * @dev: SPU driver device structure
4524 *
4525 * Return: 0 if successful
4526 * < 0 otherwise
4527 */
4528static int spu_mb_init(struct device *dev)
4529{
4530 struct mbox_client *mcl = &iproc_priv.mcl[iproc_priv.spu.num_spu];
4531 int err;
4532
4533 mcl->dev = dev;
4534 mcl->tx_block = false;
4535 mcl->tx_tout = 0;
4536 mcl->knows_txdone = false;
4537 mcl->rx_callback = spu_rx_callback;
4538 mcl->tx_done = NULL;
4539
4540 iproc_priv.mbox[iproc_priv.spu.num_spu] =
4541 mbox_request_channel(mcl, 0);
4542 if (IS_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu])) {
4543 err = (int)PTR_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu]);
4544 dev_err(dev,
4545 "Mbox channel %d request failed with err %d",
4546 iproc_priv.spu.num_spu, err);
4547 iproc_priv.mbox[iproc_priv.spu.num_spu] = NULL;
4548 return err;
4549 }
4550
4551 return 0;
4552}
4553
4554static void spu_mb_release(struct platform_device *pdev)
4555{
4556 int i;
4557
4558 for (i = 0; i < iproc_priv.spu.num_spu; i++)
4559 mbox_free_channel(iproc_priv.mbox[i]);
4560}
4561
4562static void spu_counters_init(void)
4563{
4564 int i;
4565 int j;
4566
4567 atomic_set(&iproc_priv.session_count, 0);
4568 atomic_set(&iproc_priv.stream_count, 0);
4569 atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_spu);
4570 atomic64_set(&iproc_priv.bytes_in, 0);
4571 atomic64_set(&iproc_priv.bytes_out, 0);
4572 for (i = 0; i < SPU_OP_NUM; i++) {
4573 atomic_set(&iproc_priv.op_counts[i], 0);
4574 atomic_set(&iproc_priv.setkey_cnt[i], 0);
4575 }
4576 for (i = 0; i < CIPHER_ALG_LAST; i++)
4577 for (j = 0; j < CIPHER_MODE_LAST; j++)
4578 atomic_set(&iproc_priv.cipher_cnt[i][j], 0);
4579
4580 for (i = 0; i < HASH_ALG_LAST; i++) {
4581 atomic_set(&iproc_priv.hash_cnt[i], 0);
4582 atomic_set(&iproc_priv.hmac_cnt[i], 0);
4583 }
4584 for (i = 0; i < AEAD_TYPE_LAST; i++)
4585 atomic_set(&iproc_priv.aead_cnt[i], 0);
4586
4587 atomic_set(&iproc_priv.mb_no_spc, 0);
4588 atomic_set(&iproc_priv.mb_send_fail, 0);
4589 atomic_set(&iproc_priv.bad_icv, 0);
4590}
4591
4592static int spu_register_ablkcipher(struct iproc_alg_s *driver_alg)
4593{
4594 struct spu_hw *spu = &iproc_priv.spu;
4595 struct crypto_alg *crypto = &driver_alg->alg.crypto;
4596 int err;
4597
4598 /* SPU2 does not support RC4 */
4599 if ((driver_alg->cipher_info.alg == CIPHER_ALG_RC4) &&
4600 (spu->spu_type == SPU_TYPE_SPU2))
4601 return 0;
4602
4603 crypto->cra_module = THIS_MODULE;
4604 crypto->cra_priority = cipher_pri;
4605 crypto->cra_alignmask = 0;
4606 crypto->cra_ctxsize = sizeof(struct iproc_ctx_s);
4607 INIT_LIST_HEAD(&crypto->cra_list);
4608
4609 crypto->cra_init = ablkcipher_cra_init;
4610 crypto->cra_exit = generic_cra_exit;
4611 crypto->cra_type = &crypto_ablkcipher_type;
4612 crypto->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
4613 CRYPTO_ALG_KERN_DRIVER_ONLY;
4614
4615 crypto->cra_ablkcipher.setkey = ablkcipher_setkey;
4616 crypto->cra_ablkcipher.encrypt = ablkcipher_encrypt;
4617 crypto->cra_ablkcipher.decrypt = ablkcipher_decrypt;
4618
4619 err = crypto_register_alg(crypto);
4620 /* Mark alg as having been registered, if successful */
4621 if (err == 0)
4622 driver_alg->registered = true;
4623 pr_debug(" registered ablkcipher %s\n", crypto->cra_driver_name);
4624 return err;
4625}
4626
4627static int spu_register_ahash(struct iproc_alg_s *driver_alg)
4628{
4629 struct spu_hw *spu = &iproc_priv.spu;
4630 struct ahash_alg *hash = &driver_alg->alg.hash;
4631 int err;
4632
4633 /* AES-XCBC is the only AES hash type currently supported on SPU-M */
4634 if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4635 (driver_alg->auth_info.mode != HASH_MODE_XCBC) &&
4636 (spu->spu_type == SPU_TYPE_SPUM))
4637 return 0;
4638
4639 /* SHA3 algorithm variants are not registered for SPU-M or SPU2. */
4640 if ((driver_alg->auth_info.alg >= HASH_ALG_SHA3_224) &&
4641 (spu->spu_subtype != SPU_SUBTYPE_SPU2_V2))
4642 return 0;
4643
4644 hash->halg.base.cra_module = THIS_MODULE;
4645 hash->halg.base.cra_priority = hash_pri;
4646 hash->halg.base.cra_alignmask = 0;
4647 hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4648 hash->halg.base.cra_init = ahash_cra_init;
4649 hash->halg.base.cra_exit = generic_cra_exit;
4650 hash->halg.base.cra_type = &crypto_ahash_type;
4651 hash->halg.base.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC;
4652 hash->halg.statesize = sizeof(struct spu_hash_export_s);
4653
4654 if (driver_alg->auth_info.mode != HASH_MODE_HMAC) {
4655 hash->setkey = ahash_setkey;
4656 hash->init = ahash_init;
4657 hash->update = ahash_update;
4658 hash->final = ahash_final;
4659 hash->finup = ahash_finup;
4660 hash->digest = ahash_digest;
4661 } else {
4662 hash->setkey = ahash_hmac_setkey;
4663 hash->init = ahash_hmac_init;
4664 hash->update = ahash_hmac_update;
4665 hash->final = ahash_hmac_final;
4666 hash->finup = ahash_hmac_finup;
4667 hash->digest = ahash_hmac_digest;
4668 }
4669 hash->export = ahash_export;
4670 hash->import = ahash_import;
4671
4672 err = crypto_register_ahash(hash);
4673 /* Mark alg as having been registered, if successful */
4674 if (err == 0)
4675 driver_alg->registered = true;
4676 pr_debug(" registered ahash %s\n",
4677 hash->halg.base.cra_driver_name);
4678 return err;
4679}
4680
4681static int spu_register_aead(struct iproc_alg_s *driver_alg)
4682{
4683 struct aead_alg *aead = &driver_alg->alg.aead;
4684 int err;
4685
4686 aead->base.cra_module = THIS_MODULE;
4687 aead->base.cra_priority = aead_pri;
4688 aead->base.cra_alignmask = 0;
4689 aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4690 INIT_LIST_HEAD(&aead->base.cra_list);
4691
4692 aead->base.cra_flags |= CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
4693 /* setkey set in alg initialization */
4694 aead->setauthsize = aead_setauthsize;
4695 aead->encrypt = aead_encrypt;
4696 aead->decrypt = aead_decrypt;
4697 aead->init = aead_cra_init;
4698 aead->exit = aead_cra_exit;
4699
4700 err = crypto_register_aead(aead);
4701 /* Mark alg as having been registered, if successful */
4702 if (err == 0)
4703 driver_alg->registered = true;
4704 pr_debug(" registered aead %s\n", aead->base.cra_driver_name);
4705 return err;
4706}
4707
4708/* register crypto algorithms the device supports */
4709static int spu_algs_register(struct device *dev)
4710{
4711 int i, j;
4712 int err;
4713
4714 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4715 switch (driver_algs[i].type) {
4716 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4717 err = spu_register_ablkcipher(&driver_algs[i]);
4718 break;
4719 case CRYPTO_ALG_TYPE_AHASH:
4720 err = spu_register_ahash(&driver_algs[i]);
4721 break;
4722 case CRYPTO_ALG_TYPE_AEAD:
4723 err = spu_register_aead(&driver_algs[i]);
4724 break;
4725 default:
4726 dev_err(dev,
4727 "iproc-crypto: unknown alg type: %d",
4728 driver_algs[i].type);
4729 err = -EINVAL;
4730 }
4731
4732 if (err) {
4733 dev_err(dev, "alg registration failed with error %d\n",
4734 err);
4735 goto err_algs;
4736 }
4737 }
4738
4739 return 0;
4740
4741err_algs:
4742 for (j = 0; j < i; j++) {
4743 /* Skip any algorithm not registered */
4744 if (!driver_algs[j].registered)
4745 continue;
4746 switch (driver_algs[j].type) {
4747 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4748 crypto_unregister_alg(&driver_algs[j].alg.crypto);
4749 driver_algs[j].registered = false;
4750 break;
4751 case CRYPTO_ALG_TYPE_AHASH:
4752 crypto_unregister_ahash(&driver_algs[j].alg.hash);
4753 driver_algs[j].registered = false;
4754 break;
4755 case CRYPTO_ALG_TYPE_AEAD:
4756 crypto_unregister_aead(&driver_algs[j].alg.aead);
4757 driver_algs[j].registered = false;
4758 break;
4759 }
4760 }
4761 return err;
4762}
4763
4764/* ==================== Kernel Platform API ==================== */
4765
4766static struct spu_type_subtype spum_ns2_types = {
4767 SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NS2
4768};
4769
4770static struct spu_type_subtype spum_nsp_types = {
4771 SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NSP
4772};
4773
4774static struct spu_type_subtype spu2_types = {
4775 SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V1
4776};
4777
4778static struct spu_type_subtype spu2_v2_types = {
4779 SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V2
4780};
4781
4782static const struct of_device_id bcm_spu_dt_ids[] = {
4783 {
4784 .compatible = "brcm,spum-crypto",
4785 .data = &spum_ns2_types,
4786 },
4787 {
4788 .compatible = "brcm,spum-nsp-crypto",
4789 .data = &spum_nsp_types,
4790 },
4791 {
4792 .compatible = "brcm,spu2-crypto",
4793 .data = &spu2_types,
4794 },
4795 {
4796 .compatible = "brcm,spu2-v2-crypto",
4797 .data = &spu2_v2_types,
4798 },
4799 { /* sentinel */ }
4800};
4801
4802MODULE_DEVICE_TABLE(of, bcm_spu_dt_ids);
4803
4804static int spu_dt_read(struct platform_device *pdev)
4805{
4806 struct device *dev = &pdev->dev;
4807 struct spu_hw *spu = &iproc_priv.spu;
4808 struct resource *spu_ctrl_regs;
4809 const struct of_device_id *match;
4810 const struct spu_type_subtype *matched_spu_type;
4811 void __iomem *spu_reg_vbase[MAX_SPUS];
4812 int err;
4813
4814 match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev);
4815 matched_spu_type = match->data;
4816
4817 if (iproc_priv.spu.num_spu > 1) {
4818 /* If this is 2nd or later SPU, make sure it's same type */
4819 if ((spu->spu_type != matched_spu_type->type) ||
4820 (spu->spu_subtype != matched_spu_type->subtype)) {
4821 err = -EINVAL;
4822 dev_err(&pdev->dev, "Multiple SPU types not allowed");
4823 return err;
4824 }
4825 } else {
4826 /* Record type of first SPU */
4827 spu->spu_type = matched_spu_type->type;
4828 spu->spu_subtype = matched_spu_type->subtype;
4829 }
4830
4831 /* Get and map SPU registers */
4832 spu_ctrl_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4833 if (!spu_ctrl_regs) {
4834 err = -EINVAL;
4835 dev_err(&pdev->dev, "Invalid/missing registers for SPU\n");
4836 return err;
4837 }
4838
4839 spu_reg_vbase[iproc_priv.spu.num_spu] =
4840 devm_ioremap_resource(dev, spu_ctrl_regs);
4841 if (IS_ERR(spu_reg_vbase[iproc_priv.spu.num_spu])) {
4842 err = PTR_ERR(spu_reg_vbase[iproc_priv.spu.num_spu]);
4843 dev_err(&pdev->dev, "Failed to map registers: %d\n",
4844 err);
4845 spu_reg_vbase[iproc_priv.spu.num_spu] = NULL;
4846 return err;
4847 }
4848
4849 dev_dbg(dev, "SPU %d detected.", iproc_priv.spu.num_spu);
4850
4851 spu->reg_vbase[iproc_priv.spu.num_spu] = spu_reg_vbase;
4852
4853 return 0;
4854}
4855
4856int bcm_spu_probe(struct platform_device *pdev)
4857{
4858 struct device *dev = &pdev->dev;
4859 struct spu_hw *spu = &iproc_priv.spu;
4860 int err = 0;
4861
4862 iproc_priv.pdev[iproc_priv.spu.num_spu] = pdev;
4863 platform_set_drvdata(iproc_priv.pdev[iproc_priv.spu.num_spu],
4864 &iproc_priv);
4865
4866 err = spu_dt_read(pdev);
4867 if (err < 0)
4868 goto failure;
4869
4870 err = spu_mb_init(&pdev->dev);
4871 if (err < 0)
4872 goto failure;
4873
4874 iproc_priv.spu.num_spu++;
4875
4876 /* If already initialized, we've just added another SPU and are done */
4877 if (iproc_priv.inited)
4878 return 0;
4879
4880 if (spu->spu_type == SPU_TYPE_SPUM)
4881 iproc_priv.bcm_hdr_len = 8;
4882 else if (spu->spu_type == SPU_TYPE_SPU2)
4883 iproc_priv.bcm_hdr_len = 0;
4884
4885 spu_functions_register(&pdev->dev, spu->spu_type, spu->spu_subtype);
4886
4887 spu_counters_init();
4888
4889 spu_setup_debugfs();
4890
4891 err = spu_algs_register(dev);
4892 if (err < 0)
4893 goto fail_reg;
4894
4895 iproc_priv.inited = true;
4896
4897 return 0;
4898
4899fail_reg:
4900 spu_free_debugfs();
4901failure:
4902 spu_mb_release(pdev);
4903 dev_err(dev, "%s failed with error %d.\n", __func__, err);
4904
4905 return err;
4906}
4907
4908int bcm_spu_remove(struct platform_device *pdev)
4909{
4910 int i;
4911 struct device *dev = &pdev->dev;
4912 char *cdn;
4913
4914 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4915 /*
4916 * Not all algorithms were registered, depending on whether
4917 * hardware is SPU or SPU2. So here we make sure to skip
4918 * those algorithms that were not previously registered.
4919 */
4920 if (!driver_algs[i].registered)
4921 continue;
4922
4923 switch (driver_algs[i].type) {
4924 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4925 crypto_unregister_alg(&driver_algs[i].alg.crypto);
4926 dev_dbg(dev, " unregistered cipher %s\n",
4927 driver_algs[i].alg.crypto.cra_driver_name);
4928 driver_algs[i].registered = false;
4929 break;
4930 case CRYPTO_ALG_TYPE_AHASH:
4931 crypto_unregister_ahash(&driver_algs[i].alg.hash);
4932 cdn = driver_algs[i].alg.hash.halg.base.cra_driver_name;
4933 dev_dbg(dev, " unregistered hash %s\n", cdn);
4934 driver_algs[i].registered = false;
4935 break;
4936 case CRYPTO_ALG_TYPE_AEAD:
4937 crypto_unregister_aead(&driver_algs[i].alg.aead);
4938 dev_dbg(dev, " unregistered aead %s\n",
4939 driver_algs[i].alg.aead.base.cra_driver_name);
4940 driver_algs[i].registered = false;
4941 break;
4942 }
4943 }
4944 spu_free_debugfs();
4945 spu_mb_release(pdev);
4946 return 0;
4947}
4948
4949/* ===== Kernel Module API ===== */
4950
4951static struct platform_driver bcm_spu_pdriver = {
4952 .driver = {
4953 .name = "brcm-spu-crypto",
4954 .of_match_table = of_match_ptr(bcm_spu_dt_ids),
4955 },
4956 .probe = bcm_spu_probe,
4957 .remove = bcm_spu_remove,
4958};
4959module_platform_driver(bcm_spu_pdriver);
4960
4961MODULE_AUTHOR("Rob Rice <rob.rice@broadcom.com>");
4962MODULE_DESCRIPTION("Broadcom symmetric crypto offload driver");
4963MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h
new file mode 100644
index 000000000000..51dca529ce8f
--- /dev/null
+++ b/drivers/crypto/bcm/cipher.h
@@ -0,0 +1,483 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#ifndef _CIPHER_H
18#define _CIPHER_H
19
20#include <linux/atomic.h>
21#include <linux/mailbox/brcm-message.h>
22#include <linux/mailbox_client.h>
23#include <crypto/aes.h>
24#include <crypto/internal/hash.h>
25#include <crypto/aead.h>
26#include <crypto/sha.h>
27#include <crypto/sha3.h>
28
29#include "spu.h"
30#include "spum.h"
31#include "spu2.h"
32
33/* Driver supports up to MAX_SPUS SPU blocks */
34#define MAX_SPUS 16
35
36#define ARC4_MIN_KEY_SIZE 1
37#define ARC4_MAX_KEY_SIZE 256
38#define ARC4_BLOCK_SIZE 1
39#define ARC4_STATE_SIZE 4
40
41#define CCM_AES_IV_SIZE 16
42#define GCM_AES_IV_SIZE 12
43#define GCM_ESP_IV_SIZE 8
44#define CCM_ESP_IV_SIZE 8
45#define RFC4543_ICV_SIZE 16
46
47#define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE
48#define MAX_IV_SIZE AES_BLOCK_SIZE
49#define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE
50#define MAX_ASSOC_SIZE 512
51
52/* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
53#define GCM_ESP_SALT_SIZE 4
54#define CCM_ESP_SALT_SIZE 3
55#define MAX_SALT_SIZE GCM_ESP_SALT_SIZE
56#define GCM_ESP_SALT_OFFSET 0
57#define CCM_ESP_SALT_OFFSET 1
58
59#define GCM_ESP_DIGESTSIZE 16
60
61#define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
62
63/*
64 * Maximum number of bytes from a non-final hash request that can be deferred
65 * until more data is available. With new crypto API framework, this
66 * can be no more than one block of data.
67 */
68#define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE
69
70/* Force at least 4-byte alignment of all SPU message fields */
71#define SPU_MSG_ALIGN 4
72
73/* Number of times to resend mailbox message if mb queue is full */
74#define SPU_MB_RETRY_MAX 1000
75
76/* op_counts[] indexes */
77enum op_type {
78 SPU_OP_CIPHER,
79 SPU_OP_HASH,
80 SPU_OP_HMAC,
81 SPU_OP_AEAD,
82 SPU_OP_NUM
83};
84
85enum spu_spu_type {
86 SPU_TYPE_SPUM,
87 SPU_TYPE_SPU2,
88};
89
90/*
91 * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
92 * respectively.
93 */
94enum spu_spu_subtype {
95 SPU_SUBTYPE_SPUM_NS2,
96 SPU_SUBTYPE_SPUM_NSP,
97 SPU_SUBTYPE_SPU2_V1,
98 SPU_SUBTYPE_SPU2_V2
99};
100
101struct spu_type_subtype {
102 enum spu_spu_type type;
103 enum spu_spu_subtype subtype;
104};
105
106struct cipher_op {
107 enum spu_cipher_alg alg;
108 enum spu_cipher_mode mode;
109};
110
111struct auth_op {
112 enum hash_alg alg;
113 enum hash_mode mode;
114};
115
116struct iproc_alg_s {
117 u32 type;
118 union {
119 struct crypto_alg crypto;
120 struct ahash_alg hash;
121 struct aead_alg aead;
122 } alg;
123 struct cipher_op cipher_info;
124 struct auth_op auth_info;
125 bool auth_first;
126 bool registered;
127};
128
129/*
130 * Buffers for a SPU request/reply message pair. All part of one structure to
131 * allow a single alloc per request.
132 */
133struct spu_msg_buf {
134 /* Request message fragments */
135
136 /*
137 * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
138 * and BD header. For SPU2, holds FMD, OMD.
139 */
140 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
141
142 /* IV or counter. Size to include salt. Also used for XTS tweek. */
143 u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
144
145 /* Hash digest. request and response. */
146 u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
147
148 /* SPU request message padding */
149 u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
150
151 /* SPU-M request message STATUS field */
152 u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
153
154 /* Response message fragments */
155
156 /* SPU response message header */
157 u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
158
159 /* SPU response message STATUS field padding */
160 u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
161
162 /* SPU response message STATUS field */
163 u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
164
165 union {
166 /* Buffers only used for ablkcipher */
167 struct {
168 /*
169 * Field used for either SUPDT when RC4 is used
170 * -OR- tweak value when XTS/AES is used
171 */
172 u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
173 } c;
174
175 /* Buffers only used for aead */
176 struct {
177 /* SPU response pad for GCM data */
178 u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
179
180 /* SPU request msg padding for GCM AAD */
181 u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
182
183 /* SPU response data to be discarded */
184 u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
185 SPU_MSG_ALIGN)];
186 } a;
187 };
188};
189
190struct iproc_ctx_s {
191 u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
192 unsigned int enckeylen;
193
194 u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
195 unsigned int authkeylen;
196
197 u8 salt[MAX_SALT_SIZE];
198 unsigned int salt_len;
199 unsigned int salt_offset;
200 u8 iv[MAX_IV_SIZE];
201
202 unsigned int digestsize;
203
204 struct iproc_alg_s *alg;
205 bool is_esp;
206
207 struct cipher_op cipher;
208 enum spu_cipher_type cipher_type;
209
210 struct auth_op auth;
211 bool auth_first;
212
213 /*
214 * The maximum length in bytes of the payload in a SPU message for this
215 * context. For SPU-M, the payload is the combination of AAD and data.
216 * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
217 * indicates that there is no limit to the length of the SPU message
218 * payload.
219 */
220 unsigned int max_payload;
221
222 struct crypto_aead *fallback_cipher;
223
224 /* auth_type is determined during processing of request */
225
226 u8 ipad[MAX_HASH_BLOCK_SIZE];
227 u8 opad[MAX_HASH_BLOCK_SIZE];
228
229 /*
230 * Buffer to hold SPU message header template. Template is created at
231 * setkey time for ablkcipher requests, since most of the fields in the
232 * header are known at that time. At request time, just fill in a few
233 * missing pieces related to length of data in the request and IVs, etc.
234 */
235 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
236
237 /* Length of SPU request header */
238 u16 spu_req_hdr_len;
239
240 /* Expected length of SPU response header */
241 u16 spu_resp_hdr_len;
242
243 /*
244 * shash descriptor - needed to perform incremental hashing in
245 * in software, when hw doesn't support it.
246 */
247 struct shash_desc *shash;
248
249 bool is_rfc4543; /* RFC 4543 style of GMAC */
250};
251
252/* state from iproc_reqctx_s necessary for hash state export/import */
253struct spu_hash_export_s {
254 unsigned int total_todo;
255 unsigned int total_sent;
256 u8 hash_carry[HASH_CARRY_MAX];
257 unsigned int hash_carry_len;
258 u8 incr_hash[MAX_DIGEST_SIZE];
259 bool is_sw_hmac;
260};
261
262struct iproc_reqctx_s {
263 /* general context */
264 struct crypto_async_request *parent;
265
266 /* only valid after enqueue() */
267 struct iproc_ctx_s *ctx;
268
269 u8 chan_idx; /* Mailbox channel to be used to submit this request */
270
271 /* total todo, rx'd, and sent for this request */
272 unsigned int total_todo;
273 unsigned int total_received; /* only valid for ablkcipher */
274 unsigned int total_sent;
275
276 /*
277 * num bytes sent to hw from the src sg in this request. This can differ
278 * from total_sent for incremental hashing. total_sent includes previous
279 * init() and update() data. src_sent does not.
280 */
281 unsigned int src_sent;
282
283 /*
284 * For AEAD requests, start of associated data. This will typically
285 * point to the beginning of the src scatterlist from the request,
286 * since assoc data is at the beginning of the src scatterlist rather
287 * than in its own sg.
288 */
289 struct scatterlist *assoc;
290
291 /*
292 * scatterlist entry and offset to start of data for next chunk. Crypto
293 * API src scatterlist for AEAD starts with AAD, if present. For first
294 * chunk, src_sg is sg entry at beginning of input data (after AAD).
295 * src_skip begins at the offset in that sg entry where data begins.
296 */
297 struct scatterlist *src_sg;
298 int src_nents; /* Number of src entries with data */
299 u32 src_skip; /* bytes of current sg entry already used */
300
301 /*
302 * Same for destination. For AEAD, if there is AAD, output data must
303 * be written at offset following AAD.
304 */
305 struct scatterlist *dst_sg;
306 int dst_nents; /* Number of dst entries with data */
307 u32 dst_skip; /* bytes of current sg entry already written */
308
309 /* Mailbox message used to send this request to PDC driver */
310 struct brcm_message mb_mssg;
311
312 bool bd_suppress; /* suppress BD field in SPU response? */
313
314 /* cipher context */
315 bool is_encrypt;
316
317 /*
318 * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA
319 * buffer for AEAD requests. So allocate as DMAable memory. If IV
320 * concatenated with salt, includes the salt.
321 */
322 u8 *iv_ctr;
323 /* Length of IV or counter, in bytes */
324 unsigned int iv_ctr_len;
325
326 /*
327 * Hash requests can be of any size, whether initial, update, or final.
328 * A non-final request must be submitted to the SPU as an integral
329 * number of blocks. This may leave data at the end of the request
330 * that is not a full block. Since the request is non-final, it cannot
331 * be padded. So, we write the remainder to this hash_carry buffer and
332 * hold it until the next request arrives. The carry data is then
333 * submitted at the beginning of the data in the next SPU msg.
334 * hash_carry_len is the number of bytes currently in hash_carry. These
335 * fields are only used for ahash requests.
336 */
337 u8 hash_carry[HASH_CARRY_MAX];
338 unsigned int hash_carry_len;
339 unsigned int is_final; /* is this the final for the hash op? */
340
341 /*
342 * Digest from incremental hash is saved here to include in next hash
343 * operation. Cannot be stored in req->result for truncated hashes,
344 * since result may be sized for final digest. Cannot be saved in
345 * msg_buf because that gets deleted between incremental hash ops
346 * and is not saved as part of export().
347 */
348 u8 incr_hash[MAX_DIGEST_SIZE];
349
350 /* hmac context */
351 bool is_sw_hmac;
352
353 /* aead context */
354 struct crypto_tfm *old_tfm;
355 crypto_completion_t old_complete;
356 void *old_data;
357
358 gfp_t gfp;
359
360 /* Buffers used to build SPU request and response messages */
361 struct spu_msg_buf msg_buf;
362};
363
364/*
365 * Structure encapsulates a set of function pointers specific to the type of
366 * SPU hardware running. These functions handling creation and parsing of
367 * SPU request messages and SPU response messages. Includes hardware-specific
368 * values read from device tree.
369 */
370struct spu_hw {
371 void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
372 u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
373 enum spu_cipher_mode cipher_mode,
374 unsigned int blocksize);
375 u32 (*spu_payload_length)(u8 *spu_hdr);
376 u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
377 bool is_hash);
378 u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
379 enum hash_mode hash_mode, u32 chunksize,
380 u16 hash_block_size);
381 u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
382 unsigned int data_size);
383 u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
384 unsigned int assoc_len,
385 unsigned int iv_len, bool is_encrypt);
386 u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
387 u16 iv_len);
388 enum hash_type (*spu_hash_type)(u32 src_sent);
389 u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
390 enum hash_type);
391 u32 (*spu_create_request)(u8 *spu_hdr,
392 struct spu_request_opts *req_opts,
393 struct spu_cipher_parms *cipher_parms,
394 struct spu_hash_parms *hash_parms,
395 struct spu_aead_parms *aead_parms,
396 unsigned int data_size);
397 u16 (*spu_cipher_req_init)(u8 *spu_hdr,
398 struct spu_cipher_parms *cipher_parms);
399 void (*spu_cipher_req_finish)(u8 *spu_hdr,
400 u16 spu_req_hdr_len,
401 unsigned int is_inbound,
402 struct spu_cipher_parms *cipher_parms,
403 bool update_key,
404 unsigned int data_size);
405 void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
406 u32 hash_pad_len, enum hash_alg auth_alg,
407 enum hash_mode auth_mode,
408 unsigned int total_sent, u32 status_padding);
409 u8 (*spu_xts_tweak_in_payload)(void);
410 u8 (*spu_tx_status_len)(void);
411 u8 (*spu_rx_status_len)(void);
412 int (*spu_status_process)(u8 *statp);
413 void (*spu_ccm_update_iv)(unsigned int digestsize,
414 struct spu_cipher_parms *cipher_parms,
415 unsigned int assoclen, unsigned int chunksize,
416 bool is_encrypt, bool is_esp);
417 u32 (*spu_wordalign_padlen)(u32 data_size);
418
419 /* The base virtual address of the SPU hw registers */
420 void __iomem *reg_vbase[MAX_SPUS];
421
422 /* Version of the SPU hardware */
423 enum spu_spu_type spu_type;
424
425 /* Sub-version of the SPU hardware */
426 enum spu_spu_subtype spu_subtype;
427
428 /* The number of SPUs on this platform */
429 u32 num_spu;
430};
431
432struct device_private {
433 struct platform_device *pdev[MAX_SPUS];
434
435 struct spu_hw spu;
436
437 atomic_t session_count; /* number of streams active */
438 atomic_t stream_count; /* monotonic counter for streamID's */
439
440 /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
441 u8 bcm_hdr_len;
442
443 /* The index of the channel to use for the next crypto request */
444 atomic_t next_chan;
445
446 struct dentry *debugfs_dir;
447 struct dentry *debugfs_stats;
448
449 /* Number of request bytes processed and result bytes returned */
450 atomic64_t bytes_in;
451 atomic64_t bytes_out;
452
453 /* Number of operations of each type */
454 atomic_t op_counts[SPU_OP_NUM];
455
456 atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
457 atomic_t hash_cnt[HASH_ALG_LAST];
458 atomic_t hmac_cnt[HASH_ALG_LAST];
459 atomic_t aead_cnt[AEAD_TYPE_LAST];
460
461 /* Number of calls to setkey() for each operation type */
462 atomic_t setkey_cnt[SPU_OP_NUM];
463
464 /* Number of times request was resubmitted because mb was full */
465 atomic_t mb_no_spc;
466
467 /* Number of mailbox send failures */
468 atomic_t mb_send_fail;
469
470 /* Number of ICV check failures for AEAD messages */
471 atomic_t bad_icv;
472
473 struct mbox_client mcl[MAX_SPUS];
474 /* Array of mailbox channel pointers, one for each channel */
475 struct mbox_chan *mbox[MAX_SPUS];
476
477 /* Driver initialized */
478 bool inited;
479};
480
481extern struct device_private iproc_priv;
482
483#endif
diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c
new file mode 100644
index 000000000000..dbb5c03dde49
--- /dev/null
+++ b/drivers/crypto/bcm/spu.c
@@ -0,0 +1,1251 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#include <linux/kernel.h>
18#include <linux/string.h>
19
20#include "util.h"
21#include "spu.h"
22#include "spum.h"
23#include "cipher.h"
24
25/* This array is based on the hash algo type supported in spu.h */
26char *tag_to_hash_idx[] = { "none", "md5", "sha1", "sha224", "sha256" };
27
28char *hash_alg_name[] = { "None", "md5", "sha1", "sha224", "sha256", "aes",
29 "sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", "sha3_512" };
30
31char *aead_alg_name[] = { "ccm(aes)", "gcm(aes)", "authenc" };
32
33/* Assumes SPU-M messages are in big endian */
34void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len)
35{
36 u8 *ptr = buf;
37 struct SPUHEADER *spuh = (struct SPUHEADER *)buf;
38 unsigned int hash_key_len = 0;
39 unsigned int hash_state_len = 0;
40 unsigned int cipher_key_len = 0;
41 unsigned int iv_len;
42 u32 pflags;
43 u32 cflags;
44 u32 ecf;
45 u32 cipher_alg;
46 u32 cipher_mode;
47 u32 cipher_type;
48 u32 hash_alg;
49 u32 hash_mode;
50 u32 hash_type;
51 u32 sctx_size; /* SCTX length in words */
52 u32 sctx_pl_len; /* SCTX payload length in bytes */
53
54 packet_log("\n");
55 packet_log("SPU Message header %p len: %u\n", buf, buf_len);
56
57 /* ========== Decode MH ========== */
58 packet_log(" MH 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
59 if (spuh->mh.flags & MH_SCTX_PRES)
60 packet_log(" SCTX present\n");
61 if (spuh->mh.flags & MH_BDESC_PRES)
62 packet_log(" BDESC present\n");
63 if (spuh->mh.flags & MH_MFM_PRES)
64 packet_log(" MFM present\n");
65 if (spuh->mh.flags & MH_BD_PRES)
66 packet_log(" BD present\n");
67 if (spuh->mh.flags & MH_HASH_PRES)
68 packet_log(" HASH present\n");
69 if (spuh->mh.flags & MH_SUPDT_PRES)
70 packet_log(" SUPDT present\n");
71 packet_log(" Opcode 0x%02x\n", spuh->mh.op_code);
72
73 ptr += sizeof(spuh->mh) + sizeof(spuh->emh); /* skip emh. unused */
74
75 /* ========== Decode SCTX ========== */
76 if (spuh->mh.flags & MH_SCTX_PRES) {
77 pflags = be32_to_cpu(spuh->sa.proto_flags);
78 packet_log(" SCTX[0] 0x%08x\n", pflags);
79 sctx_size = pflags & SCTX_SIZE;
80 packet_log(" Size %u words\n", sctx_size);
81
82 cflags = be32_to_cpu(spuh->sa.cipher_flags);
83 packet_log(" SCTX[1] 0x%08x\n", cflags);
84 packet_log(" Inbound:%lu (1:decrypt/vrfy 0:encrypt/auth)\n",
85 (cflags & CIPHER_INBOUND) >> CIPHER_INBOUND_SHIFT);
86 packet_log(" Order:%lu (1:AuthFirst 0:EncFirst)\n",
87 (cflags & CIPHER_ORDER) >> CIPHER_ORDER_SHIFT);
88 packet_log(" ICV_IS_512:%lx\n",
89 (cflags & ICV_IS_512) >> ICV_IS_512_SHIFT);
90 cipher_alg = (cflags & CIPHER_ALG) >> CIPHER_ALG_SHIFT;
91 cipher_mode = (cflags & CIPHER_MODE) >> CIPHER_MODE_SHIFT;
92 cipher_type = (cflags & CIPHER_TYPE) >> CIPHER_TYPE_SHIFT;
93 packet_log(" Crypto Alg:%u Mode:%u Type:%u\n",
94 cipher_alg, cipher_mode, cipher_type);
95 hash_alg = (cflags & HASH_ALG) >> HASH_ALG_SHIFT;
96 hash_mode = (cflags & HASH_MODE) >> HASH_MODE_SHIFT;
97 hash_type = (cflags & HASH_TYPE) >> HASH_TYPE_SHIFT;
98 packet_log(" Hash Alg:%x Mode:%x Type:%x\n",
99 hash_alg, hash_mode, hash_type);
100 packet_log(" UPDT_Offset:%u\n", cflags & UPDT_OFST);
101
102 ecf = be32_to_cpu(spuh->sa.ecf);
103 packet_log(" SCTX[2] 0x%08x\n", ecf);
104 packet_log(" WriteICV:%lu CheckICV:%lu ICV_SIZE:%u ",
105 (ecf & INSERT_ICV) >> INSERT_ICV_SHIFT,
106 (ecf & CHECK_ICV) >> CHECK_ICV_SHIFT,
107 (ecf & ICV_SIZE) >> ICV_SIZE_SHIFT);
108 packet_log("BD_SUPPRESS:%lu\n",
109 (ecf & BD_SUPPRESS) >> BD_SUPPRESS_SHIFT);
110 packet_log(" SCTX_IV:%lu ExplicitIV:%lu GenIV:%lu ",
111 (ecf & SCTX_IV) >> SCTX_IV_SHIFT,
112 (ecf & EXPLICIT_IV) >> EXPLICIT_IV_SHIFT,
113 (ecf & GEN_IV) >> GEN_IV_SHIFT);
114 packet_log("IV_OV_OFST:%lu EXP_IV_SIZE:%u\n",
115 (ecf & IV_OFFSET) >> IV_OFFSET_SHIFT,
116 ecf & EXP_IV_SIZE);
117
118 ptr += sizeof(struct SCTX);
119
120 if (hash_alg && hash_mode) {
121 char *name = "NONE";
122
123 switch (hash_alg) {
124 case HASH_ALG_MD5:
125 hash_key_len = 16;
126 name = "MD5";
127 break;
128 case HASH_ALG_SHA1:
129 hash_key_len = 20;
130 name = "SHA1";
131 break;
132 case HASH_ALG_SHA224:
133 hash_key_len = 28;
134 name = "SHA224";
135 break;
136 case HASH_ALG_SHA256:
137 hash_key_len = 32;
138 name = "SHA256";
139 break;
140 case HASH_ALG_SHA384:
141 hash_key_len = 48;
142 name = "SHA384";
143 break;
144 case HASH_ALG_SHA512:
145 hash_key_len = 64;
146 name = "SHA512";
147 break;
148 case HASH_ALG_AES:
149 hash_key_len = 0;
150 name = "AES";
151 break;
152 case HASH_ALG_NONE:
153 break;
154 }
155
156 packet_log(" Auth Key Type:%s Length:%u Bytes\n",
157 name, hash_key_len);
158 packet_dump(" KEY: ", ptr, hash_key_len);
159 ptr += hash_key_len;
160 } else if ((hash_alg == HASH_ALG_AES) &&
161 (hash_mode == HASH_MODE_XCBC)) {
162 char *name = "NONE";
163
164 switch (cipher_type) {
165 case CIPHER_TYPE_AES128:
166 hash_key_len = 16;
167 name = "AES128-XCBC";
168 break;
169 case CIPHER_TYPE_AES192:
170 hash_key_len = 24;
171 name = "AES192-XCBC";
172 break;
173 case CIPHER_TYPE_AES256:
174 hash_key_len = 32;
175 name = "AES256-XCBC";
176 break;
177 }
178 packet_log(" Auth Key Type:%s Length:%u Bytes\n",
179 name, hash_key_len);
180 packet_dump(" KEY: ", ptr, hash_key_len);
181 ptr += hash_key_len;
182 }
183
184 if (hash_alg && (hash_mode == HASH_MODE_NONE) &&
185 (hash_type == HASH_TYPE_UPDT)) {
186 char *name = "NONE";
187
188 switch (hash_alg) {
189 case HASH_ALG_MD5:
190 hash_state_len = 16;
191 name = "MD5";
192 break;
193 case HASH_ALG_SHA1:
194 hash_state_len = 20;
195 name = "SHA1";
196 break;
197 case HASH_ALG_SHA224:
198 hash_state_len = 32;
199 name = "SHA224";
200 break;
201 case HASH_ALG_SHA256:
202 hash_state_len = 32;
203 name = "SHA256";
204 break;
205 case HASH_ALG_SHA384:
206 hash_state_len = 48;
207 name = "SHA384";
208 break;
209 case HASH_ALG_SHA512:
210 hash_state_len = 64;
211 name = "SHA512";
212 break;
213 case HASH_ALG_AES:
214 hash_state_len = 0;
215 name = "AES";
216 break;
217 case HASH_ALG_NONE:
218 break;
219 }
220
221 packet_log(" Auth State Type:%s Length:%u Bytes\n",
222 name, hash_state_len);
223 packet_dump(" State: ", ptr, hash_state_len);
224 ptr += hash_state_len;
225 }
226
227 if (cipher_alg) {
228 char *name = "NONE";
229
230 switch (cipher_alg) {
231 case CIPHER_ALG_DES:
232 cipher_key_len = 8;
233 name = "DES";
234 break;
235 case CIPHER_ALG_3DES:
236 cipher_key_len = 24;
237 name = "3DES";
238 break;
239 case CIPHER_ALG_RC4:
240 cipher_key_len = 260;
241 name = "ARC4";
242 break;
243 case CIPHER_ALG_AES:
244 switch (cipher_type) {
245 case CIPHER_TYPE_AES128:
246 cipher_key_len = 16;
247 name = "AES128";
248 break;
249 case CIPHER_TYPE_AES192:
250 cipher_key_len = 24;
251 name = "AES192";
252 break;
253 case CIPHER_TYPE_AES256:
254 cipher_key_len = 32;
255 name = "AES256";
256 break;
257 }
258 break;
259 case CIPHER_ALG_NONE:
260 break;
261 }
262
263 packet_log(" Cipher Key Type:%s Length:%u Bytes\n",
264 name, cipher_key_len);
265
266 /* XTS has two keys */
267 if (cipher_mode == CIPHER_MODE_XTS) {
268 packet_dump(" KEY2: ", ptr, cipher_key_len);
269 ptr += cipher_key_len;
270 packet_dump(" KEY1: ", ptr, cipher_key_len);
271 ptr += cipher_key_len;
272
273 cipher_key_len *= 2;
274 } else {
275 packet_dump(" KEY: ", ptr, cipher_key_len);
276 ptr += cipher_key_len;
277 }
278
279 if (ecf & SCTX_IV) {
280 sctx_pl_len = sctx_size * sizeof(u32) -
281 sizeof(struct SCTX);
282 iv_len = sctx_pl_len -
283 (hash_key_len + hash_state_len +
284 cipher_key_len);
285 packet_log(" IV Length:%u Bytes\n", iv_len);
286 packet_dump(" IV: ", ptr, iv_len);
287 ptr += iv_len;
288 }
289 }
290 }
291
292 /* ========== Decode BDESC ========== */
293 if (spuh->mh.flags & MH_BDESC_PRES) {
294#ifdef DEBUG
295 struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr;
296#endif
297 packet_log(" BDESC[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
298 packet_log(" OffsetMAC:%u LengthMAC:%u\n",
299 be16_to_cpu(bdesc->offset_mac),
300 be16_to_cpu(bdesc->length_mac));
301 ptr += sizeof(u32);
302
303 packet_log(" BDESC[1] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
304 packet_log(" OffsetCrypto:%u LengthCrypto:%u\n",
305 be16_to_cpu(bdesc->offset_crypto),
306 be16_to_cpu(bdesc->length_crypto));
307 ptr += sizeof(u32);
308
309 packet_log(" BDESC[2] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
310 packet_log(" OffsetICV:%u OffsetIV:%u\n",
311 be16_to_cpu(bdesc->offset_icv),
312 be16_to_cpu(bdesc->offset_iv));
313 ptr += sizeof(u32);
314 }
315
316 /* ========== Decode BD ========== */
317 if (spuh->mh.flags & MH_BD_PRES) {
318#ifdef DEBUG
319 struct BD_HEADER *bd = (struct BD_HEADER *)ptr;
320#endif
321 packet_log(" BD[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
322 packet_log(" Size:%ubytes PrevLength:%u\n",
323 be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length));
324 ptr += 4;
325 }
326
327 /* Double check sanity */
328 if (buf + buf_len != ptr) {
329 packet_log(" Packet parsed incorrectly. ");
330 packet_log("buf:%p buf_len:%u buf+buf_len:%p ptr:%p\n",
331 buf, buf_len, buf + buf_len, ptr);
332 }
333
334 packet_log("\n");
335}
336
337/**
338 * spum_ns2_ctx_max_payload() - Determine the max length of the payload for a
339 * SPU message for a given cipher and hash alg context.
340 * @cipher_alg: The cipher algorithm
341 * @cipher_mode: The cipher mode
342 * @blocksize: The size of a block of data for this algo
343 *
344 * The max payload must be a multiple of the blocksize so that if a request is
345 * too large to fit in a single SPU message, the request can be broken into
346 * max_payload sized chunks. Each chunk must be a multiple of blocksize.
347 *
348 * Return: Max payload length in bytes
349 */
350u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
351 enum spu_cipher_mode cipher_mode,
352 unsigned int blocksize)
353{
354 u32 max_payload = SPUM_NS2_MAX_PAYLOAD;
355 u32 excess;
356
357 /* In XTS on SPU-M, we'll need to insert tweak before input data */
358 if (cipher_mode == CIPHER_MODE_XTS)
359 max_payload -= SPU_XTS_TWEAK_SIZE;
360
361 excess = max_payload % blocksize;
362
363 return max_payload - excess;
364}
365
366/**
367 * spum_nsp_ctx_max_payload() - Determine the max length of the payload for a
368 * SPU message for a given cipher and hash alg context.
369 * @cipher_alg: The cipher algorithm
370 * @cipher_mode: The cipher mode
371 * @blocksize: The size of a block of data for this algo
372 *
373 * The max payload must be a multiple of the blocksize so that if a request is
374 * too large to fit in a single SPU message, the request can be broken into
375 * max_payload sized chunks. Each chunk must be a multiple of blocksize.
376 *
377 * Return: Max payload length in bytes
378 */
379u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg,
380 enum spu_cipher_mode cipher_mode,
381 unsigned int blocksize)
382{
383 u32 max_payload = SPUM_NSP_MAX_PAYLOAD;
384 u32 excess;
385
386 /* In XTS on SPU-M, we'll need to insert tweak before input data */
387 if (cipher_mode == CIPHER_MODE_XTS)
388 max_payload -= SPU_XTS_TWEAK_SIZE;
389
390 excess = max_payload % blocksize;
391
392 return max_payload - excess;
393}
394
395/** spum_payload_length() - Given a SPU-M message header, extract the payload
396 * length.
397 * @spu_hdr: Start of SPU header
398 *
399 * Assumes just MH, EMH, BD (no SCTX, BDESC. Works for response frames.
400 *
401 * Return: payload length in bytes
402 */
403u32 spum_payload_length(u8 *spu_hdr)
404{
405 struct BD_HEADER *bd;
406 u32 pl_len;
407
408 /* Find BD header. skip MH, EMH */
409 bd = (struct BD_HEADER *)(spu_hdr + 8);
410 pl_len = be16_to_cpu(bd->size);
411
412 return pl_len;
413}
414
415/**
416 * spum_response_hdr_len() - Given the length of the hash key and encryption
417 * key, determine the expected length of a SPU response header.
418 * @auth_key_len: authentication key length (bytes)
419 * @enc_key_len: encryption key length (bytes)
420 * @is_hash: true if response message is for a hash operation
421 *
422 * Return: length of SPU response header (bytes)
423 */
424u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)
425{
426 if (is_hash)
427 return SPU_HASH_RESP_HDR_LEN;
428 else
429 return SPU_RESP_HDR_LEN;
430}
431
432/**
433 * spum_hash_pad_len() - Calculate the length of hash padding required to extend
434 * data to a full block size.
435 * @hash_alg: hash algorithm
436 * @hash_mode: hash mode
437 * @chunksize: length of data, in bytes
438 * @hash_block_size: size of a block of data for hash algorithm
439 *
440 * Reserve space for 1 byte (0x80) start of pad and the total length as u64
441 *
442 * Return: length of hash pad in bytes
443 */
444u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
445 u32 chunksize, u16 hash_block_size)
446{
447 unsigned int length_len;
448 unsigned int used_space_last_block;
449 int hash_pad_len;
450
451 /* AES-XCBC hash requires just padding to next block boundary */
452 if ((hash_alg == HASH_ALG_AES) && (hash_mode == HASH_MODE_XCBC)) {
453 used_space_last_block = chunksize % hash_block_size;
454 hash_pad_len = hash_block_size - used_space_last_block;
455 if (hash_pad_len >= hash_block_size)
456 hash_pad_len -= hash_block_size;
457 return hash_pad_len;
458 }
459
460 used_space_last_block = chunksize % hash_block_size + 1;
461 if ((hash_alg == HASH_ALG_SHA384) || (hash_alg == HASH_ALG_SHA512))
462 length_len = 2 * sizeof(u64);
463 else
464 length_len = sizeof(u64);
465
466 used_space_last_block += length_len;
467 hash_pad_len = hash_block_size - used_space_last_block;
468 if (hash_pad_len < 0)
469 hash_pad_len += hash_block_size;
470
471 hash_pad_len += 1 + length_len;
472 return hash_pad_len;
473}
474
475/**
476 * spum_gcm_ccm_pad_len() - Determine the required length of GCM or CCM padding.
477 * @cipher_mode: Algo type
478 * @data_size: Length of plaintext (bytes)
479 *
480 * @Return: Length of padding, in bytes
481 */
482u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
483 unsigned int data_size)
484{
485 u32 pad_len = 0;
486 u32 m1 = SPU_GCM_CCM_ALIGN - 1;
487
488 if ((cipher_mode == CIPHER_MODE_GCM) ||
489 (cipher_mode == CIPHER_MODE_CCM))
490 pad_len = ((data_size + m1) & ~m1) - data_size;
491
492 return pad_len;
493}
494
495/**
496 * spum_assoc_resp_len() - Determine the size of the receive buffer required to
497 * catch associated data.
498 * @cipher_mode: cipher mode
499 * @assoc_len: length of associated data (bytes)
500 * @iv_len: length of IV (bytes)
501 * @is_encrypt: true if encrypting. false if decrypting.
502 *
503 * Return: length of associated data in response message (bytes)
504 */
505u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode,
506 unsigned int assoc_len, unsigned int iv_len,
507 bool is_encrypt)
508{
509 u32 buflen = 0;
510 u32 pad;
511
512 if (assoc_len)
513 buflen = assoc_len;
514
515 if (cipher_mode == CIPHER_MODE_GCM) {
516 /* AAD needs to be padded in responses too */
517 pad = spum_gcm_ccm_pad_len(cipher_mode, buflen);
518 buflen += pad;
519 }
520 if (cipher_mode == CIPHER_MODE_CCM) {
521 /*
522 * AAD needs to be padded in responses too
523 * for CCM, len + 2 needs to be 128-bit aligned.
524 */
525 pad = spum_gcm_ccm_pad_len(cipher_mode, buflen + 2);
526 buflen += pad;
527 }
528
529 return buflen;
530}
531
532/**
533 * spu_aead_ivlen() - Calculate the length of the AEAD IV to be included
534 * in a SPU request after the AAD and before the payload.
535 * @cipher_mode: cipher mode
536 * @iv_ctr_len: initialization vector length in bytes
537 *
538 * In Linux ~4.2 and later, the assoc_data sg includes the IV. So no need
539 * to include the IV as a separate field in the SPU request msg.
540 *
541 * Return: Length of AEAD IV in bytes
542 */
543u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)
544{
545 return 0;
546}
547
548/**
549 * spum_hash_type() - Determine the type of hash operation.
550 * @src_sent: The number of bytes in the current request that have already
551 * been sent to the SPU to be hashed.
552 *
553 * We do not use HASH_TYPE_FULL for requests that fit in a single SPU message.
554 * Using FULL causes failures (such as when the string to be hashed is empty).
555 * For similar reasons, we never use HASH_TYPE_FIN. Instead, submit messages
556 * as INIT or UPDT and do the hash padding in sw.
557 */
558enum hash_type spum_hash_type(u32 src_sent)
559{
560 return src_sent ? HASH_TYPE_UPDT : HASH_TYPE_INIT;
561}
562
563/**
564 * spum_digest_size() - Determine the size of a hash digest to expect the SPU to
565 * return.
566 * alg_digest_size: Number of bytes in the final digest for the given algo
567 * alg: The hash algorithm
568 * htype: Type of hash operation (init, update, full, etc)
569 *
570 * When doing incremental hashing for an algorithm with a truncated hash
571 * (e.g., SHA224), the SPU returns the full digest so that it can be fed back as
572 * a partial result for the next chunk.
573 */
574u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg,
575 enum hash_type htype)
576{
577 u32 digestsize = alg_digest_size;
578
579 /* SPU returns complete digest when doing incremental hash and truncated
580 * hash algo.
581 */
582 if ((htype == HASH_TYPE_INIT) || (htype == HASH_TYPE_UPDT)) {
583 if (alg == HASH_ALG_SHA224)
584 digestsize = SHA256_DIGEST_SIZE;
585 else if (alg == HASH_ALG_SHA384)
586 digestsize = SHA512_DIGEST_SIZE;
587 }
588 return digestsize;
589}
590
591/**
592 * spum_create_request() - Build a SPU request message header, up to and
593 * including the BD header. Construct the message starting at spu_hdr. Caller
594 * should allocate this buffer in DMA-able memory at least SPU_HEADER_ALLOC_LEN
595 * bytes long.
596 * @spu_hdr: Start of buffer where SPU request header is to be written
597 * @req_opts: SPU request message options
598 * @cipher_parms: Parameters related to cipher algorithm
599 * @hash_parms: Parameters related to hash algorithm
600 * @aead_parms: Parameters related to AEAD operation
601 * @data_size: Length of data to be encrypted or authenticated. If AEAD, does
602 * not include length of AAD.
603
604 * Return: the length of the SPU header in bytes. 0 if an error occurs.
605 */
606u32 spum_create_request(u8 *spu_hdr,
607 struct spu_request_opts *req_opts,
608 struct spu_cipher_parms *cipher_parms,
609 struct spu_hash_parms *hash_parms,
610 struct spu_aead_parms *aead_parms,
611 unsigned int data_size)
612{
613 struct SPUHEADER *spuh;
614 struct BDESC_HEADER *bdesc;
615 struct BD_HEADER *bd;
616
617 u8 *ptr;
618 u32 protocol_bits = 0;
619 u32 cipher_bits = 0;
620 u32 ecf_bits = 0;
621 u8 sctx_words = 0;
622 unsigned int buf_len = 0;
623
624 /* size of the cipher payload */
625 unsigned int cipher_len = hash_parms->prebuf_len + data_size +
626 hash_parms->pad_len;
627
628 /* offset of prebuf or data from end of BD header */
629 unsigned int cipher_offset = aead_parms->assoc_size +
630 aead_parms->iv_len + aead_parms->aad_pad_len;
631
632 /* total size of the DB data (without STAT word padding) */
633 unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,
634 aead_parms->iv_len,
635 hash_parms->prebuf_len,
636 data_size,
637 aead_parms->aad_pad_len,
638 aead_parms->data_pad_len,
639 hash_parms->pad_len);
640
641 unsigned int auth_offset = 0;
642 unsigned int offset_iv = 0;
643
644 /* size/offset of the auth payload */
645 unsigned int auth_len;
646
647 auth_len = real_db_size;
648
649 if (req_opts->is_aead && req_opts->is_inbound)
650 cipher_len -= hash_parms->digestsize;
651
652 if (req_opts->is_aead && req_opts->is_inbound)
653 auth_len -= hash_parms->digestsize;
654
655 if ((hash_parms->alg == HASH_ALG_AES) &&
656 (hash_parms->mode == HASH_MODE_XCBC)) {
657 auth_len -= hash_parms->pad_len;
658 cipher_len -= hash_parms->pad_len;
659 }
660
661 flow_log("%s()\n", __func__);
662 flow_log(" in:%u authFirst:%u\n",
663 req_opts->is_inbound, req_opts->auth_first);
664 flow_log(" %s. cipher alg:%u mode:%u type %u\n",
665 spu_alg_name(cipher_parms->alg, cipher_parms->mode),
666 cipher_parms->alg, cipher_parms->mode, cipher_parms->type);
667 flow_log(" key: %d\n", cipher_parms->key_len);
668 flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
669 flow_log(" iv: %d\n", cipher_parms->iv_len);
670 flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
671 flow_log(" auth alg:%u mode:%u type %u\n",
672 hash_parms->alg, hash_parms->mode, hash_parms->type);
673 flow_log(" digestsize: %u\n", hash_parms->digestsize);
674 flow_log(" authkey: %d\n", hash_parms->key_len);
675 flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len);
676 flow_log(" assoc_size:%u\n", aead_parms->assoc_size);
677 flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len);
678 flow_log(" data_size:%u\n", data_size);
679 flow_log(" hash_pad_len:%u\n", hash_parms->pad_len);
680 flow_log(" real_db_size:%u\n", real_db_size);
681 flow_log(" auth_offset:%u auth_len:%u cipher_offset:%u cipher_len:%u\n",
682 auth_offset, auth_len, cipher_offset, cipher_len);
683 flow_log(" aead_iv: %u\n", aead_parms->iv_len);
684
685 /* starting out: zero the header (plus some) */
686 ptr = spu_hdr;
687 memset(ptr, 0, sizeof(struct SPUHEADER));
688
689 /* format master header word */
690 /* Do not set the next bit even though the datasheet says to */
691 spuh = (struct SPUHEADER *)ptr;
692 ptr += sizeof(struct SPUHEADER);
693 buf_len += sizeof(struct SPUHEADER);
694
695 spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;
696 spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);
697
698 /* Format sctx word 0 (protocol_bits) */
699 sctx_words = 3; /* size in words */
700
701 /* Format sctx word 1 (cipher_bits) */
702 if (req_opts->is_inbound)
703 cipher_bits |= CIPHER_INBOUND;
704 if (req_opts->auth_first)
705 cipher_bits |= CIPHER_ORDER;
706
707 /* Set the crypto parameters in the cipher.flags */
708 cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;
709 cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;
710 cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;
711
712 /* Set the auth parameters in the cipher.flags */
713 cipher_bits |= hash_parms->alg << HASH_ALG_SHIFT;
714 cipher_bits |= hash_parms->mode << HASH_MODE_SHIFT;
715 cipher_bits |= hash_parms->type << HASH_TYPE_SHIFT;
716
717 /*
718 * Format sctx extensions if required, and update main fields if
719 * required)
720 */
721 if (hash_parms->alg) {
722 /* Write the authentication key material if present */
723 if (hash_parms->key_len) {
724 memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);
725 ptr += hash_parms->key_len;
726 buf_len += hash_parms->key_len;
727 sctx_words += hash_parms->key_len / 4;
728 }
729
730 if ((cipher_parms->mode == CIPHER_MODE_GCM) ||
731 (cipher_parms->mode == CIPHER_MODE_CCM))
732 /* unpadded length */
733 offset_iv = aead_parms->assoc_size;
734
735 /* if GCM/CCM we need to write ICV into the payload */
736 if (!req_opts->is_inbound) {
737 if ((cipher_parms->mode == CIPHER_MODE_GCM) ||
738 (cipher_parms->mode == CIPHER_MODE_CCM))
739 ecf_bits |= 1 << INSERT_ICV_SHIFT;
740 } else {
741 ecf_bits |= CHECK_ICV;
742 }
743
744 /* Inform the SPU of the ICV size (in words) */
745 if (hash_parms->digestsize == 64)
746 cipher_bits |= ICV_IS_512;
747 else
748 ecf_bits |=
749 (hash_parms->digestsize / 4) << ICV_SIZE_SHIFT;
750 }
751
752 if (req_opts->bd_suppress)
753 ecf_bits |= BD_SUPPRESS;
754
755 /* copy the encryption keys in the SAD entry */
756 if (cipher_parms->alg) {
757 if (cipher_parms->key_len) {
758 memcpy(ptr, cipher_parms->key_buf,
759 cipher_parms->key_len);
760 ptr += cipher_parms->key_len;
761 buf_len += cipher_parms->key_len;
762 sctx_words += cipher_parms->key_len / 4;
763 }
764
765 /*
766 * if encrypting then set IV size, use SCTX IV unless no IV
767 * given here
768 */
769 if (cipher_parms->iv_buf && cipher_parms->iv_len) {
770 /* Use SCTX IV */
771 ecf_bits |= SCTX_IV;
772
773 /* cipher iv provided so put it in here */
774 memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);
775
776 ptr += cipher_parms->iv_len;
777 buf_len += cipher_parms->iv_len;
778 sctx_words += cipher_parms->iv_len / 4;
779 }
780 }
781
782 /*
783 * RFC4543 (GMAC/ESP) requires data to be sent as part of AAD
784 * so we need to override the BDESC parameters.
785 */
786 if (req_opts->is_rfc4543) {
787 if (req_opts->is_inbound)
788 data_size -= hash_parms->digestsize;
789 offset_iv = aead_parms->assoc_size + data_size;
790 cipher_len = 0;
791 cipher_offset = offset_iv;
792 auth_len = cipher_offset + aead_parms->data_pad_len;
793 }
794
795 /* write in the total sctx length now that we know it */
796 protocol_bits |= sctx_words;
797
798 /* Endian adjust the SCTX */
799 spuh->sa.proto_flags = cpu_to_be32(protocol_bits);
800 spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
801 spuh->sa.ecf = cpu_to_be32(ecf_bits);
802
803 /* === create the BDESC section === */
804 bdesc = (struct BDESC_HEADER *)ptr;
805
806 bdesc->offset_mac = cpu_to_be16(auth_offset);
807 bdesc->length_mac = cpu_to_be16(auth_len);
808 bdesc->offset_crypto = cpu_to_be16(cipher_offset);
809 bdesc->length_crypto = cpu_to_be16(cipher_len);
810
811 /*
812 * CCM in SPU-M requires that ICV not be in same 32-bit word as data or
813 * padding. So account for padding as necessary.
814 */
815 if (cipher_parms->mode == CIPHER_MODE_CCM)
816 auth_len += spum_wordalign_padlen(auth_len);
817
818 bdesc->offset_icv = cpu_to_be16(auth_len);
819 bdesc->offset_iv = cpu_to_be16(offset_iv);
820
821 ptr += sizeof(struct BDESC_HEADER);
822 buf_len += sizeof(struct BDESC_HEADER);
823
824 /* === no MFM section === */
825
826 /* === create the BD section === */
827
828 /* add the BD header */
829 bd = (struct BD_HEADER *)ptr;
830 bd->size = cpu_to_be16(real_db_size);
831 bd->prev_length = 0;
832
833 ptr += sizeof(struct BD_HEADER);
834 buf_len += sizeof(struct BD_HEADER);
835
836 packet_dump(" SPU request header: ", spu_hdr, buf_len);
837
838 return buf_len;
839}
840
841/**
842 * spum_cipher_req_init() - Build a SPU request message header, up to and
843 * including the BD header.
844 * @spu_hdr: Start of SPU request header (MH)
845 * @cipher_parms: Parameters that describe the cipher request
846 *
847 * Construct the message starting at spu_hdr. Caller should allocate this buffer
848 * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.
849 *
850 * Return: the length of the SPU header in bytes. 0 if an error occurs.
851 */
852u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)
853{
854 struct SPUHEADER *spuh;
855 u32 protocol_bits = 0;
856 u32 cipher_bits = 0;
857 u32 ecf_bits = 0;
858 u8 sctx_words = 0;
859 u8 *ptr = spu_hdr;
860
861 flow_log("%s()\n", __func__);
862 flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
863 cipher_parms->mode, cipher_parms->type);
864 flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len);
865 flow_log(" key: %d\n", cipher_parms->key_len);
866 flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
867
868 /* starting out: zero the header (plus some) */
869 memset(spu_hdr, 0, sizeof(struct SPUHEADER));
870 ptr += sizeof(struct SPUHEADER);
871
872 /* format master header word */
873 /* Do not set the next bit even though the datasheet says to */
874 spuh = (struct SPUHEADER *)spu_hdr;
875
876 spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;
877 spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);
878
879 /* Format sctx word 0 (protocol_bits) */
880 sctx_words = 3; /* size in words */
881
882 /* copy the encryption keys in the SAD entry */
883 if (cipher_parms->alg) {
884 if (cipher_parms->key_len) {
885 ptr += cipher_parms->key_len;
886 sctx_words += cipher_parms->key_len / 4;
887 }
888
889 /*
890 * if encrypting then set IV size, use SCTX IV unless no IV
891 * given here
892 */
893 if (cipher_parms->iv_len) {
894 /* Use SCTX IV */
895 ecf_bits |= SCTX_IV;
896 ptr += cipher_parms->iv_len;
897 sctx_words += cipher_parms->iv_len / 4;
898 }
899 }
900
901 /* Set the crypto parameters in the cipher.flags */
902 cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;
903 cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;
904 cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;
905
906 /* copy the encryption keys in the SAD entry */
907 if (cipher_parms->alg && cipher_parms->key_len)
908 memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);
909
910 /* write in the total sctx length now that we know it */
911 protocol_bits |= sctx_words;
912
913 /* Endian adjust the SCTX */
914 spuh->sa.proto_flags = cpu_to_be32(protocol_bits);
915
916 /* Endian adjust the SCTX */
917 spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
918 spuh->sa.ecf = cpu_to_be32(ecf_bits);
919
920 packet_dump(" SPU request header: ", spu_hdr,
921 sizeof(struct SPUHEADER));
922
923 return sizeof(struct SPUHEADER) + cipher_parms->key_len +
924 cipher_parms->iv_len + sizeof(struct BDESC_HEADER) +
925 sizeof(struct BD_HEADER);
926}
927
928/**
929 * spum_cipher_req_finish() - Finish building a SPU request message header for a
930 * block cipher request. Assumes much of the header was already filled in at
931 * setkey() time in spu_cipher_req_init().
932 * @spu_hdr: Start of the request message header (MH field)
933 * @spu_req_hdr_len: Length in bytes of the SPU request header
934 * @isInbound: 0 encrypt, 1 decrypt
935 * @cipher_parms: Parameters describing cipher operation to be performed
936 * @update_key: If true, rewrite the cipher key in SCTX
937 * @data_size: Length of the data in the BD field
938 *
939 * Assumes much of the header was already filled in at setkey() time in
940 * spum_cipher_req_init().
941 * spum_cipher_req_init() fills in the encryption key. For RC4, when submitting
942 * a request for a non-first chunk, we use the 260-byte SUPDT field from the
943 * previous response as the key. update_key is true for this case. Unused in all
944 * other cases.
945 */
946void spum_cipher_req_finish(u8 *spu_hdr,
947 u16 spu_req_hdr_len,
948 unsigned int is_inbound,
949 struct spu_cipher_parms *cipher_parms,
950 bool update_key,
951 unsigned int data_size)
952{
953 struct SPUHEADER *spuh;
954 struct BDESC_HEADER *bdesc;
955 struct BD_HEADER *bd;
956 u8 *bdesc_ptr = spu_hdr + spu_req_hdr_len -
957 (sizeof(struct BD_HEADER) + sizeof(struct BDESC_HEADER));
958
959 u32 cipher_bits;
960
961 flow_log("%s()\n", __func__);
962 flow_log(" in: %u\n", is_inbound);
963 flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
964 cipher_parms->type);
965 if (update_key) {
966 flow_log(" cipher key len: %u\n", cipher_parms->key_len);
967 flow_dump(" key: ", cipher_parms->key_buf,
968 cipher_parms->key_len);
969 }
970
971 /*
972 * In XTS mode, API puts "i" parameter (block tweak) in IV. For
973 * SPU-M, should be in start of the BD; tx_sg_create() copies it there.
974 * IV in SPU msg for SPU-M should be 0, since that's the "j" parameter
975 * (block ctr within larger data unit) - given we can send entire disk
976 * block (<= 4KB) in 1 SPU msg, don't need to use this parameter.
977 */
978 if (cipher_parms->mode == CIPHER_MODE_XTS)
979 memset(cipher_parms->iv_buf, 0, cipher_parms->iv_len);
980
981 flow_log(" iv len: %d\n", cipher_parms->iv_len);
982 flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
983 flow_log(" data_size: %u\n", data_size);
984
985 /* format master header word */
986 /* Do not set the next bit even though the datasheet says to */
987 spuh = (struct SPUHEADER *)spu_hdr;
988
989 /* cipher_bits was initialized at setkey time */
990 cipher_bits = be32_to_cpu(spuh->sa.cipher_flags);
991
992 /* Format sctx word 1 (cipher_bits) */
993 if (is_inbound)
994 cipher_bits |= CIPHER_INBOUND;
995 else
996 cipher_bits &= ~CIPHER_INBOUND;
997
998 /* update encryption key for RC4 on non-first chunk */
999 if (update_key) {
1000 spuh->sa.cipher_flags |=
1001 cipher_parms->type << CIPHER_TYPE_SHIFT;
1002 memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);
1003 }
1004
1005 if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len)
1006 /* cipher iv provided so put it in here */
1007 memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf,
1008 cipher_parms->iv_len);
1009
1010 spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
1011
1012 /* === create the BDESC section === */
1013 bdesc = (struct BDESC_HEADER *)bdesc_ptr;
1014 bdesc->offset_mac = 0;
1015 bdesc->length_mac = 0;
1016 bdesc->offset_crypto = 0;
1017
1018 /* XTS mode, data_size needs to include tweak parameter */
1019 if (cipher_parms->mode == CIPHER_MODE_XTS)
1020 bdesc->length_crypto = cpu_to_be16(data_size +
1021 SPU_XTS_TWEAK_SIZE);
1022 else
1023 bdesc->length_crypto = cpu_to_be16(data_size);
1024
1025 bdesc->offset_icv = 0;
1026 bdesc->offset_iv = 0;
1027
1028 /* === no MFM section === */
1029
1030 /* === create the BD section === */
1031 /* add the BD header */
1032 bd = (struct BD_HEADER *)(bdesc_ptr + sizeof(struct BDESC_HEADER));
1033 bd->size = cpu_to_be16(data_size);
1034
1035 /* XTS mode, data_size needs to include tweak parameter */
1036 if (cipher_parms->mode == CIPHER_MODE_XTS)
1037 bd->size = cpu_to_be16(data_size + SPU_XTS_TWEAK_SIZE);
1038 else
1039 bd->size = cpu_to_be16(data_size);
1040
1041 bd->prev_length = 0;
1042
1043 packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len);
1044}
1045
1046/**
1047 * spum_request_pad() - Create pad bytes at the end of the data.
1048 * @pad_start: Start of buffer where pad bytes are to be written
1049 * @gcm_ccm_padding: length of GCM/CCM padding, in bytes
1050 * @hash_pad_len: Number of bytes of padding extend data to full block
1051 * @auth_alg: authentication algorithm
1052 * @auth_mode: authentication mode
1053 * @total_sent: length inserted at end of hash pad
1054 * @status_padding: Number of bytes of padding to align STATUS word
1055 *
1056 * There may be three forms of pad:
1057 * 1. GCM/CCM pad - for GCM/CCM mode ciphers, pad to 16-byte alignment
1058 * 2. hash pad - pad to a block length, with 0x80 data terminator and
1059 * size at the end
1060 * 3. STAT pad - to ensure the STAT field is 4-byte aligned
1061 */
1062void spum_request_pad(u8 *pad_start,
1063 u32 gcm_ccm_padding,
1064 u32 hash_pad_len,
1065 enum hash_alg auth_alg,
1066 enum hash_mode auth_mode,
1067 unsigned int total_sent, u32 status_padding)
1068{
1069 u8 *ptr = pad_start;
1070
1071 /* fix data alignent for GCM/CCM */
1072 if (gcm_ccm_padding > 0) {
1073 flow_log(" GCM: padding to 16 byte alignment: %u bytes\n",
1074 gcm_ccm_padding);
1075 memset(ptr, 0, gcm_ccm_padding);
1076 ptr += gcm_ccm_padding;
1077 }
1078
1079 if (hash_pad_len > 0) {
1080 /* clear the padding section */
1081 memset(ptr, 0, hash_pad_len);
1082
1083 if ((auth_alg == HASH_ALG_AES) &&
1084 (auth_mode == HASH_MODE_XCBC)) {
1085 /* AES/XCBC just requires padding to be 0s */
1086 ptr += hash_pad_len;
1087 } else {
1088 /* terminate the data */
1089 *ptr = 0x80;
1090 ptr += (hash_pad_len - sizeof(u64));
1091
1092 /* add the size at the end as required per alg */
1093 if (auth_alg == HASH_ALG_MD5)
1094 *(u64 *)ptr = cpu_to_le64((u64)total_sent * 8);
1095 else /* SHA1, SHA2-224, SHA2-256 */
1096 *(u64 *)ptr = cpu_to_be64((u64)total_sent * 8);
1097 ptr += sizeof(u64);
1098 }
1099 }
1100
1101 /* pad to a 4byte alignment for STAT */
1102 if (status_padding > 0) {
1103 flow_log(" STAT: padding to 4 byte alignment: %u bytes\n",
1104 status_padding);
1105
1106 memset(ptr, 0, status_padding);
1107 ptr += status_padding;
1108 }
1109}
1110
1111/**
1112 * spum_xts_tweak_in_payload() - Indicate that SPUM DOES place the XTS tweak
1113 * field in the packet payload (rather than using IV)
1114 *
1115 * Return: 1
1116 */
1117u8 spum_xts_tweak_in_payload(void)
1118{
1119 return 1;
1120}
1121
1122/**
1123 * spum_tx_status_len() - Return the length of the STATUS field in a SPU
1124 * response message.
1125 *
1126 * Return: Length of STATUS field in bytes.
1127 */
1128u8 spum_tx_status_len(void)
1129{
1130 return SPU_TX_STATUS_LEN;
1131}
1132
1133/**
1134 * spum_rx_status_len() - Return the length of the STATUS field in a SPU
1135 * response message.
1136 *
1137 * Return: Length of STATUS field in bytes.
1138 */
1139u8 spum_rx_status_len(void)
1140{
1141 return SPU_RX_STATUS_LEN;
1142}
1143
1144/**
1145 * spum_status_process() - Process the status from a SPU response message.
1146 * @statp: start of STATUS word
1147 * Return:
1148 * 0 - if status is good and response should be processed
1149 * !0 - status indicates an error and response is invalid
1150 */
1151int spum_status_process(u8 *statp)
1152{
1153 u32 status;
1154
1155 status = __be32_to_cpu(*(__be32 *)statp);
1156 flow_log("SPU response STATUS %#08x\n", status);
1157 if (status & SPU_STATUS_ERROR_FLAG) {
1158 pr_err("%s() Warning: Error result from SPU: %#08x\n",
1159 __func__, status);
1160 if (status & SPU_STATUS_INVALID_ICV)
1161 return SPU_INVALID_ICV;
1162 return -EBADMSG;
1163 }
1164 return 0;
1165}
1166
1167/**
1168 * spum_ccm_update_iv() - Update the IV as per the requirements for CCM mode.
1169 *
1170 * @digestsize: Digest size of this request
1171 * @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len
1172 * @assoclen: Length of AAD data
1173 * @chunksize: length of input data to be sent in this req
1174 * @is_encrypt: true if this is an output/encrypt operation
1175 * @is_esp: true if this is an ESP / RFC4309 operation
1176 *
1177 */
1178void spum_ccm_update_iv(unsigned int digestsize,
1179 struct spu_cipher_parms *cipher_parms,
1180 unsigned int assoclen,
1181 unsigned int chunksize,
1182 bool is_encrypt,
1183 bool is_esp)
1184{
1185 u8 L; /* L from CCM algorithm, length of plaintext data */
1186 u8 mprime; /* M' from CCM algo, (M - 2) / 2, where M=authsize */
1187 u8 adata;
1188
1189 if (cipher_parms->iv_len != CCM_AES_IV_SIZE) {
1190 pr_err("%s(): Invalid IV len %d for CCM mode, should be %d\n",
1191 __func__, cipher_parms->iv_len, CCM_AES_IV_SIZE);
1192 return;
1193 }
1194
1195 /*
1196 * IV needs to be formatted as follows:
1197 *
1198 * | Byte 0 | Bytes 1 - N | Bytes (N+1) - 15 |
1199 * | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | Bits 7 - 0 | Bits 7 - 0 |
1200 * | 0 |Ad?|(M - 2) / 2| L - 1 | Nonce | Plaintext Length |
1201 *
1202 * Ad? = 1 if AAD present, 0 if not present
1203 * M = size of auth field, 8, 12, or 16 bytes (SPU-M) -or-
1204 * 4, 6, 8, 10, 12, 14, 16 bytes (SPU2)
1205 * L = Size of Plaintext Length field; Nonce size = 15 - L
1206 *
1207 * It appears that the crypto API already expects the L-1 portion
1208 * to be set in the first byte of the IV, which implicitly determines
1209 * the nonce size, and also fills in the nonce. But the other bits
1210 * in byte 0 as well as the plaintext length need to be filled in.
1211 *
1212 * In rfc4309/esp mode, L is not already in the supplied IV and
1213 * we need to fill it in, as well as move the IV data to be after
1214 * the salt
1215 */
1216 if (is_esp) {
1217 L = CCM_ESP_L_VALUE; /* RFC4309 has fixed L */
1218 } else {
1219 /* L' = plaintext length - 1 so Plaintext length is L' + 1 */
1220 L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>
1221 CCM_B0_L_PRIME_SHIFT) + 1;
1222 }
1223
1224 mprime = (digestsize - 2) >> 1; /* M' = (M - 2) / 2 */
1225 adata = (assoclen > 0); /* adata = 1 if any associated data */
1226
1227 cipher_parms->iv_buf[0] = (adata << CCM_B0_ADATA_SHIFT) |
1228 (mprime << CCM_B0_M_PRIME_SHIFT) |
1229 ((L - 1) << CCM_B0_L_PRIME_SHIFT);
1230
1231 /* Nonce is already filled in by crypto API, and is 15 - L bytes */
1232
1233 /* Don't include digest in plaintext size when decrypting */
1234 if (!is_encrypt)
1235 chunksize -= digestsize;
1236
1237 /* Fill in length of plaintext, formatted to be L bytes long */
1238 format_value_ccm(chunksize, &cipher_parms->iv_buf[15 - L + 1], L);
1239}
1240
1241/**
1242 * spum_wordalign_padlen() - Given the length of a data field, determine the
1243 * padding required to align the data following this field on a 4-byte boundary.
1244 * @data_size: length of data field in bytes
1245 *
1246 * Return: length of status field padding, in bytes
1247 */
1248u32 spum_wordalign_padlen(u32 data_size)
1249{
1250 return ((data_size + 3) & ~3) - data_size;
1251}
diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h
new file mode 100644
index 000000000000..aa6fc38db263
--- /dev/null
+++ b/drivers/crypto/bcm/spu.h
@@ -0,0 +1,287 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17/*
18 * This file contains the definition of SPU messages. There are currently two
19 * SPU message formats: SPU-M and SPU2. The hardware uses different values to
20 * identify the same things in SPU-M vs SPU2. So this file defines values that
21 * are hardware independent. Software can use these values for any version of
22 * SPU hardware. These values are used in APIs in spu.c. Functions internal to
23 * spu.c and spu2.c convert these to hardware-specific values.
24 */
25
26#ifndef _SPU_H
27#define _SPU_H
28
29#include <linux/types.h>
30#include <linux/scatterlist.h>
31#include <crypto/sha.h>
32
33enum spu_cipher_alg {
34 CIPHER_ALG_NONE = 0x0,
35 CIPHER_ALG_RC4 = 0x1,
36 CIPHER_ALG_DES = 0x2,
37 CIPHER_ALG_3DES = 0x3,
38 CIPHER_ALG_AES = 0x4,
39 CIPHER_ALG_LAST = 0x5
40};
41
42enum spu_cipher_mode {
43 CIPHER_MODE_NONE = 0x0,
44 CIPHER_MODE_ECB = 0x0,
45 CIPHER_MODE_CBC = 0x1,
46 CIPHER_MODE_OFB = 0x2,
47 CIPHER_MODE_CFB = 0x3,
48 CIPHER_MODE_CTR = 0x4,
49 CIPHER_MODE_CCM = 0x5,
50 CIPHER_MODE_GCM = 0x6,
51 CIPHER_MODE_XTS = 0x7,
52 CIPHER_MODE_LAST = 0x8
53};
54
55enum spu_cipher_type {
56 CIPHER_TYPE_NONE = 0x0,
57 CIPHER_TYPE_DES = 0x0,
58 CIPHER_TYPE_3DES = 0x0,
59 CIPHER_TYPE_INIT = 0x0, /* used for ARC4 */
60 CIPHER_TYPE_AES128 = 0x0,
61 CIPHER_TYPE_AES192 = 0x1,
62 CIPHER_TYPE_UPDT = 0x1, /* used for ARC4 */
63 CIPHER_TYPE_AES256 = 0x2,
64};
65
66enum hash_alg {
67 HASH_ALG_NONE = 0x0,
68 HASH_ALG_MD5 = 0x1,
69 HASH_ALG_SHA1 = 0x2,
70 HASH_ALG_SHA224 = 0x3,
71 HASH_ALG_SHA256 = 0x4,
72 HASH_ALG_AES = 0x5,
73 HASH_ALG_SHA384 = 0x6,
74 HASH_ALG_SHA512 = 0x7,
75 /* Keep SHA3 algorithms at the end always */
76 HASH_ALG_SHA3_224 = 0x8,
77 HASH_ALG_SHA3_256 = 0x9,
78 HASH_ALG_SHA3_384 = 0xa,
79 HASH_ALG_SHA3_512 = 0xb,
80 HASH_ALG_LAST
81};
82
83enum hash_mode {
84 HASH_MODE_NONE = 0x0,
85 HASH_MODE_HASH = 0x0,
86 HASH_MODE_XCBC = 0x0,
87 HASH_MODE_CMAC = 0x1,
88 HASH_MODE_CTXT = 0x1,
89 HASH_MODE_HMAC = 0x2,
90 HASH_MODE_RABIN = 0x4,
91 HASH_MODE_FHMAC = 0x6,
92 HASH_MODE_CCM = 0x5,
93 HASH_MODE_GCM = 0x6,
94};
95
96enum hash_type {
97 HASH_TYPE_NONE = 0x0,
98 HASH_TYPE_FULL = 0x0,
99 HASH_TYPE_INIT = 0x1,
100 HASH_TYPE_UPDT = 0x2,
101 HASH_TYPE_FIN = 0x3,
102 HASH_TYPE_AES128 = 0x0,
103 HASH_TYPE_AES192 = 0x1,
104 HASH_TYPE_AES256 = 0x2
105};
106
107enum aead_type {
108 AES_CCM,
109 AES_GCM,
110 AUTHENC,
111 AEAD_TYPE_LAST
112};
113
114extern char *hash_alg_name[HASH_ALG_LAST];
115extern char *aead_alg_name[AEAD_TYPE_LAST];
116
117struct spu_request_opts {
118 bool is_inbound;
119 bool auth_first;
120 bool is_aead;
121 bool is_esp;
122 bool bd_suppress;
123 bool is_rfc4543;
124};
125
126struct spu_cipher_parms {
127 enum spu_cipher_alg alg;
128 enum spu_cipher_mode mode;
129 enum spu_cipher_type type;
130 u8 *key_buf;
131 u16 key_len;
132 /* iv_buf and iv_len include salt, if applicable */
133 u8 *iv_buf;
134 u16 iv_len;
135};
136
137struct spu_hash_parms {
138 enum hash_alg alg;
139 enum hash_mode mode;
140 enum hash_type type;
141 u8 digestsize;
142 u8 *key_buf;
143 u16 key_len;
144 u16 prebuf_len;
145 /* length of hash pad. signed, needs to handle roll-overs */
146 int pad_len;
147};
148
149struct spu_aead_parms {
150 u32 assoc_size;
151 u16 iv_len; /* length of IV field between assoc data and data */
152 u8 aad_pad_len; /* For AES GCM/CCM, length of padding after AAD */
153 u8 data_pad_len;/* For AES GCM/CCM, length of padding after data */
154 bool return_iv; /* True if SPU should return an IV */
155 u32 ret_iv_len; /* Length in bytes of returned IV */
156 u32 ret_iv_off; /* Offset into full IV if partial IV returned */
157};
158
159/************** SPU sizes ***************/
160
161#define SPU_RX_STATUS_LEN 4
162
163/* Max length of padding for 4-byte alignment of STATUS field */
164#define SPU_STAT_PAD_MAX 4
165
166/* Max length of pad fragment. 4 is for 4-byte alignment of STATUS field */
167#define SPU_PAD_LEN_MAX (SPU_GCM_CCM_ALIGN + MAX_HASH_BLOCK_SIZE + \
168 SPU_STAT_PAD_MAX)
169
170/* GCM and CCM require 16-byte alignment */
171#define SPU_GCM_CCM_ALIGN 16
172
173/* Length up SUPDT field in SPU response message for RC4 */
174#define SPU_SUPDT_LEN 260
175
176/* SPU status error codes. These used as common error codes across all
177 * SPU variants.
178 */
179#define SPU_INVALID_ICV 1
180
181/* Indicates no limit to the length of the payload in a SPU message */
182#define SPU_MAX_PAYLOAD_INF 0xFFFFFFFF
183
184/* Size of XTS tweak ("i" parameter), in bytes */
185#define SPU_XTS_TWEAK_SIZE 16
186
187/* CCM B_0 field definitions, common for SPU-M and SPU2 */
188#define CCM_B0_ADATA 0x40
189#define CCM_B0_ADATA_SHIFT 6
190#define CCM_B0_M_PRIME 0x38
191#define CCM_B0_M_PRIME_SHIFT 3
192#define CCM_B0_L_PRIME 0x07
193#define CCM_B0_L_PRIME_SHIFT 0
194#define CCM_ESP_L_VALUE 4
195
196/**
197 * spu_req_incl_icv() - Return true if SPU request message should include the
198 * ICV as a separate buffer.
199 * @cipher_mode: the cipher mode being requested
200 * @is_encrypt: true if encrypting. false if decrypting.
201 *
202 * Return: true if ICV to be included as separate buffer
203 */
204static __always_inline bool spu_req_incl_icv(enum spu_cipher_mode cipher_mode,
205 bool is_encrypt)
206{
207 if ((cipher_mode == CIPHER_MODE_GCM) && !is_encrypt)
208 return true;
209 if ((cipher_mode == CIPHER_MODE_CCM) && !is_encrypt)
210 return true;
211
212 return false;
213}
214
215static __always_inline u32 spu_real_db_size(u32 assoc_size,
216 u32 aead_iv_buf_len,
217 u32 prebuf_len,
218 u32 data_size,
219 u32 aad_pad_len,
220 u32 gcm_pad_len,
221 u32 hash_pad_len)
222{
223 return assoc_size + aead_iv_buf_len + prebuf_len + data_size +
224 aad_pad_len + gcm_pad_len + hash_pad_len;
225}
226
227/************** SPU Functions Prototypes **************/
228
229void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len);
230
231u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
232 enum spu_cipher_mode cipher_mode,
233 unsigned int blocksize);
234u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg,
235 enum spu_cipher_mode cipher_mode,
236 unsigned int blocksize);
237u32 spum_payload_length(u8 *spu_hdr);
238u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash);
239u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
240 u32 chunksize, u16 hash_block_size);
241u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
242 unsigned int data_size);
243u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode,
244 unsigned int assoc_len, unsigned int iv_len,
245 bool is_encrypt);
246u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len);
247bool spu_req_incl_icv(enum spu_cipher_mode cipher_mode, bool is_encrypt);
248enum hash_type spum_hash_type(u32 src_sent);
249u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg,
250 enum hash_type htype);
251
252u32 spum_create_request(u8 *spu_hdr,
253 struct spu_request_opts *req_opts,
254 struct spu_cipher_parms *cipher_parms,
255 struct spu_hash_parms *hash_parms,
256 struct spu_aead_parms *aead_parms,
257 unsigned int data_size);
258
259u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms);
260
261void spum_cipher_req_finish(u8 *spu_hdr,
262 u16 spu_req_hdr_len,
263 unsigned int is_inbound,
264 struct spu_cipher_parms *cipher_parms,
265 bool update_key,
266 unsigned int data_size);
267
268void spum_request_pad(u8 *pad_start,
269 u32 gcm_padding,
270 u32 hash_pad_len,
271 enum hash_alg auth_alg,
272 enum hash_mode auth_mode,
273 unsigned int total_sent, u32 status_padding);
274
275u8 spum_xts_tweak_in_payload(void);
276u8 spum_tx_status_len(void);
277u8 spum_rx_status_len(void);
278int spum_status_process(u8 *statp);
279
280void spum_ccm_update_iv(unsigned int digestsize,
281 struct spu_cipher_parms *cipher_parms,
282 unsigned int assoclen,
283 unsigned int chunksize,
284 bool is_encrypt,
285 bool is_esp);
286u32 spum_wordalign_padlen(u32 data_size);
287#endif
diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c
new file mode 100644
index 000000000000..ef04c9748317
--- /dev/null
+++ b/drivers/crypto/bcm/spu2.c
@@ -0,0 +1,1401 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17/*
18 * This file works with the SPU2 version of the SPU. SPU2 has different message
19 * formats than the previous version of the SPU. All SPU message format
20 * differences should be hidden in the spux.c,h files.
21 */
22
23#include <linux/kernel.h>
24#include <linux/string.h>
25
26#include "util.h"
27#include "spu.h"
28#include "spu2.h"
29
30#define SPU2_TX_STATUS_LEN 0 /* SPU2 has no STATUS in input packet */
31
32/*
33 * Controlled by pkt_stat_cnt field in CRYPTO_SS_SPU0_CORE_SPU2_CONTROL0
34 * register. Defaults to 2.
35 */
36#define SPU2_RX_STATUS_LEN 2
37
38enum spu2_proto_sel {
39 SPU2_PROTO_RESV = 0,
40 SPU2_MACSEC_SECTAG8_ECB = 1,
41 SPU2_MACSEC_SECTAG8_SCB = 2,
42 SPU2_MACSEC_SECTAG16 = 3,
43 SPU2_MACSEC_SECTAG16_8_XPN = 4,
44 SPU2_IPSEC = 5,
45 SPU2_IPSEC_ESN = 6,
46 SPU2_TLS_CIPHER = 7,
47 SPU2_TLS_AEAD = 8,
48 SPU2_DTLS_CIPHER = 9,
49 SPU2_DTLS_AEAD = 10
50};
51
52char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
53 "DES", "3DES"
54};
55
56char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS",
57 "CCM", "GCM"
58};
59
60char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
61 "Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384",
62 "SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256",
63 "SHA3-384", "SHA3-512"
64};
65
66char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
67 "Rabin", "CCM", "GCM", "Reserved"
68};
69
70static char *spu2_ciph_type_name(enum spu2_cipher_type cipher_type)
71{
72 if (cipher_type >= SPU2_CIPHER_TYPE_LAST)
73 return "Reserved";
74 return spu2_cipher_type_names[cipher_type];
75}
76
77static char *spu2_ciph_mode_name(enum spu2_cipher_mode cipher_mode)
78{
79 if (cipher_mode >= SPU2_CIPHER_MODE_LAST)
80 return "Reserved";
81 return spu2_cipher_mode_names[cipher_mode];
82}
83
84static char *spu2_hash_type_name(enum spu2_hash_type hash_type)
85{
86 if (hash_type >= SPU2_HASH_TYPE_LAST)
87 return "Reserved";
88 return spu2_hash_type_names[hash_type];
89}
90
91static char *spu2_hash_mode_name(enum spu2_hash_mode hash_mode)
92{
93 if (hash_mode >= SPU2_HASH_MODE_LAST)
94 return "Reserved";
95 return spu2_hash_mode_names[hash_mode];
96}
97
98/*
99 * Convert from a software cipher mode value to the corresponding value
100 * for SPU2.
101 */
102static int spu2_cipher_mode_xlate(enum spu_cipher_mode cipher_mode,
103 enum spu2_cipher_mode *spu2_mode)
104{
105 switch (cipher_mode) {
106 case CIPHER_MODE_ECB:
107 *spu2_mode = SPU2_CIPHER_MODE_ECB;
108 break;
109 case CIPHER_MODE_CBC:
110 *spu2_mode = SPU2_CIPHER_MODE_CBC;
111 break;
112 case CIPHER_MODE_OFB:
113 *spu2_mode = SPU2_CIPHER_MODE_OFB;
114 break;
115 case CIPHER_MODE_CFB:
116 *spu2_mode = SPU2_CIPHER_MODE_CFB;
117 break;
118 case CIPHER_MODE_CTR:
119 *spu2_mode = SPU2_CIPHER_MODE_CTR;
120 break;
121 case CIPHER_MODE_CCM:
122 *spu2_mode = SPU2_CIPHER_MODE_CCM;
123 break;
124 case CIPHER_MODE_GCM:
125 *spu2_mode = SPU2_CIPHER_MODE_GCM;
126 break;
127 case CIPHER_MODE_XTS:
128 *spu2_mode = SPU2_CIPHER_MODE_XTS;
129 break;
130 default:
131 return -EINVAL;
132 }
133 return 0;
134}
135
136/**
137 * spu2_cipher_xlate() - Convert a cipher {alg/mode/type} triple to a SPU2
138 * cipher type and mode.
139 * @cipher_alg: [in] cipher algorithm value from software enumeration
140 * @cipher_mode: [in] cipher mode value from software enumeration
141 * @cipher_type: [in] cipher type value from software enumeration
142 * @spu2_type: [out] cipher type value used by spu2 hardware
143 * @spu2_mode: [out] cipher mode value used by spu2 hardware
144 *
145 * Return: 0 if successful
146 */
147static int spu2_cipher_xlate(enum spu_cipher_alg cipher_alg,
148 enum spu_cipher_mode cipher_mode,
149 enum spu_cipher_type cipher_type,
150 enum spu2_cipher_type *spu2_type,
151 enum spu2_cipher_mode *spu2_mode)
152{
153 int err;
154
155 err = spu2_cipher_mode_xlate(cipher_mode, spu2_mode);
156 if (err) {
157 flow_log("Invalid cipher mode %d\n", cipher_mode);
158 return err;
159 }
160
161 switch (cipher_alg) {
162 case CIPHER_ALG_NONE:
163 *spu2_type = SPU2_CIPHER_TYPE_NONE;
164 break;
165 case CIPHER_ALG_RC4:
166 /* SPU2 does not support RC4 */
167 err = -EINVAL;
168 *spu2_type = SPU2_CIPHER_TYPE_NONE;
169 break;
170 case CIPHER_ALG_DES:
171 *spu2_type = SPU2_CIPHER_TYPE_DES;
172 break;
173 case CIPHER_ALG_3DES:
174 *spu2_type = SPU2_CIPHER_TYPE_3DES;
175 break;
176 case CIPHER_ALG_AES:
177 switch (cipher_type) {
178 case CIPHER_TYPE_AES128:
179 *spu2_type = SPU2_CIPHER_TYPE_AES128;
180 break;
181 case CIPHER_TYPE_AES192:
182 *spu2_type = SPU2_CIPHER_TYPE_AES192;
183 break;
184 case CIPHER_TYPE_AES256:
185 *spu2_type = SPU2_CIPHER_TYPE_AES256;
186 break;
187 default:
188 err = -EINVAL;
189 }
190 break;
191 case CIPHER_ALG_LAST:
192 default:
193 err = -EINVAL;
194 break;
195 }
196
197 if (err)
198 flow_log("Invalid cipher alg %d or type %d\n",
199 cipher_alg, cipher_type);
200 return err;
201}
202
203/*
204 * Convert from a software hash mode value to the corresponding value
205 * for SPU2. Note that HASH_MODE_NONE and HASH_MODE_XCBC have the same value.
206 */
207static int spu2_hash_mode_xlate(enum hash_mode hash_mode,
208 enum spu2_hash_mode *spu2_mode)
209{
210 switch (hash_mode) {
211 case HASH_MODE_XCBC:
212 *spu2_mode = SPU2_HASH_MODE_XCBC_MAC;
213 break;
214 case HASH_MODE_CMAC:
215 *spu2_mode = SPU2_HASH_MODE_CMAC;
216 break;
217 case HASH_MODE_HMAC:
218 *spu2_mode = SPU2_HASH_MODE_HMAC;
219 break;
220 case HASH_MODE_CCM:
221 *spu2_mode = SPU2_HASH_MODE_CCM;
222 break;
223 case HASH_MODE_GCM:
224 *spu2_mode = SPU2_HASH_MODE_GCM;
225 break;
226 default:
227 return -EINVAL;
228 }
229 return 0;
230}
231
232/**
233 * spu2_hash_xlate() - Convert a hash {alg/mode/type} triple to a SPU2 hash type
234 * and mode.
235 * @hash_alg: [in] hash algorithm value from software enumeration
236 * @hash_mode: [in] hash mode value from software enumeration
237 * @hash_type: [in] hash type value from software enumeration
238 * @ciph_type: [in] cipher type value from software enumeration
239 * @spu2_type: [out] hash type value used by SPU2 hardware
240 * @spu2_mode: [out] hash mode value used by SPU2 hardware
241 *
242 * Return: 0 if successful
243 */
244static int
245spu2_hash_xlate(enum hash_alg hash_alg, enum hash_mode hash_mode,
246 enum hash_type hash_type, enum spu_cipher_type ciph_type,
247 enum spu2_hash_type *spu2_type, enum spu2_hash_mode *spu2_mode)
248{
249 int err;
250
251 err = spu2_hash_mode_xlate(hash_mode, spu2_mode);
252 if (err) {
253 flow_log("Invalid hash mode %d\n", hash_mode);
254 return err;
255 }
256
257 switch (hash_alg) {
258 case HASH_ALG_NONE:
259 *spu2_type = SPU2_HASH_TYPE_NONE;
260 break;
261 case HASH_ALG_MD5:
262 *spu2_type = SPU2_HASH_TYPE_MD5;
263 break;
264 case HASH_ALG_SHA1:
265 *spu2_type = SPU2_HASH_TYPE_SHA1;
266 break;
267 case HASH_ALG_SHA224:
268 *spu2_type = SPU2_HASH_TYPE_SHA224;
269 break;
270 case HASH_ALG_SHA256:
271 *spu2_type = SPU2_HASH_TYPE_SHA256;
272 break;
273 case HASH_ALG_SHA384:
274 *spu2_type = SPU2_HASH_TYPE_SHA384;
275 break;
276 case HASH_ALG_SHA512:
277 *spu2_type = SPU2_HASH_TYPE_SHA512;
278 break;
279 case HASH_ALG_AES:
280 switch (ciph_type) {
281 case CIPHER_TYPE_AES128:
282 *spu2_type = SPU2_HASH_TYPE_AES128;
283 break;
284 case CIPHER_TYPE_AES192:
285 *spu2_type = SPU2_HASH_TYPE_AES192;
286 break;
287 case CIPHER_TYPE_AES256:
288 *spu2_type = SPU2_HASH_TYPE_AES256;
289 break;
290 default:
291 err = -EINVAL;
292 }
293 break;
294 case HASH_ALG_SHA3_224:
295 *spu2_type = SPU2_HASH_TYPE_SHA3_224;
296 break;
297 case HASH_ALG_SHA3_256:
298 *spu2_type = SPU2_HASH_TYPE_SHA3_256;
299 break;
300 case HASH_ALG_SHA3_384:
301 *spu2_type = SPU2_HASH_TYPE_SHA3_384;
302 break;
303 case HASH_ALG_SHA3_512:
304 *spu2_type = SPU2_HASH_TYPE_SHA3_512;
305 case HASH_ALG_LAST:
306 default:
307 err = -EINVAL;
308 break;
309 }
310
311 if (err)
312 flow_log("Invalid hash alg %d or type %d\n",
313 hash_alg, hash_type);
314 return err;
315}
316
317/* Dump FMD ctrl0. The ctrl0 input is in host byte order */
318static void spu2_dump_fmd_ctrl0(u64 ctrl0)
319{
320 enum spu2_cipher_type ciph_type;
321 enum spu2_cipher_mode ciph_mode;
322 enum spu2_hash_type hash_type;
323 enum spu2_hash_mode hash_mode;
324 char *ciph_name;
325 char *ciph_mode_name;
326 char *hash_name;
327 char *hash_mode_name;
328 u8 cfb;
329 u8 proto;
330
331 packet_log(" FMD CTRL0 %#16llx\n", ctrl0);
332 if (ctrl0 & SPU2_CIPH_ENCRYPT_EN)
333 packet_log(" encrypt\n");
334 else
335 packet_log(" decrypt\n");
336
337 ciph_type = (ctrl0 & SPU2_CIPH_TYPE) >> SPU2_CIPH_TYPE_SHIFT;
338 ciph_name = spu2_ciph_type_name(ciph_type);
339 packet_log(" Cipher type: %s\n", ciph_name);
340
341 if (ciph_type != SPU2_CIPHER_TYPE_NONE) {
342 ciph_mode = (ctrl0 & SPU2_CIPH_MODE) >> SPU2_CIPH_MODE_SHIFT;
343 ciph_mode_name = spu2_ciph_mode_name(ciph_mode);
344 packet_log(" Cipher mode: %s\n", ciph_mode_name);
345 }
346
347 cfb = (ctrl0 & SPU2_CFB_MASK) >> SPU2_CFB_MASK_SHIFT;
348 packet_log(" CFB %#x\n", cfb);
349
350 proto = (ctrl0 & SPU2_PROTO_SEL) >> SPU2_PROTO_SEL_SHIFT;
351 packet_log(" protocol %#x\n", proto);
352
353 if (ctrl0 & SPU2_HASH_FIRST)
354 packet_log(" hash first\n");
355 else
356 packet_log(" cipher first\n");
357
358 if (ctrl0 & SPU2_CHK_TAG)
359 packet_log(" check tag\n");
360
361 hash_type = (ctrl0 & SPU2_HASH_TYPE) >> SPU2_HASH_TYPE_SHIFT;
362 hash_name = spu2_hash_type_name(hash_type);
363 packet_log(" Hash type: %s\n", hash_name);
364
365 if (hash_type != SPU2_HASH_TYPE_NONE) {
366 hash_mode = (ctrl0 & SPU2_HASH_MODE) >> SPU2_HASH_MODE_SHIFT;
367 hash_mode_name = spu2_hash_mode_name(hash_mode);
368 packet_log(" Hash mode: %s\n", hash_mode_name);
369 }
370
371 if (ctrl0 & SPU2_CIPH_PAD_EN) {
372 packet_log(" Cipher pad: %#2llx\n",
373 (ctrl0 & SPU2_CIPH_PAD) >> SPU2_CIPH_PAD_SHIFT);
374 }
375}
376
377/* Dump FMD ctrl1. The ctrl1 input is in host byte order */
378static void spu2_dump_fmd_ctrl1(u64 ctrl1)
379{
380 u8 hash_key_len;
381 u8 ciph_key_len;
382 u8 ret_iv_len;
383 u8 iv_offset;
384 u8 iv_len;
385 u8 hash_tag_len;
386 u8 ret_md;
387
388 packet_log(" FMD CTRL1 %#16llx\n", ctrl1);
389 if (ctrl1 & SPU2_TAG_LOC)
390 packet_log(" Tag after payload\n");
391
392 packet_log(" Msg includes ");
393 if (ctrl1 & SPU2_HAS_FR_DATA)
394 packet_log("FD ");
395 if (ctrl1 & SPU2_HAS_AAD1)
396 packet_log("AAD1 ");
397 if (ctrl1 & SPU2_HAS_NAAD)
398 packet_log("NAAD ");
399 if (ctrl1 & SPU2_HAS_AAD2)
400 packet_log("AAD2 ");
401 if (ctrl1 & SPU2_HAS_ESN)
402 packet_log("ESN ");
403 packet_log("\n");
404
405 hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT;
406 packet_log(" Hash key len %u\n", hash_key_len);
407
408 ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT;
409 packet_log(" Cipher key len %u\n", ciph_key_len);
410
411 if (ctrl1 & SPU2_GENIV)
412 packet_log(" Generate IV\n");
413
414 if (ctrl1 & SPU2_HASH_IV)
415 packet_log(" IV included in hash\n");
416
417 if (ctrl1 & SPU2_RET_IV)
418 packet_log(" Return IV in output before payload\n");
419
420 ret_iv_len = (ctrl1 & SPU2_RET_IV_LEN) >> SPU2_RET_IV_LEN_SHIFT;
421 packet_log(" Length of returned IV %u bytes\n",
422 ret_iv_len ? ret_iv_len : 16);
423
424 iv_offset = (ctrl1 & SPU2_IV_OFFSET) >> SPU2_IV_OFFSET_SHIFT;
425 packet_log(" IV offset %u\n", iv_offset);
426
427 iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT;
428 packet_log(" Input IV len %u bytes\n", iv_len);
429
430 hash_tag_len = (ctrl1 & SPU2_HASH_TAG_LEN) >> SPU2_HASH_TAG_LEN_SHIFT;
431 packet_log(" Hash tag length %u bytes\n", hash_tag_len);
432
433 packet_log(" Return ");
434 ret_md = (ctrl1 & SPU2_RETURN_MD) >> SPU2_RETURN_MD_SHIFT;
435 if (ret_md)
436 packet_log("FMD ");
437 if (ret_md == SPU2_RET_FMD_OMD)
438 packet_log("OMD ");
439 else if (ret_md == SPU2_RET_FMD_OMD_IV)
440 packet_log("OMD IV ");
441 if (ctrl1 & SPU2_RETURN_FD)
442 packet_log("FD ");
443 if (ctrl1 & SPU2_RETURN_AAD1)
444 packet_log("AAD1 ");
445 if (ctrl1 & SPU2_RETURN_NAAD)
446 packet_log("NAAD ");
447 if (ctrl1 & SPU2_RETURN_AAD2)
448 packet_log("AAD2 ");
449 if (ctrl1 & SPU2_RETURN_PAY)
450 packet_log("Payload");
451 packet_log("\n");
452}
453
454/* Dump FMD ctrl2. The ctrl2 input is in host byte order */
455static void spu2_dump_fmd_ctrl2(u64 ctrl2)
456{
457 packet_log(" FMD CTRL2 %#16llx\n", ctrl2);
458
459 packet_log(" AAD1 offset %llu length %llu bytes\n",
460 ctrl2 & SPU2_AAD1_OFFSET,
461 (ctrl2 & SPU2_AAD1_LEN) >> SPU2_AAD1_LEN_SHIFT);
462 packet_log(" AAD2 offset %llu\n",
463 (ctrl2 & SPU2_AAD2_OFFSET) >> SPU2_AAD2_OFFSET_SHIFT);
464 packet_log(" Payload offset %llu\n",
465 (ctrl2 & SPU2_PL_OFFSET) >> SPU2_PL_OFFSET_SHIFT);
466}
467
468/* Dump FMD ctrl3. The ctrl3 input is in host byte order */
469static void spu2_dump_fmd_ctrl3(u64 ctrl3)
470{
471 packet_log(" FMD CTRL3 %#16llx\n", ctrl3);
472
473 packet_log(" Payload length %llu bytes\n", ctrl3 & SPU2_PL_LEN);
474 packet_log(" TLS length %llu bytes\n",
475 (ctrl3 & SPU2_TLS_LEN) >> SPU2_TLS_LEN_SHIFT);
476}
477
478static void spu2_dump_fmd(struct SPU2_FMD *fmd)
479{
480 spu2_dump_fmd_ctrl0(le64_to_cpu(fmd->ctrl0));
481 spu2_dump_fmd_ctrl1(le64_to_cpu(fmd->ctrl1));
482 spu2_dump_fmd_ctrl2(le64_to_cpu(fmd->ctrl2));
483 spu2_dump_fmd_ctrl3(le64_to_cpu(fmd->ctrl3));
484}
485
486static void spu2_dump_omd(u8 *omd, u16 hash_key_len, u16 ciph_key_len,
487 u16 hash_iv_len, u16 ciph_iv_len)
488{
489 u8 *ptr = omd;
490
491 packet_log(" OMD:\n");
492
493 if (hash_key_len) {
494 packet_log(" Hash Key Length %u bytes\n", hash_key_len);
495 packet_dump(" KEY: ", ptr, hash_key_len);
496 ptr += hash_key_len;
497 }
498
499 if (ciph_key_len) {
500 packet_log(" Cipher Key Length %u bytes\n", ciph_key_len);
501 packet_dump(" KEY: ", ptr, ciph_key_len);
502 ptr += ciph_key_len;
503 }
504
505 if (hash_iv_len) {
506 packet_log(" Hash IV Length %u bytes\n", hash_iv_len);
507 packet_dump(" hash IV: ", ptr, hash_iv_len);
508 ptr += ciph_key_len;
509 }
510
511 if (ciph_iv_len) {
512 packet_log(" Cipher IV Length %u bytes\n", ciph_iv_len);
513 packet_dump(" cipher IV: ", ptr, ciph_iv_len);
514 }
515}
516
517/* Dump a SPU2 header for debug */
518void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len)
519{
520 struct SPU2_FMD *fmd = (struct SPU2_FMD *)buf;
521 u8 *omd;
522 u64 ctrl1;
523 u16 hash_key_len;
524 u16 ciph_key_len;
525 u16 hash_iv_len;
526 u16 ciph_iv_len;
527 u16 omd_len;
528
529 packet_log("\n");
530 packet_log("SPU2 message header %p len: %u\n", buf, buf_len);
531
532 spu2_dump_fmd(fmd);
533 omd = (u8 *)(fmd + 1);
534
535 ctrl1 = le64_to_cpu(fmd->ctrl1);
536 hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT;
537 ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT;
538 hash_iv_len = 0;
539 ciph_iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT;
540 spu2_dump_omd(omd, hash_key_len, ciph_key_len, hash_iv_len,
541 ciph_iv_len);
542
543 /* Double check sanity */
544 omd_len = hash_key_len + ciph_key_len + hash_iv_len + ciph_iv_len;
545 if (FMD_SIZE + omd_len != buf_len) {
546 packet_log
547 (" Packet parsed incorrectly. buf_len %u, sum of MD %zu\n",
548 buf_len, FMD_SIZE + omd_len);
549 }
550 packet_log("\n");
551}
552
553/**
554 * spu2_fmd_init() - At setkey time, initialize the fixed meta data for
555 * subsequent ablkcipher requests for this context.
556 * @spu2_cipher_type: Cipher algorithm
557 * @spu2_mode: Cipher mode
558 * @cipher_key_len: Length of cipher key, in bytes
559 * @cipher_iv_len: Length of cipher initialization vector, in bytes
560 *
561 * Return: 0 (success)
562 */
563static int spu2_fmd_init(struct SPU2_FMD *fmd,
564 enum spu2_cipher_type spu2_type,
565 enum spu2_cipher_mode spu2_mode,
566 u32 cipher_key_len, u32 cipher_iv_len)
567{
568 u64 ctrl0;
569 u64 ctrl1;
570 u64 ctrl2;
571 u64 ctrl3;
572 u32 aad1_offset;
573 u32 aad2_offset;
574 u16 aad1_len = 0;
575 u64 payload_offset;
576
577 ctrl0 = (spu2_type << SPU2_CIPH_TYPE_SHIFT) |
578 (spu2_mode << SPU2_CIPH_MODE_SHIFT);
579
580 ctrl1 = (cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) |
581 ((u64)cipher_iv_len << SPU2_IV_LEN_SHIFT) |
582 ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT) | SPU2_RETURN_PAY;
583
584 /*
585 * AAD1 offset is from start of FD. FD length is always 0 for this
586 * driver. So AAD1_offset is always 0.
587 */
588 aad1_offset = 0;
589 aad2_offset = aad1_offset;
590 payload_offset = 0;
591 ctrl2 = aad1_offset |
592 (aad1_len << SPU2_AAD1_LEN_SHIFT) |
593 (aad2_offset << SPU2_AAD2_OFFSET_SHIFT) |
594 (payload_offset << SPU2_PL_OFFSET_SHIFT);
595
596 ctrl3 = 0;
597
598 fmd->ctrl0 = cpu_to_le64(ctrl0);
599 fmd->ctrl1 = cpu_to_le64(ctrl1);
600 fmd->ctrl2 = cpu_to_le64(ctrl2);
601 fmd->ctrl3 = cpu_to_le64(ctrl3);
602
603 return 0;
604}
605
606/**
607 * spu2_fmd_ctrl0_write() - Write ctrl0 field in fixed metadata (FMD) field of
608 * SPU request packet.
609 * @fmd: Start of FMD field to be written
610 * @is_inbound: true if decrypting. false if encrypting.
611 * @authFirst: true if alg authenticates before encrypting
612 * @protocol: protocol selector
613 * @cipher_type: cipher algorithm
614 * @cipher_mode: cipher mode
615 * @auth_type: authentication type
616 * @auth_mode: authentication mode
617 */
618static void spu2_fmd_ctrl0_write(struct SPU2_FMD *fmd,
619 bool is_inbound, bool auth_first,
620 enum spu2_proto_sel protocol,
621 enum spu2_cipher_type cipher_type,
622 enum spu2_cipher_mode cipher_mode,
623 enum spu2_hash_type auth_type,
624 enum spu2_hash_mode auth_mode)
625{
626 u64 ctrl0 = 0;
627
628 if ((cipher_type != SPU2_CIPHER_TYPE_NONE) && !is_inbound)
629 ctrl0 |= SPU2_CIPH_ENCRYPT_EN;
630
631 ctrl0 |= ((u64)cipher_type << SPU2_CIPH_TYPE_SHIFT) |
632 ((u64)cipher_mode << SPU2_CIPH_MODE_SHIFT);
633
634 if (protocol)
635 ctrl0 |= (u64)protocol << SPU2_PROTO_SEL_SHIFT;
636
637 if (auth_first)
638 ctrl0 |= SPU2_HASH_FIRST;
639
640 if (is_inbound && (auth_type != SPU2_HASH_TYPE_NONE))
641 ctrl0 |= SPU2_CHK_TAG;
642
643 ctrl0 |= (((u64)auth_type << SPU2_HASH_TYPE_SHIFT) |
644 ((u64)auth_mode << SPU2_HASH_MODE_SHIFT));
645
646 fmd->ctrl0 = cpu_to_le64(ctrl0);
647}
648
649/**
650 * spu2_fmd_ctrl1_write() - Write ctrl1 field in fixed metadata (FMD) field of
651 * SPU request packet.
652 * @fmd: Start of FMD field to be written
653 * @assoc_size: Length of additional associated data, in bytes
654 * @auth_key_len: Length of authentication key, in bytes
655 * @cipher_key_len: Length of cipher key, in bytes
656 * @gen_iv: If true, hw generates IV and returns in response
657 * @hash_iv: IV participates in hash. Used for IPSEC and TLS.
658 * @return_iv: Return IV in output packet before payload
659 * @ret_iv_len: Length of IV returned from SPU, in bytes
660 * @ret_iv_offset: Offset into full IV of start of returned IV
661 * @cipher_iv_len: Length of input cipher IV, in bytes
662 * @digest_size: Length of digest (aka, hash tag or ICV), in bytes
663 * @return_payload: Return payload in SPU response
664 * @return_md : return metadata in SPU response
665 *
666 * Packet can have AAD2 w/o AAD1. For algorithms currently supported,
667 * associated data goes in AAD2.
668 */
669static void spu2_fmd_ctrl1_write(struct SPU2_FMD *fmd, bool is_inbound,
670 u64 assoc_size,
671 u64 auth_key_len, u64 cipher_key_len,
672 bool gen_iv, bool hash_iv, bool return_iv,
673 u64 ret_iv_len, u64 ret_iv_offset,
674 u64 cipher_iv_len, u64 digest_size,
675 bool return_payload, bool return_md)
676{
677 u64 ctrl1 = 0;
678
679 if (is_inbound && digest_size)
680 ctrl1 |= SPU2_TAG_LOC;
681
682 if (assoc_size) {
683 ctrl1 |= SPU2_HAS_AAD2;
684 ctrl1 |= SPU2_RETURN_AAD2; /* need aad2 for gcm aes esp */
685 }
686
687 if (auth_key_len)
688 ctrl1 |= ((auth_key_len << SPU2_HASH_KEY_LEN_SHIFT) &
689 SPU2_HASH_KEY_LEN);
690
691 if (cipher_key_len)
692 ctrl1 |= ((cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) &
693 SPU2_CIPH_KEY_LEN);
694
695 if (gen_iv)
696 ctrl1 |= SPU2_GENIV;
697
698 if (hash_iv)
699 ctrl1 |= SPU2_HASH_IV;
700
701 if (return_iv) {
702 ctrl1 |= SPU2_RET_IV;
703 ctrl1 |= ret_iv_len << SPU2_RET_IV_LEN_SHIFT;
704 ctrl1 |= ret_iv_offset << SPU2_IV_OFFSET_SHIFT;
705 }
706
707 ctrl1 |= ((cipher_iv_len << SPU2_IV_LEN_SHIFT) & SPU2_IV_LEN);
708
709 if (digest_size)
710 ctrl1 |= ((digest_size << SPU2_HASH_TAG_LEN_SHIFT) &
711 SPU2_HASH_TAG_LEN);
712
713 /* Let's ask for the output pkt to include FMD, but don't need to
714 * get keys and IVs back in OMD.
715 */
716 if (return_md)
717 ctrl1 |= ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT);
718 else
719 ctrl1 |= ((u64)SPU2_RET_NO_MD << SPU2_RETURN_MD_SHIFT);
720
721 /* Crypto API does not get assoc data back. So no need for AAD2. */
722
723 if (return_payload)
724 ctrl1 |= SPU2_RETURN_PAY;
725
726 fmd->ctrl1 = cpu_to_le64(ctrl1);
727}
728
729/**
730 * spu2_fmd_ctrl2_write() - Set the ctrl2 field in the fixed metadata field of
731 * SPU2 header.
732 * @fmd: Start of FMD field to be written
733 * @cipher_offset: Number of bytes from Start of Packet (end of FD field) where
734 * data to be encrypted or decrypted begins
735 * @auth_key_len: Length of authentication key, in bytes
736 * @auth_iv_len: Length of authentication initialization vector, in bytes
737 * @cipher_key_len: Length of cipher key, in bytes
738 * @cipher_iv_len: Length of cipher IV, in bytes
739 */
740static void spu2_fmd_ctrl2_write(struct SPU2_FMD *fmd, u64 cipher_offset,
741 u64 auth_key_len, u64 auth_iv_len,
742 u64 cipher_key_len, u64 cipher_iv_len)
743{
744 u64 ctrl2;
745 u64 aad1_offset;
746 u64 aad2_offset;
747 u16 aad1_len = 0;
748 u64 payload_offset;
749
750 /* AAD1 offset is from start of FD. FD length always 0. */
751 aad1_offset = 0;
752
753 aad2_offset = aad1_offset;
754 payload_offset = cipher_offset;
755 ctrl2 = aad1_offset |
756 (aad1_len << SPU2_AAD1_LEN_SHIFT) |
757 (aad2_offset << SPU2_AAD2_OFFSET_SHIFT) |
758 (payload_offset << SPU2_PL_OFFSET_SHIFT);
759
760 fmd->ctrl2 = cpu_to_le64(ctrl2);
761}
762
763/**
764 * spu2_fmd_ctrl3_write() - Set the ctrl3 field in FMD
765 * @fmd: Fixed meta data. First field in SPU2 msg header.
766 * @payload_len: Length of payload, in bytes
767 */
768static void spu2_fmd_ctrl3_write(struct SPU2_FMD *fmd, u64 payload_len)
769{
770 u64 ctrl3;
771
772 ctrl3 = payload_len & SPU2_PL_LEN;
773
774 fmd->ctrl3 = cpu_to_le64(ctrl3);
775}
776
777/**
778 * spu2_ctx_max_payload() - Determine the maximum length of the payload for a
779 * SPU message for a given cipher and hash alg context.
780 * @cipher_alg: The cipher algorithm
781 * @cipher_mode: The cipher mode
782 * @blocksize: The size of a block of data for this algo
783 *
784 * For SPU2, the hardware generally ignores the PayloadLen field in ctrl3 of
785 * FMD and just keeps computing until it receives a DMA descriptor with the EOF
786 * flag set. So we consider the max payload to be infinite. AES CCM is an
787 * exception.
788 *
789 * Return: Max payload length in bytes
790 */
791u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
792 enum spu_cipher_mode cipher_mode,
793 unsigned int blocksize)
794{
795 if ((cipher_alg == CIPHER_ALG_AES) &&
796 (cipher_mode == CIPHER_MODE_CCM)) {
797 u32 excess = SPU2_MAX_PAYLOAD % blocksize;
798
799 return SPU2_MAX_PAYLOAD - excess;
800 } else {
801 return SPU_MAX_PAYLOAD_INF;
802 }
803}
804
805/**
806 * spu_payload_length() - Given a SPU2 message header, extract the payload
807 * length.
808 * @spu_hdr: Start of SPU message header (FMD)
809 *
810 * Return: payload length, in bytes
811 */
812u32 spu2_payload_length(u8 *spu_hdr)
813{
814 struct SPU2_FMD *fmd = (struct SPU2_FMD *)spu_hdr;
815 u32 pl_len;
816 u64 ctrl3;
817
818 ctrl3 = le64_to_cpu(fmd->ctrl3);
819 pl_len = ctrl3 & SPU2_PL_LEN;
820
821 return pl_len;
822}
823
824/**
825 * spu_response_hdr_len() - Determine the expected length of a SPU response
826 * header.
827 * @auth_key_len: Length of authentication key, in bytes
828 * @enc_key_len: Length of encryption key, in bytes
829 *
830 * For SPU2, includes just FMD. OMD is never requested.
831 *
832 * Return: Length of FMD, in bytes
833 */
834u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)
835{
836 return FMD_SIZE;
837}
838
839/**
840 * spu_hash_pad_len() - Calculate the length of hash padding required to extend
841 * data to a full block size.
842 * @hash_alg: hash algorithm
843 * @hash_mode: hash mode
844 * @chunksize: length of data, in bytes
845 * @hash_block_size: size of a hash block, in bytes
846 *
847 * SPU2 hardware does all hash padding
848 *
849 * Return: length of hash pad in bytes
850 */
851u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
852 u32 chunksize, u16 hash_block_size)
853{
854 return 0;
855}
856
857/**
858 * spu2_gcm_ccm_padlen() - Determine the length of GCM/CCM padding for either
859 * the AAD field or the data.
860 *
861 * Return: 0. Unlike SPU-M, SPU2 hardware does any GCM/CCM padding required.
862 */
863u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
864 unsigned int data_size)
865{
866 return 0;
867}
868
869/**
870 * spu_assoc_resp_len() - Determine the size of the AAD2 buffer needed to catch
871 * associated data in a SPU2 output packet.
872 * @cipher_mode: cipher mode
873 * @assoc_len: length of additional associated data, in bytes
874 * @iv_len: length of initialization vector, in bytes
875 * @is_encrypt: true if encrypting. false if decrypt.
876 *
877 * Return: Length of buffer to catch associated data in response
878 */
879u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode,
880 unsigned int assoc_len, unsigned int iv_len,
881 bool is_encrypt)
882{
883 u32 resp_len = assoc_len;
884
885 if (is_encrypt)
886 /* gcm aes esp has to write 8-byte IV in response */
887 resp_len += iv_len;
888 return resp_len;
889}
890
891/*
892 * spu_aead_ivlen() - Calculate the length of the AEAD IV to be included
893 * in a SPU request after the AAD and before the payload.
894 * @cipher_mode: cipher mode
895 * @iv_ctr_len: initialization vector length in bytes
896 *
897 * For SPU2, AEAD IV is included in OMD and does not need to be repeated
898 * prior to the payload.
899 *
900 * Return: Length of AEAD IV in bytes
901 */
902u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)
903{
904 return 0;
905}
906
907/**
908 * spu2_hash_type() - Determine the type of hash operation.
909 * @src_sent: The number of bytes in the current request that have already
910 * been sent to the SPU to be hashed.
911 *
912 * SPU2 always does a FULL hash operation
913 */
914enum hash_type spu2_hash_type(u32 src_sent)
915{
916 return HASH_TYPE_FULL;
917}
918
919/**
920 * spu2_digest_size() - Determine the size of a hash digest to expect the SPU to
921 * return.
922 * alg_digest_size: Number of bytes in the final digest for the given algo
923 * alg: The hash algorithm
924 * htype: Type of hash operation (init, update, full, etc)
925 *
926 */
927u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg,
928 enum hash_type htype)
929{
930 return alg_digest_size;
931}
932
933/**
934 * spu_create_request() - Build a SPU2 request message header, includint FMD and
935 * OMD.
936 * @spu_hdr: Start of buffer where SPU request header is to be written
937 * @req_opts: SPU request message options
938 * @cipher_parms: Parameters related to cipher algorithm
939 * @hash_parms: Parameters related to hash algorithm
940 * @aead_parms: Parameters related to AEAD operation
941 * @data_size: Length of data to be encrypted or authenticated. If AEAD, does
942 * not include length of AAD.
943 *
944 * Construct the message starting at spu_hdr. Caller should allocate this buffer
945 * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.
946 *
947 * Return: the length of the SPU header in bytes. 0 if an error occurs.
948 */
949u32 spu2_create_request(u8 *spu_hdr,
950 struct spu_request_opts *req_opts,
951 struct spu_cipher_parms *cipher_parms,
952 struct spu_hash_parms *hash_parms,
953 struct spu_aead_parms *aead_parms,
954 unsigned int data_size)
955{
956 struct SPU2_FMD *fmd;
957 u8 *ptr;
958 unsigned int buf_len;
959 int err;
960 enum spu2_cipher_type spu2_ciph_type = SPU2_CIPHER_TYPE_NONE;
961 enum spu2_cipher_mode spu2_ciph_mode;
962 enum spu2_hash_type spu2_auth_type = SPU2_HASH_TYPE_NONE;
963 enum spu2_hash_mode spu2_auth_mode;
964 bool return_md = true;
965 enum spu2_proto_sel proto = SPU2_PROTO_RESV;
966
967 /* size of the payload */
968 unsigned int payload_len =
969 hash_parms->prebuf_len + data_size + hash_parms->pad_len -
970 ((req_opts->is_aead && req_opts->is_inbound) ?
971 hash_parms->digestsize : 0);
972
973 /* offset of prebuf or data from start of AAD2 */
974 unsigned int cipher_offset = aead_parms->assoc_size +
975 aead_parms->aad_pad_len + aead_parms->iv_len;
976
977#ifdef DEBUG
978 /* total size of the data following OMD (without STAT word padding) */
979 unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,
980 aead_parms->iv_len,
981 hash_parms->prebuf_len,
982 data_size,
983 aead_parms->aad_pad_len,
984 aead_parms->data_pad_len,
985 hash_parms->pad_len);
986#endif
987 unsigned int assoc_size = aead_parms->assoc_size;
988
989 if (req_opts->is_aead &&
990 (cipher_parms->alg == CIPHER_ALG_AES) &&
991 (cipher_parms->mode == CIPHER_MODE_GCM))
992 /*
993 * On SPU 2, aes gcm cipher first on encrypt, auth first on
994 * decrypt
995 */
996 req_opts->auth_first = req_opts->is_inbound;
997
998 /* and do opposite for ccm (auth 1st on encrypt) */
999 if (req_opts->is_aead &&
1000 (cipher_parms->alg == CIPHER_ALG_AES) &&
1001 (cipher_parms->mode == CIPHER_MODE_CCM))
1002 req_opts->auth_first = !req_opts->is_inbound;
1003
1004 flow_log("%s()\n", __func__);
1005 flow_log(" in:%u authFirst:%u\n",
1006 req_opts->is_inbound, req_opts->auth_first);
1007 flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
1008 cipher_parms->mode, cipher_parms->type);
1009 flow_log(" is_esp: %s\n", req_opts->is_esp ? "yes" : "no");
1010 flow_log(" key: %d\n", cipher_parms->key_len);
1011 flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
1012 flow_log(" iv: %d\n", cipher_parms->iv_len);
1013 flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
1014 flow_log(" auth alg:%u mode:%u type %u\n",
1015 hash_parms->alg, hash_parms->mode, hash_parms->type);
1016 flow_log(" digestsize: %u\n", hash_parms->digestsize);
1017 flow_log(" authkey: %d\n", hash_parms->key_len);
1018 flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len);
1019 flow_log(" assoc_size:%u\n", assoc_size);
1020 flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len);
1021 flow_log(" data_size:%u\n", data_size);
1022 flow_log(" hash_pad_len:%u\n", hash_parms->pad_len);
1023 flow_log(" real_db_size:%u\n", real_db_size);
1024 flow_log(" cipher_offset:%u payload_len:%u\n",
1025 cipher_offset, payload_len);
1026 flow_log(" aead_iv: %u\n", aead_parms->iv_len);
1027
1028 /* Convert to spu2 values for cipher alg, hash alg */
1029 err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode,
1030 cipher_parms->type,
1031 &spu2_ciph_type, &spu2_ciph_mode);
1032
1033 /* If we are doing GCM hashing only - either via rfc4543 transform
1034 * or because we happen to do GCM with AAD only and no payload - we
1035 * need to configure hardware to use hash key rather than cipher key
1036 * and put data into payload. This is because unlike SPU-M, running
1037 * GCM cipher with 0 size payload is not permitted.
1038 */
1039 if ((req_opts->is_rfc4543) ||
1040 ((spu2_ciph_mode == SPU2_CIPHER_MODE_GCM) &&
1041 (payload_len == 0))) {
1042 /* Use hashing (only) and set up hash key */
1043 spu2_ciph_type = SPU2_CIPHER_TYPE_NONE;
1044 hash_parms->key_len = cipher_parms->key_len;
1045 memcpy(hash_parms->key_buf, cipher_parms->key_buf,
1046 cipher_parms->key_len);
1047 cipher_parms->key_len = 0;
1048
1049 if (req_opts->is_rfc4543)
1050 payload_len += assoc_size;
1051 else
1052 payload_len = assoc_size;
1053 cipher_offset = 0;
1054 assoc_size = 0;
1055 }
1056
1057 if (err)
1058 return 0;
1059
1060 flow_log("spu2 cipher type %s, cipher mode %s\n",
1061 spu2_ciph_type_name(spu2_ciph_type),
1062 spu2_ciph_mode_name(spu2_ciph_mode));
1063
1064 err = spu2_hash_xlate(hash_parms->alg, hash_parms->mode,
1065 hash_parms->type,
1066 cipher_parms->type,
1067 &spu2_auth_type, &spu2_auth_mode);
1068 if (err)
1069 return 0;
1070
1071 flow_log("spu2 hash type %s, hash mode %s\n",
1072 spu2_hash_type_name(spu2_auth_type),
1073 spu2_hash_mode_name(spu2_auth_mode));
1074
1075 fmd = (struct SPU2_FMD *)spu_hdr;
1076
1077 spu2_fmd_ctrl0_write(fmd, req_opts->is_inbound, req_opts->auth_first,
1078 proto, spu2_ciph_type, spu2_ciph_mode,
1079 spu2_auth_type, spu2_auth_mode);
1080
1081 spu2_fmd_ctrl1_write(fmd, req_opts->is_inbound, assoc_size,
1082 hash_parms->key_len, cipher_parms->key_len,
1083 false, false,
1084 aead_parms->return_iv, aead_parms->ret_iv_len,
1085 aead_parms->ret_iv_off,
1086 cipher_parms->iv_len, hash_parms->digestsize,
1087 !req_opts->bd_suppress, return_md);
1088
1089 spu2_fmd_ctrl2_write(fmd, cipher_offset, hash_parms->key_len, 0,
1090 cipher_parms->key_len, cipher_parms->iv_len);
1091
1092 spu2_fmd_ctrl3_write(fmd, payload_len);
1093
1094 ptr = (u8 *)(fmd + 1);
1095 buf_len = sizeof(struct SPU2_FMD);
1096
1097 /* Write OMD */
1098 if (hash_parms->key_len) {
1099 memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);
1100 ptr += hash_parms->key_len;
1101 buf_len += hash_parms->key_len;
1102 }
1103 if (cipher_parms->key_len) {
1104 memcpy(ptr, cipher_parms->key_buf, cipher_parms->key_len);
1105 ptr += cipher_parms->key_len;
1106 buf_len += cipher_parms->key_len;
1107 }
1108 if (cipher_parms->iv_len) {
1109 memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);
1110 ptr += cipher_parms->iv_len;
1111 buf_len += cipher_parms->iv_len;
1112 }
1113
1114 packet_dump(" SPU request header: ", spu_hdr, buf_len);
1115
1116 return buf_len;
1117}
1118
1119/**
1120 * spu_cipher_req_init() - Build an ablkcipher SPU2 request message header,
1121 * including FMD and OMD.
1122 * @spu_hdr: Location of start of SPU request (FMD field)
1123 * @cipher_parms: Parameters describing cipher request
1124 *
1125 * Called at setkey time to initialize a msg header that can be reused for all
1126 * subsequent ablkcipher requests. Construct the message starting at spu_hdr.
1127 * Caller should allocate this buffer in DMA-able memory at least
1128 * SPU_HEADER_ALLOC_LEN bytes long.
1129 *
1130 * Return: the total length of the SPU header (FMD and OMD) in bytes. 0 if an
1131 * error occurs.
1132 */
1133u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)
1134{
1135 struct SPU2_FMD *fmd;
1136 u8 *omd;
1137 enum spu2_cipher_type spu2_type = SPU2_CIPHER_TYPE_NONE;
1138 enum spu2_cipher_mode spu2_mode;
1139 int err;
1140
1141 flow_log("%s()\n", __func__);
1142 flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
1143 cipher_parms->mode, cipher_parms->type);
1144 flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len);
1145 flow_log(" key: %d\n", cipher_parms->key_len);
1146 flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
1147
1148 /* Convert to spu2 values */
1149 err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode,
1150 cipher_parms->type, &spu2_type, &spu2_mode);
1151 if (err)
1152 return 0;
1153
1154 flow_log("spu2 cipher type %s, cipher mode %s\n",
1155 spu2_ciph_type_name(spu2_type),
1156 spu2_ciph_mode_name(spu2_mode));
1157
1158 /* Construct the FMD header */
1159 fmd = (struct SPU2_FMD *)spu_hdr;
1160 err = spu2_fmd_init(fmd, spu2_type, spu2_mode, cipher_parms->key_len,
1161 cipher_parms->iv_len);
1162 if (err)
1163 return 0;
1164
1165 /* Write cipher key to OMD */
1166 omd = (u8 *)(fmd + 1);
1167 if (cipher_parms->key_buf && cipher_parms->key_len)
1168 memcpy(omd, cipher_parms->key_buf, cipher_parms->key_len);
1169
1170 packet_dump(" SPU request header: ", spu_hdr,
1171 FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len);
1172
1173 return FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len;
1174}
1175
1176/**
1177 * spu_cipher_req_finish() - Finish building a SPU request message header for a
1178 * block cipher request.
1179 * @spu_hdr: Start of the request message header (MH field)
1180 * @spu_req_hdr_len: Length in bytes of the SPU request header
1181 * @isInbound: 0 encrypt, 1 decrypt
1182 * @cipher_parms: Parameters describing cipher operation to be performed
1183 * @update_key: If true, rewrite the cipher key in SCTX
1184 * @data_size: Length of the data in the BD field
1185 *
1186 * Assumes much of the header was already filled in at setkey() time in
1187 * spu_cipher_req_init().
1188 * spu_cipher_req_init() fills in the encryption key. For RC4, when submitting a
1189 * request for a non-first chunk, we use the 260-byte SUPDT field from the
1190 * previous response as the key. update_key is true for this case. Unused in all
1191 * other cases.
1192 */
1193void spu2_cipher_req_finish(u8 *spu_hdr,
1194 u16 spu_req_hdr_len,
1195 unsigned int is_inbound,
1196 struct spu_cipher_parms *cipher_parms,
1197 bool update_key,
1198 unsigned int data_size)
1199{
1200 struct SPU2_FMD *fmd;
1201 u8 *omd; /* start of optional metadata */
1202 u64 ctrl0;
1203 u64 ctrl3;
1204
1205 flow_log("%s()\n", __func__);
1206 flow_log(" in: %u\n", is_inbound);
1207 flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
1208 cipher_parms->type);
1209 if (update_key) {
1210 flow_log(" cipher key len: %u\n", cipher_parms->key_len);
1211 flow_dump(" key: ", cipher_parms->key_buf,
1212 cipher_parms->key_len);
1213 }
1214 flow_log(" iv len: %d\n", cipher_parms->iv_len);
1215 flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
1216 flow_log(" data_size: %u\n", data_size);
1217
1218 fmd = (struct SPU2_FMD *)spu_hdr;
1219 omd = (u8 *)(fmd + 1);
1220
1221 /*
1222 * FMD ctrl0 was initialized at setkey time. update it to indicate
1223 * whether we are encrypting or decrypting.
1224 */
1225 ctrl0 = le64_to_cpu(fmd->ctrl0);
1226 if (is_inbound)
1227 ctrl0 &= ~SPU2_CIPH_ENCRYPT_EN; /* decrypt */
1228 else
1229 ctrl0 |= SPU2_CIPH_ENCRYPT_EN; /* encrypt */
1230 fmd->ctrl0 = cpu_to_le64(ctrl0);
1231
1232 if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) {
1233 /* cipher iv provided so put it in here */
1234 memcpy(omd + cipher_parms->key_len, cipher_parms->iv_buf,
1235 cipher_parms->iv_len);
1236 }
1237
1238 ctrl3 = le64_to_cpu(fmd->ctrl3);
1239 data_size &= SPU2_PL_LEN;
1240 ctrl3 |= data_size;
1241 fmd->ctrl3 = cpu_to_le64(ctrl3);
1242
1243 packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len);
1244}
1245
1246/**
1247 * spu_request_pad() - Create pad bytes at the end of the data.
1248 * @pad_start: Start of buffer where pad bytes are to be written
1249 * @gcm_padding: Length of GCM padding, in bytes
1250 * @hash_pad_len: Number of bytes of padding extend data to full block
1251 * @auth_alg: Authentication algorithm
1252 * @auth_mode: Authentication mode
1253 * @total_sent: Length inserted at end of hash pad
1254 * @status_padding: Number of bytes of padding to align STATUS word
1255 *
1256 * There may be three forms of pad:
1257 * 1. GCM pad - for GCM mode ciphers, pad to 16-byte alignment
1258 * 2. hash pad - pad to a block length, with 0x80 data terminator and
1259 * size at the end
1260 * 3. STAT pad - to ensure the STAT field is 4-byte aligned
1261 */
1262void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len,
1263 enum hash_alg auth_alg, enum hash_mode auth_mode,
1264 unsigned int total_sent, u32 status_padding)
1265{
1266 u8 *ptr = pad_start;
1267
1268 /* fix data alignent for GCM */
1269 if (gcm_padding > 0) {
1270 flow_log(" GCM: padding to 16 byte alignment: %u bytes\n",
1271 gcm_padding);
1272 memset(ptr, 0, gcm_padding);
1273 ptr += gcm_padding;
1274 }
1275
1276 if (hash_pad_len > 0) {
1277 /* clear the padding section */
1278 memset(ptr, 0, hash_pad_len);
1279
1280 /* terminate the data */
1281 *ptr = 0x80;
1282 ptr += (hash_pad_len - sizeof(u64));
1283
1284 /* add the size at the end as required per alg */
1285 if (auth_alg == HASH_ALG_MD5)
1286 *(u64 *)ptr = cpu_to_le64((u64)total_sent * 8);
1287 else /* SHA1, SHA2-224, SHA2-256 */
1288 *(u64 *)ptr = cpu_to_be64((u64)total_sent * 8);
1289 ptr += sizeof(u64);
1290 }
1291
1292 /* pad to a 4byte alignment for STAT */
1293 if (status_padding > 0) {
1294 flow_log(" STAT: padding to 4 byte alignment: %u bytes\n",
1295 status_padding);
1296
1297 memset(ptr, 0, status_padding);
1298 ptr += status_padding;
1299 }
1300}
1301
1302/**
1303 * spu2_xts_tweak_in_payload() - Indicate that SPU2 does NOT place the XTS
1304 * tweak field in the packet payload (it uses IV instead)
1305 *
1306 * Return: 0
1307 */
1308u8 spu2_xts_tweak_in_payload(void)
1309{
1310 return 0;
1311}
1312
1313/**
1314 * spu2_tx_status_len() - Return the length of the STATUS field in a SPU
1315 * response message.
1316 *
1317 * Return: Length of STATUS field in bytes.
1318 */
1319u8 spu2_tx_status_len(void)
1320{
1321 return SPU2_TX_STATUS_LEN;
1322}
1323
1324/**
1325 * spu2_rx_status_len() - Return the length of the STATUS field in a SPU
1326 * response message.
1327 *
1328 * Return: Length of STATUS field in bytes.
1329 */
1330u8 spu2_rx_status_len(void)
1331{
1332 return SPU2_RX_STATUS_LEN;
1333}
1334
1335/**
1336 * spu_status_process() - Process the status from a SPU response message.
1337 * @statp: start of STATUS word
1338 *
1339 * Return: 0 - if status is good and response should be processed
1340 * !0 - status indicates an error and response is invalid
1341 */
1342int spu2_status_process(u8 *statp)
1343{
1344 /* SPU2 status is 2 bytes by default - SPU_RX_STATUS_LEN */
1345 u16 status = le16_to_cpu(*(__le16 *)statp);
1346
1347 if (status == 0)
1348 return 0;
1349
1350 flow_log("rx status is %#x\n", status);
1351 if (status == SPU2_INVALID_ICV)
1352 return SPU_INVALID_ICV;
1353
1354 return -EBADMSG;
1355}
1356
1357/**
1358 * spu2_ccm_update_iv() - Update the IV as per the requirements for CCM mode.
1359 *
1360 * @digestsize: Digest size of this request
1361 * @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len
1362 * @assoclen: Length of AAD data
1363 * @chunksize: length of input data to be sent in this req
1364 * @is_encrypt: true if this is an output/encrypt operation
1365 * @is_esp: true if this is an ESP / RFC4309 operation
1366 *
1367 */
1368void spu2_ccm_update_iv(unsigned int digestsize,
1369 struct spu_cipher_parms *cipher_parms,
1370 unsigned int assoclen, unsigned int chunksize,
1371 bool is_encrypt, bool is_esp)
1372{
1373 int L; /* size of length field, in bytes */
1374
1375 /*
1376 * In RFC4309 mode, L is fixed at 4 bytes; otherwise, IV from
1377 * testmgr contains (L-1) in bottom 3 bits of first byte,
1378 * per RFC 3610.
1379 */
1380 if (is_esp)
1381 L = CCM_ESP_L_VALUE;
1382 else
1383 L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>
1384 CCM_B0_L_PRIME_SHIFT) + 1;
1385
1386 /* SPU2 doesn't want these length bytes nor the first byte... */
1387 cipher_parms->iv_len -= (1 + L);
1388 memmove(cipher_parms->iv_buf, &cipher_parms->iv_buf[1],
1389 cipher_parms->iv_len);
1390}
1391
1392/**
1393 * spu2_wordalign_padlen() - SPU2 does not require padding.
1394 * @data_size: length of data field in bytes
1395 *
1396 * Return: length of status field padding, in bytes (always 0 on SPU2)
1397 */
1398u32 spu2_wordalign_padlen(u32 data_size)
1399{
1400 return 0;
1401}
diff --git a/drivers/crypto/bcm/spu2.h b/drivers/crypto/bcm/spu2.h
new file mode 100644
index 000000000000..ab1f59934828
--- /dev/null
+++ b/drivers/crypto/bcm/spu2.h
@@ -0,0 +1,228 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17/*
18 * This file contains SPU message definitions specific to SPU2.
19 */
20
21#ifndef _SPU2_H
22#define _SPU2_H
23
24enum spu2_cipher_type {
25 SPU2_CIPHER_TYPE_NONE = 0x0,
26 SPU2_CIPHER_TYPE_AES128 = 0x1,
27 SPU2_CIPHER_TYPE_AES192 = 0x2,
28 SPU2_CIPHER_TYPE_AES256 = 0x3,
29 SPU2_CIPHER_TYPE_DES = 0x4,
30 SPU2_CIPHER_TYPE_3DES = 0x5,
31 SPU2_CIPHER_TYPE_LAST
32};
33
34enum spu2_cipher_mode {
35 SPU2_CIPHER_MODE_ECB = 0x0,
36 SPU2_CIPHER_MODE_CBC = 0x1,
37 SPU2_CIPHER_MODE_CTR = 0x2,
38 SPU2_CIPHER_MODE_CFB = 0x3,
39 SPU2_CIPHER_MODE_OFB = 0x4,
40 SPU2_CIPHER_MODE_XTS = 0x5,
41 SPU2_CIPHER_MODE_CCM = 0x6,
42 SPU2_CIPHER_MODE_GCM = 0x7,
43 SPU2_CIPHER_MODE_LAST
44};
45
46enum spu2_hash_type {
47 SPU2_HASH_TYPE_NONE = 0x0,
48 SPU2_HASH_TYPE_AES128 = 0x1,
49 SPU2_HASH_TYPE_AES192 = 0x2,
50 SPU2_HASH_TYPE_AES256 = 0x3,
51 SPU2_HASH_TYPE_MD5 = 0x6,
52 SPU2_HASH_TYPE_SHA1 = 0x7,
53 SPU2_HASH_TYPE_SHA224 = 0x8,
54 SPU2_HASH_TYPE_SHA256 = 0x9,
55 SPU2_HASH_TYPE_SHA384 = 0xa,
56 SPU2_HASH_TYPE_SHA512 = 0xb,
57 SPU2_HASH_TYPE_SHA512_224 = 0xc,
58 SPU2_HASH_TYPE_SHA512_256 = 0xd,
59 SPU2_HASH_TYPE_SHA3_224 = 0xe,
60 SPU2_HASH_TYPE_SHA3_256 = 0xf,
61 SPU2_HASH_TYPE_SHA3_384 = 0x10,
62 SPU2_HASH_TYPE_SHA3_512 = 0x11,
63 SPU2_HASH_TYPE_LAST
64};
65
66enum spu2_hash_mode {
67 SPU2_HASH_MODE_CMAC = 0x0,
68 SPU2_HASH_MODE_CBC_MAC = 0x1,
69 SPU2_HASH_MODE_XCBC_MAC = 0x2,
70 SPU2_HASH_MODE_HMAC = 0x3,
71 SPU2_HASH_MODE_RABIN = 0x4,
72 SPU2_HASH_MODE_CCM = 0x5,
73 SPU2_HASH_MODE_GCM = 0x6,
74 SPU2_HASH_MODE_RESERVED = 0x7,
75 SPU2_HASH_MODE_LAST
76};
77
78enum spu2_ret_md_opts {
79 SPU2_RET_NO_MD = 0, /* return no metadata */
80 SPU2_RET_FMD_OMD = 1, /* return both FMD and OMD */
81 SPU2_RET_FMD_ONLY = 2, /* return only FMD */
82 SPU2_RET_FMD_OMD_IV = 3, /* return FMD and OMD with just IVs */
83};
84
85/* Fixed Metadata format */
86struct SPU2_FMD {
87 u64 ctrl0;
88 u64 ctrl1;
89 u64 ctrl2;
90 u64 ctrl3;
91};
92
93#define FMD_SIZE sizeof(struct SPU2_FMD)
94
95/* Fixed part of request message header length in bytes. Just FMD. */
96#define SPU2_REQ_FIXED_LEN FMD_SIZE
97#define SPU2_HEADER_ALLOC_LEN (SPU_REQ_FIXED_LEN + \
98 2 * MAX_KEY_SIZE + 2 * MAX_IV_SIZE)
99
100/* FMD ctrl0 field masks */
101#define SPU2_CIPH_ENCRYPT_EN 0x1 /* 0: decrypt, 1: encrypt */
102#define SPU2_CIPH_TYPE 0xF0 /* one of spu2_cipher_type */
103#define SPU2_CIPH_TYPE_SHIFT 4
104#define SPU2_CIPH_MODE 0xF00 /* one of spu2_cipher_mode */
105#define SPU2_CIPH_MODE_SHIFT 8
106#define SPU2_CFB_MASK 0x7000 /* cipher feedback mask */
107#define SPU2_CFB_MASK_SHIFT 12
108#define SPU2_PROTO_SEL 0xF00000 /* MACsec, IPsec, TLS... */
109#define SPU2_PROTO_SEL_SHIFT 20
110#define SPU2_HASH_FIRST 0x1000000 /* 1: hash input is input pkt
111 * data
112 */
113#define SPU2_CHK_TAG 0x2000000 /* 1: check digest provided */
114#define SPU2_HASH_TYPE 0x1F0000000 /* one of spu2_hash_type */
115#define SPU2_HASH_TYPE_SHIFT 28
116#define SPU2_HASH_MODE 0xF000000000 /* one of spu2_hash_mode */
117#define SPU2_HASH_MODE_SHIFT 36
118#define SPU2_CIPH_PAD_EN 0x100000000000 /* 1: Add pad to end of payload for
119 * enc
120 */
121#define SPU2_CIPH_PAD 0xFF000000000000 /* cipher pad value */
122#define SPU2_CIPH_PAD_SHIFT 48
123
124/* FMD ctrl1 field masks */
125#define SPU2_TAG_LOC 0x1 /* 1: end of payload, 0: undef */
126#define SPU2_HAS_FR_DATA 0x2 /* 1: msg has frame data */
127#define SPU2_HAS_AAD1 0x4 /* 1: msg has AAD1 field */
128#define SPU2_HAS_NAAD 0x8 /* 1: msg has NAAD field */
129#define SPU2_HAS_AAD2 0x10 /* 1: msg has AAD2 field */
130#define SPU2_HAS_ESN 0x20 /* 1: msg has ESN field */
131#define SPU2_HASH_KEY_LEN 0xFF00 /* len of hash key in bytes.
132 * HMAC only.
133 */
134#define SPU2_HASH_KEY_LEN_SHIFT 8
135#define SPU2_CIPH_KEY_LEN 0xFF00000 /* len of cipher key in bytes */
136#define SPU2_CIPH_KEY_LEN_SHIFT 20
137#define SPU2_GENIV 0x10000000 /* 1: hw generates IV */
138#define SPU2_HASH_IV 0x20000000 /* 1: IV incl in hash */
139#define SPU2_RET_IV 0x40000000 /* 1: return IV in output msg
140 * b4 payload
141 */
142#define SPU2_RET_IV_LEN 0xF00000000 /* length in bytes of IV returned.
143 * 0 = 16 bytes
144 */
145#define SPU2_RET_IV_LEN_SHIFT 32
146#define SPU2_IV_OFFSET 0xF000000000 /* gen IV offset */
147#define SPU2_IV_OFFSET_SHIFT 36
148#define SPU2_IV_LEN 0x1F0000000000 /* length of input IV in bytes */
149#define SPU2_IV_LEN_SHIFT 40
150#define SPU2_HASH_TAG_LEN 0x7F000000000000 /* hash tag length in bytes */
151#define SPU2_HASH_TAG_LEN_SHIFT 48
152#define SPU2_RETURN_MD 0x300000000000000 /* return metadata */
153#define SPU2_RETURN_MD_SHIFT 56
154#define SPU2_RETURN_FD 0x400000000000000
155#define SPU2_RETURN_AAD1 0x800000000000000
156#define SPU2_RETURN_NAAD 0x1000000000000000
157#define SPU2_RETURN_AAD2 0x2000000000000000
158#define SPU2_RETURN_PAY 0x4000000000000000 /* return payload */
159
160/* FMD ctrl2 field masks */
161#define SPU2_AAD1_OFFSET 0xFFF /* byte offset of AAD1 field */
162#define SPU2_AAD1_LEN 0xFF000 /* length of AAD1 in bytes */
163#define SPU2_AAD1_LEN_SHIFT 12
164#define SPU2_AAD2_OFFSET 0xFFF00000 /* byte offset of AAD2 field */
165#define SPU2_AAD2_OFFSET_SHIFT 20
166#define SPU2_PL_OFFSET 0xFFFFFFFF00000000 /* payload offset from AAD2 */
167#define SPU2_PL_OFFSET_SHIFT 32
168
169/* FMD ctrl3 field masks */
170#define SPU2_PL_LEN 0xFFFFFFFF /* payload length in bytes */
171#define SPU2_TLS_LEN 0xFFFF00000000 /* TLS encrypt: cipher len
172 * TLS decrypt: compressed len
173 */
174#define SPU2_TLS_LEN_SHIFT 32
175
176/*
177 * Max value that can be represented in the Payload Length field of the
178 * ctrl3 word of FMD.
179 */
180#define SPU2_MAX_PAYLOAD SPU2_PL_LEN
181
182/* Error values returned in STATUS field of response messages */
183#define SPU2_INVALID_ICV 1
184
185void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len);
186u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
187 enum spu_cipher_mode cipher_mode,
188 unsigned int blocksize);
189u32 spu2_payload_length(u8 *spu_hdr);
190u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash);
191u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
192 u32 chunksize, u16 hash_block_size);
193u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
194 unsigned int data_size);
195u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode,
196 unsigned int assoc_len, unsigned int iv_len,
197 bool is_encrypt);
198u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode,
199 u16 iv_len);
200enum hash_type spu2_hash_type(u32 src_sent);
201u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg,
202 enum hash_type htype);
203u32 spu2_create_request(u8 *spu_hdr,
204 struct spu_request_opts *req_opts,
205 struct spu_cipher_parms *cipher_parms,
206 struct spu_hash_parms *hash_parms,
207 struct spu_aead_parms *aead_parms,
208 unsigned int data_size);
209u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms);
210void spu2_cipher_req_finish(u8 *spu_hdr,
211 u16 spu_req_hdr_len,
212 unsigned int is_inbound,
213 struct spu_cipher_parms *cipher_parms,
214 bool update_key,
215 unsigned int data_size);
216void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len,
217 enum hash_alg auth_alg, enum hash_mode auth_mode,
218 unsigned int total_sent, u32 status_padding);
219u8 spu2_xts_tweak_in_payload(void);
220u8 spu2_tx_status_len(void);
221u8 spu2_rx_status_len(void);
222int spu2_status_process(u8 *statp);
223void spu2_ccm_update_iv(unsigned int digestsize,
224 struct spu_cipher_parms *cipher_parms,
225 unsigned int assoclen, unsigned int chunksize,
226 bool is_encrypt, bool is_esp);
227u32 spu2_wordalign_padlen(u32 data_size);
228#endif
diff --git a/drivers/crypto/bcm/spum.h b/drivers/crypto/bcm/spum.h
new file mode 100644
index 000000000000..d0a5b5828638
--- /dev/null
+++ b/drivers/crypto/bcm/spum.h
@@ -0,0 +1,174 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17/*
18 * This file contains SPU message definitions specific to SPU-M.
19 */
20
21#ifndef _SPUM_H_
22#define _SPUM_H_
23
24#define SPU_CRYPTO_OPERATION_GENERIC 0x1
25
26/* Length of STATUS field in tx and rx packets */
27#define SPU_TX_STATUS_LEN 4
28
29/* SPU-M error codes */
30#define SPU_STATUS_MASK 0x0000FF00
31#define SPU_STATUS_SUCCESS 0x00000000
32#define SPU_STATUS_INVALID_ICV 0x00000100
33
34#define SPU_STATUS_ERROR_FLAG 0x00020000
35
36/* Request message. MH + EMH + BDESC + BD header */
37#define SPU_REQ_FIXED_LEN 24
38
39/*
40 * Max length of a SPU message header. Used to allocate a buffer where
41 * the SPU message header is constructed. Can be used for either a SPU-M
42 * header or a SPU2 header.
43 * For SPU-M, sum of the following:
44 * MH - 4 bytes
45 * EMH - 4
46 * SCTX - 3 +
47 * max auth key len - 64
48 * max cipher key len - 264 (RC4)
49 * max IV len - 16
50 * BDESC - 12
51 * BD header - 4
52 * Total: 371
53 *
54 * For SPU2, FMD_SIZE (32) plus lengths of hash and cipher keys,
55 * hash and cipher IVs. If SPU2 does not support RC4, then
56 */
57#define SPU_HEADER_ALLOC_LEN (SPU_REQ_FIXED_LEN + MAX_KEY_SIZE + \
58 MAX_KEY_SIZE + MAX_IV_SIZE)
59
60/*
61 * Response message header length. Normally MH, EMH, BD header, but when
62 * BD_SUPPRESS is used for hash requests, there is no BD header.
63 */
64#define SPU_RESP_HDR_LEN 12
65#define SPU_HASH_RESP_HDR_LEN 8
66
67/*
68 * Max value that can be represented in the Payload Length field of the BD
69 * header. This is a 16-bit field.
70 */
71#define SPUM_NS2_MAX_PAYLOAD (BIT(16) - 1)
72
73/*
74 * NSP SPU is limited to ~9KB because of FA2 FIFO size limitations;
75 * Set MAX_PAYLOAD to 8k to allow for addition of header, digest, etc.
76 * and stay within limitation.
77 */
78
79#define SPUM_NSP_MAX_PAYLOAD 8192
80
81/* Buffer Descriptor Header [BDESC]. SPU in big-endian mode. */
82struct BDESC_HEADER {
83 u16 offset_mac; /* word 0 [31-16] */
84 u16 length_mac; /* word 0 [15-0] */
85 u16 offset_crypto; /* word 1 [31-16] */
86 u16 length_crypto; /* word 1 [15-0] */
87 u16 offset_icv; /* word 2 [31-16] */
88 u16 offset_iv; /* word 2 [15-0] */
89};
90
91/* Buffer Data Header [BD]. SPU in big-endian mode. */
92struct BD_HEADER {
93 u16 size;
94 u16 prev_length;
95};
96
97/* Command Context Header. SPU-M in big endian mode. */
98struct MHEADER {
99 u8 flags; /* [31:24] */
100 u8 op_code; /* [23:16] */
101 u16 reserved; /* [15:0] */
102};
103
104/* MH header flags bits */
105#define MH_SUPDT_PRES BIT(0)
106#define MH_HASH_PRES BIT(2)
107#define MH_BD_PRES BIT(3)
108#define MH_MFM_PRES BIT(4)
109#define MH_BDESC_PRES BIT(5)
110#define MH_SCTX_PRES BIT(7)
111
112/* SCTX word 0 bit offsets and fields masks */
113#define SCTX_SIZE 0x000000FF
114
115/* SCTX word 1 bit shifts and field masks */
116#define UPDT_OFST 0x000000FF /* offset of SCTX updateable fld */
117#define HASH_TYPE 0x00000300 /* hash alg operation type */
118#define HASH_TYPE_SHIFT 8
119#define HASH_MODE 0x00001C00 /* one of spu2_hash_mode */
120#define HASH_MODE_SHIFT 10
121#define HASH_ALG 0x0000E000 /* hash algorithm */
122#define HASH_ALG_SHIFT 13
123#define CIPHER_TYPE 0x00030000 /* encryption operation type */
124#define CIPHER_TYPE_SHIFT 16
125#define CIPHER_MODE 0x001C0000 /* encryption mode */
126#define CIPHER_MODE_SHIFT 18
127#define CIPHER_ALG 0x00E00000 /* encryption algo */
128#define CIPHER_ALG_SHIFT 21
129#define ICV_IS_512 BIT(27)
130#define ICV_IS_512_SHIFT 27
131#define CIPHER_ORDER BIT(30)
132#define CIPHER_ORDER_SHIFT 30
133#define CIPHER_INBOUND BIT(31)
134#define CIPHER_INBOUND_SHIFT 31
135
136/* SCTX word 2 bit shifts and field masks */
137#define EXP_IV_SIZE 0x7
138#define IV_OFFSET BIT(3)
139#define IV_OFFSET_SHIFT 3
140#define GEN_IV BIT(5)
141#define GEN_IV_SHIFT 5
142#define EXPLICIT_IV BIT(6)
143#define EXPLICIT_IV_SHIFT 6
144#define SCTX_IV BIT(7)
145#define SCTX_IV_SHIFT 7
146#define ICV_SIZE 0x0F00
147#define ICV_SIZE_SHIFT 8
148#define CHECK_ICV BIT(12)
149#define CHECK_ICV_SHIFT 12
150#define INSERT_ICV BIT(13)
151#define INSERT_ICV_SHIFT 13
152#define BD_SUPPRESS BIT(19)
153#define BD_SUPPRESS_SHIFT 19
154
155/* Generic Mode Security Context Structure [SCTX] */
156struct SCTX {
157/* word 0: protocol flags */
158 u32 proto_flags;
159
160/* word 1: cipher flags */
161 u32 cipher_flags;
162
163/* word 2: Extended cipher flags */
164 u32 ecf;
165
166};
167
168struct SPUHEADER {
169 struct MHEADER mh;
170 u32 emh;
171 struct SCTX sa;
172};
173
174#endif /* _SPUM_H_ */
diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c
new file mode 100644
index 000000000000..0502f460dacd
--- /dev/null
+++ b/drivers/crypto/bcm/util.c
@@ -0,0 +1,581 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#include <linux/debugfs.h>
18
19#include "cipher.h"
20#include "util.h"
21
22/* offset of SPU_OFIFO_CTRL register */
23#define SPU_OFIFO_CTRL 0x40
24#define SPU_FIFO_WATERMARK 0x1FF
25
26/**
27 * spu_sg_at_offset() - Find the scatterlist entry at a given distance from the
28 * start of a scatterlist.
29 * @sg: [in] Start of a scatterlist
30 * @skip: [in] Distance from the start of the scatterlist, in bytes
31 * @sge: [out] Scatterlist entry at skip bytes from start
32 * @sge_offset: [out] Number of bytes from start of sge buffer to get to
33 * requested distance.
34 *
35 * Return: 0 if entry found at requested distance
36 * < 0 otherwise
37 */
38int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip,
39 struct scatterlist **sge, unsigned int *sge_offset)
40{
41 /* byte index from start of sg to the end of the previous entry */
42 unsigned int index = 0;
43 /* byte index from start of sg to the end of the current entry */
44 unsigned int next_index;
45
46 next_index = sg->length;
47 while (next_index <= skip) {
48 sg = sg_next(sg);
49 index = next_index;
50 if (!sg)
51 return -EINVAL;
52 next_index += sg->length;
53 }
54
55 *sge_offset = skip - index;
56 *sge = sg;
57 return 0;
58}
59
60/* Copy len bytes of sg data, starting at offset skip, to a dest buffer */
61void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest,
62 unsigned int len, unsigned int skip)
63{
64 size_t copied;
65 unsigned int nents = sg_nents(src);
66
67 copied = sg_pcopy_to_buffer(src, nents, dest, len, skip);
68 if (copied != len) {
69 flow_log("%s copied %u bytes of %u requested. ",
70 __func__, (u32)copied, len);
71 flow_log("sg with %u entries and skip %u\n", nents, skip);
72 }
73}
74
75/*
76 * Copy data into a scatterlist starting at a specified offset in the
77 * scatterlist. Specifically, copy len bytes of data in the buffer src
78 * into the scatterlist dest, starting skip bytes into the scatterlist.
79 */
80void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src,
81 unsigned int len, unsigned int skip)
82{
83 size_t copied;
84 unsigned int nents = sg_nents(dest);
85
86 copied = sg_pcopy_from_buffer(dest, nents, src, len, skip);
87 if (copied != len) {
88 flow_log("%s copied %u bytes of %u requested. ",
89 __func__, (u32)copied, len);
90 flow_log("sg with %u entries and skip %u\n", nents, skip);
91 }
92}
93
94/**
95 * spu_sg_count() - Determine number of elements in scatterlist to provide a
96 * specified number of bytes.
97 * @sg_list: scatterlist to examine
98 * @skip: index of starting point
99 * @nbytes: consider elements of scatterlist until reaching this number of
100 * bytes
101 *
102 * Return: the number of sg entries contributing to nbytes of data
103 */
104int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes)
105{
106 struct scatterlist *sg;
107 int sg_nents = 0;
108 unsigned int offset;
109
110 if (!sg_list)
111 return 0;
112
113 if (spu_sg_at_offset(sg_list, skip, &sg, &offset) < 0)
114 return 0;
115
116 while (sg && (nbytes > 0)) {
117 sg_nents++;
118 nbytes -= (sg->length - offset);
119 offset = 0;
120 sg = sg_next(sg);
121 }
122 return sg_nents;
123}
124
125/**
126 * spu_msg_sg_add() - Copy scatterlist entries from one sg to another, up to a
127 * given length.
128 * @to_sg: scatterlist to copy to
129 * @from_sg: scatterlist to copy from
130 * @from_skip: number of bytes to skip in from_sg. Non-zero when previous
131 * request included part of the buffer in entry in from_sg.
132 * Assumes from_skip < from_sg->length.
133 * @from_nents number of entries in from_sg
134 * @length number of bytes to copy. may reach this limit before exhausting
135 * from_sg.
136 *
137 * Copies the entries themselves, not the data in the entries. Assumes to_sg has
138 * enough entries. Does not limit the size of an individual buffer in to_sg.
139 *
140 * to_sg, from_sg, skip are all updated to end of copy
141 *
142 * Return: Number of bytes copied
143 */
144u32 spu_msg_sg_add(struct scatterlist **to_sg,
145 struct scatterlist **from_sg, u32 *from_skip,
146 u8 from_nents, u32 length)
147{
148 struct scatterlist *sg; /* an entry in from_sg */
149 struct scatterlist *to = *to_sg;
150 struct scatterlist *from = *from_sg;
151 u32 skip = *from_skip;
152 u32 offset;
153 int i;
154 u32 entry_len = 0;
155 u32 frag_len = 0; /* length of entry added to to_sg */
156 u32 copied = 0; /* number of bytes copied so far */
157
158 if (length == 0)
159 return 0;
160
161 for_each_sg(from, sg, from_nents, i) {
162 /* number of bytes in this from entry not yet used */
163 entry_len = sg->length - skip;
164 frag_len = min(entry_len, length - copied);
165 offset = sg->offset + skip;
166 if (frag_len)
167 sg_set_page(to++, sg_page(sg), frag_len, offset);
168 copied += frag_len;
169 if (copied == entry_len) {
170 /* used up all of from entry */
171 skip = 0; /* start at beginning of next entry */
172 }
173 if (copied == length)
174 break;
175 }
176 *to_sg = to;
177 *from_sg = sg;
178 if (frag_len < entry_len)
179 *from_skip = skip + frag_len;
180 else
181 *from_skip = 0;
182
183 return copied;
184}
185
186void add_to_ctr(u8 *ctr_pos, unsigned int increment)
187{
188 __be64 *high_be = (__be64 *)ctr_pos;
189 __be64 *low_be = high_be + 1;
190 u64 orig_low = __be64_to_cpu(*low_be);
191 u64 new_low = orig_low + (u64)increment;
192
193 *low_be = __cpu_to_be64(new_low);
194 if (new_low < orig_low)
195 /* there was a carry from the low 8 bytes */
196 *high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1);
197}
198
199struct sdesc {
200 struct shash_desc shash;
201 char ctx[];
202};
203
204/* do a synchronous decrypt operation */
205int do_decrypt(char *alg_name,
206 void *key_ptr, unsigned int key_len,
207 void *iv_ptr, void *src_ptr, void *dst_ptr,
208 unsigned int block_len)
209{
210 struct scatterlist sg_in[1], sg_out[1];
211 struct crypto_blkcipher *tfm =
212 crypto_alloc_blkcipher(alg_name, 0, CRYPTO_ALG_ASYNC);
213 struct blkcipher_desc desc = {.tfm = tfm, .flags = 0 };
214 int ret = 0;
215 void *iv;
216 int ivsize;
217
218 flow_log("%s() name:%s block_len:%u\n", __func__, alg_name, block_len);
219
220 if (IS_ERR(tfm))
221 return PTR_ERR(tfm);
222
223 crypto_blkcipher_setkey((void *)tfm, key_ptr, key_len);
224
225 sg_init_table(sg_in, 1);
226 sg_set_buf(sg_in, src_ptr, block_len);
227
228 sg_init_table(sg_out, 1);
229 sg_set_buf(sg_out, dst_ptr, block_len);
230
231 iv = crypto_blkcipher_crt(tfm)->iv;
232 ivsize = crypto_blkcipher_ivsize(tfm);
233 memcpy(iv, iv_ptr, ivsize);
234
235 ret = crypto_blkcipher_decrypt(&desc, sg_out, sg_in, block_len);
236 crypto_free_blkcipher(tfm);
237
238 if (ret < 0)
239 pr_err("aes_decrypt failed %d\n", ret);
240
241 return ret;
242}
243
244/**
245 * do_shash() - Do a synchronous hash operation in software
246 * @name: The name of the hash algorithm
247 * @result: Buffer where digest is to be written
248 * @data1: First part of data to hash. May be NULL.
249 * @data1_len: Length of data1, in bytes
250 * @data2: Second part of data to hash. May be NULL.
251 * @data2_len: Length of data2, in bytes
252 * @key: Key (if keyed hash)
253 * @key_len: Length of key, in bytes (or 0 if non-keyed hash)
254 *
255 * Note that the crypto API will not select this driver's own transform because
256 * this driver only registers asynchronous algos.
257 *
258 * Return: 0 if hash successfully stored in result
259 * < 0 otherwise
260 */
261int do_shash(unsigned char *name, unsigned char *result,
262 const u8 *data1, unsigned int data1_len,
263 const u8 *data2, unsigned int data2_len,
264 const u8 *key, unsigned int key_len)
265{
266 int rc;
267 unsigned int size;
268 struct crypto_shash *hash;
269 struct sdesc *sdesc;
270
271 hash = crypto_alloc_shash(name, 0, 0);
272 if (IS_ERR(hash)) {
273 rc = PTR_ERR(hash);
274 pr_err("%s: Crypto %s allocation error %d", __func__, name, rc);
275 return rc;
276 }
277
278 size = sizeof(struct shash_desc) + crypto_shash_descsize(hash);
279 sdesc = kmalloc(size, GFP_KERNEL);
280 if (!sdesc) {
281 rc = -ENOMEM;
282 pr_err("%s: Memory allocation failure", __func__);
283 goto do_shash_err;
284 }
285 sdesc->shash.tfm = hash;
286 sdesc->shash.flags = 0x0;
287
288 if (key_len > 0) {
289 rc = crypto_shash_setkey(hash, key, key_len);
290 if (rc) {
291 pr_err("%s: Could not setkey %s shash", __func__, name);
292 goto do_shash_err;
293 }
294 }
295
296 rc = crypto_shash_init(&sdesc->shash);
297 if (rc) {
298 pr_err("%s: Could not init %s shash", __func__, name);
299 goto do_shash_err;
300 }
301 rc = crypto_shash_update(&sdesc->shash, data1, data1_len);
302 if (rc) {
303 pr_err("%s: Could not update1", __func__);
304 goto do_shash_err;
305 }
306 if (data2 && data2_len) {
307 rc = crypto_shash_update(&sdesc->shash, data2, data2_len);
308 if (rc) {
309 pr_err("%s: Could not update2", __func__);
310 goto do_shash_err;
311 }
312 }
313 rc = crypto_shash_final(&sdesc->shash, result);
314 if (rc)
315 pr_err("%s: Could not genereate %s hash", __func__, name);
316
317do_shash_err:
318 crypto_free_shash(hash);
319 kfree(sdesc);
320
321 return rc;
322}
323
324/* Dump len bytes of a scatterlist starting at skip bytes into the sg */
325void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len)
326{
327 u8 dbuf[16];
328 unsigned int idx = skip;
329 unsigned int num_out = 0; /* number of bytes dumped so far */
330 unsigned int count;
331
332 if (packet_debug_logging) {
333 while (num_out < len) {
334 count = (len - num_out > 16) ? 16 : len - num_out;
335 sg_copy_part_to_buf(sg, dbuf, count, idx);
336 num_out += count;
337 print_hex_dump(KERN_ALERT, " sg: ", DUMP_PREFIX_NONE,
338 4, 1, dbuf, count, false);
339 idx += 16;
340 }
341 }
342 if (debug_logging_sleep)
343 msleep(debug_logging_sleep);
344}
345
346/* Returns the name for a given cipher alg/mode */
347char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode)
348{
349 switch (alg) {
350 case CIPHER_ALG_RC4:
351 return "rc4";
352 case CIPHER_ALG_AES:
353 switch (mode) {
354 case CIPHER_MODE_CBC:
355 return "cbc(aes)";
356 case CIPHER_MODE_ECB:
357 return "ecb(aes)";
358 case CIPHER_MODE_OFB:
359 return "ofb(aes)";
360 case CIPHER_MODE_CFB:
361 return "cfb(aes)";
362 case CIPHER_MODE_CTR:
363 return "ctr(aes)";
364 case CIPHER_MODE_XTS:
365 return "xts(aes)";
366 case CIPHER_MODE_GCM:
367 return "gcm(aes)";
368 default:
369 return "aes";
370 }
371 break;
372 case CIPHER_ALG_DES:
373 switch (mode) {
374 case CIPHER_MODE_CBC:
375 return "cbc(des)";
376 case CIPHER_MODE_ECB:
377 return "ecb(des)";
378 case CIPHER_MODE_CTR:
379 return "ctr(des)";
380 default:
381 return "des";
382 }
383 break;
384 case CIPHER_ALG_3DES:
385 switch (mode) {
386 case CIPHER_MODE_CBC:
387 return "cbc(des3_ede)";
388 case CIPHER_MODE_ECB:
389 return "ecb(des3_ede)";
390 case CIPHER_MODE_CTR:
391 return "ctr(des3_ede)";
392 default:
393 return "3des";
394 }
395 break;
396 default:
397 return "other";
398 }
399}
400
401static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf,
402 size_t count, loff_t *offp)
403{
404 struct device_private *ipriv;
405 char *buf;
406 ssize_t ret, out_offset, out_count;
407 int i;
408 u32 fifo_len;
409 u32 spu_ofifo_ctrl;
410 u32 alg;
411 u32 mode;
412 u32 op_cnt;
413
414 out_count = 2048;
415
416 buf = kmalloc(out_count, GFP_KERNEL);
417 if (!buf)
418 return -ENOMEM;
419
420 ipriv = filp->private_data;
421 out_offset = 0;
422 out_offset += snprintf(buf + out_offset, out_count - out_offset,
423 "Number of SPUs.........%u\n",
424 ipriv->spu.num_spu);
425 out_offset += snprintf(buf + out_offset, out_count - out_offset,
426 "Current sessions.......%u\n",
427 atomic_read(&ipriv->session_count));
428 out_offset += snprintf(buf + out_offset, out_count - out_offset,
429 "Session count..........%u\n",
430 atomic_read(&ipriv->stream_count));
431 out_offset += snprintf(buf + out_offset, out_count - out_offset,
432 "Cipher setkey..........%u\n",
433 atomic_read(&ipriv->setkey_cnt[SPU_OP_CIPHER]));
434 out_offset += snprintf(buf + out_offset, out_count - out_offset,
435 "Cipher Ops.............%u\n",
436 atomic_read(&ipriv->op_counts[SPU_OP_CIPHER]));
437 for (alg = 0; alg < CIPHER_ALG_LAST; alg++) {
438 for (mode = 0; mode < CIPHER_MODE_LAST; mode++) {
439 op_cnt = atomic_read(&ipriv->cipher_cnt[alg][mode]);
440 if (op_cnt) {
441 out_offset += snprintf(buf + out_offset,
442 out_count - out_offset,
443 " %-13s%11u\n",
444 spu_alg_name(alg, mode), op_cnt);
445 }
446 }
447 }
448 out_offset += snprintf(buf + out_offset, out_count - out_offset,
449 "Hash Ops...............%u\n",
450 atomic_read(&ipriv->op_counts[SPU_OP_HASH]));
451 for (alg = 0; alg < HASH_ALG_LAST; alg++) {
452 op_cnt = atomic_read(&ipriv->hash_cnt[alg]);
453 if (op_cnt) {
454 out_offset += snprintf(buf + out_offset,
455 out_count - out_offset,
456 " %-13s%11u\n",
457 hash_alg_name[alg], op_cnt);
458 }
459 }
460 out_offset += snprintf(buf + out_offset, out_count - out_offset,
461 "HMAC setkey............%u\n",
462 atomic_read(&ipriv->setkey_cnt[SPU_OP_HMAC]));
463 out_offset += snprintf(buf + out_offset, out_count - out_offset,
464 "HMAC Ops...............%u\n",
465 atomic_read(&ipriv->op_counts[SPU_OP_HMAC]));
466 for (alg = 0; alg < HASH_ALG_LAST; alg++) {
467 op_cnt = atomic_read(&ipriv->hmac_cnt[alg]);
468 if (op_cnt) {
469 out_offset += snprintf(buf + out_offset,
470 out_count - out_offset,
471 " %-13s%11u\n",
472 hash_alg_name[alg], op_cnt);
473 }
474 }
475 out_offset += snprintf(buf + out_offset, out_count - out_offset,
476 "AEAD setkey............%u\n",
477 atomic_read(&ipriv->setkey_cnt[SPU_OP_AEAD]));
478
479 out_offset += snprintf(buf + out_offset, out_count - out_offset,
480 "AEAD Ops...............%u\n",
481 atomic_read(&ipriv->op_counts[SPU_OP_AEAD]));
482 for (alg = 0; alg < AEAD_TYPE_LAST; alg++) {
483 op_cnt = atomic_read(&ipriv->aead_cnt[alg]);
484 if (op_cnt) {
485 out_offset += snprintf(buf + out_offset,
486 out_count - out_offset,
487 " %-13s%11u\n",
488 aead_alg_name[alg], op_cnt);
489 }
490 }
491 out_offset += snprintf(buf + out_offset, out_count - out_offset,
492 "Bytes of req data......%llu\n",
493 (u64)atomic64_read(&ipriv->bytes_out));
494 out_offset += snprintf(buf + out_offset, out_count - out_offset,
495 "Bytes of resp data.....%llu\n",
496 (u64)atomic64_read(&ipriv->bytes_in));
497 out_offset += snprintf(buf + out_offset, out_count - out_offset,
498 "Mailbox full...........%u\n",
499 atomic_read(&ipriv->mb_no_spc));
500 out_offset += snprintf(buf + out_offset, out_count - out_offset,
501 "Mailbox send failures..%u\n",
502 atomic_read(&ipriv->mb_send_fail));
503 out_offset += snprintf(buf + out_offset, out_count - out_offset,
504 "Check ICV errors.......%u\n",
505 atomic_read(&ipriv->bad_icv));
506 if (ipriv->spu.spu_type == SPU_TYPE_SPUM)
507 for (i = 0; i < ipriv->spu.num_spu; i++) {
508 spu_ofifo_ctrl = ioread32(ipriv->spu.reg_vbase[i] +
509 SPU_OFIFO_CTRL);
510 fifo_len = spu_ofifo_ctrl & SPU_FIFO_WATERMARK;
511 out_offset += snprintf(buf + out_offset,
512 out_count - out_offset,
513 "SPU %d output FIFO high water.....%u\n",
514 i, fifo_len);
515 }
516
517 if (out_offset > out_count)
518 out_offset = out_count;
519
520 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
521 kfree(buf);
522 return ret;
523}
524
525static const struct file_operations spu_debugfs_stats = {
526 .owner = THIS_MODULE,
527 .open = simple_open,
528 .read = spu_debugfs_read,
529};
530
531/*
532 * Create the debug FS directories. If the top-level directory has not yet
533 * been created, create it now. Create a stats file in this directory for
534 * a SPU.
535 */
536void spu_setup_debugfs(void)
537{
538 if (!debugfs_initialized())
539 return;
540
541 if (!iproc_priv.debugfs_dir)
542 iproc_priv.debugfs_dir = debugfs_create_dir(KBUILD_MODNAME,
543 NULL);
544
545 if (!iproc_priv.debugfs_stats)
546 /* Create file with permissions S_IRUSR */
547 debugfs_create_file("stats", 0400, iproc_priv.debugfs_dir,
548 &iproc_priv, &spu_debugfs_stats);
549}
550
551void spu_free_debugfs(void)
552{
553 debugfs_remove_recursive(iproc_priv.debugfs_dir);
554 iproc_priv.debugfs_dir = NULL;
555}
556
557/**
558 * format_value_ccm() - Format a value into a buffer, using a specified number
559 * of bytes (i.e. maybe writing value X into a 4 byte
560 * buffer, or maybe into a 12 byte buffer), as per the
561 * SPU CCM spec.
562 *
563 * @val: value to write (up to max of unsigned int)
564 * @buf: (pointer to) buffer to write the value
565 * @len: number of bytes to use (0 to 255)
566 *
567 */
568void format_value_ccm(unsigned int val, u8 *buf, u8 len)
569{
570 int i;
571
572 /* First clear full output buffer */
573 memset(buf, 0, len);
574
575 /* Then, starting from right side, fill in with data */
576 for (i = 0; i < len; i++) {
577 buf[len - i - 1] = (val >> (8 * i)) & 0xff;
578 if (i >= 3)
579 break; /* Only handle up to 32 bits of 'val' */
580 }
581}
diff --git a/drivers/crypto/bcm/util.h b/drivers/crypto/bcm/util.h
new file mode 100644
index 000000000000..712e029795f8
--- /dev/null
+++ b/drivers/crypto/bcm/util.h
@@ -0,0 +1,116 @@
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#ifndef _UTIL_H
18#define _UTIL_H
19
20#include <linux/kernel.h>
21#include <linux/delay.h>
22
23#include "spu.h"
24
25extern int flow_debug_logging;
26extern int packet_debug_logging;
27extern int debug_logging_sleep;
28
29#ifdef DEBUG
30#define flow_log(...) \
31 do { \
32 if (flow_debug_logging) { \
33 printk(__VA_ARGS__); \
34 if (debug_logging_sleep) \
35 msleep(debug_logging_sleep); \
36 } \
37 } while (0)
38#define flow_dump(msg, var, var_len) \
39 do { \
40 if (flow_debug_logging) { \
41 print_hex_dump(KERN_ALERT, msg, DUMP_PREFIX_NONE, \
42 16, 1, var, var_len, false); \
43 if (debug_logging_sleep) \
44 msleep(debug_logging_sleep); \
45 } \
46 } while (0)
47
48#define packet_log(...) \
49 do { \
50 if (packet_debug_logging) { \
51 printk(__VA_ARGS__); \
52 if (debug_logging_sleep) \
53 msleep(debug_logging_sleep); \
54 } \
55 } while (0)
56#define packet_dump(msg, var, var_len) \
57 do { \
58 if (packet_debug_logging) { \
59 print_hex_dump(KERN_ALERT, msg, DUMP_PREFIX_NONE, \
60 16, 1, var, var_len, false); \
61 if (debug_logging_sleep) \
62 msleep(debug_logging_sleep); \
63 } \
64 } while (0)
65
66void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len);
67
68#define dump_sg(sg, skip, len) __dump_sg(sg, skip, len)
69
70#else /* !DEBUG_ON */
71
72#define flow_log(...) do {} while (0)
73#define flow_dump(msg, var, var_len) do {} while (0)
74#define packet_log(...) do {} while (0)
75#define packet_dump(msg, var, var_len) do {} while (0)
76
77#define dump_sg(sg, skip, len) do {} while (0)
78
79#endif /* DEBUG_ON */
80
81int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip,
82 struct scatterlist **sge, unsigned int *sge_offset);
83
84/* Copy sg data, from skip, length len, to dest */
85void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest,
86 unsigned int len, unsigned int skip);
87/* Copy src into scatterlist from offset, length len */
88void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src,
89 unsigned int len, unsigned int skip);
90
91int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes);
92u32 spu_msg_sg_add(struct scatterlist **to_sg,
93 struct scatterlist **from_sg, u32 *skip,
94 u8 from_nents, u32 tot_len);
95
96void add_to_ctr(u8 *ctr_pos, unsigned int increment);
97
98/* do a synchronous decrypt operation */
99int do_decrypt(char *alg_name,
100 void *key_ptr, unsigned int key_len,
101 void *iv_ptr, void *src_ptr, void *dst_ptr,
102 unsigned int block_len);
103
104/* produce a message digest from data of length n bytes */
105int do_shash(unsigned char *name, unsigned char *result,
106 const u8 *data1, unsigned int data1_len,
107 const u8 *data2, unsigned int data2_len,
108 const u8 *key, unsigned int key_len);
109
110char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode);
111
112void spu_setup_debugfs(void);
113void spu_free_debugfs(void);
114void format_value_ccm(unsigned int val, u8 *buf, u8 len);
115
116#endif
diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c
index 10db7df366c8..a118b9bed669 100644
--- a/drivers/crypto/bfin_crc.c
+++ b/drivers/crypto/bfin_crc.c
@@ -203,7 +203,7 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc)
203 crc->sg_cpu[i].x_count = 1; 203 crc->sg_cpu[i].x_count = 1;
204 crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE; 204 crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE;
205 dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, " 205 dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, "
206 "cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n", 206 "cfg:0x%x, x_count:0x%x, x_modify:0x%x\n",
207 i, crc->sg_cpu[i].start_addr, 207 i, crc->sg_cpu[i].start_addr,
208 crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count, 208 crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count,
209 crc->sg_cpu[i].x_modify); 209 crc->sg_cpu[i].x_modify);
@@ -233,7 +233,7 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc)
233 crc->sg_cpu[i].x_count = dma_count; 233 crc->sg_cpu[i].x_count = dma_count;
234 crc->sg_cpu[i].x_modify = dma_mod; 234 crc->sg_cpu[i].x_modify = dma_mod;
235 dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, " 235 dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, "
236 "cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n", 236 "cfg:0x%x, x_count:0x%x, x_modify:0x%x\n",
237 i, crc->sg_cpu[i].start_addr, 237 i, crc->sg_cpu[i].start_addr,
238 crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count, 238 crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count,
239 crc->sg_cpu[i].x_modify); 239 crc->sg_cpu[i].x_modify);
@@ -257,7 +257,7 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc)
257 crc->sg_cpu[i].x_count = 1; 257 crc->sg_cpu[i].x_count = 1;
258 crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE; 258 crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE;
259 dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, " 259 dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, "
260 "cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n", 260 "cfg:0x%x, x_count:0x%x, x_modify:0x%x\n",
261 i, crc->sg_cpu[i].start_addr, 261 i, crc->sg_cpu[i].start_addr,
262 crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count, 262 crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count,
263 crc->sg_cpu[i].x_modify); 263 crc->sg_cpu[i].x_modify);
diff --git a/drivers/crypto/bfin_crc.h b/drivers/crypto/bfin_crc.h
index 75cef4dc85a1..786ef746d109 100644
--- a/drivers/crypto/bfin_crc.h
+++ b/drivers/crypto/bfin_crc.h
@@ -55,7 +55,6 @@ struct crc_info {
55 55
56#include <linux/types.h> 56#include <linux/types.h>
57#include <linux/spinlock.h> 57#include <linux/spinlock.h>
58#include <linux/miscdevice.h>
59 58
60struct crc_register { 59struct crc_register {
61 u32 control; 60 u32 control;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 662fe94cb2f8..9bc80eb06934 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -134,15 +134,15 @@ struct caam_aead_alg {
134 * per-session context 134 * per-session context
135 */ 135 */
136struct caam_ctx { 136struct caam_ctx {
137 struct device *jrdev;
138 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 137 u32 sh_desc_enc[DESC_MAX_USED_LEN];
139 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 138 u32 sh_desc_dec[DESC_MAX_USED_LEN];
140 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 139 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
140 u8 key[CAAM_MAX_KEY_SIZE];
141 dma_addr_t sh_desc_enc_dma; 141 dma_addr_t sh_desc_enc_dma;
142 dma_addr_t sh_desc_dec_dma; 142 dma_addr_t sh_desc_dec_dma;
143 dma_addr_t sh_desc_givenc_dma; 143 dma_addr_t sh_desc_givenc_dma;
144 u8 key[CAAM_MAX_KEY_SIZE];
145 dma_addr_t key_dma; 144 dma_addr_t key_dma;
145 struct device *jrdev;
146 struct alginfo adata; 146 struct alginfo adata;
147 struct alginfo cdata; 147 struct alginfo cdata;
148 unsigned int authsize; 148 unsigned int authsize;
@@ -171,13 +171,8 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
171 /* aead_encrypt shared descriptor */ 171 /* aead_encrypt shared descriptor */
172 desc = ctx->sh_desc_enc; 172 desc = ctx->sh_desc_enc;
173 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize); 173 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize);
174 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 174 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
175 desc_bytes(desc), 175 desc_bytes(desc), DMA_TO_DEVICE);
176 DMA_TO_DEVICE);
177 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
178 dev_err(jrdev, "unable to map shared descriptor\n");
179 return -ENOMEM;
180 }
181 176
182 /* 177 /*
183 * Job Descriptor and Shared Descriptors 178 * Job Descriptor and Shared Descriptors
@@ -194,13 +189,8 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
194 /* aead_decrypt shared descriptor */ 189 /* aead_decrypt shared descriptor */
195 desc = ctx->sh_desc_dec; 190 desc = ctx->sh_desc_dec;
196 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize); 191 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize);
197 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 192 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
198 desc_bytes(desc), 193 desc_bytes(desc), DMA_TO_DEVICE);
199 DMA_TO_DEVICE);
200 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
201 dev_err(jrdev, "unable to map shared descriptor\n");
202 return -ENOMEM;
203 }
204 194
205 return 0; 195 return 0;
206} 196}
@@ -278,13 +268,8 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
278 desc = ctx->sh_desc_enc; 268 desc = ctx->sh_desc_enc;
279 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ctx->authsize, 269 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ctx->authsize,
280 is_rfc3686, nonce, ctx1_iv_off); 270 is_rfc3686, nonce, ctx1_iv_off);
281 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 271 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
282 desc_bytes(desc), 272 desc_bytes(desc), DMA_TO_DEVICE);
283 DMA_TO_DEVICE);
284 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
285 dev_err(jrdev, "unable to map shared descriptor\n");
286 return -ENOMEM;
287 }
288 273
289skip_enc: 274skip_enc:
290 /* 275 /*
@@ -315,13 +300,8 @@ skip_enc:
315 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 300 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
316 ctx->authsize, alg->caam.geniv, is_rfc3686, 301 ctx->authsize, alg->caam.geniv, is_rfc3686,
317 nonce, ctx1_iv_off); 302 nonce, ctx1_iv_off);
318 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 303 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
319 desc_bytes(desc), 304 desc_bytes(desc), DMA_TO_DEVICE);
320 DMA_TO_DEVICE);
321 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
322 dev_err(jrdev, "unable to map shared descriptor\n");
323 return -ENOMEM;
324 }
325 305
326 if (!alg->caam.geniv) 306 if (!alg->caam.geniv)
327 goto skip_givenc; 307 goto skip_givenc;
@@ -354,13 +334,8 @@ skip_enc:
354 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 334 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
355 ctx->authsize, is_rfc3686, nonce, 335 ctx->authsize, is_rfc3686, nonce,
356 ctx1_iv_off); 336 ctx1_iv_off);
357 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 337 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
358 desc_bytes(desc), 338 desc_bytes(desc), DMA_TO_DEVICE);
359 DMA_TO_DEVICE);
360 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
361 dev_err(jrdev, "unable to map shared descriptor\n");
362 return -ENOMEM;
363 }
364 339
365skip_givenc: 340skip_givenc:
366 return 0; 341 return 0;
@@ -403,13 +378,8 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
403 378
404 desc = ctx->sh_desc_enc; 379 desc = ctx->sh_desc_enc;
405 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize); 380 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize);
406 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 381 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
407 desc_bytes(desc), 382 desc_bytes(desc), DMA_TO_DEVICE);
408 DMA_TO_DEVICE);
409 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
410 dev_err(jrdev, "unable to map shared descriptor\n");
411 return -ENOMEM;
412 }
413 383
414 /* 384 /*
415 * Job Descriptor and Shared Descriptors 385 * Job Descriptor and Shared Descriptors
@@ -425,13 +395,8 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
425 395
426 desc = ctx->sh_desc_dec; 396 desc = ctx->sh_desc_dec;
427 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize); 397 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize);
428 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 398 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
429 desc_bytes(desc), 399 desc_bytes(desc), DMA_TO_DEVICE);
430 DMA_TO_DEVICE);
431 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
432 dev_err(jrdev, "unable to map shared descriptor\n");
433 return -ENOMEM;
434 }
435 400
436 return 0; 401 return 0;
437} 402}
@@ -472,13 +437,8 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
472 437
473 desc = ctx->sh_desc_enc; 438 desc = ctx->sh_desc_enc;
474 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize); 439 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize);
475 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 440 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
476 desc_bytes(desc), 441 desc_bytes(desc), DMA_TO_DEVICE);
477 DMA_TO_DEVICE);
478 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
479 dev_err(jrdev, "unable to map shared descriptor\n");
480 return -ENOMEM;
481 }
482 442
483 /* 443 /*
484 * Job Descriptor and Shared Descriptors 444 * Job Descriptor and Shared Descriptors
@@ -494,13 +454,8 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
494 454
495 desc = ctx->sh_desc_dec; 455 desc = ctx->sh_desc_dec;
496 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize); 456 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize);
497 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 457 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
498 desc_bytes(desc), 458 desc_bytes(desc), DMA_TO_DEVICE);
499 DMA_TO_DEVICE);
500 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
501 dev_err(jrdev, "unable to map shared descriptor\n");
502 return -ENOMEM;
503 }
504 459
505 return 0; 460 return 0;
506} 461}
@@ -542,13 +497,8 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
542 497
543 desc = ctx->sh_desc_enc; 498 desc = ctx->sh_desc_enc;
544 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize); 499 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize);
545 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 500 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
546 desc_bytes(desc), 501 desc_bytes(desc), DMA_TO_DEVICE);
547 DMA_TO_DEVICE);
548 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
549 dev_err(jrdev, "unable to map shared descriptor\n");
550 return -ENOMEM;
551 }
552 502
553 /* 503 /*
554 * Job Descriptor and Shared Descriptors 504 * Job Descriptor and Shared Descriptors
@@ -564,13 +514,8 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
564 514
565 desc = ctx->sh_desc_dec; 515 desc = ctx->sh_desc_dec;
566 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize); 516 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize);
567 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 517 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
568 desc_bytes(desc), 518 desc_bytes(desc), DMA_TO_DEVICE);
569 DMA_TO_DEVICE);
570 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
571 dev_err(jrdev, "unable to map shared descriptor\n");
572 return -ENOMEM;
573 }
574 519
575 return 0; 520 return 0;
576} 521}
@@ -614,28 +559,15 @@ static int aead_setkey(struct crypto_aead *aead,
614 559
615 /* postpend encryption key to auth split key */ 560 /* postpend encryption key to auth split key */
616 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 561 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
617 562 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
618 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->adata.keylen_pad + 563 keys.enckeylen, DMA_TO_DEVICE);
619 keys.enckeylen, DMA_TO_DEVICE);
620 if (dma_mapping_error(jrdev, ctx->key_dma)) {
621 dev_err(jrdev, "unable to map key i/o memory\n");
622 return -ENOMEM;
623 }
624#ifdef DEBUG 564#ifdef DEBUG
625 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 565 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
626 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 566 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
627 ctx->adata.keylen_pad + keys.enckeylen, 1); 567 ctx->adata.keylen_pad + keys.enckeylen, 1);
628#endif 568#endif
629
630 ctx->cdata.keylen = keys.enckeylen; 569 ctx->cdata.keylen = keys.enckeylen;
631 570 return aead_set_sh_desc(aead);
632 ret = aead_set_sh_desc(aead);
633 if (ret) {
634 dma_unmap_single(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
635 keys.enckeylen, DMA_TO_DEVICE);
636 }
637
638 return ret;
639badkey: 571badkey:
640 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 572 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
641 return -EINVAL; 573 return -EINVAL;
@@ -646,7 +578,6 @@ static int gcm_setkey(struct crypto_aead *aead,
646{ 578{
647 struct caam_ctx *ctx = crypto_aead_ctx(aead); 579 struct caam_ctx *ctx = crypto_aead_ctx(aead);
648 struct device *jrdev = ctx->jrdev; 580 struct device *jrdev = ctx->jrdev;
649 int ret = 0;
650 581
651#ifdef DEBUG 582#ifdef DEBUG
652 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 583 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
@@ -654,21 +585,10 @@ static int gcm_setkey(struct crypto_aead *aead,
654#endif 585#endif
655 586
656 memcpy(ctx->key, key, keylen); 587 memcpy(ctx->key, key, keylen);
657 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 588 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
658 DMA_TO_DEVICE);
659 if (dma_mapping_error(jrdev, ctx->key_dma)) {
660 dev_err(jrdev, "unable to map key i/o memory\n");
661 return -ENOMEM;
662 }
663 ctx->cdata.keylen = keylen; 589 ctx->cdata.keylen = keylen;
664 590
665 ret = gcm_set_sh_desc(aead); 591 return gcm_set_sh_desc(aead);
666 if (ret) {
667 dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
668 DMA_TO_DEVICE);
669 }
670
671 return ret;
672} 592}
673 593
674static int rfc4106_setkey(struct crypto_aead *aead, 594static int rfc4106_setkey(struct crypto_aead *aead,
@@ -676,7 +596,6 @@ static int rfc4106_setkey(struct crypto_aead *aead,
676{ 596{
677 struct caam_ctx *ctx = crypto_aead_ctx(aead); 597 struct caam_ctx *ctx = crypto_aead_ctx(aead);
678 struct device *jrdev = ctx->jrdev; 598 struct device *jrdev = ctx->jrdev;
679 int ret = 0;
680 599
681 if (keylen < 4) 600 if (keylen < 4)
682 return -EINVAL; 601 return -EINVAL;
@@ -693,21 +612,9 @@ static int rfc4106_setkey(struct crypto_aead *aead,
693 * in the nonce. Update the AES key length. 612 * in the nonce. Update the AES key length.
694 */ 613 */
695 ctx->cdata.keylen = keylen - 4; 614 ctx->cdata.keylen = keylen - 4;
696 615 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
697 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->cdata.keylen, 616 DMA_TO_DEVICE);
698 DMA_TO_DEVICE); 617 return rfc4106_set_sh_desc(aead);
699 if (dma_mapping_error(jrdev, ctx->key_dma)) {
700 dev_err(jrdev, "unable to map key i/o memory\n");
701 return -ENOMEM;
702 }
703
704 ret = rfc4106_set_sh_desc(aead);
705 if (ret) {
706 dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
707 DMA_TO_DEVICE);
708 }
709
710 return ret;
711} 618}
712 619
713static int rfc4543_setkey(struct crypto_aead *aead, 620static int rfc4543_setkey(struct crypto_aead *aead,
@@ -715,7 +622,6 @@ static int rfc4543_setkey(struct crypto_aead *aead,
715{ 622{
716 struct caam_ctx *ctx = crypto_aead_ctx(aead); 623 struct caam_ctx *ctx = crypto_aead_ctx(aead);
717 struct device *jrdev = ctx->jrdev; 624 struct device *jrdev = ctx->jrdev;
718 int ret = 0;
719 625
720 if (keylen < 4) 626 if (keylen < 4)
721 return -EINVAL; 627 return -EINVAL;
@@ -732,21 +638,9 @@ static int rfc4543_setkey(struct crypto_aead *aead,
732 * in the nonce. Update the AES key length. 638 * in the nonce. Update the AES key length.
733 */ 639 */
734 ctx->cdata.keylen = keylen - 4; 640 ctx->cdata.keylen = keylen - 4;
735 641 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
736 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->cdata.keylen, 642 DMA_TO_DEVICE);
737 DMA_TO_DEVICE); 643 return rfc4543_set_sh_desc(aead);
738 if (dma_mapping_error(jrdev, ctx->key_dma)) {
739 dev_err(jrdev, "unable to map key i/o memory\n");
740 return -ENOMEM;
741 }
742
743 ret = rfc4543_set_sh_desc(aead);
744 if (ret) {
745 dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
746 DMA_TO_DEVICE);
747 }
748
749 return ret;
750} 644}
751 645
752static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 646static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
@@ -787,12 +681,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
787 keylen -= CTR_RFC3686_NONCE_SIZE; 681 keylen -= CTR_RFC3686_NONCE_SIZE;
788 } 682 }
789 683
790 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 684 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
791 DMA_TO_DEVICE);
792 if (dma_mapping_error(jrdev, ctx->key_dma)) {
793 dev_err(jrdev, "unable to map key i/o memory\n");
794 return -ENOMEM;
795 }
796 ctx->cdata.keylen = keylen; 685 ctx->cdata.keylen = keylen;
797 ctx->cdata.key_virt = ctx->key; 686 ctx->cdata.key_virt = ctx->key;
798 ctx->cdata.key_inline = true; 687 ctx->cdata.key_inline = true;
@@ -801,37 +690,22 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
801 desc = ctx->sh_desc_enc; 690 desc = ctx->sh_desc_enc;
802 cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 691 cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
803 ctx1_iv_off); 692 ctx1_iv_off);
804 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 693 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
805 desc_bytes(desc), 694 desc_bytes(desc), DMA_TO_DEVICE);
806 DMA_TO_DEVICE);
807 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
808 dev_err(jrdev, "unable to map shared descriptor\n");
809 return -ENOMEM;
810 }
811 695
812 /* ablkcipher_decrypt shared descriptor */ 696 /* ablkcipher_decrypt shared descriptor */
813 desc = ctx->sh_desc_dec; 697 desc = ctx->sh_desc_dec;
814 cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 698 cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
815 ctx1_iv_off); 699 ctx1_iv_off);
816 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 700 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
817 desc_bytes(desc), 701 desc_bytes(desc), DMA_TO_DEVICE);
818 DMA_TO_DEVICE);
819 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
820 dev_err(jrdev, "unable to map shared descriptor\n");
821 return -ENOMEM;
822 }
823 702
824 /* ablkcipher_givencrypt shared descriptor */ 703 /* ablkcipher_givencrypt shared descriptor */
825 desc = ctx->sh_desc_givenc; 704 desc = ctx->sh_desc_givenc;
826 cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, 705 cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
827 ctx1_iv_off); 706 ctx1_iv_off);
828 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 707 dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma,
829 desc_bytes(desc), 708 desc_bytes(desc), DMA_TO_DEVICE);
830 DMA_TO_DEVICE);
831 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
832 dev_err(jrdev, "unable to map shared descriptor\n");
833 return -ENOMEM;
834 }
835 709
836 return 0; 710 return 0;
837} 711}
@@ -851,11 +725,7 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
851 } 725 }
852 726
853 memcpy(ctx->key, key, keylen); 727 memcpy(ctx->key, key, keylen);
854 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); 728 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
855 if (dma_mapping_error(jrdev, ctx->key_dma)) {
856 dev_err(jrdev, "unable to map key i/o memory\n");
857 return -ENOMEM;
858 }
859 ctx->cdata.keylen = keylen; 729 ctx->cdata.keylen = keylen;
860 ctx->cdata.key_virt = ctx->key; 730 ctx->cdata.key_virt = ctx->key;
861 ctx->cdata.key_inline = true; 731 ctx->cdata.key_inline = true;
@@ -863,32 +733,22 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
863 /* xts_ablkcipher_encrypt shared descriptor */ 733 /* xts_ablkcipher_encrypt shared descriptor */
864 desc = ctx->sh_desc_enc; 734 desc = ctx->sh_desc_enc;
865 cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); 735 cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
866 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 736 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
867 DMA_TO_DEVICE); 737 desc_bytes(desc), DMA_TO_DEVICE);
868 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
869 dev_err(jrdev, "unable to map shared descriptor\n");
870 return -ENOMEM;
871 }
872 738
873 /* xts_ablkcipher_decrypt shared descriptor */ 739 /* xts_ablkcipher_decrypt shared descriptor */
874 desc = ctx->sh_desc_dec; 740 desc = ctx->sh_desc_dec;
875 cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); 741 cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
876 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 742 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
877 DMA_TO_DEVICE); 743 desc_bytes(desc), DMA_TO_DEVICE);
878 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
879 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma,
880 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
881 dev_err(jrdev, "unable to map shared descriptor\n");
882 return -ENOMEM;
883 }
884 744
885 return 0; 745 return 0;
886} 746}
887 747
888/* 748/*
889 * aead_edesc - s/w-extended aead descriptor 749 * aead_edesc - s/w-extended aead descriptor
890 * @src_nents: number of segments in input scatterlist 750 * @src_nents: number of segments in input s/w scatterlist
891 * @dst_nents: number of segments in output scatterlist 751 * @dst_nents: number of segments in output s/w scatterlist
892 * @sec4_sg_bytes: length of dma mapped sec4_sg space 752 * @sec4_sg_bytes: length of dma mapped sec4_sg space
893 * @sec4_sg_dma: bus physical mapped address of h/w link table 753 * @sec4_sg_dma: bus physical mapped address of h/w link table
894 * @sec4_sg: pointer to h/w link table 754 * @sec4_sg: pointer to h/w link table
@@ -905,8 +765,8 @@ struct aead_edesc {
905 765
906/* 766/*
907 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 767 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
908 * @src_nents: number of segments in input scatterlist 768 * @src_nents: number of segments in input s/w scatterlist
909 * @dst_nents: number of segments in output scatterlist 769 * @dst_nents: number of segments in output s/w scatterlist
910 * @iv_dma: dma address of iv for checking continuity and link table 770 * @iv_dma: dma address of iv for checking continuity and link table
911 * @sec4_sg_bytes: length of dma mapped sec4_sg space 771 * @sec4_sg_bytes: length of dma mapped sec4_sg space
912 * @sec4_sg_dma: bus physical mapped address of h/w link table 772 * @sec4_sg_dma: bus physical mapped address of h/w link table
@@ -930,10 +790,11 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
930 int sec4_sg_bytes) 790 int sec4_sg_bytes)
931{ 791{
932 if (dst != src) { 792 if (dst != src) {
933 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); 793 if (src_nents)
934 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); 794 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
795 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
935 } else { 796 } else {
936 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); 797 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
937 } 798 }
938 799
939 if (iv_dma) 800 if (iv_dma)
@@ -1102,7 +963,7 @@ static void init_aead_job(struct aead_request *req,
1102 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 963 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1103 964
1104 if (all_contig) { 965 if (all_contig) {
1105 src_dma = sg_dma_address(req->src); 966 src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0;
1106 in_options = 0; 967 in_options = 0;
1107 } else { 968 } else {
1108 src_dma = edesc->sec4_sg_dma; 969 src_dma = edesc->sec4_sg_dma;
@@ -1117,7 +978,7 @@ static void init_aead_job(struct aead_request *req,
1117 out_options = in_options; 978 out_options = in_options;
1118 979
1119 if (unlikely(req->src != req->dst)) { 980 if (unlikely(req->src != req->dst)) {
1120 if (!edesc->dst_nents) { 981 if (edesc->dst_nents == 1) {
1121 dst_dma = sg_dma_address(req->dst); 982 dst_dma = sg_dma_address(req->dst);
1122 } else { 983 } else {
1123 dst_dma = edesc->sec4_sg_dma + 984 dst_dma = edesc->sec4_sg_dma +
@@ -1227,10 +1088,11 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1227 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1088 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1228 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1089 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1229 ivsize, 1); 1090 ivsize, 1);
1230 printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes); 1091 pr_err("asked=%d, nbytes%d\n",
1092 (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes);
1231 dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ", 1093 dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ",
1232 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1094 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1233 edesc->src_nents ? 100 : req->nbytes, 1); 1095 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1234#endif 1096#endif
1235 1097
1236 len = desc_len(sh_desc); 1098 len = desc_len(sh_desc);
@@ -1247,7 +1109,7 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1247 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 1109 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
1248 1110
1249 if (likely(req->src == req->dst)) { 1111 if (likely(req->src == req->dst)) {
1250 if (!edesc->src_nents && iv_contig) { 1112 if (edesc->src_nents == 1 && iv_contig) {
1251 dst_dma = sg_dma_address(req->src); 1113 dst_dma = sg_dma_address(req->src);
1252 } else { 1114 } else {
1253 dst_dma = edesc->sec4_sg_dma + 1115 dst_dma = edesc->sec4_sg_dma +
@@ -1255,7 +1117,7 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1255 out_options = LDST_SGF; 1117 out_options = LDST_SGF;
1256 } 1118 }
1257 } else { 1119 } else {
1258 if (!edesc->dst_nents) { 1120 if (edesc->dst_nents == 1) {
1259 dst_dma = sg_dma_address(req->dst); 1121 dst_dma = sg_dma_address(req->dst);
1260 } else { 1122 } else {
1261 dst_dma = edesc->sec4_sg_dma + 1123 dst_dma = edesc->sec4_sg_dma +
@@ -1287,13 +1149,13 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
1287 ivsize, 1); 1149 ivsize, 1);
1288 dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", 1150 dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ",
1289 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1151 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1290 edesc->src_nents ? 100 : req->nbytes, 1); 1152 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1291#endif 1153#endif
1292 1154
1293 len = desc_len(sh_desc); 1155 len = desc_len(sh_desc);
1294 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1156 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1295 1157
1296 if (!edesc->src_nents) { 1158 if (edesc->src_nents == 1) {
1297 src_dma = sg_dma_address(req->src); 1159 src_dma = sg_dma_address(req->src);
1298 in_options = 0; 1160 in_options = 0;
1299 } else { 1161 } else {
@@ -1326,83 +1188,98 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1326 struct device *jrdev = ctx->jrdev; 1188 struct device *jrdev = ctx->jrdev;
1327 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1189 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1328 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1190 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1329 int src_nents, dst_nents = 0; 1191 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1330 struct aead_edesc *edesc; 1192 struct aead_edesc *edesc;
1331 int sgc; 1193 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
1332 bool all_contig = true;
1333 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
1334 unsigned int authsize = ctx->authsize; 1194 unsigned int authsize = ctx->authsize;
1335 1195
1336 if (unlikely(req->dst != req->src)) { 1196 if (unlikely(req->dst != req->src)) {
1337 src_nents = sg_count(req->src, req->assoclen + req->cryptlen); 1197 src_nents = sg_nents_for_len(req->src, req->assoclen +
1338 dst_nents = sg_count(req->dst, 1198 req->cryptlen);
1339 req->assoclen + req->cryptlen + 1199 if (unlikely(src_nents < 0)) {
1340 (encrypt ? authsize : (-authsize))); 1200 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1341 } else { 1201 req->assoclen + req->cryptlen);
1342 src_nents = sg_count(req->src, 1202 return ERR_PTR(src_nents);
1343 req->assoclen + req->cryptlen + 1203 }
1344 (encrypt ? authsize : 0));
1345 }
1346
1347 /* Check if data are contiguous. */
1348 all_contig = !src_nents;
1349 if (!all_contig)
1350 sec4_sg_len = src_nents;
1351
1352 sec4_sg_len += dst_nents;
1353
1354 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1355 1204
1356 /* allocate space for base edesc and hw desc commands, link tables */ 1205 dst_nents = sg_nents_for_len(req->dst, req->assoclen +
1357 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1206 req->cryptlen +
1358 GFP_DMA | flags); 1207 (encrypt ? authsize :
1359 if (!edesc) { 1208 (-authsize)));
1360 dev_err(jrdev, "could not allocate extended descriptor\n"); 1209 if (unlikely(dst_nents < 0)) {
1361 return ERR_PTR(-ENOMEM); 1210 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1211 req->assoclen + req->cryptlen +
1212 (encrypt ? authsize : (-authsize)));
1213 return ERR_PTR(dst_nents);
1214 }
1215 } else {
1216 src_nents = sg_nents_for_len(req->src, req->assoclen +
1217 req->cryptlen +
1218 (encrypt ? authsize : 0));
1219 if (unlikely(src_nents < 0)) {
1220 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1221 req->assoclen + req->cryptlen +
1222 (encrypt ? authsize : 0));
1223 return ERR_PTR(src_nents);
1224 }
1362 } 1225 }
1363 1226
1364 if (likely(req->src == req->dst)) { 1227 if (likely(req->src == req->dst)) {
1365 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1228 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1366 DMA_BIDIRECTIONAL); 1229 DMA_BIDIRECTIONAL);
1367 if (unlikely(!sgc)) { 1230 if (unlikely(!mapped_src_nents)) {
1368 dev_err(jrdev, "unable to map source\n"); 1231 dev_err(jrdev, "unable to map source\n");
1369 kfree(edesc);
1370 return ERR_PTR(-ENOMEM); 1232 return ERR_PTR(-ENOMEM);
1371 } 1233 }
1372 } else { 1234 } else {
1373 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1235 /* Cover also the case of null (zero length) input data */
1374 DMA_TO_DEVICE); 1236 if (src_nents) {
1375 if (unlikely(!sgc)) { 1237 mapped_src_nents = dma_map_sg(jrdev, req->src,
1376 dev_err(jrdev, "unable to map source\n"); 1238 src_nents, DMA_TO_DEVICE);
1377 kfree(edesc); 1239 if (unlikely(!mapped_src_nents)) {
1378 return ERR_PTR(-ENOMEM); 1240 dev_err(jrdev, "unable to map source\n");
1241 return ERR_PTR(-ENOMEM);
1242 }
1243 } else {
1244 mapped_src_nents = 0;
1379 } 1245 }
1380 1246
1381 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 1247 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1382 DMA_FROM_DEVICE); 1248 DMA_FROM_DEVICE);
1383 if (unlikely(!sgc)) { 1249 if (unlikely(!mapped_dst_nents)) {
1384 dev_err(jrdev, "unable to map destination\n"); 1250 dev_err(jrdev, "unable to map destination\n");
1385 dma_unmap_sg(jrdev, req->src, src_nents ? : 1, 1251 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1386 DMA_TO_DEVICE);
1387 kfree(edesc);
1388 return ERR_PTR(-ENOMEM); 1252 return ERR_PTR(-ENOMEM);
1389 } 1253 }
1390 } 1254 }
1391 1255
1256 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
1257 sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1258 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1259
1260 /* allocate space for base edesc and hw desc commands, link tables */
1261 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1262 GFP_DMA | flags);
1263 if (!edesc) {
1264 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1265 0, 0, 0);
1266 return ERR_PTR(-ENOMEM);
1267 }
1268
1392 edesc->src_nents = src_nents; 1269 edesc->src_nents = src_nents;
1393 edesc->dst_nents = dst_nents; 1270 edesc->dst_nents = dst_nents;
1394 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1271 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1395 desc_bytes; 1272 desc_bytes;
1396 *all_contig_ptr = all_contig; 1273 *all_contig_ptr = !(mapped_src_nents > 1);
1397 1274
1398 sec4_sg_index = 0; 1275 sec4_sg_index = 0;
1399 if (!all_contig) { 1276 if (mapped_src_nents > 1) {
1400 sg_to_sec4_sg_last(req->src, src_nents, 1277 sg_to_sec4_sg_last(req->src, mapped_src_nents,
1401 edesc->sec4_sg + sec4_sg_index, 0); 1278 edesc->sec4_sg + sec4_sg_index, 0);
1402 sec4_sg_index += src_nents; 1279 sec4_sg_index += mapped_src_nents;
1403 } 1280 }
1404 if (dst_nents) { 1281 if (mapped_dst_nents > 1) {
1405 sg_to_sec4_sg_last(req->dst, dst_nents, 1282 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1406 edesc->sec4_sg + sec4_sg_index, 0); 1283 edesc->sec4_sg + sec4_sg_index, 0);
1407 } 1284 }
1408 1285
@@ -1600,40 +1477,49 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1600 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1477 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1601 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 1478 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1602 GFP_KERNEL : GFP_ATOMIC; 1479 GFP_KERNEL : GFP_ATOMIC;
1603 int src_nents, dst_nents = 0, sec4_sg_bytes; 1480 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1604 struct ablkcipher_edesc *edesc; 1481 struct ablkcipher_edesc *edesc;
1605 dma_addr_t iv_dma = 0; 1482 dma_addr_t iv_dma = 0;
1606 bool iv_contig = false; 1483 bool in_contig;
1607 int sgc;
1608 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1484 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1609 int sec4_sg_index; 1485 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1610 1486
1611 src_nents = sg_count(req->src, req->nbytes); 1487 src_nents = sg_nents_for_len(req->src, req->nbytes);
1488 if (unlikely(src_nents < 0)) {
1489 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1490 req->nbytes);
1491 return ERR_PTR(src_nents);
1492 }
1612 1493
1613 if (req->dst != req->src) 1494 if (req->dst != req->src) {
1614 dst_nents = sg_count(req->dst, req->nbytes); 1495 dst_nents = sg_nents_for_len(req->dst, req->nbytes);
1496 if (unlikely(dst_nents < 0)) {
1497 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1498 req->nbytes);
1499 return ERR_PTR(dst_nents);
1500 }
1501 }
1615 1502
1616 if (likely(req->src == req->dst)) { 1503 if (likely(req->src == req->dst)) {
1617 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1504 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1618 DMA_BIDIRECTIONAL); 1505 DMA_BIDIRECTIONAL);
1619 if (unlikely(!sgc)) { 1506 if (unlikely(!mapped_src_nents)) {
1620 dev_err(jrdev, "unable to map source\n"); 1507 dev_err(jrdev, "unable to map source\n");
1621 return ERR_PTR(-ENOMEM); 1508 return ERR_PTR(-ENOMEM);
1622 } 1509 }
1623 } else { 1510 } else {
1624 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1511 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1625 DMA_TO_DEVICE); 1512 DMA_TO_DEVICE);
1626 if (unlikely(!sgc)) { 1513 if (unlikely(!mapped_src_nents)) {
1627 dev_err(jrdev, "unable to map source\n"); 1514 dev_err(jrdev, "unable to map source\n");
1628 return ERR_PTR(-ENOMEM); 1515 return ERR_PTR(-ENOMEM);
1629 } 1516 }
1630 1517
1631 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 1518 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1632 DMA_FROM_DEVICE); 1519 DMA_FROM_DEVICE);
1633 if (unlikely(!sgc)) { 1520 if (unlikely(!mapped_dst_nents)) {
1634 dev_err(jrdev, "unable to map destination\n"); 1521 dev_err(jrdev, "unable to map destination\n");
1635 dma_unmap_sg(jrdev, req->src, src_nents ? : 1, 1522 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1636 DMA_TO_DEVICE);
1637 return ERR_PTR(-ENOMEM); 1523 return ERR_PTR(-ENOMEM);
1638 } 1524 }
1639 } 1525 }
@@ -1646,16 +1532,17 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1646 return ERR_PTR(-ENOMEM); 1532 return ERR_PTR(-ENOMEM);
1647 } 1533 }
1648 1534
1649 /* 1535 if (mapped_src_nents == 1 &&
1650 * Check if iv can be contiguous with source and destination. 1536 iv_dma + ivsize == sg_dma_address(req->src)) {
1651 * If so, include it. If not, create scatterlist. 1537 in_contig = true;
1652 */ 1538 sec4_sg_ents = 0;
1653 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 1539 } else {
1654 iv_contig = true; 1540 in_contig = false;
1655 else 1541 sec4_sg_ents = 1 + mapped_src_nents;
1656 src_nents = src_nents ? : 1; 1542 }
1657 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 1543 dst_sg_idx = sec4_sg_ents;
1658 sizeof(struct sec4_sg_entry); 1544 sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1545 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1659 1546
1660 /* allocate space for base edesc and hw desc commands, link tables */ 1547 /* allocate space for base edesc and hw desc commands, link tables */
1661 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1548 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
@@ -1673,17 +1560,15 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1673 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1560 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1674 desc_bytes; 1561 desc_bytes;
1675 1562
1676 sec4_sg_index = 0; 1563 if (!in_contig) {
1677 if (!iv_contig) {
1678 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1564 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1679 sg_to_sec4_sg_last(req->src, src_nents, 1565 sg_to_sec4_sg_last(req->src, mapped_src_nents,
1680 edesc->sec4_sg + 1, 0); 1566 edesc->sec4_sg + 1, 0);
1681 sec4_sg_index += 1 + src_nents;
1682 } 1567 }
1683 1568
1684 if (dst_nents) { 1569 if (mapped_dst_nents > 1) {
1685 sg_to_sec4_sg_last(req->dst, dst_nents, 1570 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1686 edesc->sec4_sg + sec4_sg_index, 0); 1571 edesc->sec4_sg + dst_sg_idx, 0);
1687 } 1572 }
1688 1573
1689 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1574 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@@ -1704,7 +1589,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1704 sec4_sg_bytes, 1); 1589 sec4_sg_bytes, 1);
1705#endif 1590#endif
1706 1591
1707 *iv_contig_out = iv_contig; 1592 *iv_contig_out = in_contig;
1708 return edesc; 1593 return edesc;
1709} 1594}
1710 1595
@@ -1798,40 +1683,50 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1798 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1683 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1799 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 1684 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1800 GFP_KERNEL : GFP_ATOMIC; 1685 GFP_KERNEL : GFP_ATOMIC;
1801 int src_nents, dst_nents = 0, sec4_sg_bytes; 1686 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
1802 struct ablkcipher_edesc *edesc; 1687 struct ablkcipher_edesc *edesc;
1803 dma_addr_t iv_dma = 0; 1688 dma_addr_t iv_dma = 0;
1804 bool iv_contig = false; 1689 bool out_contig;
1805 int sgc;
1806 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1690 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1807 int sec4_sg_index; 1691 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1808
1809 src_nents = sg_count(req->src, req->nbytes);
1810 1692
1811 if (unlikely(req->dst != req->src)) 1693 src_nents = sg_nents_for_len(req->src, req->nbytes);
1812 dst_nents = sg_count(req->dst, req->nbytes); 1694 if (unlikely(src_nents < 0)) {
1695 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1696 req->nbytes);
1697 return ERR_PTR(src_nents);
1698 }
1813 1699
1814 if (likely(req->src == req->dst)) { 1700 if (likely(req->src == req->dst)) {
1815 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1701 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1816 DMA_BIDIRECTIONAL); 1702 DMA_BIDIRECTIONAL);
1817 if (unlikely(!sgc)) { 1703 if (unlikely(!mapped_src_nents)) {
1818 dev_err(jrdev, "unable to map source\n"); 1704 dev_err(jrdev, "unable to map source\n");
1819 return ERR_PTR(-ENOMEM); 1705 return ERR_PTR(-ENOMEM);
1820 } 1706 }
1707
1708 dst_nents = src_nents;
1709 mapped_dst_nents = src_nents;
1821 } else { 1710 } else {
1822 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1711 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1823 DMA_TO_DEVICE); 1712 DMA_TO_DEVICE);
1824 if (unlikely(!sgc)) { 1713 if (unlikely(!mapped_src_nents)) {
1825 dev_err(jrdev, "unable to map source\n"); 1714 dev_err(jrdev, "unable to map source\n");
1826 return ERR_PTR(-ENOMEM); 1715 return ERR_PTR(-ENOMEM);
1827 } 1716 }
1828 1717
1829 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 1718 dst_nents = sg_nents_for_len(req->dst, req->nbytes);
1830 DMA_FROM_DEVICE); 1719 if (unlikely(dst_nents < 0)) {
1831 if (unlikely(!sgc)) { 1720 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1721 req->nbytes);
1722 return ERR_PTR(dst_nents);
1723 }
1724
1725 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1726 DMA_FROM_DEVICE);
1727 if (unlikely(!mapped_dst_nents)) {
1832 dev_err(jrdev, "unable to map destination\n"); 1728 dev_err(jrdev, "unable to map destination\n");
1833 dma_unmap_sg(jrdev, req->src, src_nents ? : 1, 1729 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1834 DMA_TO_DEVICE);
1835 return ERR_PTR(-ENOMEM); 1730 return ERR_PTR(-ENOMEM);
1836 } 1731 }
1837 } 1732 }
@@ -1848,14 +1743,18 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1848 return ERR_PTR(-ENOMEM); 1743 return ERR_PTR(-ENOMEM);
1849 } 1744 }
1850 1745
1851 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) 1746 sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0;
1852 iv_contig = true; 1747 dst_sg_idx = sec4_sg_ents;
1853 else 1748 if (mapped_dst_nents == 1 &&
1854 dst_nents = dst_nents ? : 1; 1749 iv_dma + ivsize == sg_dma_address(req->dst)) {
1855 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 1750 out_contig = true;
1856 sizeof(struct sec4_sg_entry); 1751 } else {
1752 out_contig = false;
1753 sec4_sg_ents += 1 + mapped_dst_nents;
1754 }
1857 1755
1858 /* allocate space for base edesc and hw desc commands, link tables */ 1756 /* allocate space for base edesc and hw desc commands, link tables */
1757 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1859 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1758 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1860 GFP_DMA | flags); 1759 GFP_DMA | flags);
1861 if (!edesc) { 1760 if (!edesc) {
@@ -1871,18 +1770,15 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1871 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1770 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1872 desc_bytes; 1771 desc_bytes;
1873 1772
1874 sec4_sg_index = 0; 1773 if (mapped_src_nents > 1)
1875 if (src_nents) { 1774 sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg,
1876 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 1775 0);
1877 sec4_sg_index += src_nents;
1878 }
1879 1776
1880 if (!iv_contig) { 1777 if (!out_contig) {
1881 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1778 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx,
1882 iv_dma, ivsize, 0); 1779 iv_dma, ivsize, 0);
1883 sec4_sg_index += 1; 1780 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1884 sg_to_sec4_sg_last(req->dst, dst_nents, 1781 edesc->sec4_sg + dst_sg_idx + 1, 0);
1885 edesc->sec4_sg + sec4_sg_index, 0);
1886 } 1782 }
1887 1783
1888 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1784 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@@ -1903,7 +1799,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1903 sec4_sg_bytes, 1); 1799 sec4_sg_bytes, 1);
1904#endif 1800#endif
1905 1801
1906 *iv_contig_out = iv_contig; 1802 *iv_contig_out = out_contig;
1907 return edesc; 1803 return edesc;
1908} 1804}
1909 1805
@@ -1914,7 +1810,7 @@ static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
1914 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1810 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1915 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1811 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1916 struct device *jrdev = ctx->jrdev; 1812 struct device *jrdev = ctx->jrdev;
1917 bool iv_contig; 1813 bool iv_contig = false;
1918 u32 *desc; 1814 u32 *desc;
1919 int ret = 0; 1815 int ret = 0;
1920 1816
@@ -3355,12 +3251,31 @@ struct caam_crypto_alg {
3355 3251
3356static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 3252static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
3357{ 3253{
3254 dma_addr_t dma_addr;
3255
3358 ctx->jrdev = caam_jr_alloc(); 3256 ctx->jrdev = caam_jr_alloc();
3359 if (IS_ERR(ctx->jrdev)) { 3257 if (IS_ERR(ctx->jrdev)) {
3360 pr_err("Job Ring Device allocation for transform failed\n"); 3258 pr_err("Job Ring Device allocation for transform failed\n");
3361 return PTR_ERR(ctx->jrdev); 3259 return PTR_ERR(ctx->jrdev);
3362 } 3260 }
3363 3261
3262 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
3263 offsetof(struct caam_ctx,
3264 sh_desc_enc_dma),
3265 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
3266 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
3267 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3268 caam_jr_free(ctx->jrdev);
3269 return -ENOMEM;
3270 }
3271
3272 ctx->sh_desc_enc_dma = dma_addr;
3273 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
3274 sh_desc_dec);
3275 ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx,
3276 sh_desc_givenc);
3277 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key);
3278
3364 /* copy descriptor header template value */ 3279 /* copy descriptor header template value */
3365 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3280 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3366 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3281 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
@@ -3390,25 +3305,9 @@ static int caam_aead_init(struct crypto_aead *tfm)
3390 3305
3391static void caam_exit_common(struct caam_ctx *ctx) 3306static void caam_exit_common(struct caam_ctx *ctx)
3392{ 3307{
3393 if (ctx->sh_desc_enc_dma && 3308 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
3394 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 3309 offsetof(struct caam_ctx, sh_desc_enc_dma),
3395 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 3310 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
3396 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
3397 if (ctx->sh_desc_dec_dma &&
3398 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
3399 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
3400 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
3401 if (ctx->sh_desc_givenc_dma &&
3402 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
3403 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
3404 desc_bytes(ctx->sh_desc_givenc),
3405 DMA_TO_DEVICE);
3406 if (ctx->key_dma &&
3407 !dma_mapping_error(ctx->jrdev, ctx->key_dma))
3408 dma_unmap_single(ctx->jrdev, ctx->key_dma,
3409 ctx->cdata.keylen + ctx->adata.keylen_pad,
3410 DMA_TO_DEVICE);
3411
3412 caam_jr_free(ctx->jrdev); 3311 caam_jr_free(ctx->jrdev);
3413} 3312}
3414 3313
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index e58639ea53b1..da4f94eab3da 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -109,7 +109,6 @@ struct caam_hash_ctx {
109 dma_addr_t sh_desc_digest_dma; 109 dma_addr_t sh_desc_digest_dma;
110 struct device *jrdev; 110 struct device *jrdev;
111 u8 key[CAAM_MAX_HASH_KEY_SIZE]; 111 u8 key[CAAM_MAX_HASH_KEY_SIZE];
112 dma_addr_t key_dma;
113 int ctx_len; 112 int ctx_len;
114 struct alginfo adata; 113 struct alginfo adata;
115}; 114};
@@ -138,6 +137,31 @@ struct caam_export_state {
138 int (*finup)(struct ahash_request *req); 137 int (*finup)(struct ahash_request *req);
139}; 138};
140 139
140static inline void switch_buf(struct caam_hash_state *state)
141{
142 state->current_buf ^= 1;
143}
144
145static inline u8 *current_buf(struct caam_hash_state *state)
146{
147 return state->current_buf ? state->buf_1 : state->buf_0;
148}
149
150static inline u8 *alt_buf(struct caam_hash_state *state)
151{
152 return state->current_buf ? state->buf_0 : state->buf_1;
153}
154
155static inline int *current_buflen(struct caam_hash_state *state)
156{
157 return state->current_buf ? &state->buflen_1 : &state->buflen_0;
158}
159
160static inline int *alt_buflen(struct caam_hash_state *state)
161{
162 return state->current_buf ? &state->buflen_0 : &state->buflen_1;
163}
164
141/* Common job descriptor seq in/out ptr routines */ 165/* Common job descriptor seq in/out ptr routines */
142 166
143/* Map state->caam_ctx, and append seq_out_ptr command that points to it */ 167/* Map state->caam_ctx, and append seq_out_ptr command that points to it */
@@ -149,6 +173,7 @@ static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
149 ctx_len, DMA_FROM_DEVICE); 173 ctx_len, DMA_FROM_DEVICE);
150 if (dma_mapping_error(jrdev, state->ctx_dma)) { 174 if (dma_mapping_error(jrdev, state->ctx_dma)) {
151 dev_err(jrdev, "unable to map ctx\n"); 175 dev_err(jrdev, "unable to map ctx\n");
176 state->ctx_dma = 0;
152 return -ENOMEM; 177 return -ENOMEM;
153 } 178 }
154 179
@@ -169,36 +194,27 @@ static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
169 return dst_dma; 194 return dst_dma;
170} 195}
171 196
172/* Map current buffer in state and put it in link table */ 197/* Map current buffer in state (if length > 0) and put it in link table */
173static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev, 198static inline int buf_map_to_sec4_sg(struct device *jrdev,
174 struct sec4_sg_entry *sec4_sg, 199 struct sec4_sg_entry *sec4_sg,
175 u8 *buf, int buflen) 200 struct caam_hash_state *state)
176{ 201{
177 dma_addr_t buf_dma; 202 int buflen = *current_buflen(state);
178 203
179 buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 204 if (!buflen)
180 dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0); 205 return 0;
181 206
182 return buf_dma; 207 state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen,
183} 208 DMA_TO_DEVICE);
209 if (dma_mapping_error(jrdev, state->buf_dma)) {
210 dev_err(jrdev, "unable to map buf\n");
211 state->buf_dma = 0;
212 return -ENOMEM;
213 }
184 214
185/* 215 dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
186 * Only put buffer in link table if it contains data, which is possible, 216
187 * since a buffer has previously been used, and needs to be unmapped, 217 return 0;
188 */
189static inline dma_addr_t
190try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg,
191 u8 *buf, dma_addr_t buf_dma, int buflen,
192 int last_buflen)
193{
194 if (buf_dma && !dma_mapping_error(jrdev, buf_dma))
195 dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE);
196 if (buflen)
197 buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen);
198 else
199 buf_dma = 0;
200
201 return buf_dma;
202} 218}
203 219
204/* Map state->caam_ctx, and add it to link table */ 220/* Map state->caam_ctx, and add it to link table */
@@ -209,6 +225,7 @@ static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
209 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); 225 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
210 if (dma_mapping_error(jrdev, state->ctx_dma)) { 226 if (dma_mapping_error(jrdev, state->ctx_dma)) {
211 dev_err(jrdev, "unable to map ctx\n"); 227 dev_err(jrdev, "unable to map ctx\n");
228 state->ctx_dma = 0;
212 return -ENOMEM; 229 return -ENOMEM;
213 } 230 }
214 231
@@ -277,12 +294,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
277 /* ahash_update shared descriptor */ 294 /* ahash_update shared descriptor */
278 desc = ctx->sh_desc_update; 295 desc = ctx->sh_desc_update;
279 ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true); 296 ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
280 ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 297 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
281 DMA_TO_DEVICE); 298 desc_bytes(desc), DMA_TO_DEVICE);
282 if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) {
283 dev_err(jrdev, "unable to map shared descriptor\n");
284 return -ENOMEM;
285 }
286#ifdef DEBUG 299#ifdef DEBUG
287 print_hex_dump(KERN_ERR, 300 print_hex_dump(KERN_ERR,
288 "ahash update shdesc@"__stringify(__LINE__)": ", 301 "ahash update shdesc@"__stringify(__LINE__)": ",
@@ -292,13 +305,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
292 /* ahash_update_first shared descriptor */ 305 /* ahash_update_first shared descriptor */
293 desc = ctx->sh_desc_update_first; 306 desc = ctx->sh_desc_update_first;
294 ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false); 307 ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
295 ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc, 308 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
296 desc_bytes(desc), 309 desc_bytes(desc), DMA_TO_DEVICE);
297 DMA_TO_DEVICE);
298 if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) {
299 dev_err(jrdev, "unable to map shared descriptor\n");
300 return -ENOMEM;
301 }
302#ifdef DEBUG 310#ifdef DEBUG
303 print_hex_dump(KERN_ERR, 311 print_hex_dump(KERN_ERR,
304 "ahash update first shdesc@"__stringify(__LINE__)": ", 312 "ahash update first shdesc@"__stringify(__LINE__)": ",
@@ -308,12 +316,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
308 /* ahash_final shared descriptor */ 316 /* ahash_final shared descriptor */
309 desc = ctx->sh_desc_fin; 317 desc = ctx->sh_desc_fin;
310 ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true); 318 ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
311 ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 319 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
312 DMA_TO_DEVICE); 320 desc_bytes(desc), DMA_TO_DEVICE);
313 if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) {
314 dev_err(jrdev, "unable to map shared descriptor\n");
315 return -ENOMEM;
316 }
317#ifdef DEBUG 321#ifdef DEBUG
318 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", 322 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
319 DUMP_PREFIX_ADDRESS, 16, 4, desc, 323 DUMP_PREFIX_ADDRESS, 16, 4, desc,
@@ -323,13 +327,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
323 /* ahash_digest shared descriptor */ 327 /* ahash_digest shared descriptor */
324 desc = ctx->sh_desc_digest; 328 desc = ctx->sh_desc_digest;
325 ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false); 329 ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
326 ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc, 330 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
327 desc_bytes(desc), 331 desc_bytes(desc), DMA_TO_DEVICE);
328 DMA_TO_DEVICE);
329 if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) {
330 dev_err(jrdev, "unable to map shared descriptor\n");
331 return -ENOMEM;
332 }
333#ifdef DEBUG 332#ifdef DEBUG
334 print_hex_dump(KERN_ERR, 333 print_hex_dump(KERN_ERR,
335 "ahash digest shdesc@"__stringify(__LINE__)": ", 334 "ahash digest shdesc@"__stringify(__LINE__)": ",
@@ -420,7 +419,6 @@ static int ahash_setkey(struct crypto_ahash *ahash,
420 const u8 *key, unsigned int keylen) 419 const u8 *key, unsigned int keylen)
421{ 420{
422 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 421 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
423 struct device *jrdev = ctx->jrdev;
424 int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 422 int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
425 int digestsize = crypto_ahash_digestsize(ahash); 423 int digestsize = crypto_ahash_digestsize(ahash);
426 int ret; 424 int ret;
@@ -448,28 +446,14 @@ static int ahash_setkey(struct crypto_ahash *ahash,
448 if (ret) 446 if (ret)
449 goto bad_free_key; 447 goto bad_free_key;
450 448
451 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->adata.keylen_pad,
452 DMA_TO_DEVICE);
453 if (dma_mapping_error(jrdev, ctx->key_dma)) {
454 dev_err(jrdev, "unable to map key i/o memory\n");
455 ret = -ENOMEM;
456 goto error_free_key;
457 }
458#ifdef DEBUG 449#ifdef DEBUG
459 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 450 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
460 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 451 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
461 ctx->adata.keylen_pad, 1); 452 ctx->adata.keylen_pad, 1);
462#endif 453#endif
463 454
464 ret = ahash_set_sh_desc(ahash);
465 if (ret) {
466 dma_unmap_single(jrdev, ctx->key_dma, ctx->adata.keylen_pad,
467 DMA_TO_DEVICE);
468 }
469
470 error_free_key:
471 kfree(hashed_key); 455 kfree(hashed_key);
472 return ret; 456 return ahash_set_sh_desc(ahash);
473 bad_free_key: 457 bad_free_key:
474 kfree(hashed_key); 458 kfree(hashed_key);
475 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 459 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
@@ -498,6 +482,8 @@ static inline void ahash_unmap(struct device *dev,
498 struct ahash_edesc *edesc, 482 struct ahash_edesc *edesc,
499 struct ahash_request *req, int dst_len) 483 struct ahash_request *req, int dst_len)
500{ 484{
485 struct caam_hash_state *state = ahash_request_ctx(req);
486
501 if (edesc->src_nents) 487 if (edesc->src_nents)
502 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 488 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
503 if (edesc->dst_dma) 489 if (edesc->dst_dma)
@@ -506,6 +492,12 @@ static inline void ahash_unmap(struct device *dev,
506 if (edesc->sec4_sg_bytes) 492 if (edesc->sec4_sg_bytes)
507 dma_unmap_single(dev, edesc->sec4_sg_dma, 493 dma_unmap_single(dev, edesc->sec4_sg_dma,
508 edesc->sec4_sg_bytes, DMA_TO_DEVICE); 494 edesc->sec4_sg_bytes, DMA_TO_DEVICE);
495
496 if (state->buf_dma) {
497 dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
498 DMA_TO_DEVICE);
499 state->buf_dma = 0;
500 }
509} 501}
510 502
511static inline void ahash_unmap_ctx(struct device *dev, 503static inline void ahash_unmap_ctx(struct device *dev,
@@ -516,8 +508,10 @@ static inline void ahash_unmap_ctx(struct device *dev,
516 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 508 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
517 struct caam_hash_state *state = ahash_request_ctx(req); 509 struct caam_hash_state *state = ahash_request_ctx(req);
518 510
519 if (state->ctx_dma) 511 if (state->ctx_dma) {
520 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); 512 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
513 state->ctx_dma = 0;
514 }
521 ahash_unmap(dev, edesc, req, dst_len); 515 ahash_unmap(dev, edesc, req, dst_len);
522} 516}
523 517
@@ -562,8 +556,8 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
562 struct ahash_edesc *edesc; 556 struct ahash_edesc *edesc;
563 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 557 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
564 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 558 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
565#ifdef DEBUG
566 struct caam_hash_state *state = ahash_request_ctx(req); 559 struct caam_hash_state *state = ahash_request_ctx(req);
560#ifdef DEBUG
567 int digestsize = crypto_ahash_digestsize(ahash); 561 int digestsize = crypto_ahash_digestsize(ahash);
568 562
569 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 563 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
@@ -574,6 +568,7 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
574 caam_jr_strstatus(jrdev, err); 568 caam_jr_strstatus(jrdev, err);
575 569
576 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 570 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
571 switch_buf(state);
577 kfree(edesc); 572 kfree(edesc);
578 573
579#ifdef DEBUG 574#ifdef DEBUG
@@ -630,8 +625,8 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
630 struct ahash_edesc *edesc; 625 struct ahash_edesc *edesc;
631 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 626 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
632 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 627 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
633#ifdef DEBUG
634 struct caam_hash_state *state = ahash_request_ctx(req); 628 struct caam_hash_state *state = ahash_request_ctx(req);
629#ifdef DEBUG
635 int digestsize = crypto_ahash_digestsize(ahash); 630 int digestsize = crypto_ahash_digestsize(ahash);
636 631
637 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 632 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
@@ -642,6 +637,7 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
642 caam_jr_strstatus(jrdev, err); 637 caam_jr_strstatus(jrdev, err);
643 638
644 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 639 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
640 switch_buf(state);
645 kfree(edesc); 641 kfree(edesc);
646 642
647#ifdef DEBUG 643#ifdef DEBUG
@@ -725,11 +721,10 @@ static int ahash_update_ctx(struct ahash_request *req)
725 struct device *jrdev = ctx->jrdev; 721 struct device *jrdev = ctx->jrdev;
726 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 722 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
727 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 723 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
728 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 724 u8 *buf = current_buf(state);
729 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; 725 int *buflen = current_buflen(state);
730 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; 726 u8 *next_buf = alt_buf(state);
731 int *next_buflen = state->current_buf ? &state->buflen_0 : 727 int *next_buflen = alt_buflen(state), last_buflen;
732 &state->buflen_1, last_buflen;
733 int in_len = *buflen + req->nbytes, to_hash; 728 int in_len = *buflen + req->nbytes, to_hash;
734 u32 *desc; 729 u32 *desc;
735 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; 730 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
@@ -783,10 +778,9 @@ static int ahash_update_ctx(struct ahash_request *req)
783 if (ret) 778 if (ret)
784 goto unmap_ctx; 779 goto unmap_ctx;
785 780
786 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, 781 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
787 edesc->sec4_sg + 1, 782 if (ret)
788 buf, state->buf_dma, 783 goto unmap_ctx;
789 *buflen, last_buflen);
790 784
791 if (mapped_nents) { 785 if (mapped_nents) {
792 sg_to_sec4_sg_last(req->src, mapped_nents, 786 sg_to_sec4_sg_last(req->src, mapped_nents,
@@ -801,8 +795,6 @@ static int ahash_update_ctx(struct ahash_request *req)
801 cpu_to_caam32(SEC4_SG_LEN_FIN); 795 cpu_to_caam32(SEC4_SG_LEN_FIN);
802 } 796 }
803 797
804 state->current_buf = !state->current_buf;
805
806 desc = edesc->hw_desc; 798 desc = edesc->hw_desc;
807 799
808 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 800 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@@ -859,10 +851,7 @@ static int ahash_final_ctx(struct ahash_request *req)
859 struct device *jrdev = ctx->jrdev; 851 struct device *jrdev = ctx->jrdev;
860 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 852 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
861 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 853 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
862 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 854 int buflen = *current_buflen(state);
863 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
864 int last_buflen = state->current_buf ? state->buflen_0 :
865 state->buflen_1;
866 u32 *desc; 855 u32 *desc;
867 int sec4_sg_bytes, sec4_sg_src_index; 856 int sec4_sg_bytes, sec4_sg_src_index;
868 int digestsize = crypto_ahash_digestsize(ahash); 857 int digestsize = crypto_ahash_digestsize(ahash);
@@ -889,9 +878,10 @@ static int ahash_final_ctx(struct ahash_request *req)
889 if (ret) 878 if (ret)
890 goto unmap_ctx; 879 goto unmap_ctx;
891 880
892 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, 881 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
893 buf, state->buf_dma, buflen, 882 if (ret)
894 last_buflen); 883 goto unmap_ctx;
884
895 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 885 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
896 cpu_to_caam32(SEC4_SG_LEN_FIN); 886 cpu_to_caam32(SEC4_SG_LEN_FIN);
897 887
@@ -938,10 +928,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
938 struct device *jrdev = ctx->jrdev; 928 struct device *jrdev = ctx->jrdev;
939 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 929 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
940 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 930 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
941 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 931 int buflen = *current_buflen(state);
942 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
943 int last_buflen = state->current_buf ? state->buflen_0 :
944 state->buflen_1;
945 u32 *desc; 932 u32 *desc;
946 int sec4_sg_src_index; 933 int sec4_sg_src_index;
947 int src_nents, mapped_nents; 934 int src_nents, mapped_nents;
@@ -986,9 +973,9 @@ static int ahash_finup_ctx(struct ahash_request *req)
986 if (ret) 973 if (ret)
987 goto unmap_ctx; 974 goto unmap_ctx;
988 975
989 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, 976 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
990 buf, state->buf_dma, buflen, 977 if (ret)
991 last_buflen); 978 goto unmap_ctx;
992 979
993 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 980 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
994 sec4_sg_src_index, ctx->ctx_len + buflen, 981 sec4_sg_src_index, ctx->ctx_len + buflen,
@@ -1024,6 +1011,7 @@ static int ahash_digest(struct ahash_request *req)
1024{ 1011{
1025 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1012 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1026 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1013 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1014 struct caam_hash_state *state = ahash_request_ctx(req);
1027 struct device *jrdev = ctx->jrdev; 1015 struct device *jrdev = ctx->jrdev;
1028 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1016 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1029 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1017 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
@@ -1033,6 +1021,8 @@ static int ahash_digest(struct ahash_request *req)
1033 struct ahash_edesc *edesc; 1021 struct ahash_edesc *edesc;
1034 int ret; 1022 int ret;
1035 1023
1024 state->buf_dma = 0;
1025
1036 src_nents = sg_nents_for_len(req->src, req->nbytes); 1026 src_nents = sg_nents_for_len(req->src, req->nbytes);
1037 if (src_nents < 0) { 1027 if (src_nents < 0) {
1038 dev_err(jrdev, "Invalid number of src SG.\n"); 1028 dev_err(jrdev, "Invalid number of src SG.\n");
@@ -1105,8 +1095,8 @@ static int ahash_final_no_ctx(struct ahash_request *req)
1105 struct device *jrdev = ctx->jrdev; 1095 struct device *jrdev = ctx->jrdev;
1106 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1096 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1107 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1097 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1108 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1098 u8 *buf = current_buf(state);
1109 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 1099 int buflen = *current_buflen(state);
1110 u32 *desc; 1100 u32 *desc;
1111 int digestsize = crypto_ahash_digestsize(ahash); 1101 int digestsize = crypto_ahash_digestsize(ahash);
1112 struct ahash_edesc *edesc; 1102 struct ahash_edesc *edesc;
@@ -1166,11 +1156,10 @@ static int ahash_update_no_ctx(struct ahash_request *req)
1166 struct device *jrdev = ctx->jrdev; 1156 struct device *jrdev = ctx->jrdev;
1167 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1157 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1168 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1158 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1169 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1159 u8 *buf = current_buf(state);
1170 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; 1160 int *buflen = current_buflen(state);
1171 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; 1161 u8 *next_buf = alt_buf(state);
1172 int *next_buflen = state->current_buf ? &state->buflen_0 : 1162 int *next_buflen = alt_buflen(state);
1173 &state->buflen_1;
1174 int in_len = *buflen + req->nbytes, to_hash; 1163 int in_len = *buflen + req->nbytes, to_hash;
1175 int sec4_sg_bytes, src_nents, mapped_nents; 1164 int sec4_sg_bytes, src_nents, mapped_nents;
1176 struct ahash_edesc *edesc; 1165 struct ahash_edesc *edesc;
@@ -1219,8 +1208,10 @@ static int ahash_update_no_ctx(struct ahash_request *req)
1219 edesc->sec4_sg_bytes = sec4_sg_bytes; 1208 edesc->sec4_sg_bytes = sec4_sg_bytes;
1220 edesc->dst_dma = 0; 1209 edesc->dst_dma = 0;
1221 1210
1222 state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, 1211 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1223 buf, *buflen); 1212 if (ret)
1213 goto unmap_ctx;
1214
1224 sg_to_sec4_sg_last(req->src, mapped_nents, 1215 sg_to_sec4_sg_last(req->src, mapped_nents,
1225 edesc->sec4_sg + 1, 0); 1216 edesc->sec4_sg + 1, 0);
1226 1217
@@ -1230,8 +1221,6 @@ static int ahash_update_no_ctx(struct ahash_request *req)
1230 *next_buflen, 0); 1221 *next_buflen, 0);
1231 } 1222 }
1232 1223
1233 state->current_buf = !state->current_buf;
1234
1235 desc = edesc->hw_desc; 1224 desc = edesc->hw_desc;
1236 1225
1237 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1226 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@@ -1293,10 +1282,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
1293 struct device *jrdev = ctx->jrdev; 1282 struct device *jrdev = ctx->jrdev;
1294 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1283 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1295 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1284 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1296 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1285 int buflen = *current_buflen(state);
1297 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1298 int last_buflen = state->current_buf ? state->buflen_0 :
1299 state->buflen_1;
1300 u32 *desc; 1286 u32 *desc;
1301 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; 1287 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1302 int digestsize = crypto_ahash_digestsize(ahash); 1288 int digestsize = crypto_ahash_digestsize(ahash);
@@ -1338,9 +1324,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
1338 edesc->src_nents = src_nents; 1324 edesc->src_nents = src_nents;
1339 edesc->sec4_sg_bytes = sec4_sg_bytes; 1325 edesc->sec4_sg_bytes = sec4_sg_bytes;
1340 1326
1341 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, 1327 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1342 state->buf_dma, buflen, 1328 if (ret)
1343 last_buflen); 1329 goto unmap;
1344 1330
1345 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, 1331 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1346 req->nbytes); 1332 req->nbytes);
@@ -1386,9 +1372,8 @@ static int ahash_update_first(struct ahash_request *req)
1386 struct device *jrdev = ctx->jrdev; 1372 struct device *jrdev = ctx->jrdev;
1387 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1373 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1388 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1374 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1389 u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0; 1375 u8 *next_buf = alt_buf(state);
1390 int *next_buflen = state->current_buf ? 1376 int *next_buflen = alt_buflen(state);
1391 &state->buflen_1 : &state->buflen_0;
1392 int to_hash; 1377 int to_hash;
1393 u32 *desc; 1378 u32 *desc;
1394 int src_nents, mapped_nents; 1379 int src_nents, mapped_nents;
@@ -1470,6 +1455,7 @@ static int ahash_update_first(struct ahash_request *req)
1470 state->final = ahash_final_no_ctx; 1455 state->final = ahash_final_no_ctx;
1471 scatterwalk_map_and_copy(next_buf, req->src, 0, 1456 scatterwalk_map_and_copy(next_buf, req->src, 0,
1472 req->nbytes, 0); 1457 req->nbytes, 0);
1458 switch_buf(state);
1473 } 1459 }
1474#ifdef DEBUG 1460#ifdef DEBUG
1475 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1461 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
@@ -1497,6 +1483,7 @@ static int ahash_init(struct ahash_request *req)
1497 state->finup = ahash_finup_first; 1483 state->finup = ahash_finup_first;
1498 state->final = ahash_final_no_ctx; 1484 state->final = ahash_final_no_ctx;
1499 1485
1486 state->ctx_dma = 0;
1500 state->current_buf = 0; 1487 state->current_buf = 0;
1501 state->buf_dma = 0; 1488 state->buf_dma = 0;
1502 state->buflen_0 = 0; 1489 state->buflen_0 = 0;
@@ -1732,6 +1719,7 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
1732 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 1719 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1733 HASH_MSG_LEN + 64, 1720 HASH_MSG_LEN + 64,
1734 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 1721 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1722 dma_addr_t dma_addr;
1735 1723
1736 /* 1724 /*
1737 * Get a Job ring from Job Ring driver to ensure in-order 1725 * Get a Job ring from Job Ring driver to ensure in-order
@@ -1742,6 +1730,26 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
1742 pr_err("Job Ring Device allocation for transform failed\n"); 1730 pr_err("Job Ring Device allocation for transform failed\n");
1743 return PTR_ERR(ctx->jrdev); 1731 return PTR_ERR(ctx->jrdev);
1744 } 1732 }
1733
1734 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
1735 offsetof(struct caam_hash_ctx,
1736 sh_desc_update_dma),
1737 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1738 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
1739 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
1740 caam_jr_free(ctx->jrdev);
1741 return -ENOMEM;
1742 }
1743
1744 ctx->sh_desc_update_dma = dma_addr;
1745 ctx->sh_desc_update_first_dma = dma_addr +
1746 offsetof(struct caam_hash_ctx,
1747 sh_desc_update_first);
1748 ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
1749 sh_desc_fin);
1750 ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
1751 sh_desc_digest);
1752
1745 /* copy descriptor header template value */ 1753 /* copy descriptor header template value */
1746 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 1754 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1747 1755
@@ -1758,26 +1766,10 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1758{ 1766{
1759 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1767 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1760 1768
1761 if (ctx->sh_desc_update_dma && 1769 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
1762 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma)) 1770 offsetof(struct caam_hash_ctx,
1763 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma, 1771 sh_desc_update_dma),
1764 desc_bytes(ctx->sh_desc_update), 1772 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1765 DMA_TO_DEVICE);
1766 if (ctx->sh_desc_update_first_dma &&
1767 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma))
1768 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma,
1769 desc_bytes(ctx->sh_desc_update_first),
1770 DMA_TO_DEVICE);
1771 if (ctx->sh_desc_fin_dma &&
1772 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma))
1773 dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma,
1774 desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE);
1775 if (ctx->sh_desc_digest_dma &&
1776 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma))
1777 dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma,
1778 desc_bytes(ctx->sh_desc_digest),
1779 DMA_TO_DEVICE);
1780
1781 caam_jr_free(ctx->jrdev); 1773 caam_jr_free(ctx->jrdev);
1782} 1774}
1783 1775
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 755109841cfd..579f8263c479 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -13,7 +13,6 @@
13#include "intern.h" 13#include "intern.h"
14#include "jr.h" 14#include "jr.h"
15#include "desc_constr.h" 15#include "desc_constr.h"
16#include "error.h"
17#include "ctrl.h" 16#include "ctrl.h"
18 17
19bool caam_little_end; 18bool caam_little_end;
@@ -309,10 +308,8 @@ static int caam_remove(struct platform_device *pdev)
309 ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; 308 ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
310 309
311 /* Remove platform devices for JobRs */ 310 /* Remove platform devices for JobRs */
312 for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { 311 for (ring = 0; ring < ctrlpriv->total_jobrs; ring++)
313 if (ctrlpriv->jrpdev[ring]) 312 of_device_unregister(ctrlpriv->jrpdev[ring]);
314 of_device_unregister(ctrlpriv->jrpdev[ring]);
315 }
316 313
317 /* De-initialize RNG state handles initialized by this driver. */ 314 /* De-initialize RNG state handles initialized by this driver. */
318 if (ctrlpriv->rng4_sh_init) 315 if (ctrlpriv->rng4_sh_init)
@@ -424,7 +421,7 @@ DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n");
424/* Probe routine for CAAM top (controller) level */ 421/* Probe routine for CAAM top (controller) level */
425static int caam_probe(struct platform_device *pdev) 422static int caam_probe(struct platform_device *pdev)
426{ 423{
427 int ret, ring, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; 424 int ret, ring, ridx, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN;
428 u64 caam_id; 425 u64 caam_id;
429 struct device *dev; 426 struct device *dev;
430 struct device_node *nprop, *np; 427 struct device_node *nprop, *np;
@@ -587,13 +584,18 @@ static int caam_probe(struct platform_device *pdev)
587 JRSTART_JR1_START | JRSTART_JR2_START | 584 JRSTART_JR1_START | JRSTART_JR2_START |
588 JRSTART_JR3_START); 585 JRSTART_JR3_START);
589 586
590 if (sizeof(dma_addr_t) == sizeof(u64)) 587 if (sizeof(dma_addr_t) == sizeof(u64)) {
591 if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) 588 if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
592 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); 589 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
593 else 590 else
594 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); 591 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36));
595 else 592 } else {
596 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 593 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
594 }
595 if (ret) {
596 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
597 goto iounmap_ctrl;
598 }
597 599
598 /* 600 /*
599 * Detect and enable JobRs 601 * Detect and enable JobRs
@@ -614,6 +616,7 @@ static int caam_probe(struct platform_device *pdev)
614 } 616 }
615 617
616 ring = 0; 618 ring = 0;
619 ridx = 0;
617 ctrlpriv->total_jobrs = 0; 620 ctrlpriv->total_jobrs = 0;
618 for_each_available_child_of_node(nprop, np) 621 for_each_available_child_of_node(nprop, np)
619 if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || 622 if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
@@ -621,17 +624,19 @@ static int caam_probe(struct platform_device *pdev)
621 ctrlpriv->jrpdev[ring] = 624 ctrlpriv->jrpdev[ring] =
622 of_platform_device_create(np, NULL, dev); 625 of_platform_device_create(np, NULL, dev);
623 if (!ctrlpriv->jrpdev[ring]) { 626 if (!ctrlpriv->jrpdev[ring]) {
624 pr_warn("JR%d Platform device creation error\n", 627 pr_warn("JR physical index %d: Platform device creation error\n",
625 ring); 628 ridx);
629 ridx++;
626 continue; 630 continue;
627 } 631 }
628 ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) 632 ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
629 ((__force uint8_t *)ctrl + 633 ((__force uint8_t *)ctrl +
630 (ring + JR_BLOCK_NUMBER) * 634 (ridx + JR_BLOCK_NUMBER) *
631 BLOCK_OFFSET 635 BLOCK_OFFSET
632 ); 636 );
633 ctrlpriv->total_jobrs++; 637 ctrlpriv->total_jobrs++;
634 ring++; 638 ring++;
639 ridx++;
635 } 640 }
636 641
637 /* Check to see if QI present. If so, enable */ 642 /* Check to see if QI present. If so, enable */
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 79a0cc70717f..6f44ccb55c63 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -6,9 +6,7 @@
6 6
7#include "compat.h" 7#include "compat.h"
8#include "regs.h" 8#include "regs.h"
9#include "intern.h"
10#include "desc.h" 9#include "desc.h"
11#include "jr.h"
12#include "error.h" 10#include "error.h"
13 11
14static const struct { 12static const struct {
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index c8604dfadbf5..27631000b9f8 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -498,13 +498,22 @@ static int caam_jr_probe(struct platform_device *pdev)
498 498
499 jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl; 499 jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
500 500
501 if (sizeof(dma_addr_t) == sizeof(u64)) 501 if (sizeof(dma_addr_t) == sizeof(u64)) {
502 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) 502 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
503 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40)); 503 error = dma_set_mask_and_coherent(jrdev,
504 DMA_BIT_MASK(40));
504 else 505 else
505 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36)); 506 error = dma_set_mask_and_coherent(jrdev,
506 else 507 DMA_BIT_MASK(36));
507 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); 508 } else {
509 error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
510 }
511 if (error) {
512 dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
513 error);
514 iounmap(ctrl);
515 return error;
516 }
508 517
509 /* Identify the interrupt */ 518 /* Identify the interrupt */
510 jrpriv->irq = irq_of_parse_and_map(nprop, 0); 519 jrpriv->irq = irq_of_parse_and_map(nprop, 0);
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index 6afa20c4a013..c6adad09c972 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -73,14 +73,3 @@ static inline struct sec4_sg_entry *sg_to_sec4_sg_len(
73 } while (total); 73 } while (total);
74 return sec4_sg_ptr - 1; 74 return sec4_sg_ptr - 1;
75} 75}
76
77/* derive number of elements in scatterlist, but return 0 for 1 */
78static inline int sg_count(struct scatterlist *sg_list, int nbytes)
79{
80 int sg_nents = sg_nents_for_len(sg_list, nbytes);
81
82 if (likely(sg_nents == 1))
83 return 0;
84
85 return sg_nents;
86}
diff --git a/drivers/crypto/cavium/cpt/Kconfig b/drivers/crypto/cavium/cpt/Kconfig
new file mode 100644
index 000000000000..cbd51b1aa046
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/Kconfig
@@ -0,0 +1,17 @@
1#
2# Cavium crypto device configuration
3#
4
5config CRYPTO_DEV_CPT
6 tristate
7
8config CAVIUM_CPT
9 tristate "Cavium Cryptographic Accelerator driver"
10 depends on ARCH_THUNDER || COMPILE_TEST
11 depends on PCI_MSI && 64BIT
12 select CRYPTO_DEV_CPT
13 help
14 Support for Cavium CPT block found in octeon-tx series of
15 processors.
16
17 To compile this as a module, choose M here.
diff --git a/drivers/crypto/cavium/cpt/Makefile b/drivers/crypto/cavium/cpt/Makefile
new file mode 100644
index 000000000000..dbf055e14622
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/Makefile
@@ -0,0 +1,3 @@
1obj-$(CONFIG_CAVIUM_CPT) += cptpf.o cptvf.o
2cptpf-objs := cptpf_main.o cptpf_mbox.o
3cptvf-objs := cptvf_main.o cptvf_reqmanager.o cptvf_mbox.o cptvf_algs.o
diff --git a/drivers/crypto/cavium/cpt/cpt_common.h b/drivers/crypto/cavium/cpt/cpt_common.h
new file mode 100644
index 000000000000..225078d03773
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cpt_common.h
@@ -0,0 +1,156 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#ifndef __CPT_COMMON_H
10#define __CPT_COMMON_H
11
12#include <asm/byteorder.h>
13#include <linux/delay.h>
14#include <linux/pci.h>
15
16#include "cpt_hw_types.h"
17
18/* Device ID */
19#define CPT_81XX_PCI_PF_DEVICE_ID 0xa040
20#define CPT_81XX_PCI_VF_DEVICE_ID 0xa041
21
22/* flags to indicate the features supported */
23#define CPT_FLAG_SRIOV_ENABLED BIT(1)
24#define CPT_FLAG_VF_DRIVER BIT(2)
25#define CPT_FLAG_DEVICE_READY BIT(3)
26
27#define cpt_sriov_enabled(cpt) ((cpt)->flags & CPT_FLAG_SRIOV_ENABLED)
28#define cpt_vf_driver(cpt) ((cpt)->flags & CPT_FLAG_VF_DRIVER)
29#define cpt_device_ready(cpt) ((cpt)->flags & CPT_FLAG_DEVICE_READY)
30
31#define CPT_MBOX_MSG_TYPE_ACK 1
32#define CPT_MBOX_MSG_TYPE_NACK 2
33#define CPT_MBOX_MSG_TIMEOUT 2000
34#define VF_STATE_DOWN 0
35#define VF_STATE_UP 1
36
37/*
38 * CPT Registers map for 81xx
39 */
40
41/* PF registers */
42#define CPTX_PF_CONSTANTS(a) (0x0ll + ((u64)(a) << 36))
43#define CPTX_PF_RESET(a) (0x100ll + ((u64)(a) << 36))
44#define CPTX_PF_DIAG(a) (0x120ll + ((u64)(a) << 36))
45#define CPTX_PF_BIST_STATUS(a) (0x160ll + ((u64)(a) << 36))
46#define CPTX_PF_ECC0_CTL(a) (0x200ll + ((u64)(a) << 36))
47#define CPTX_PF_ECC0_FLIP(a) (0x210ll + ((u64)(a) << 36))
48#define CPTX_PF_ECC0_INT(a) (0x220ll + ((u64)(a) << 36))
49#define CPTX_PF_ECC0_INT_W1S(a) (0x230ll + ((u64)(a) << 36))
50#define CPTX_PF_ECC0_ENA_W1S(a) (0x240ll + ((u64)(a) << 36))
51#define CPTX_PF_ECC0_ENA_W1C(a) (0x250ll + ((u64)(a) << 36))
52#define CPTX_PF_MBOX_INTX(a, b) \
53 (0x400ll + ((u64)(a) << 36) + ((b) << 3))
54#define CPTX_PF_MBOX_INT_W1SX(a, b) \
55 (0x420ll + ((u64)(a) << 36) + ((b) << 3))
56#define CPTX_PF_MBOX_ENA_W1CX(a, b) \
57 (0x440ll + ((u64)(a) << 36) + ((b) << 3))
58#define CPTX_PF_MBOX_ENA_W1SX(a, b) \
59 (0x460ll + ((u64)(a) << 36) + ((b) << 3))
60#define CPTX_PF_EXEC_INT(a) (0x500ll + 0x1000000000ll * ((a) & 0x1))
61#define CPTX_PF_EXEC_INT_W1S(a) (0x520ll + ((u64)(a) << 36))
62#define CPTX_PF_EXEC_ENA_W1C(a) (0x540ll + ((u64)(a) << 36))
63#define CPTX_PF_EXEC_ENA_W1S(a) (0x560ll + ((u64)(a) << 36))
64#define CPTX_PF_GX_EN(a, b) \
65 (0x600ll + ((u64)(a) << 36) + ((b) << 3))
66#define CPTX_PF_EXEC_INFO(a) (0x700ll + ((u64)(a) << 36))
67#define CPTX_PF_EXEC_BUSY(a) (0x800ll + ((u64)(a) << 36))
68#define CPTX_PF_EXEC_INFO0(a) (0x900ll + ((u64)(a) << 36))
69#define CPTX_PF_EXEC_INFO1(a) (0x910ll + ((u64)(a) << 36))
70#define CPTX_PF_INST_REQ_PC(a) (0x10000ll + ((u64)(a) << 36))
71#define CPTX_PF_INST_LATENCY_PC(a) \
72 (0x10020ll + ((u64)(a) << 36))
73#define CPTX_PF_RD_REQ_PC(a) (0x10040ll + ((u64)(a) << 36))
74#define CPTX_PF_RD_LATENCY_PC(a) (0x10060ll + ((u64)(a) << 36))
75#define CPTX_PF_RD_UC_PC(a) (0x10080ll + ((u64)(a) << 36))
76#define CPTX_PF_ACTIVE_CYCLES_PC(a) (0x10100ll + ((u64)(a) << 36))
77#define CPTX_PF_EXE_CTL(a) (0x4000000ll + ((u64)(a) << 36))
78#define CPTX_PF_EXE_STATUS(a) (0x4000008ll + ((u64)(a) << 36))
79#define CPTX_PF_EXE_CLK(a) (0x4000010ll + ((u64)(a) << 36))
80#define CPTX_PF_EXE_DBG_CTL(a) (0x4000018ll + ((u64)(a) << 36))
81#define CPTX_PF_EXE_DBG_DATA(a) (0x4000020ll + ((u64)(a) << 36))
82#define CPTX_PF_EXE_BIST_STATUS(a) (0x4000028ll + ((u64)(a) << 36))
83#define CPTX_PF_EXE_REQ_TIMER(a) (0x4000030ll + ((u64)(a) << 36))
84#define CPTX_PF_EXE_MEM_CTL(a) (0x4000038ll + ((u64)(a) << 36))
85#define CPTX_PF_EXE_PERF_CTL(a) (0x4001000ll + ((u64)(a) << 36))
86#define CPTX_PF_EXE_DBG_CNTX(a, b) \
87 (0x4001100ll + ((u64)(a) << 36) + ((b) << 3))
88#define CPTX_PF_EXE_PERF_EVENT_CNT(a) (0x4001180ll + ((u64)(a) << 36))
89#define CPTX_PF_EXE_EPCI_INBX_CNT(a, b) \
90 (0x4001200ll + ((u64)(a) << 36) + ((b) << 3))
91#define CPTX_PF_EXE_EPCI_OUTBX_CNT(a, b) \
92 (0x4001240ll + ((u64)(a) << 36) + ((b) << 3))
93#define CPTX_PF_ENGX_UCODE_BASE(a, b) \
94 (0x4002000ll + ((u64)(a) << 36) + ((b) << 3))
95#define CPTX_PF_QX_CTL(a, b) \
96 (0x8000000ll + ((u64)(a) << 36) + ((b) << 20))
97#define CPTX_PF_QX_GMCTL(a, b) \
98 (0x8000020ll + ((u64)(a) << 36) + ((b) << 20))
99#define CPTX_PF_QX_CTL2(a, b) \
100 (0x8000100ll + ((u64)(a) << 36) + ((b) << 20))
101#define CPTX_PF_VFX_MBOXX(a, b, c) \
102 (0x8001000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 8))
103
104/* VF registers */
105#define CPTX_VQX_CTL(a, b) (0x100ll + ((u64)(a) << 36) + ((b) << 20))
106#define CPTX_VQX_SADDR(a, b) (0x200ll + ((u64)(a) << 36) + ((b) << 20))
107#define CPTX_VQX_DONE_WAIT(a, b) (0x400ll + ((u64)(a) << 36) + ((b) << 20))
108#define CPTX_VQX_INPROG(a, b) (0x410ll + ((u64)(a) << 36) + ((b) << 20))
109#define CPTX_VQX_DONE(a, b) (0x420ll + ((u64)(a) << 36) + ((b) << 20))
110#define CPTX_VQX_DONE_ACK(a, b) (0x440ll + ((u64)(a) << 36) + ((b) << 20))
111#define CPTX_VQX_DONE_INT_W1S(a, b) (0x460ll + ((u64)(a) << 36) + ((b) << 20))
112#define CPTX_VQX_DONE_INT_W1C(a, b) (0x468ll + ((u64)(a) << 36) + ((b) << 20))
113#define CPTX_VQX_DONE_ENA_W1S(a, b) (0x470ll + ((u64)(a) << 36) + ((b) << 20))
114#define CPTX_VQX_DONE_ENA_W1C(a, b) (0x478ll + ((u64)(a) << 36) + ((b) << 20))
115#define CPTX_VQX_MISC_INT(a, b) (0x500ll + ((u64)(a) << 36) + ((b) << 20))
116#define CPTX_VQX_MISC_INT_W1S(a, b) (0x508ll + ((u64)(a) << 36) + ((b) << 20))
117#define CPTX_VQX_MISC_ENA_W1S(a, b) (0x510ll + ((u64)(a) << 36) + ((b) << 20))
118#define CPTX_VQX_MISC_ENA_W1C(a, b) (0x518ll + ((u64)(a) << 36) + ((b) << 20))
119#define CPTX_VQX_DOORBELL(a, b) (0x600ll + ((u64)(a) << 36) + ((b) << 20))
120#define CPTX_VFX_PF_MBOXX(a, b, c) \
121 (0x1000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 3))
122
123enum vftype {
124 AE_TYPES = 1,
125 SE_TYPES = 2,
126 BAD_CPT_TYPES,
127};
128
129/* Max CPT devices supported */
130enum cpt_mbox_opcode {
131 CPT_MSG_VF_UP = 1,
132 CPT_MSG_VF_DOWN,
133 CPT_MSG_READY,
134 CPT_MSG_QLEN,
135 CPT_MSG_QBIND_GRP,
136 CPT_MSG_VQ_PRIORITY,
137};
138
139/* CPT mailbox structure */
140struct cpt_mbox {
141 u64 msg; /* Message type MBOX[0] */
142 u64 data;/* Data MBOX[1] */
143};
144
145/* Register read/write APIs */
146static inline void cpt_write_csr64(u8 __iomem *hw_addr, u64 offset,
147 u64 val)
148{
149 writeq(val, hw_addr + offset);
150}
151
152static inline u64 cpt_read_csr64(u8 __iomem *hw_addr, u64 offset)
153{
154 return readq(hw_addr + offset);
155}
156#endif /* __CPT_COMMON_H */
diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h
new file mode 100644
index 000000000000..279669494196
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h
@@ -0,0 +1,658 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#ifndef __CPT_HW_TYPES_H
10#define __CPT_HW_TYPES_H
11
12#include "cpt_common.h"
13
14/**
15 * Enumeration cpt_comp_e
16 *
17 * CPT Completion Enumeration
18 * Enumerates the values of CPT_RES_S[COMPCODE].
19 */
20enum cpt_comp_e {
21 CPT_COMP_E_NOTDONE = 0x00,
22 CPT_COMP_E_GOOD = 0x01,
23 CPT_COMP_E_FAULT = 0x02,
24 CPT_COMP_E_SWERR = 0x03,
25 CPT_COMP_E_LAST_ENTRY = 0xFF
26};
27
28/**
29 * Structure cpt_inst_s
30 *
31 * CPT Instruction Structure
32 * This structure specifies the instruction layout. Instructions are
33 * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set.
34 * cpt_inst_s_s
35 * Word 0
36 * doneint:1 Done interrupt.
37 * 0 = No interrupts related to this instruction.
38 * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be
39 * incremented,and based on the rules described there an interrupt may
40 * occur.
41 * Word 1
42 * res_addr [127: 64] Result IOVA.
43 * If nonzero, specifies where to write CPT_RES_S.
44 * If zero, no result structure will be written.
45 * Address must be 16-byte aligned.
46 * Bits <63:49> are ignored by hardware; software should use a
47 * sign-extended bit <48> for forward compatibility.
48 * Word 2
49 * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when
50 * CPT submits work SSO.
51 * For the SSO to not discard the add-work request, FPA_PF_MAP() must map
52 * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid.
53 * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT
54 * submits work to SSO
55 * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT
56 * submits work to SSO.
57 * Word 3
58 * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a
59 * work-queue entry that CPT submits work to SSO after all context,
60 * output data, and result write operations are visible to other
61 * CNXXXX units and the cores. Bits <2:0> must be zero.
62 * Bits <63:49> are ignored by hardware; software should
63 * use a sign-extended bit <48> for forward compatibility.
64 * Internal:
65 * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0.
66 * Word 4
67 * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE.
68 * Word 5
69 * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE.
70 * Word 6
71 * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE.
72 * Word 7
73 * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE.
74 *
75 */
76union cpt_inst_s {
77 u64 u[8];
78 struct cpt_inst_s_s {
79#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
80 u64 reserved_17_63:47;
81 u64 doneint:1;
82 u64 reserved_0_1:16;
83#else /* Word 0 - Little Endian */
84 u64 reserved_0_15:16;
85 u64 doneint:1;
86 u64 reserved_17_63:47;
87#endif /* Word 0 - End */
88 u64 res_addr;
89#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */
90 u64 reserved_172_19:20;
91 u64 grp:10;
92 u64 tt:2;
93 u64 tag:32;
94#else /* Word 2 - Little Endian */
95 u64 tag:32;
96 u64 tt:2;
97 u64 grp:10;
98 u64 reserved_172_191:20;
99#endif /* Word 2 - End */
100 u64 wq_ptr;
101 u64 ei0;
102 u64 ei1;
103 u64 ei2;
104 u64 ei3;
105 } s;
106};
107
108/**
109 * Structure cpt_res_s
110 *
111 * CPT Result Structure
112 * The CPT coprocessor writes the result structure after it completes a
113 * CPT_INST_S instruction. The result structure is exactly 16 bytes, and
114 * each instruction completion produces exactly one result structure.
115 *
116 * This structure is stored in memory as little-endian unless
117 * CPT()_PF_Q()_CTL[INST_BE] is set.
118 * cpt_res_s_s
119 * Word 0
120 * doneint:1 [16:16] Done interrupt. This bit is copied from the
121 * corresponding instruction's CPT_INST_S[DONEINT].
122 * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor
123 * for the associated instruction, as enumerated by CPT_COMP_E.
124 * Core software may write the memory location containing [COMPCODE] to
125 * 0x0 before ringing the doorbell, and then poll for completion by
126 * checking for a nonzero value.
127 * Once the core observes a nonzero [COMPCODE] value in this case,the CPT
128 * coprocessor will have also completed L2/DRAM write operations.
129 * Word 1
130 * reserved
131 *
132 */
133union cpt_res_s {
134 u64 u[2];
135 struct cpt_res_s_s {
136#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
137 u64 reserved_17_63:47;
138 u64 doneint:1;
139 u64 reserved_8_15:8;
140 u64 compcode:8;
141#else /* Word 0 - Little Endian */
142 u64 compcode:8;
143 u64 reserved_8_15:8;
144 u64 doneint:1;
145 u64 reserved_17_63:47;
146#endif /* Word 0 - End */
147 u64 reserved_64_127;
148 } s;
149};
150
151/**
152 * Register (NCB) cpt#_pf_bist_status
153 *
154 * CPT PF Control Bist Status Register
155 * This register has the BIST status of memories. Each bit is the BIST result
156 * of an individual memory (per bit, 0 = pass and 1 = fail).
157 * cptx_pf_bist_status_s
158 * Word0
159 * bstatus [29:0](RO/H) BIST status. One bit per memory, enumerated by
160 * CPT_RAMS_E.
161 */
162union cptx_pf_bist_status {
163 u64 u;
164 struct cptx_pf_bist_status_s {
165#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
166 u64 reserved_30_63:34;
167 u64 bstatus:30;
168#else /* Word 0 - Little Endian */
169 u64 bstatus:30;
170 u64 reserved_30_63:34;
171#endif /* Word 0 - End */
172 } s;
173};
174
175/**
176 * Register (NCB) cpt#_pf_constants
177 *
178 * CPT PF Constants Register
179 * This register contains implementation-related parameters of CPT in CNXXXX.
180 * cptx_pf_constants_s
181 * Word 0
182 * reserved_40_63:24 [63:40] Reserved.
183 * epcis:8 [39:32](RO) Number of EPCI busses.
184 * grps:8 [31:24](RO) Number of engine groups implemented.
185 * ae:8 [23:16](RO/H) Number of AEs. In CNXXXX, for CPT0 returns 0x0,
186 * for CPT1 returns 0x18, or less if there are fuse-disables.
187 * se:8 [15:8](RO/H) Number of SEs. In CNXXXX, for CPT0 returns 0x30,
188 * or less if there are fuse-disables, for CPT1 returns 0x0.
189 * vq:8 [7:0](RO) Number of VQs.
190 */
191union cptx_pf_constants {
192 u64 u;
193 struct cptx_pf_constants_s {
194#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
195 u64 reserved_40_63:24;
196 u64 epcis:8;
197 u64 grps:8;
198 u64 ae:8;
199 u64 se:8;
200 u64 vq:8;
201#else /* Word 0 - Little Endian */
202 u64 vq:8;
203 u64 se:8;
204 u64 ae:8;
205 u64 grps:8;
206 u64 epcis:8;
207 u64 reserved_40_63:24;
208#endif /* Word 0 - End */
209 } s;
210};
211
212/**
213 * Register (NCB) cpt#_pf_exe_bist_status
214 *
215 * CPT PF Engine Bist Status Register
216 * This register has the BIST status of each engine. Each bit is the
217 * BIST result of an individual engine (per bit, 0 = pass and 1 = fail).
218 * cptx_pf_exe_bist_status_s
219 * Word0
220 * reserved_48_63:16 [63:48] reserved
221 * bstatus:48 [47:0](RO/H) BIST status. One bit per engine.
222 *
223 */
224union cptx_pf_exe_bist_status {
225 u64 u;
226 struct cptx_pf_exe_bist_status_s {
227#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
228 u64 reserved_48_63:16;
229 u64 bstatus:48;
230#else /* Word 0 - Little Endian */
231 u64 bstatus:48;
232 u64 reserved_48_63:16;
233#endif /* Word 0 - End */
234 } s;
235};
236
237/**
238 * Register (NCB) cpt#_pf_q#_ctl
239 *
240 * CPT Queue Control Register
241 * This register configures queues. This register should be changed only
242 * when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
243 * cptx_pf_qx_ctl_s
244 * Word0
245 * reserved_60_63:4 [63:60] reserved.
246 * aura:12; [59:48](R/W) Guest-aura for returning this queue's
247 * instruction-chunk buffers to FPA. Only used when [INST_FREE] is set.
248 * For the FPA to not discard the request, FPA_PF_MAP() must map
249 * [AURA] and CPT()_PF_Q()_GMCTL[GMID] as valid.
250 * reserved_45_47:3 [47:45] reserved.
251 * size:13 [44:32](R/W) Command-buffer size, in number of 64-bit words per
252 * command buffer segment. Must be 8*n + 1, where n is the number of
253 * instructions per buffer segment.
254 * reserved_11_31:21 [31:11] Reserved.
255 * cont_err:1 [10:10](R/W) Continue on error.
256 * 0 = When CPT()_VQ()_MISC_INT[NWRP], CPT()_VQ()_MISC_INT[IRDE] or
257 * CPT()_VQ()_MISC_INT[DOVF] are set by hardware or software via
258 * CPT()_VQ()_MISC_INT_W1S, then CPT()_VQ()_CTL[ENA] is cleared. Due to
259 * pipelining, additional instructions may have been processed between the
260 * instruction causing the error and the next instruction in the disabled
261 * queue (the instruction at CPT()_VQ()_SADDR).
262 * 1 = Ignore errors and continue processing instructions.
263 * For diagnostic use only.
264 * inst_free:1 [9:9](R/W) Instruction FPA free. When set, when CPT reaches the
265 * end of an instruction chunk, that chunk will be freed to the FPA.
266 * inst_be:1 [8:8](R/W) Instruction big-endian control. When set, instructions,
267 * instruction next chunk pointers, and result structures are stored in
268 * big-endian format in memory.
269 * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back.
270 * 0 = The hardware issues NCB transient load (LDT) towards the cache,
271 * which if the line hits and is is dirty will cause the line to be
272 * written back before being replaced.
273 * 1 = The hardware issues NCB LDWB read-and-invalidate command towards
274 * the cache when fetching the last word of instructions; as a result the
275 * line will not be written back when replaced. This improves
276 * performance, but software must not read the instructions after they are
277 * posted to the hardware. Reads that do not consume the last word of a
278 * cache line always use LDI.
279 * reserved_4_6:3 [6:4] Reserved.
280 * grp:3; [3:1](R/W) Engine group.
281 * pri:1; [0:0](R/W) Queue priority.
282 * 1 = This queue has higher priority. Round-robin between higher
283 * priority queues.
284 * 0 = This queue has lower priority. Round-robin between lower
285 * priority queues.
286 */
287union cptx_pf_qx_ctl {
288 u64 u;
289 struct cptx_pf_qx_ctl_s {
290#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
291 u64 reserved_60_63:4;
292 u64 aura:12;
293 u64 reserved_45_47:3;
294 u64 size:13;
295 u64 reserved_11_31:21;
296 u64 cont_err:1;
297 u64 inst_free:1;
298 u64 inst_be:1;
299 u64 iqb_ldwb:1;
300 u64 reserved_4_6:3;
301 u64 grp:3;
302 u64 pri:1;
303#else /* Word 0 - Little Endian */
304 u64 pri:1;
305 u64 grp:3;
306 u64 reserved_4_6:3;
307 u64 iqb_ldwb:1;
308 u64 inst_be:1;
309 u64 inst_free:1;
310 u64 cont_err:1;
311 u64 reserved_11_31:21;
312 u64 size:13;
313 u64 reserved_45_47:3;
314 u64 aura:12;
315 u64 reserved_60_63:4;
316#endif /* Word 0 - End */
317 } s;
318};
319
320/**
321 * Register (NCB) cpt#_vq#_saddr
322 *
323 * CPT Queue Starting Buffer Address Registers
324 * These registers set the instruction buffer starting address.
325 * cptx_vqx_saddr_s
326 * Word0
327 * reserved_49_63:15 [63:49] Reserved.
328 * ptr:43 [48:6](R/W/H) Instruction buffer IOVA <48:6> (64-byte aligned).
329 * When written, it is the initial buffer starting address; when read,
330 * it is the next read pointer to be requested from L2C. The PTR field
331 * is overwritten with the next pointer each time that the command buffer
332 * segment is exhausted. New commands will then be read from the newly
333 * specified command buffer pointer.
334 * reserved_0_5:6 [5:0] Reserved.
335 *
336 */
337union cptx_vqx_saddr {
338 u64 u;
339 struct cptx_vqx_saddr_s {
340#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
341 u64 reserved_49_63:15;
342 u64 ptr:43;
343 u64 reserved_0_5:6;
344#else /* Word 0 - Little Endian */
345 u64 reserved_0_5:6;
346 u64 ptr:43;
347 u64 reserved_49_63:15;
348#endif /* Word 0 - End */
349 } s;
350};
351
352/**
353 * Register (NCB) cpt#_vq#_misc_ena_w1s
354 *
355 * CPT Queue Misc Interrupt Enable Set Register
356 * This register sets interrupt enable bits.
357 * cptx_vqx_misc_ena_w1s_s
358 * Word0
359 * reserved_5_63:59 [63:5] Reserved.
360 * swerr:1 [4:4](R/W1S/H) Reads or sets enable for
361 * CPT(0..1)_VQ(0..63)_MISC_INT[SWERR].
362 * nwrp:1 [3:3](R/W1S/H) Reads or sets enable for
363 * CPT(0..1)_VQ(0..63)_MISC_INT[NWRP].
364 * irde:1 [2:2](R/W1S/H) Reads or sets enable for
365 * CPT(0..1)_VQ(0..63)_MISC_INT[IRDE].
366 * dovf:1 [1:1](R/W1S/H) Reads or sets enable for
367 * CPT(0..1)_VQ(0..63)_MISC_INT[DOVF].
368 * mbox:1 [0:0](R/W1S/H) Reads or sets enable for
369 * CPT(0..1)_VQ(0..63)_MISC_INT[MBOX].
370 *
371 */
372union cptx_vqx_misc_ena_w1s {
373 u64 u;
374 struct cptx_vqx_misc_ena_w1s_s {
375#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
376 u64 reserved_5_63:59;
377 u64 swerr:1;
378 u64 nwrp:1;
379 u64 irde:1;
380 u64 dovf:1;
381 u64 mbox:1;
382#else /* Word 0 - Little Endian */
383 u64 mbox:1;
384 u64 dovf:1;
385 u64 irde:1;
386 u64 nwrp:1;
387 u64 swerr:1;
388 u64 reserved_5_63:59;
389#endif /* Word 0 - End */
390 } s;
391};
392
393/**
394 * Register (NCB) cpt#_vq#_doorbell
395 *
396 * CPT Queue Doorbell Registers
397 * Doorbells for the CPT instruction queues.
398 * cptx_vqx_doorbell_s
399 * Word0
400 * reserved_20_63:44 [63:20] Reserved.
401 * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add
402 * to the CPT instruction doorbell count. Readback value is the the
403 * current number of pending doorbell requests. If counter overflows
404 * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to
405 * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF],
406 * then write a value of 2^20 minus the read [DBELL_CNT], then write one
407 * to CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and
408 * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF]. Must be a multiple of 8.
409 * All CPT instructions are 8 words and require a doorbell count of
410 * multiple of 8.
411 */
412union cptx_vqx_doorbell {
413 u64 u;
414 struct cptx_vqx_doorbell_s {
415#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
416 u64 reserved_20_63:44;
417 u64 dbell_cnt:20;
418#else /* Word 0 - Little Endian */
419 u64 dbell_cnt:20;
420 u64 reserved_20_63:44;
421#endif /* Word 0 - End */
422 } s;
423};
424
425/**
426 * Register (NCB) cpt#_vq#_inprog
427 *
428 * CPT Queue In Progress Count Registers
429 * These registers contain the per-queue instruction in flight registers.
430 * cptx_vqx_inprog_s
431 * Word0
432 * reserved_8_63:56 [63:8] Reserved.
433 * inflight:8 [7:0](RO/H) Inflight count. Counts the number of instructions
434 * for the VF for which CPT is fetching, executing or responding to
435 * instructions. However this does not include any interrupts that are
436 * awaiting software handling (CPT()_VQ()_DONE[DONE] != 0x0).
437 * A queue may not be reconfigured until:
438 * 1. CPT()_VQ()_CTL[ENA] is cleared by software.
439 * 2. [INFLIGHT] is polled until equals to zero.
440 */
441union cptx_vqx_inprog {
442 u64 u;
443 struct cptx_vqx_inprog_s {
444#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
445 u64 reserved_8_63:56;
446 u64 inflight:8;
447#else /* Word 0 - Little Endian */
448 u64 inflight:8;
449 u64 reserved_8_63:56;
450#endif /* Word 0 - End */
451 } s;
452};
453
454/**
455 * Register (NCB) cpt#_vq#_misc_int
456 *
457 * CPT Queue Misc Interrupt Register
458 * These registers contain the per-queue miscellaneous interrupts.
459 * cptx_vqx_misc_int_s
460 * Word 0
461 * reserved_5_63:59 [63:5] Reserved.
462 * swerr:1 [4:4](R/W1C/H) Software error from engines.
463 * nwrp:1 [3:3](R/W1C/H) NCB result write response error.
464 * irde:1 [2:2](R/W1C/H) Instruction NCB read response error.
465 * dovf:1 [1:1](R/W1C/H) Doorbell overflow.
466 * mbox:1 [0:0](R/W1C/H) PF to VF mailbox interrupt. Set when
467 * CPT()_VF()_PF_MBOX(0) is written.
468 *
469 */
470union cptx_vqx_misc_int {
471 u64 u;
472 struct cptx_vqx_misc_int_s {
473#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
474 u64 reserved_5_63:59;
475 u64 swerr:1;
476 u64 nwrp:1;
477 u64 irde:1;
478 u64 dovf:1;
479 u64 mbox:1;
480#else /* Word 0 - Little Endian */
481 u64 mbox:1;
482 u64 dovf:1;
483 u64 irde:1;
484 u64 nwrp:1;
485 u64 swerr:1;
486 u64 reserved_5_63:59;
487#endif /* Word 0 - End */
488 } s;
489};
490
491/**
492 * Register (NCB) cpt#_vq#_done_ack
493 *
494 * CPT Queue Done Count Ack Registers
495 * This register is written by software to acknowledge interrupts.
496 * cptx_vqx_done_ack_s
497 * Word0
498 * reserved_20_63:44 [63:20] Reserved.
499 * done_ack:20 [19:0](R/W/H) Number of decrements to CPT()_VQ()_DONE[DONE].
500 * Reads CPT()_VQ()_DONE[DONE]. Written by software to acknowledge
501 * interrupts. If CPT()_VQ()_DONE[DONE] is still nonzero the interrupt
502 * will be re-sent if the conditions described in CPT()_VQ()_DONE[DONE]
503 * are satisfied.
504 *
505 */
506union cptx_vqx_done_ack {
507 u64 u;
508 struct cptx_vqx_done_ack_s {
509#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
510 u64 reserved_20_63:44;
511 u64 done_ack:20;
512#else /* Word 0 - Little Endian */
513 u64 done_ack:20;
514 u64 reserved_20_63:44;
515#endif /* Word 0 - End */
516 } s;
517};
518
519/**
520 * Register (NCB) cpt#_vq#_done
521 *
522 * CPT Queue Done Count Registers
523 * These registers contain the per-queue instruction done count.
524 * cptx_vqx_done_s
525 * Word0
526 * reserved_20_63:44 [63:20] Reserved.
527 * done:20 [19:0](R/W/H) Done count. When CPT_INST_S[DONEINT] set and that
528 * instruction completes, CPT()_VQ()_DONE[DONE] is incremented when the
529 * instruction finishes. Write to this field are for diagnostic use only;
530 * instead software writes CPT()_VQ()_DONE_ACK with the number of
531 * decrements for this field.
532 * Interrupts are sent as follows:
533 * * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending, the
534 * interrupt coalescing timer is held to zero, and an interrupt is not
535 * sent.
536 * * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt coalescing timer
537 * counts. If the counter is >= CPT()_VQ()_DONE_WAIT[TIME_WAIT]*1024, or
538 * CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT], i.e. enough
539 * time has passed or enough results have arrived, then the interrupt is
540 * sent.
541 * * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is written
542 * but this is not typical), the interrupt coalescing timer restarts.
543 * Note after decrementing this interrupt equation is recomputed,
544 * for example if CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT]
545 * and because the timer is zero, the interrupt will be resent immediately.
546 * (This covers the race case between software acknowledging an interrupt
547 * and a result returning.)
548 * * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not sent,
549 * but the counting described above still occurs.
550 * Since CPT instructions complete out-of-order, if software is using
551 * completion interrupts the suggested scheme is to request a DONEINT on
552 * each request, and when an interrupt arrives perform a "greedy" scan for
553 * completions; even if a later command is acknowledged first this will
554 * not result in missing a completion.
555 * Software is responsible for making sure [DONE] does not overflow;
556 * for example by insuring there are not more than 2^20-1 instructions in
557 * flight that may request interrupts.
558 *
559 */
560union cptx_vqx_done {
561 u64 u;
562 struct cptx_vqx_done_s {
563#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
564 u64 reserved_20_63:44;
565 u64 done:20;
566#else /* Word 0 - Little Endian */
567 u64 done:20;
568 u64 reserved_20_63:44;
569#endif /* Word 0 - End */
570 } s;
571};
572
573/**
574 * Register (NCB) cpt#_vq#_done_wait
575 *
576 * CPT Queue Done Interrupt Coalescing Wait Registers
577 * Specifies the per queue interrupt coalescing settings.
578 * cptx_vqx_done_wait_s
579 * Word0
580 * reserved_48_63:16 [63:48] Reserved.
581 * time_wait:16; [47:32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] = 0
582 * or CPT()_VQ()_DONE_ACK is written a timer is cleared. When the timer
583 * reaches [TIME_WAIT]*1024 then interrupt coalescing ends.
584 * see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is disabled.
585 * reserved_20_31:12 [31:20] Reserved.
586 * num_wait:20 [19:0](R/W) Number of messages hold-off.
587 * When CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing ends
588 * see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as 0x1.
589 *
590 */
591union cptx_vqx_done_wait {
592 u64 u;
593 struct cptx_vqx_done_wait_s {
594#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
595 u64 reserved_48_63:16;
596 u64 time_wait:16;
597 u64 reserved_20_31:12;
598 u64 num_wait:20;
599#else /* Word 0 - Little Endian */
600 u64 num_wait:20;
601 u64 reserved_20_31:12;
602 u64 time_wait:16;
603 u64 reserved_48_63:16;
604#endif /* Word 0 - End */
605 } s;
606};
607
608/**
609 * Register (NCB) cpt#_vq#_done_ena_w1s
610 *
611 * CPT Queue Done Interrupt Enable Set Registers
612 * Write 1 to these registers will enable the DONEINT interrupt for the queue.
613 * cptx_vqx_done_ena_w1s_s
614 * Word0
615 * reserved_1_63:63 [63:1] Reserved.
616 * done:1 [0:0](R/W1S/H) Write 1 will enable DONEINT for this queue.
617 * Write 0 has no effect. Read will return the enable bit.
618 */
619union cptx_vqx_done_ena_w1s {
620 u64 u;
621 struct cptx_vqx_done_ena_w1s_s {
622#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
623 u64 reserved_1_63:63;
624 u64 done:1;
625#else /* Word 0 - Little Endian */
626 u64 done:1;
627 u64 reserved_1_63:63;
628#endif /* Word 0 - End */
629 } s;
630};
631
632/**
633 * Register (NCB) cpt#_vq#_ctl
634 *
635 * CPT VF Queue Control Registers
636 * This register configures queues. This register should be changed (other than
637 * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
638 * cptx_vqx_ctl_s
639 * Word0
640 * reserved_1_63:63 [63:1] Reserved.
641 * ena:1 [0:0](R/W/H) Enables the logical instruction queue.
642 * See also CPT()_PF_Q()_CTL[CONT_ERR] and CPT()_VQ()_INPROG[INFLIGHT].
643 * 1 = Queue is enabled.
644 * 0 = Queue is disabled.
645 */
646union cptx_vqx_ctl {
647 u64 u;
648 struct cptx_vqx_ctl_s {
649#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
650 u64 reserved_1_63:63;
651 u64 ena:1;
652#else /* Word 0 - Little Endian */
653 u64 ena:1;
654 u64 reserved_1_63:63;
655#endif /* Word 0 - End */
656 } s;
657};
658#endif /*__CPT_HW_TYPES_H*/
diff --git a/drivers/crypto/cavium/cpt/cptpf.h b/drivers/crypto/cavium/cpt/cptpf.h
new file mode 100644
index 000000000000..c0556c5f63c9
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptpf.h
@@ -0,0 +1,64 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#ifndef __CPTPF_H
10#define __CPTPF_H
11
12#include "cpt_common.h"
13
14#define CSR_DELAY 30
15#define CPT_MAX_CORE_GROUPS 8
16#define CPT_MAX_SE_CORES 10
17#define CPT_MAX_AE_CORES 6
18#define CPT_MAX_TOTAL_CORES (CPT_MAX_SE_CORES + CPT_MAX_AE_CORES)
19#define CPT_MAX_VF_NUM 16
20#define CPT_PF_MSIX_VECTORS 3
21#define CPT_PF_INT_VEC_E_MBOXX(a) (0x02 + (a))
22#define CPT_UCODE_VERSION_SZ 32
23struct cpt_device;
24
25struct microcode {
26 u8 is_mc_valid;
27 u8 is_ae;
28 u8 group;
29 u8 num_cores;
30 u32 code_size;
31 u64 core_mask;
32 u8 version[CPT_UCODE_VERSION_SZ];
33 /* Base info */
34 dma_addr_t phys_base;
35 void *code;
36};
37
38struct cpt_vf_info {
39 u8 state;
40 u8 priority;
41 u8 id;
42 u32 qlen;
43};
44
45/**
46 * cpt device structure
47 */
48struct cpt_device {
49 u16 flags; /* Flags to hold device status bits */
50 u8 num_vf_en; /* Number of VFs enabled (0...CPT_MAX_VF_NUM) */
51 struct cpt_vf_info vfinfo[CPT_MAX_VF_NUM]; /* Per VF info */
52
53 void __iomem *reg_base; /* Register start address */
54 struct pci_dev *pdev; /* pci device handle */
55
56 struct microcode mcode[CPT_MAX_CORE_GROUPS];
57 u8 next_mc_idx; /* next microcode index */
58 u8 next_group;
59 u8 max_se_cores;
60 u8 max_ae_cores;
61};
62
63void cpt_mbox_intr_handler(struct cpt_device *cpt, int mbx);
64#endif /* __CPTPF_H */
diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c
new file mode 100644
index 000000000000..4119c40e7c4b
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptpf_main.c
@@ -0,0 +1,670 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#include <linux/device.h>
10#include <linux/firmware.h>
11#include <linux/interrupt.h>
12#include <linux/module.h>
13#include <linux/moduleparam.h>
14#include <linux/pci.h>
15#include <linux/printk.h>
16#include <linux/version.h>
17
18#include "cptpf.h"
19
20#define DRV_NAME "thunder-cpt"
21#define DRV_VERSION "1.0"
22
23static u32 num_vfs = 4; /* Default 4 VF enabled */
24module_param(num_vfs, uint, 0444);
25MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)");
26
27/*
28 * Disable cores specified by coremask
29 */
30static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
31 u8 type, u8 grp)
32{
33 u64 pf_exe_ctl;
34 u32 timeout = 100;
35 u64 grpmask = 0;
36 struct device *dev = &cpt->pdev->dev;
37
38 if (type == AE_TYPES)
39 coremask = (coremask << cpt->max_se_cores);
40
41 /* Disengage the cores from groups */
42 grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
43 cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
44 (grpmask & ~coremask));
45 udelay(CSR_DELAY);
46 grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
47 while (grp & coremask) {
48 dev_err(dev, "Cores still busy %llx", coremask);
49 grp = cpt_read_csr64(cpt->reg_base,
50 CPTX_PF_EXEC_BUSY(0));
51 if (timeout--)
52 break;
53
54 udelay(CSR_DELAY);
55 }
56
57 /* Disable the cores */
58 pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
59 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
60 (pf_exe_ctl & ~coremask));
61 udelay(CSR_DELAY);
62}
63
64/*
65 * Enable cores specified by coremask
66 */
67static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
68 u8 type)
69{
70 u64 pf_exe_ctl;
71
72 if (type == AE_TYPES)
73 coremask = (coremask << cpt->max_se_cores);
74
75 pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
76 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
77 (pf_exe_ctl | coremask));
78 udelay(CSR_DELAY);
79}
80
81static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
82 u64 coremask, u8 type)
83{
84 u64 pf_gx_en = 0;
85
86 if (type == AE_TYPES)
87 coremask = (coremask << cpt->max_se_cores);
88
89 pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
90 cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
91 (pf_gx_en | coremask));
92 udelay(CSR_DELAY);
93}
94
95static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
96{
97 /* Clear mbox(0) interupts for all vfs */
98 cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
99}
100
101static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
102{
103 /* Clear ecc(0) interupts for all vfs */
104 cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
105}
106
107static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
108{
109 /* Clear exec interupts for all vfs */
110 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
111}
112
113static void cpt_disable_all_interrupts(struct cpt_device *cpt)
114{
115 cpt_disable_mbox_interrupts(cpt);
116 cpt_disable_ecc_interrupts(cpt);
117 cpt_disable_exec_interrupts(cpt);
118}
119
120static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
121{
122 /* Set mbox(0) interupts for all vfs */
123 cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
124}
125
126static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
127{
128 int ret = 0, core = 0, shift = 0;
129 u32 total_cores = 0;
130 struct device *dev = &cpt->pdev->dev;
131
132 if (!mcode || !mcode->code) {
133 dev_err(dev, "Either the mcode is null or data is NULL\n");
134 return -EINVAL;
135 }
136
137 if (mcode->code_size == 0) {
138 dev_err(dev, "microcode size is 0\n");
139 return -EINVAL;
140 }
141
142 /* Assumes 0-9 are SE cores for UCODE_BASE registers and
143 * AE core bases follow
144 */
145 if (mcode->is_ae) {
146 core = CPT_MAX_SE_CORES; /* start couting from 10 */
147 total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
148 } else {
149 core = 0; /* start couting from 0 */
150 total_cores = CPT_MAX_SE_CORES; /* upto 9 */
151 }
152
153 /* Point to microcode for each core of the group */
154 for (; core < total_cores ; core++, shift++) {
155 if (mcode->core_mask & (1 << shift)) {
156 cpt_write_csr64(cpt->reg_base,
157 CPTX_PF_ENGX_UCODE_BASE(0, core),
158 (u64)mcode->phys_base);
159 }
160 }
161 return ret;
162}
163
164static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
165{
166 int ret = 0;
167 struct device *dev = &cpt->pdev->dev;
168
169 /* Make device not ready */
170 cpt->flags &= ~CPT_FLAG_DEVICE_READY;
171 /* Disable All PF interrupts */
172 cpt_disable_all_interrupts(cpt);
173 /* Calculate mcode group and coremasks */
174 if (mcode->is_ae) {
175 if (mcode->num_cores > cpt->max_ae_cores) {
176 dev_err(dev, "Requested for more cores than available AE cores\n");
177 ret = -EINVAL;
178 goto cpt_init_fail;
179 }
180
181 if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
182 dev_err(dev, "Can't load, all eight microcode groups in use");
183 return -ENFILE;
184 }
185
186 mcode->group = cpt->next_group;
187 /* Convert requested cores to mask */
188 mcode->core_mask = GENMASK(mcode->num_cores, 0);
189 cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
190 mcode->group);
191 /* Load microcode for AE engines */
192 ret = cpt_load_microcode(cpt, mcode);
193 if (ret) {
194 dev_err(dev, "Microcode load Failed for %s\n",
195 mcode->version);
196 goto cpt_init_fail;
197 }
198 cpt->next_group++;
199 /* Configure group mask for the mcode */
200 cpt_configure_group(cpt, mcode->group, mcode->core_mask,
201 AE_TYPES);
202 /* Enable AE cores for the group mask */
203 cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
204 } else {
205 if (mcode->num_cores > cpt->max_se_cores) {
206 dev_err(dev, "Requested for more cores than available SE cores\n");
207 ret = -EINVAL;
208 goto cpt_init_fail;
209 }
210 if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
211 dev_err(dev, "Can't load, all eight microcode groups in use");
212 return -ENFILE;
213 }
214
215 mcode->group = cpt->next_group;
216 /* Covert requested cores to mask */
217 mcode->core_mask = GENMASK(mcode->num_cores, 0);
218 cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
219 mcode->group);
220 /* Load microcode for SE engines */
221 ret = cpt_load_microcode(cpt, mcode);
222 if (ret) {
223 dev_err(dev, "Microcode load Failed for %s\n",
224 mcode->version);
225 goto cpt_init_fail;
226 }
227 cpt->next_group++;
228 /* Configure group mask for the mcode */
229 cpt_configure_group(cpt, mcode->group, mcode->core_mask,
230 SE_TYPES);
231 /* Enable SE cores for the group mask */
232 cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
233 }
234
235 /* Enabled PF mailbox interrupts */
236 cpt_enable_mbox_interrupts(cpt);
237 cpt->flags |= CPT_FLAG_DEVICE_READY;
238
239 return ret;
240
241cpt_init_fail:
242 /* Enabled PF mailbox interrupts */
243 cpt_enable_mbox_interrupts(cpt);
244
245 return ret;
246}
247
248struct ucode_header {
249 u8 version[CPT_UCODE_VERSION_SZ];
250 u32 code_length;
251 u32 data_length;
252 u64 sram_address;
253};
254
255static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
256{
257 const struct firmware *fw_entry;
258 struct device *dev = &cpt->pdev->dev;
259 struct ucode_header *ucode;
260 struct microcode *mcode;
261 int j, ret = 0;
262
263 ret = request_firmware(&fw_entry, fw, dev);
264 if (ret)
265 return ret;
266
267 ucode = (struct ucode_header *)fw_entry->data;
268 mcode = &cpt->mcode[cpt->next_mc_idx];
269 memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
270 mcode->code_size = ntohl(ucode->code_length) * 2;
271 if (!mcode->code_size)
272 return -EINVAL;
273
274 mcode->is_ae = is_ae;
275 mcode->core_mask = 0ULL;
276 mcode->num_cores = is_ae ? 6 : 10;
277
278 /* Allocate DMAable space */
279 mcode->code = dma_zalloc_coherent(&cpt->pdev->dev, mcode->code_size,
280 &mcode->phys_base, GFP_KERNEL);
281 if (!mcode->code) {
282 dev_err(dev, "Unable to allocate space for microcode");
283 return -ENOMEM;
284 }
285
286 memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
287 mcode->code_size);
288
289 /* Byte swap 64-bit */
290 for (j = 0; j < (mcode->code_size / 8); j++)
291 ((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]);
292 /* MC needs 16-bit swap */
293 for (j = 0; j < (mcode->code_size / 2); j++)
294 ((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]);
295
296 dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size);
297 dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae);
298 dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores);
299 dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code);
300 dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base);
301
302 ret = do_cpt_init(cpt, mcode);
303 if (ret) {
304 dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
305 return ret;
306 }
307
308 dev_info(dev, "Microcode Loaded %s\n", mcode->version);
309 mcode->is_mc_valid = 1;
310 cpt->next_mc_idx++;
311 release_firmware(fw_entry);
312
313 return ret;
314}
315
316static int cpt_ucode_load(struct cpt_device *cpt)
317{
318 int ret = 0;
319 struct device *dev = &cpt->pdev->dev;
320
321 ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true);
322 if (ret) {
323 dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret);
324 return ret;
325 }
326 ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false);
327 if (ret) {
328 dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret);
329 return ret;
330 }
331
332 return ret;
333}
334
335static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq)
336{
337 struct cpt_device *cpt = (struct cpt_device *)cpt_irq;
338
339 cpt_mbox_intr_handler(cpt, 0);
340
341 return IRQ_HANDLED;
342}
343
344static void cpt_reset(struct cpt_device *cpt)
345{
346 cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1);
347}
348
349static void cpt_find_max_enabled_cores(struct cpt_device *cpt)
350{
351 union cptx_pf_constants pf_cnsts = {0};
352
353 pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0));
354 cpt->max_se_cores = pf_cnsts.s.se;
355 cpt->max_ae_cores = pf_cnsts.s.ae;
356}
357
358static u32 cpt_check_bist_status(struct cpt_device *cpt)
359{
360 union cptx_pf_bist_status bist_sts = {0};
361
362 bist_sts.u = cpt_read_csr64(cpt->reg_base,
363 CPTX_PF_BIST_STATUS(0));
364
365 return bist_sts.u;
366}
367
368static u64 cpt_check_exe_bist_status(struct cpt_device *cpt)
369{
370 union cptx_pf_exe_bist_status bist_sts = {0};
371
372 bist_sts.u = cpt_read_csr64(cpt->reg_base,
373 CPTX_PF_EXE_BIST_STATUS(0));
374
375 return bist_sts.u;
376}
377
378static void cpt_disable_all_cores(struct cpt_device *cpt)
379{
380 u32 grp, timeout = 100;
381 struct device *dev = &cpt->pdev->dev;
382
383 /* Disengage the cores from groups */
384 for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
385 cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
386 udelay(CSR_DELAY);
387 }
388
389 grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
390 while (grp) {
391 dev_err(dev, "Cores still busy");
392 grp = cpt_read_csr64(cpt->reg_base,
393 CPTX_PF_EXEC_BUSY(0));
394 if (timeout--)
395 break;
396
397 udelay(CSR_DELAY);
398 }
399 /* Disable the cores */
400 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
401}
402
403/**
404 * Ensure all cores are disengaged from all groups by
405 * calling cpt_disable_all_cores() before calling this
406 * function.
407 */
408static void cpt_unload_microcode(struct cpt_device *cpt)
409{
410 u32 grp = 0, core;
411
412 /* Free microcode bases and reset group masks */
413 for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
414 struct microcode *mcode = &cpt->mcode[grp];
415
416 if (cpt->mcode[grp].code)
417 dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
418 mcode->code, mcode->phys_base);
419 mcode->code = NULL;
420 }
421 /* Clear UCODE_BASE registers for all engines */
422 for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
423 cpt_write_csr64(cpt->reg_base,
424 CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
425}
426
427static int cpt_device_init(struct cpt_device *cpt)
428{
429 u64 bist;
430 struct device *dev = &cpt->pdev->dev;
431
432 /* Reset the PF when probed first */
433 cpt_reset(cpt);
434 mdelay(100);
435
436 /*Check BIST status*/
437 bist = (u64)cpt_check_bist_status(cpt);
438 if (bist) {
439 dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
440 return -ENODEV;
441 }
442
443 bist = cpt_check_exe_bist_status(cpt);
444 if (bist) {
445 dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
446 return -ENODEV;
447 }
448
449 /*Get CLK frequency*/
450 /*Get max enabled cores */
451 cpt_find_max_enabled_cores(cpt);
452 /*Disable all cores*/
453 cpt_disable_all_cores(cpt);
454 /*Reset device parameters*/
455 cpt->next_mc_idx = 0;
456 cpt->next_group = 0;
457 /* PF is ready */
458 cpt->flags |= CPT_FLAG_DEVICE_READY;
459
460 return 0;
461}
462
463static int cpt_register_interrupts(struct cpt_device *cpt)
464{
465 int ret;
466 struct device *dev = &cpt->pdev->dev;
467
468 /* Enable MSI-X */
469 ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS,
470 CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX);
471 if (ret < 0) {
472 dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n",
473 CPT_PF_MSIX_VECTORS);
474 return ret;
475 }
476
477 /* Register mailbox interrupt handlers */
478 ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)),
479 cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
480 if (ret)
481 goto fail;
482
483 /* Enable mailbox interrupt */
484 cpt_enable_mbox_interrupts(cpt);
485 return 0;
486
487fail:
488 dev_err(dev, "Request irq failed\n");
489 pci_disable_msix(cpt->pdev);
490 return ret;
491}
492
493static void cpt_unregister_interrupts(struct cpt_device *cpt)
494{
495 free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt);
496 pci_disable_msix(cpt->pdev);
497}
498
499static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
500{
501 int pos = 0;
502 int err;
503 u16 total_vf_cnt;
504 struct pci_dev *pdev = cpt->pdev;
505
506 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
507 if (!pos) {
508 dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
509 return -ENODEV;
510 }
511
512 cpt->num_vf_en = num_vfs; /* User requested VFs */
513 pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
514 if (total_vf_cnt < cpt->num_vf_en)
515 cpt->num_vf_en = total_vf_cnt;
516
517 if (!total_vf_cnt)
518 return 0;
519
520 /*Enabled the available VFs */
521 err = pci_enable_sriov(pdev, cpt->num_vf_en);
522 if (err) {
523 dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
524 cpt->num_vf_en);
525 cpt->num_vf_en = 0;
526 return err;
527 }
528
529 /* TODO: Optionally enable static VQ priorities feature */
530
531 dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
532 cpt->num_vf_en);
533
534 cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
535
536 return 0;
537}
538
539static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
540{
541 struct device *dev = &pdev->dev;
542 struct cpt_device *cpt;
543 int err;
544
545 if (num_vfs > 16 || num_vfs < 4) {
546 dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n",
547 num_vfs);
548 num_vfs = 4;
549 }
550
551 cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL);
552 if (!cpt)
553 return -ENOMEM;
554
555 pci_set_drvdata(pdev, cpt);
556 cpt->pdev = pdev;
557 err = pci_enable_device(pdev);
558 if (err) {
559 dev_err(dev, "Failed to enable PCI device\n");
560 pci_set_drvdata(pdev, NULL);
561 return err;
562 }
563
564 err = pci_request_regions(pdev, DRV_NAME);
565 if (err) {
566 dev_err(dev, "PCI request regions failed 0x%x\n", err);
567 goto cpt_err_disable_device;
568 }
569
570 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
571 if (err) {
572 dev_err(dev, "Unable to get usable DMA configuration\n");
573 goto cpt_err_release_regions;
574 }
575
576 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
577 if (err) {
578 dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
579 goto cpt_err_release_regions;
580 }
581
582 /* MAP PF's configuration registers */
583 cpt->reg_base = pcim_iomap(pdev, 0, 0);
584 if (!cpt->reg_base) {
585 dev_err(dev, "Cannot map config register space, aborting\n");
586 err = -ENOMEM;
587 goto cpt_err_release_regions;
588 }
589
590 /* CPT device HW initialization */
591 cpt_device_init(cpt);
592
593 /* Register interrupts */
594 err = cpt_register_interrupts(cpt);
595 if (err)
596 goto cpt_err_release_regions;
597
598 err = cpt_ucode_load(cpt);
599 if (err)
600 goto cpt_err_unregister_interrupts;
601
602 /* Configure SRIOV */
603 err = cpt_sriov_init(cpt, num_vfs);
604 if (err)
605 goto cpt_err_unregister_interrupts;
606
607 return 0;
608
609cpt_err_unregister_interrupts:
610 cpt_unregister_interrupts(cpt);
611cpt_err_release_regions:
612 pci_release_regions(pdev);
613cpt_err_disable_device:
614 pci_disable_device(pdev);
615 pci_set_drvdata(pdev, NULL);
616 return err;
617}
618
619static void cpt_remove(struct pci_dev *pdev)
620{
621 struct cpt_device *cpt = pci_get_drvdata(pdev);
622
623 /* Disengage SE and AE cores from all groups*/
624 cpt_disable_all_cores(cpt);
625 /* Unload microcodes */
626 cpt_unload_microcode(cpt);
627 cpt_unregister_interrupts(cpt);
628 pci_disable_sriov(pdev);
629 pci_release_regions(pdev);
630 pci_disable_device(pdev);
631 pci_set_drvdata(pdev, NULL);
632}
633
634static void cpt_shutdown(struct pci_dev *pdev)
635{
636 struct cpt_device *cpt = pci_get_drvdata(pdev);
637
638 if (!cpt)
639 return;
640
641 dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
642 (u32)pdev->vendor, (u32)pdev->device);
643
644 cpt_unregister_interrupts(cpt);
645 pci_release_regions(pdev);
646 pci_disable_device(pdev);
647 pci_set_drvdata(pdev, NULL);
648}
649
650/* Supported devices */
651static const struct pci_device_id cpt_id_table[] = {
652 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
653 { 0, } /* end of table */
654};
655
656static struct pci_driver cpt_pci_driver = {
657 .name = DRV_NAME,
658 .id_table = cpt_id_table,
659 .probe = cpt_probe,
660 .remove = cpt_remove,
661 .shutdown = cpt_shutdown,
662};
663
664module_pci_driver(cpt_pci_driver);
665
666MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
667MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
668MODULE_LICENSE("GPL v2");
669MODULE_VERSION(DRV_VERSION);
670MODULE_DEVICE_TABLE(pci, cpt_id_table);
diff --git a/drivers/crypto/cavium/cpt/cptpf_mbox.c b/drivers/crypto/cavium/cpt/cptpf_mbox.c
new file mode 100644
index 000000000000..20f2c6ee46a5
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptpf_mbox.c
@@ -0,0 +1,163 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8#include <linux/module.h>
9#include "cptpf.h"
10
11static void cpt_send_msg_to_vf(struct cpt_device *cpt, int vf,
12 struct cpt_mbox *mbx)
13{
14 /* Writing mbox(0) causes interrupt */
15 cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1),
16 mbx->data);
17 cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0), mbx->msg);
18}
19
20/* ACKs VF's mailbox message
21 * @vf: VF to which ACK to be sent
22 */
23static void cpt_mbox_send_ack(struct cpt_device *cpt, int vf,
24 struct cpt_mbox *mbx)
25{
26 mbx->data = 0ull;
27 mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
28 cpt_send_msg_to_vf(cpt, vf, mbx);
29}
30
31static void cpt_clear_mbox_intr(struct cpt_device *cpt, u32 vf)
32{
33 /* W1C for the VF */
34 cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0), (1 << vf));
35}
36
37/*
38 * Configure QLEN/Chunk sizes for VF
39 */
40static void cpt_cfg_qlen_for_vf(struct cpt_device *cpt, int vf, u32 size)
41{
42 union cptx_pf_qx_ctl pf_qx_ctl;
43
44 pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf));
45 pf_qx_ctl.s.size = size;
46 pf_qx_ctl.s.cont_err = true;
47 cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u);
48}
49
50/*
51 * Configure VQ priority
52 */
53static void cpt_cfg_vq_priority(struct cpt_device *cpt, int vf, u32 pri)
54{
55 union cptx_pf_qx_ctl pf_qx_ctl;
56
57 pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf));
58 pf_qx_ctl.s.pri = pri;
59 cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u);
60}
61
62static int cpt_bind_vq_to_grp(struct cpt_device *cpt, u8 q, u8 grp)
63{
64 struct microcode *mcode = cpt->mcode;
65 union cptx_pf_qx_ctl pf_qx_ctl;
66 struct device *dev = &cpt->pdev->dev;
67
68 if (q >= CPT_MAX_VF_NUM) {
69 dev_err(dev, "Queues are more than cores in the group");
70 return -EINVAL;
71 }
72 if (grp >= CPT_MAX_CORE_GROUPS) {
73 dev_err(dev, "Request group is more than possible groups");
74 return -EINVAL;
75 }
76 if (grp >= cpt->next_mc_idx) {
77 dev_err(dev, "Request group is higher than available functional groups");
78 return -EINVAL;
79 }
80 pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q));
81 pf_qx_ctl.s.grp = mcode[grp].group;
82 cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q), pf_qx_ctl.u);
83 dev_dbg(dev, "VF %d TYPE %s", q, (mcode[grp].is_ae ? "AE" : "SE"));
84
85 return mcode[grp].is_ae ? AE_TYPES : SE_TYPES;
86}
87
88/* Interrupt handler to handle mailbox messages from VFs */
89static void cpt_handle_mbox_intr(struct cpt_device *cpt, int vf)
90{
91 struct cpt_vf_info *vfx = &cpt->vfinfo[vf];
92 struct cpt_mbox mbx = {};
93 int vftype;
94 struct device *dev = &cpt->pdev->dev;
95 /*
96 * MBOX[0] contains msg
97 * MBOX[1] contains data
98 */
99 mbx.msg = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0));
100 mbx.data = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1));
101 dev_dbg(dev, "%s: Mailbox msg 0x%llx from VF%d", __func__, mbx.msg, vf);
102 switch (mbx.msg) {
103 case CPT_MSG_VF_UP:
104 vfx->state = VF_STATE_UP;
105 try_module_get(THIS_MODULE);
106 cpt_mbox_send_ack(cpt, vf, &mbx);
107 break;
108 case CPT_MSG_READY:
109 mbx.msg = CPT_MSG_READY;
110 mbx.data = vf;
111 cpt_send_msg_to_vf(cpt, vf, &mbx);
112 break;
113 case CPT_MSG_VF_DOWN:
114 /* First msg in VF teardown sequence */
115 vfx->state = VF_STATE_DOWN;
116 module_put(THIS_MODULE);
117 cpt_mbox_send_ack(cpt, vf, &mbx);
118 break;
119 case CPT_MSG_QLEN:
120 vfx->qlen = mbx.data;
121 cpt_cfg_qlen_for_vf(cpt, vf, vfx->qlen);
122 cpt_mbox_send_ack(cpt, vf, &mbx);
123 break;
124 case CPT_MSG_QBIND_GRP:
125 vftype = cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data);
126 if ((vftype != AE_TYPES) && (vftype != SE_TYPES))
127 dev_err(dev, "Queue %d binding to group %llu failed",
128 vf, mbx.data);
129 else {
130 dev_dbg(dev, "Queue %d binding to group %llu successful",
131 vf, mbx.data);
132 mbx.msg = CPT_MSG_QBIND_GRP;
133 mbx.data = vftype;
134 cpt_send_msg_to_vf(cpt, vf, &mbx);
135 }
136 break;
137 case CPT_MSG_VQ_PRIORITY:
138 vfx->priority = mbx.data;
139 cpt_cfg_vq_priority(cpt, vf, vfx->priority);
140 cpt_mbox_send_ack(cpt, vf, &mbx);
141 break;
142 default:
143 dev_err(&cpt->pdev->dev, "Invalid msg from VF%d, msg 0x%llx\n",
144 vf, mbx.msg);
145 break;
146 }
147}
148
149void cpt_mbox_intr_handler (struct cpt_device *cpt, int mbx)
150{
151 u64 intr;
152 u8 vf;
153
154 intr = cpt_read_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0));
155 dev_dbg(&cpt->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr);
156 for (vf = 0; vf < CPT_MAX_VF_NUM; vf++) {
157 if (intr & (1ULL << vf)) {
158 dev_dbg(&cpt->pdev->dev, "Intr from VF %d\n", vf);
159 cpt_handle_mbox_intr(cpt, vf);
160 cpt_clear_mbox_intr(cpt, vf);
161 }
162 }
163}
diff --git a/drivers/crypto/cavium/cpt/cptvf.h b/drivers/crypto/cavium/cpt/cptvf.h
new file mode 100644
index 000000000000..0a835a07d4f2
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf.h
@@ -0,0 +1,132 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#ifndef __CPTVF_H
10#define __CPTVF_H
11
12#include <linux/list.h>
13#include "cpt_common.h"
14
15/* Default command queue length */
16#define CPT_CMD_QLEN 2046
17#define CPT_CMD_QCHUNK_SIZE 1023
18
19/* Default command timeout in seconds */
20#define CPT_COMMAND_TIMEOUT 4
21#define CPT_TIMER_THOLD 0xFFFF
22#define CPT_NUM_QS_PER_VF 1
23#define CPT_INST_SIZE 64
24#define CPT_NEXT_CHUNK_PTR_SIZE 8
25
26#define CPT_VF_MSIX_VECTORS 2
27#define CPT_VF_INTR_MBOX_MASK BIT(0)
28#define CPT_VF_INTR_DOVF_MASK BIT(1)
29#define CPT_VF_INTR_IRDE_MASK BIT(2)
30#define CPT_VF_INTR_NWRP_MASK BIT(3)
31#define CPT_VF_INTR_SERR_MASK BIT(4)
32#define DMA_DIRECT_DIRECT 0 /* Input DIRECT, Output DIRECT */
33#define DMA_GATHER_SCATTER 1
34#define FROM_DPTR 1
35
36/**
37 * Enumeration cpt_vf_int_vec_e
38 *
39 * CPT VF MSI-X Vector Enumeration
40 * Enumerates the MSI-X interrupt vectors.
41 */
42enum cpt_vf_int_vec_e {
43 CPT_VF_INT_VEC_E_MISC = 0x00,
44 CPT_VF_INT_VEC_E_DONE = 0x01
45};
46
47struct command_chunk {
48 u8 *head;
49 dma_addr_t dma_addr;
50 u32 size; /* Chunk size, max CPT_INST_CHUNK_MAX_SIZE */
51 struct hlist_node nextchunk;
52};
53
54struct command_queue {
55 spinlock_t lock; /* command queue lock */
56 u32 idx; /* Command queue host write idx */
57 u32 nchunks; /* Number of command chunks */
58 struct command_chunk *qhead; /* Command queue head, instructions
59 * are inserted here
60 */
61 struct hlist_head chead;
62};
63
64struct command_qinfo {
65 u32 cmd_size;
66 u32 qchunksize; /* Command queue chunk size */
67 struct command_queue queue[CPT_NUM_QS_PER_VF];
68};
69
70struct pending_entry {
71 u8 busy; /* Entry status (free/busy) */
72
73 volatile u64 *completion_addr; /* Completion address */
74 void *post_arg;
75 void (*callback)(int, void *); /* Kernel ASYNC request callabck */
76 void *callback_arg; /* Kernel ASYNC request callabck arg */
77};
78
79struct pending_queue {
80 struct pending_entry *head; /* head of the queue */
81 u32 front; /* Process work from here */
82 u32 rear; /* Append new work here */
83 atomic64_t pending_count;
84 spinlock_t lock; /* Queue lock */
85};
86
87struct pending_qinfo {
88 u32 nr_queues; /* Number of queues supported */
89 u32 qlen; /* Queue length */
90 struct pending_queue queue[CPT_NUM_QS_PER_VF];
91};
92
93#define for_each_pending_queue(qinfo, q, i) \
94 for (i = 0, q = &qinfo->queue[i]; i < qinfo->nr_queues; i++, \
95 q = &qinfo->queue[i])
96
97struct cpt_vf {
98 u16 flags; /* Flags to hold device status bits */
99 u8 vfid; /* Device Index 0...CPT_MAX_VF_NUM */
100 u8 vftype; /* VF type of SE_TYPE(1) or AE_TYPE(1) */
101 u8 vfgrp; /* VF group (0 - 8) */
102 u8 node; /* Operating node: Bits (46:44) in BAR0 address */
103 u8 priority; /* VF priority ring: 1-High proirity round
104 * robin ring;0-Low priority round robin ring;
105 */
106 struct pci_dev *pdev; /* pci device handle */
107 void __iomem *reg_base; /* Register start address */
108 void *wqe_info; /* BH worker info */
109 /* MSI-X */
110 cpumask_var_t affinity_mask[CPT_VF_MSIX_VECTORS];
111 /* Command and Pending queues */
112 u32 qsize;
113 u32 nr_queues;
114 struct command_qinfo cqinfo; /* Command queue information */
115 struct pending_qinfo pqinfo; /* Pending queue information */
116 /* VF-PF mailbox communication */
117 bool pf_acked;
118 bool pf_nacked;
119};
120
121int cptvf_send_vf_up(struct cpt_vf *cptvf);
122int cptvf_send_vf_down(struct cpt_vf *cptvf);
123int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf);
124int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf);
125int cptvf_send_vq_size_msg(struct cpt_vf *cptvf);
126int cptvf_check_pf_ready(struct cpt_vf *cptvf);
127void cptvf_handle_mbox_intr(struct cpt_vf *cptvf);
128void cvm_crypto_exit(void);
129int cvm_crypto_init(struct cpt_vf *cptvf);
130void vq_post_process(struct cpt_vf *cptvf, u32 qno);
131void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val);
132#endif /* __CPTVF_H */
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c
new file mode 100644
index 000000000000..cc853f913d4b
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.c
@@ -0,0 +1,444 @@
1
2/*
3 * Copyright (C) 2016 Cavium, Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of version 2 of the GNU General Public License
7 * as published by the Free Software Foundation.
8 */
9
10#include <crypto/aes.h>
11#include <crypto/algapi.h>
12#include <crypto/authenc.h>
13#include <crypto/cryptd.h>
14#include <crypto/crypto_wq.h>
15#include <crypto/des.h>
16#include <crypto/xts.h>
17#include <linux/crypto.h>
18#include <linux/err.h>
19#include <linux/list.h>
20#include <linux/scatterlist.h>
21
22#include "cptvf.h"
23#include "cptvf_algs.h"
24
25struct cpt_device_handle {
26 void *cdev[MAX_DEVICES];
27 u32 dev_count;
28};
29
30static struct cpt_device_handle dev_handle;
31
32static void cvm_callback(u32 status, void *arg)
33{
34 struct crypto_async_request *req = (struct crypto_async_request *)arg;
35
36 req->complete(req, !status);
37}
38
39static inline void update_input_iv(struct cpt_request_info *req_info,
40 u8 *iv, u32 enc_iv_len,
41 u32 *argcnt)
42{
43 /* Setting the iv information */
44 req_info->in[*argcnt].vptr = (void *)iv;
45 req_info->in[*argcnt].size = enc_iv_len;
46 req_info->req.dlen += enc_iv_len;
47
48 ++(*argcnt);
49}
50
51static inline void update_output_iv(struct cpt_request_info *req_info,
52 u8 *iv, u32 enc_iv_len,
53 u32 *argcnt)
54{
55 /* Setting the iv information */
56 req_info->out[*argcnt].vptr = (void *)iv;
57 req_info->out[*argcnt].size = enc_iv_len;
58 req_info->rlen += enc_iv_len;
59
60 ++(*argcnt);
61}
62
63static inline void update_input_data(struct cpt_request_info *req_info,
64 struct scatterlist *inp_sg,
65 u32 nbytes, u32 *argcnt)
66{
67 req_info->req.dlen += nbytes;
68
69 while (nbytes) {
70 u32 len = min(nbytes, inp_sg->length);
71 u8 *ptr = sg_virt(inp_sg);
72
73 req_info->in[*argcnt].vptr = (void *)ptr;
74 req_info->in[*argcnt].size = len;
75 nbytes -= len;
76
77 ++(*argcnt);
78 ++inp_sg;
79 }
80}
81
82static inline void update_output_data(struct cpt_request_info *req_info,
83 struct scatterlist *outp_sg,
84 u32 nbytes, u32 *argcnt)
85{
86 req_info->rlen += nbytes;
87
88 while (nbytes) {
89 u32 len = min(nbytes, outp_sg->length);
90 u8 *ptr = sg_virt(outp_sg);
91
92 req_info->out[*argcnt].vptr = (void *)ptr;
93 req_info->out[*argcnt].size = len;
94 nbytes -= len;
95 ++(*argcnt);
96 ++outp_sg;
97 }
98}
99
100static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
101 u32 cipher_type, u32 aes_key_type,
102 u32 *argcnt)
103{
104 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
105 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
106 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
107 struct fc_context *fctx = &rctx->fctx;
108 u64 *offset_control = &rctx->control_word;
109 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
110 struct cpt_request_info *req_info = &rctx->cpt_req;
111 u64 *ctrl_flags = NULL;
112
113 req_info->ctrl.s.grp = 0;
114 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
115 req_info->ctrl.s.se_req = SE_CORE_REQ;
116
117 req_info->req.opcode.s.major = MAJOR_OP_FC |
118 DMA_MODE_FLAG(DMA_GATHER_SCATTER);
119 if (enc)
120 req_info->req.opcode.s.minor = 2;
121 else
122 req_info->req.opcode.s.minor = 3;
123
124 req_info->req.param1 = req->nbytes; /* Encryption Data length */
125 req_info->req.param2 = 0; /*Auth data length */
126
127 fctx->enc.enc_ctrl.e.enc_cipher = cipher_type;
128 fctx->enc.enc_ctrl.e.aes_key = aes_key_type;
129 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
130
131 if (cipher_type == AES_XTS)
132 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
133 else
134 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
135 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
136 *ctrl_flags = cpu_to_be64(*ctrl_flags);
137
138 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
139 /* Storing Packet Data Information in offset
140 * Control Word First 8 bytes
141 */
142 req_info->in[*argcnt].vptr = (u8 *)offset_control;
143 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
144 req_info->req.dlen += CONTROL_WORD_LEN;
145 ++(*argcnt);
146
147 req_info->in[*argcnt].vptr = (u8 *)fctx;
148 req_info->in[*argcnt].size = sizeof(struct fc_context);
149 req_info->req.dlen += sizeof(struct fc_context);
150
151 ++(*argcnt);
152
153 return 0;
154}
155
156static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc,
157 u32 cipher_type, u32 aes_key_type,
158 u32 enc_iv_len)
159{
160 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
161 struct cpt_request_info *req_info = &rctx->cpt_req;
162 u32 argcnt = 0;
163
164 create_ctx_hdr(req, enc, cipher_type, aes_key_type, &argcnt);
165 update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
166 update_input_data(req_info, req->src, req->nbytes, &argcnt);
167 req_info->incnt = argcnt;
168
169 return 0;
170}
171
172static inline void store_cb_info(struct ablkcipher_request *req,
173 struct cpt_request_info *req_info)
174{
175 req_info->callback = (void *)cvm_callback;
176 req_info->callback_arg = (void *)&req->base;
177}
178
179static inline void create_output_list(struct ablkcipher_request *req,
180 u32 cipher_type,
181 u32 enc_iv_len)
182{
183 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
184 struct cpt_request_info *req_info = &rctx->cpt_req;
185 u32 argcnt = 0;
186
187 /* OUTPUT Buffer Processing
188 * AES encryption/decryption output would be
189 * received in the following format
190 *
191 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
192 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
193 */
194 /* Reading IV information */
195 update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
196 update_output_data(req_info, req->dst, req->nbytes, &argcnt);
197 req_info->outcnt = argcnt;
198}
199
200static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
201 u32 cipher_type)
202{
203 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
204 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
205 u32 key_type = AES_128_BIT;
206 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
207 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
208 struct fc_context *fctx = &rctx->fctx;
209 struct cpt_request_info *req_info = &rctx->cpt_req;
210 void *cdev = NULL;
211 int status;
212
213 switch (ctx->key_len) {
214 case 16:
215 key_type = AES_128_BIT;
216 break;
217 case 24:
218 key_type = AES_192_BIT;
219 break;
220 case 32:
221 if (cipher_type == AES_XTS)
222 key_type = AES_128_BIT;
223 else
224 key_type = AES_256_BIT;
225 break;
226 case 64:
227 if (cipher_type == AES_XTS)
228 key_type = AES_256_BIT;
229 else
230 return -EINVAL;
231 break;
232 default:
233 return -EINVAL;
234 }
235
236 if (cipher_type == DES3_CBC)
237 key_type = 0;
238
239 memset(req_info, 0, sizeof(struct cpt_request_info));
240 memset(fctx, 0, sizeof(struct fc_context));
241 create_input_list(req, enc, cipher_type, key_type, enc_iv_len);
242 create_output_list(req, cipher_type, enc_iv_len);
243 store_cb_info(req, req_info);
244 cdev = dev_handle.cdev[smp_processor_id()];
245 status = cptvf_do_request(cdev, req_info);
246 /* We perform an asynchronous send and once
247 * the request is completed the driver would
248 * intimate through registered call back functions
249 */
250
251 if (status)
252 return status;
253 else
254 return -EINPROGRESS;
255}
256
257int cvm_des3_encrypt_cbc(struct ablkcipher_request *req)
258{
259 return cvm_enc_dec(req, true, DES3_CBC);
260}
261
262int cvm_des3_decrypt_cbc(struct ablkcipher_request *req)
263{
264 return cvm_enc_dec(req, false, DES3_CBC);
265}
266
267int cvm_aes_encrypt_xts(struct ablkcipher_request *req)
268{
269 return cvm_enc_dec(req, true, AES_XTS);
270}
271
272int cvm_aes_decrypt_xts(struct ablkcipher_request *req)
273{
274 return cvm_enc_dec(req, false, AES_XTS);
275}
276
277int cvm_aes_encrypt_cbc(struct ablkcipher_request *req)
278{
279 return cvm_enc_dec(req, true, AES_CBC);
280}
281
282int cvm_aes_decrypt_cbc(struct ablkcipher_request *req)
283{
284 return cvm_enc_dec(req, false, AES_CBC);
285}
286
287int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
288 u32 keylen)
289{
290 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
291 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
292 int err;
293 const u8 *key1 = key;
294 const u8 *key2 = key + (keylen / 2);
295
296 err = xts_check_key(tfm, key, keylen);
297 if (err)
298 return err;
299 ctx->key_len = keylen;
300 memcpy(ctx->enc_key, key1, keylen / 2);
301 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
302
303 return 0;
304}
305
306int cvm_enc_dec_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
307 u32 keylen)
308{
309 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
310 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
311
312 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
313 ctx->key_len = keylen;
314 memcpy(ctx->enc_key, key, keylen);
315 return 0;
316 }
317 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
318
319 return -EINVAL;
320}
321
322int cvm_enc_dec_init(struct crypto_tfm *tfm)
323{
324 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
325
326 memset(ctx, 0, sizeof(*ctx));
327 tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) +
328 sizeof(struct ablkcipher_request);
329 /* Additional memory for ablkcipher_request is
330 * allocated since the cryptd daemon uses
331 * this memory for request_ctx information
332 */
333
334 return 0;
335}
336
337struct crypto_alg algs[] = { {
338 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
339 .cra_blocksize = AES_BLOCK_SIZE,
340 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
341 .cra_alignmask = 7,
342 .cra_priority = 4001,
343 .cra_name = "xts(aes)",
344 .cra_driver_name = "cavium-xts-aes",
345 .cra_type = &crypto_ablkcipher_type,
346 .cra_u = {
347 .ablkcipher = {
348 .ivsize = AES_BLOCK_SIZE,
349 .min_keysize = 2 * AES_MIN_KEY_SIZE,
350 .max_keysize = 2 * AES_MAX_KEY_SIZE,
351 .setkey = cvm_xts_setkey,
352 .encrypt = cvm_aes_encrypt_xts,
353 .decrypt = cvm_aes_decrypt_xts,
354 },
355 },
356 .cra_init = cvm_enc_dec_init,
357 .cra_module = THIS_MODULE,
358}, {
359 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
360 .cra_blocksize = AES_BLOCK_SIZE,
361 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
362 .cra_alignmask = 7,
363 .cra_priority = 4001,
364 .cra_name = "cbc(aes)",
365 .cra_driver_name = "cavium-cbc-aes",
366 .cra_type = &crypto_ablkcipher_type,
367 .cra_u = {
368 .ablkcipher = {
369 .ivsize = AES_BLOCK_SIZE,
370 .min_keysize = AES_MIN_KEY_SIZE,
371 .max_keysize = AES_MAX_KEY_SIZE,
372 .setkey = cvm_enc_dec_setkey,
373 .encrypt = cvm_aes_encrypt_cbc,
374 .decrypt = cvm_aes_decrypt_cbc,
375 },
376 },
377 .cra_init = cvm_enc_dec_init,
378 .cra_module = THIS_MODULE,
379}, {
380 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
381 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
382 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
383 .cra_alignmask = 7,
384 .cra_priority = 4001,
385 .cra_name = "cbc(des3_ede)",
386 .cra_driver_name = "cavium-cbc-des3_ede",
387 .cra_type = &crypto_ablkcipher_type,
388 .cra_u = {
389 .ablkcipher = {
390 .min_keysize = DES3_EDE_KEY_SIZE,
391 .max_keysize = DES3_EDE_KEY_SIZE,
392 .ivsize = DES_BLOCK_SIZE,
393 .setkey = cvm_enc_dec_setkey,
394 .encrypt = cvm_des3_encrypt_cbc,
395 .decrypt = cvm_des3_decrypt_cbc,
396 },
397 },
398 .cra_init = cvm_enc_dec_init,
399 .cra_module = THIS_MODULE,
400} };
401
402static inline int cav_register_algs(void)
403{
404 int err = 0;
405
406 err = crypto_register_algs(algs, ARRAY_SIZE(algs));
407 if (err)
408 return err;
409
410 return 0;
411}
412
413static inline void cav_unregister_algs(void)
414{
415 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
416}
417
418int cvm_crypto_init(struct cpt_vf *cptvf)
419{
420 struct pci_dev *pdev = cptvf->pdev;
421 u32 dev_count;
422
423 dev_count = dev_handle.dev_count;
424 dev_handle.cdev[dev_count] = cptvf;
425 dev_handle.dev_count++;
426
427 if (dev_count == 3) {
428 if (cav_register_algs()) {
429 dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
430 return -EINVAL;
431 }
432 }
433
434 return 0;
435}
436
437void cvm_crypto_exit(void)
438{
439 u32 dev_count;
440
441 dev_count = --dev_handle.dev_count;
442 if (!dev_count)
443 cav_unregister_algs();
444}
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h
new file mode 100644
index 000000000000..a12050d11b0c
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.h
@@ -0,0 +1,113 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#ifndef _CPTVF_ALGS_H_
10#define _CPTVF_ALGS_H_
11
12#include "request_manager.h"
13
14#define MAX_DEVICES 16
15#define MAJOR_OP_FC 0x33
16#define MAX_ENC_KEY_SIZE 32
17#define MAX_HASH_KEY_SIZE 64
18#define MAX_KEY_SIZE (MAX_ENC_KEY_SIZE + MAX_HASH_KEY_SIZE)
19#define CONTROL_WORD_LEN 8
20#define KEY2_OFFSET 48
21
22#define DMA_MODE_FLAG(dma_mode) \
23 (((dma_mode) == DMA_GATHER_SCATTER) ? (1 << 7) : 0)
24
25enum req_type {
26 AE_CORE_REQ,
27 SE_CORE_REQ,
28};
29
30enum cipher_type {
31 DES3_CBC = 0x1,
32 DES3_ECB = 0x2,
33 AES_CBC = 0x3,
34 AES_ECB = 0x4,
35 AES_CFB = 0x5,
36 AES_CTR = 0x6,
37 AES_GCM = 0x7,
38 AES_XTS = 0x8
39};
40
41enum aes_type {
42 AES_128_BIT = 0x1,
43 AES_192_BIT = 0x2,
44 AES_256_BIT = 0x3
45};
46
47union encr_ctrl {
48 u64 flags;
49 struct {
50#if defined(__BIG_ENDIAN_BITFIELD)
51 u64 enc_cipher:4;
52 u64 reserved1:1;
53 u64 aes_key:2;
54 u64 iv_source:1;
55 u64 hash_type:4;
56 u64 reserved2:3;
57 u64 auth_input_type:1;
58 u64 mac_len:8;
59 u64 reserved3:8;
60 u64 encr_offset:16;
61 u64 iv_offset:8;
62 u64 auth_offset:8;
63#else
64 u64 auth_offset:8;
65 u64 iv_offset:8;
66 u64 encr_offset:16;
67 u64 reserved3:8;
68 u64 mac_len:8;
69 u64 auth_input_type:1;
70 u64 reserved2:3;
71 u64 hash_type:4;
72 u64 iv_source:1;
73 u64 aes_key:2;
74 u64 reserved1:1;
75 u64 enc_cipher:4;
76#endif
77 } e;
78};
79
80struct enc_context {
81 union encr_ctrl enc_ctrl;
82 u8 encr_key[32];
83 u8 encr_iv[16];
84};
85
86struct fchmac_context {
87 u8 ipad[64];
88 u8 opad[64]; /* or OPAD */
89};
90
91struct fc_context {
92 struct enc_context enc;
93 struct fchmac_context hmac;
94};
95
96struct cvm_enc_ctx {
97 u32 key_len;
98 u8 enc_key[MAX_KEY_SIZE];
99};
100
101struct cvm_des3_ctx {
102 u32 key_len;
103 u8 des3_key[MAX_KEY_SIZE];
104};
105
106struct cvm_req_ctx {
107 struct cpt_request_info cpt_req;
108 u64 control_word;
109 struct fc_context fctx;
110};
111
112int cptvf_do_request(void *cptvf, struct cpt_request_info *req);
113#endif /*_CPTVF_ALGS_H_*/
diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
new file mode 100644
index 000000000000..aac2966ff8d9
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_main.c
@@ -0,0 +1,863 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#include <linux/interrupt.h>
10#include <linux/module.h>
11
12#include "cptvf.h"
13
14#define DRV_NAME "thunder-cptvf"
15#define DRV_VERSION "1.0"
16
17struct cptvf_wqe {
18 struct tasklet_struct twork;
19 void *cptvf;
20 u32 qno;
21};
22
23struct cptvf_wqe_info {
24 struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
25};
26
27static void vq_work_handler(unsigned long data)
28{
29 struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
30 struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
31
32 vq_post_process(cwqe->cptvf, cwqe->qno);
33}
34
35static int init_worker_threads(struct cpt_vf *cptvf)
36{
37 struct pci_dev *pdev = cptvf->pdev;
38 struct cptvf_wqe_info *cwqe_info;
39 int i;
40
41 cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
42 if (!cwqe_info)
43 return -ENOMEM;
44
45 if (cptvf->nr_queues) {
46 dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
47 cptvf->nr_queues);
48 }
49
50 for (i = 0; i < cptvf->nr_queues; i++) {
51 tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
52 (u64)cwqe_info);
53 cwqe_info->vq_wqe[i].qno = i;
54 cwqe_info->vq_wqe[i].cptvf = cptvf;
55 }
56
57 cptvf->wqe_info = cwqe_info;
58
59 return 0;
60}
61
62static void cleanup_worker_threads(struct cpt_vf *cptvf)
63{
64 struct cptvf_wqe_info *cwqe_info;
65 struct pci_dev *pdev = cptvf->pdev;
66 int i;
67
68 cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
69 if (!cwqe_info)
70 return;
71
72 if (cptvf->nr_queues) {
73 dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
74 cptvf->nr_queues);
75 }
76
77 for (i = 0; i < cptvf->nr_queues; i++)
78 tasklet_kill(&cwqe_info->vq_wqe[i].twork);
79
80 kzfree(cwqe_info);
81 cptvf->wqe_info = NULL;
82}
83
84static void free_pending_queues(struct pending_qinfo *pqinfo)
85{
86 int i;
87 struct pending_queue *queue;
88
89 for_each_pending_queue(pqinfo, queue, i) {
90 if (!queue->head)
91 continue;
92
93 /* free single queue */
94 kzfree((queue->head));
95
96 queue->front = 0;
97 queue->rear = 0;
98
99 return;
100 }
101
102 pqinfo->qlen = 0;
103 pqinfo->nr_queues = 0;
104}
105
106static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
107 u32 nr_queues)
108{
109 u32 i;
110 size_t size;
111 int ret;
112 struct pending_queue *queue = NULL;
113
114 pqinfo->nr_queues = nr_queues;
115 pqinfo->qlen = qlen;
116
117 size = (qlen * sizeof(struct pending_entry));
118
119 for_each_pending_queue(pqinfo, queue, i) {
120 queue->head = kzalloc((size), GFP_KERNEL);
121 if (!queue->head) {
122 ret = -ENOMEM;
123 goto pending_qfail;
124 }
125
126 queue->front = 0;
127 queue->rear = 0;
128 atomic64_set((&queue->pending_count), (0));
129
130 /* init queue spin lock */
131 spin_lock_init(&queue->lock);
132 }
133
134 return 0;
135
136pending_qfail:
137 free_pending_queues(pqinfo);
138
139 return ret;
140}
141
142static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
143{
144 struct pci_dev *pdev = cptvf->pdev;
145 int ret;
146
147 if (!nr_queues)
148 return 0;
149
150 ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
151 if (ret) {
152 dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
153 nr_queues);
154 return ret;
155 }
156
157 return 0;
158}
159
160static void cleanup_pending_queues(struct cpt_vf *cptvf)
161{
162 struct pci_dev *pdev = cptvf->pdev;
163
164 if (!cptvf->nr_queues)
165 return;
166
167 dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
168 cptvf->nr_queues);
169 free_pending_queues(&cptvf->pqinfo);
170}
171
172static void free_command_queues(struct cpt_vf *cptvf,
173 struct command_qinfo *cqinfo)
174{
175 int i;
176 struct command_queue *queue = NULL;
177 struct command_chunk *chunk = NULL;
178 struct pci_dev *pdev = cptvf->pdev;
179 struct hlist_node *node;
180
181 /* clean up for each queue */
182 for (i = 0; i < cptvf->nr_queues; i++) {
183 queue = &cqinfo->queue[i];
184 if (hlist_empty(&cqinfo->queue[i].chead))
185 continue;
186
187 hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
188 nextchunk) {
189 dma_free_coherent(&pdev->dev, chunk->size,
190 chunk->head,
191 chunk->dma_addr);
192 chunk->head = NULL;
193 chunk->dma_addr = 0;
194 hlist_del(&chunk->nextchunk);
195 kzfree(chunk);
196 }
197
198 queue->nchunks = 0;
199 queue->idx = 0;
200 }
201
202 /* common cleanup */
203 cqinfo->cmd_size = 0;
204}
205
206static int alloc_command_queues(struct cpt_vf *cptvf,
207 struct command_qinfo *cqinfo, size_t cmd_size,
208 u32 qlen)
209{
210 int i;
211 size_t q_size;
212 struct command_queue *queue = NULL;
213 struct pci_dev *pdev = cptvf->pdev;
214
215 /* common init */
216 cqinfo->cmd_size = cmd_size;
217 /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
218 cptvf->qsize = min(qlen, cqinfo->qchunksize) *
219 CPT_NEXT_CHUNK_PTR_SIZE + 1;
220 /* Qsize in bytes to create space for alignment */
221 q_size = qlen * cqinfo->cmd_size;
222
223 /* per queue initialization */
224 for (i = 0; i < cptvf->nr_queues; i++) {
225 size_t c_size = 0;
226 size_t rem_q_size = q_size;
227 struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
228 u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
229
230 queue = &cqinfo->queue[i];
231 INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
232 do {
233 curr = kzalloc(sizeof(*curr), GFP_KERNEL);
234 if (!curr)
235 goto cmd_qfail;
236
237 c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
238 rem_q_size;
239 curr->head = (u8 *)dma_zalloc_coherent(&pdev->dev,
240 c_size + CPT_NEXT_CHUNK_PTR_SIZE,
241 &curr->dma_addr, GFP_KERNEL);
242 if (!curr->head) {
243 dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
244 i, queue->nchunks);
245 goto cmd_qfail;
246 }
247
248 curr->size = c_size;
249 if (queue->nchunks == 0) {
250 hlist_add_head(&curr->nextchunk,
251 &cqinfo->queue[i].chead);
252 first = curr;
253 } else {
254 hlist_add_behind(&curr->nextchunk,
255 &last->nextchunk);
256 }
257
258 queue->nchunks++;
259 rem_q_size -= c_size;
260 if (last)
261 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
262
263 last = curr;
264 } while (rem_q_size);
265
266 /* Make the queue circular */
267 /* Tie back last chunk entry to head */
268 curr = first;
269 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
270 queue->qhead = curr;
271 spin_lock_init(&queue->lock);
272 }
273 return 0;
274
275cmd_qfail:
276 free_command_queues(cptvf, cqinfo);
277 return -ENOMEM;
278}
279
280static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
281{
282 struct pci_dev *pdev = cptvf->pdev;
283 int ret;
284
285 /* setup AE command queues */
286 ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
287 qlen);
288 if (ret) {
289 dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
290 cptvf->nr_queues);
291 return ret;
292 }
293
294 return ret;
295}
296
297static void cleanup_command_queues(struct cpt_vf *cptvf)
298{
299 struct pci_dev *pdev = cptvf->pdev;
300
301 if (!cptvf->nr_queues)
302 return;
303
304 dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
305 cptvf->nr_queues);
306 free_command_queues(cptvf, &cptvf->cqinfo);
307}
308
309static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
310{
311 cleanup_worker_threads(cptvf);
312 cleanup_pending_queues(cptvf);
313 cleanup_command_queues(cptvf);
314}
315
316static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
317{
318 struct pci_dev *pdev = cptvf->pdev;
319 int ret = 0;
320 u32 max_dev_queues = 0;
321
322 max_dev_queues = CPT_NUM_QS_PER_VF;
323 /* possible cpus */
324 nr_queues = min_t(u32, nr_queues, max_dev_queues);
325 cptvf->nr_queues = nr_queues;
326
327 ret = init_command_queues(cptvf, qlen);
328 if (ret) {
329 dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
330 nr_queues);
331 return ret;
332 }
333
334 ret = init_pending_queues(cptvf, qlen, nr_queues);
335 if (ret) {
336 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
337 nr_queues);
338 goto setup_pqfail;
339 }
340
341 /* Create worker threads for BH processing */
342 ret = init_worker_threads(cptvf);
343 if (ret) {
344 dev_err(&pdev->dev, "Failed to setup worker threads\n");
345 goto init_work_fail;
346 }
347
348 return 0;
349
350init_work_fail:
351 cleanup_worker_threads(cptvf);
352 cleanup_pending_queues(cptvf);
353
354setup_pqfail:
355 cleanup_command_queues(cptvf);
356
357 return ret;
358}
359
360static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec)
361{
362 irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
363 free_cpumask_var(cptvf->affinity_mask[vec]);
364}
365
366static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
367{
368 union cptx_vqx_ctl vqx_ctl;
369
370 vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
371 vqx_ctl.s.ena = val;
372 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
373}
374
375void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
376{
377 union cptx_vqx_doorbell vqx_dbell;
378
379 vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
380 CPTX_VQX_DOORBELL(0, 0));
381 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
382 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
383 vqx_dbell.u);
384}
385
386static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
387{
388 union cptx_vqx_inprog vqx_inprg;
389
390 vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
391 vqx_inprg.s.inflight = val;
392 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
393}
394
395static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
396{
397 union cptx_vqx_done_wait vqx_dwait;
398
399 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
400 CPTX_VQX_DONE_WAIT(0, 0));
401 vqx_dwait.s.num_wait = val;
402 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
403 vqx_dwait.u);
404}
405
406static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
407{
408 union cptx_vqx_done_wait vqx_dwait;
409
410 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
411 CPTX_VQX_DONE_WAIT(0, 0));
412 vqx_dwait.s.time_wait = time;
413 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
414 vqx_dwait.u);
415}
416
417static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
418{
419 union cptx_vqx_misc_ena_w1s vqx_misc_ena;
420
421 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
422 CPTX_VQX_MISC_ENA_W1S(0, 0));
423 /* Set mbox(0) interupts for the requested vf */
424 vqx_misc_ena.s.swerr = 1;
425 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
426 vqx_misc_ena.u);
427}
428
429static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
430{
431 union cptx_vqx_misc_ena_w1s vqx_misc_ena;
432
433 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
434 CPTX_VQX_MISC_ENA_W1S(0, 0));
435 /* Set mbox(0) interupts for the requested vf */
436 vqx_misc_ena.s.mbox = 1;
437 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
438 vqx_misc_ena.u);
439}
440
441static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
442{
443 union cptx_vqx_done_ena_w1s vqx_done_ena;
444
445 vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
446 CPTX_VQX_DONE_ENA_W1S(0, 0));
447 /* Set DONE interrupt for the requested vf */
448 vqx_done_ena.s.done = 1;
449 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
450 vqx_done_ena.u);
451}
452
453static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
454{
455 union cptx_vqx_misc_int vqx_misc_int;
456
457 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
458 CPTX_VQX_MISC_INT(0, 0));
459 /* W1C for the VF */
460 vqx_misc_int.s.dovf = 1;
461 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
462 vqx_misc_int.u);
463}
464
465static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
466{
467 union cptx_vqx_misc_int vqx_misc_int;
468
469 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
470 CPTX_VQX_MISC_INT(0, 0));
471 /* W1C for the VF */
472 vqx_misc_int.s.irde = 1;
473 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
474 vqx_misc_int.u);
475}
476
477static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
478{
479 union cptx_vqx_misc_int vqx_misc_int;
480
481 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
482 CPTX_VQX_MISC_INT(0, 0));
483 /* W1C for the VF */
484 vqx_misc_int.s.nwrp = 1;
485 cpt_write_csr64(cptvf->reg_base,
486 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
487}
488
489static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
490{
491 union cptx_vqx_misc_int vqx_misc_int;
492
493 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
494 CPTX_VQX_MISC_INT(0, 0));
495 /* W1C for the VF */
496 vqx_misc_int.s.mbox = 1;
497 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
498 vqx_misc_int.u);
499}
500
501static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
502{
503 union cptx_vqx_misc_int vqx_misc_int;
504
505 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
506 CPTX_VQX_MISC_INT(0, 0));
507 /* W1C for the VF */
508 vqx_misc_int.s.swerr = 1;
509 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
510 vqx_misc_int.u);
511}
512
513static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
514{
515 return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
516}
517
518static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
519{
520 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
521 struct pci_dev *pdev = cptvf->pdev;
522 u64 intr;
523
524 intr = cptvf_read_vf_misc_intr_status(cptvf);
525 /*Check for MISC interrupt types*/
526 if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
527 dev_err(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
528 intr, cptvf->vfid);
529 cptvf_handle_mbox_intr(cptvf);
530 cptvf_clear_mbox_intr(cptvf);
531 } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
532 cptvf_clear_dovf_intr(cptvf);
533 /*Clear doorbell count*/
534 cptvf_write_vq_doorbell(cptvf, 0);
535 dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
536 intr, cptvf->vfid);
537 } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
538 cptvf_clear_irde_intr(cptvf);
539 dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
540 intr, cptvf->vfid);
541 } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
542 cptvf_clear_nwrp_intr(cptvf);
543 dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
544 intr, cptvf->vfid);
545 } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
546 cptvf_clear_swerr_intr(cptvf);
547 dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
548 intr, cptvf->vfid);
549 } else {
550 dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
551 cptvf->vfid);
552 }
553
554 return IRQ_HANDLED;
555}
556
557static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
558 int qno)
559{
560 struct cptvf_wqe_info *nwqe_info;
561
562 if (unlikely(qno >= cptvf->nr_queues))
563 return NULL;
564 nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
565
566 return &nwqe_info->vq_wqe[qno];
567}
568
569static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
570{
571 union cptx_vqx_done vqx_done;
572
573 vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
574 return vqx_done.s.done;
575}
576
577static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
578 u32 ackcnt)
579{
580 union cptx_vqx_done_ack vqx_dack_cnt;
581
582 vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
583 CPTX_VQX_DONE_ACK(0, 0));
584 vqx_dack_cnt.s.done_ack = ackcnt;
585 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
586 vqx_dack_cnt.u);
587}
588
589static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
590{
591 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
592 struct pci_dev *pdev = cptvf->pdev;
593 /* Read the number of completions */
594 u32 intr = cptvf_read_vq_done_count(cptvf);
595
596 if (intr) {
597 struct cptvf_wqe *wqe;
598
599 /* Acknowledge the number of
600 * scheduled completions for processing
601 */
602 cptvf_write_vq_done_ack(cptvf, intr);
603 wqe = get_cptvf_vq_wqe(cptvf, 0);
604 if (unlikely(!wqe)) {
605 dev_err(&pdev->dev, "No work to schedule for VF (%d)",
606 cptvf->vfid);
607 return IRQ_NONE;
608 }
609 tasklet_hi_schedule(&wqe->twork);
610 }
611
612 return IRQ_HANDLED;
613}
614
615static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec)
616{
617 struct pci_dev *pdev = cptvf->pdev;
618 int cpu;
619
620 if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
621 GFP_KERNEL)) {
622 dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
623 cptvf->vfid);
624 return;
625 }
626
627 cpu = cptvf->vfid % num_online_cpus();
628 cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
629 cptvf->affinity_mask[vec]);
630 irq_set_affinity_hint(pci_irq_vector(pdev, vec),
631 cptvf->affinity_mask[vec]);
632}
633
634static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
635{
636 union cptx_vqx_saddr vqx_saddr;
637
638 vqx_saddr.u = val;
639 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
640}
641
642void cptvf_device_init(struct cpt_vf *cptvf)
643{
644 u64 base_addr = 0;
645
646 /* Disable the VQ */
647 cptvf_write_vq_ctl(cptvf, 0);
648 /* Reset the doorbell */
649 cptvf_write_vq_doorbell(cptvf, 0);
650 /* Clear inflight */
651 cptvf_write_vq_inprog(cptvf, 0);
652 /* Write VQ SADDR */
653 /* TODO: for now only one queue, so hard coded */
654 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
655 cptvf_write_vq_saddr(cptvf, base_addr);
656 /* Configure timerhold / coalescence */
657 cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
658 cptvf_write_vq_done_numwait(cptvf, 1);
659 /* Enable the VQ */
660 cptvf_write_vq_ctl(cptvf, 1);
661 /* Flag the VF ready */
662 cptvf->flags |= CPT_FLAG_DEVICE_READY;
663}
664
665static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
666{
667 struct device *dev = &pdev->dev;
668 struct cpt_vf *cptvf;
669 int err;
670
671 cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
672 if (!cptvf)
673 return -ENOMEM;
674
675 pci_set_drvdata(pdev, cptvf);
676 cptvf->pdev = pdev;
677 err = pci_enable_device(pdev);
678 if (err) {
679 dev_err(dev, "Failed to enable PCI device\n");
680 pci_set_drvdata(pdev, NULL);
681 return err;
682 }
683
684 err = pci_request_regions(pdev, DRV_NAME);
685 if (err) {
686 dev_err(dev, "PCI request regions failed 0x%x\n", err);
687 goto cptvf_err_disable_device;
688 }
689 /* Mark as VF driver */
690 cptvf->flags |= CPT_FLAG_VF_DRIVER;
691 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
692 if (err) {
693 dev_err(dev, "Unable to get usable DMA configuration\n");
694 goto cptvf_err_release_regions;
695 }
696
697 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
698 if (err) {
699 dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
700 goto cptvf_err_release_regions;
701 }
702
703 /* MAP PF's configuration registers */
704 cptvf->reg_base = pcim_iomap(pdev, 0, 0);
705 if (!cptvf->reg_base) {
706 dev_err(dev, "Cannot map config register space, aborting\n");
707 err = -ENOMEM;
708 goto cptvf_err_release_regions;
709 }
710
711 cptvf->node = dev_to_node(&pdev->dev);
712 err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS,
713 CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
714 if (err < 0) {
715 dev_err(dev, "Request for #%d msix vectors failed\n",
716 CPT_VF_MSIX_VECTORS);
717 goto cptvf_err_release_regions;
718 }
719
720 err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
721 cptvf_misc_intr_handler, 0, "CPT VF misc intr",
722 cptvf);
723 if (err) {
724 dev_err(dev, "Request misc irq failed");
725 goto cptvf_free_vectors;
726 }
727
728 /* Enable mailbox interrupt */
729 cptvf_enable_mbox_interrupts(cptvf);
730 cptvf_enable_swerr_interrupts(cptvf);
731
732 /* Check ready with PF */
733 /* Gets chip ID / device Id from PF if ready */
734 err = cptvf_check_pf_ready(cptvf);
735 if (err) {
736 dev_err(dev, "PF not responding to READY msg");
737 goto cptvf_free_misc_irq;
738 }
739
740 /* CPT VF software resources initialization */
741 cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
742 err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
743 if (err) {
744 dev_err(dev, "cptvf_sw_init() failed");
745 goto cptvf_free_misc_irq;
746 }
747 /* Convey VQ LEN to PF */
748 err = cptvf_send_vq_size_msg(cptvf);
749 if (err) {
750 dev_err(dev, "PF not responding to QLEN msg");
751 goto cptvf_free_misc_irq;
752 }
753
754 /* CPT VF device initialization */
755 cptvf_device_init(cptvf);
756 /* Send msg to PF to assign currnet Q to required group */
757 cptvf->vfgrp = 1;
758 err = cptvf_send_vf_to_grp_msg(cptvf);
759 if (err) {
760 dev_err(dev, "PF not responding to VF_GRP msg");
761 goto cptvf_free_misc_irq;
762 }
763
764 cptvf->priority = 1;
765 err = cptvf_send_vf_priority_msg(cptvf);
766 if (err) {
767 dev_err(dev, "PF not responding to VF_PRIO msg");
768 goto cptvf_free_misc_irq;
769 }
770
771 err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
772 cptvf_done_intr_handler, 0, "CPT VF done intr",
773 cptvf);
774 if (err) {
775 dev_err(dev, "Request done irq failed\n");
776 goto cptvf_free_misc_irq;
777 }
778
779 /* Enable mailbox interrupt */
780 cptvf_enable_done_interrupts(cptvf);
781
782 /* Set irq affinity masks */
783 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
784 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
785
786 err = cptvf_send_vf_up(cptvf);
787 if (err) {
788 dev_err(dev, "PF not responding to UP msg");
789 goto cptvf_free_irq_affinity;
790 }
791 err = cvm_crypto_init(cptvf);
792 if (err) {
793 dev_err(dev, "Algorithm register failed\n");
794 goto cptvf_free_irq_affinity;
795 }
796 return 0;
797
798cptvf_free_irq_affinity:
799 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
800 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
801cptvf_free_misc_irq:
802 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
803cptvf_free_vectors:
804 pci_free_irq_vectors(cptvf->pdev);
805cptvf_err_release_regions:
806 pci_release_regions(pdev);
807cptvf_err_disable_device:
808 pci_disable_device(pdev);
809 pci_set_drvdata(pdev, NULL);
810
811 return err;
812}
813
814static void cptvf_remove(struct pci_dev *pdev)
815{
816 struct cpt_vf *cptvf = pci_get_drvdata(pdev);
817
818 if (!cptvf)
819 dev_err(&pdev->dev, "Invalid CPT-VF device\n");
820
821 /* Convey DOWN to PF */
822 if (cptvf_send_vf_down(cptvf)) {
823 dev_err(&pdev->dev, "PF not responding to DOWN msg");
824 } else {
825 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
826 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
827 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
828 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
829 pci_free_irq_vectors(cptvf->pdev);
830 cptvf_sw_cleanup(cptvf);
831 pci_set_drvdata(pdev, NULL);
832 pci_release_regions(pdev);
833 pci_disable_device(pdev);
834 cvm_crypto_exit();
835 }
836}
837
838static void cptvf_shutdown(struct pci_dev *pdev)
839{
840 cptvf_remove(pdev);
841}
842
843/* Supported devices */
844static const struct pci_device_id cptvf_id_table[] = {
845 {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
846 { 0, } /* end of table */
847};
848
849static struct pci_driver cptvf_pci_driver = {
850 .name = DRV_NAME,
851 .id_table = cptvf_id_table,
852 .probe = cptvf_probe,
853 .remove = cptvf_remove,
854 .shutdown = cptvf_shutdown,
855};
856
857module_pci_driver(cptvf_pci_driver);
858
859MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
860MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
861MODULE_LICENSE("GPL v2");
862MODULE_VERSION(DRV_VERSION);
863MODULE_DEVICE_TABLE(pci, cptvf_id_table);
diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c
new file mode 100644
index 000000000000..d5ec3b8a9e61
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c
@@ -0,0 +1,211 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#include "cptvf.h"
10
11static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
12{
13 /* Writing mbox(1) causes interrupt */
14 cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0),
15 mbx->msg);
16 cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1),
17 mbx->data);
18}
19
20/* ACKs PF's mailbox message
21 */
22void cptvf_mbox_send_ack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
23{
24 mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
25 cptvf_send_msg_to_pf(cptvf, mbx);
26}
27
28/* NACKs PF's mailbox message that VF is not able to
29 * complete the action
30 */
31void cptvf_mbox_send_nack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
32{
33 mbx->msg = CPT_MBOX_MSG_TYPE_NACK;
34 cptvf_send_msg_to_pf(cptvf, mbx);
35}
36
37/* Interrupt handler to handle mailbox messages from VFs */
38void cptvf_handle_mbox_intr(struct cpt_vf *cptvf)
39{
40 struct cpt_mbox mbx = {};
41
42 /*
43 * MBOX[0] contains msg
44 * MBOX[1] contains data
45 */
46 mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0));
47 mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1));
48 dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n",
49 __func__, mbx.msg);
50 switch (mbx.msg) {
51 case CPT_MSG_READY:
52 {
53 cptvf->pf_acked = true;
54 cptvf->vfid = mbx.data;
55 dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid);
56 break;
57 }
58 case CPT_MSG_QBIND_GRP:
59 cptvf->pf_acked = true;
60 cptvf->vftype = mbx.data;
61 dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n",
62 cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"),
63 cptvf->vfgrp);
64 break;
65 case CPT_MBOX_MSG_TYPE_ACK:
66 cptvf->pf_acked = true;
67 break;
68 case CPT_MBOX_MSG_TYPE_NACK:
69 cptvf->pf_nacked = true;
70 break;
71 default:
72 dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n",
73 mbx.msg);
74 break;
75 }
76}
77
78static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf,
79 struct cpt_mbox *mbx)
80{
81 int timeout = CPT_MBOX_MSG_TIMEOUT;
82 int sleep = 10;
83
84 cptvf->pf_acked = false;
85 cptvf->pf_nacked = false;
86 cptvf_send_msg_to_pf(cptvf, mbx);
87 /* Wait for previous message to be acked, timeout 2sec */
88 while (!cptvf->pf_acked) {
89 if (cptvf->pf_nacked)
90 return -EINVAL;
91 msleep(sleep);
92 if (cptvf->pf_acked)
93 break;
94 timeout -= sleep;
95 if (!timeout) {
96 dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n",
97 (mbx->msg & 0xFF), cptvf->vfid);
98 return -EBUSY;
99 }
100 }
101
102 return 0;
103}
104
105/*
106 * Checks if VF is able to comminicate with PF
107 * and also gets the CPT number this VF is associated to.
108 */
109int cptvf_check_pf_ready(struct cpt_vf *cptvf)
110{
111 struct pci_dev *pdev = cptvf->pdev;
112 struct cpt_mbox mbx = {};
113
114 mbx.msg = CPT_MSG_READY;
115 if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
116 dev_err(&pdev->dev, "PF didn't respond to READY msg\n");
117 return -EBUSY;
118 }
119
120 return 0;
121}
122
123/*
124 * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
125 * Must be ACKed.
126 */
127int cptvf_send_vq_size_msg(struct cpt_vf *cptvf)
128{
129 struct pci_dev *pdev = cptvf->pdev;
130 struct cpt_mbox mbx = {};
131
132 mbx.msg = CPT_MSG_QLEN;
133 mbx.data = cptvf->qsize;
134 if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
135 dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n");
136 return -EBUSY;
137 }
138
139 return 0;
140}
141
142/*
143 * Communicate VF group required to PF and get the VQ binded to that group
144 */
145int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
146{
147 struct pci_dev *pdev = cptvf->pdev;
148 struct cpt_mbox mbx = {};
149
150 mbx.msg = CPT_MSG_QBIND_GRP;
151 /* Convey group of the VF */
152 mbx.data = cptvf->vfgrp;
153 if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
154 dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
155 return -EBUSY;
156 }
157
158 return 0;
159}
160
161/*
162 * Communicate VF group required to PF and get the VQ binded to that group
163 */
164int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf)
165{
166 struct pci_dev *pdev = cptvf->pdev;
167 struct cpt_mbox mbx = {};
168
169 mbx.msg = CPT_MSG_VQ_PRIORITY;
170 /* Convey group of the VF */
171 mbx.data = cptvf->priority;
172 if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
173 dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
174 return -EBUSY;
175 }
176 return 0;
177}
178
179/*
180 * Communicate to PF that VF is UP and running
181 */
182int cptvf_send_vf_up(struct cpt_vf *cptvf)
183{
184 struct pci_dev *pdev = cptvf->pdev;
185 struct cpt_mbox mbx = {};
186
187 mbx.msg = CPT_MSG_VF_UP;
188 if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
189 dev_err(&pdev->dev, "PF didn't respond to UP msg\n");
190 return -EBUSY;
191 }
192
193 return 0;
194}
195
196/*
197 * Communicate to PF that VF is DOWN and running
198 */
199int cptvf_send_vf_down(struct cpt_vf *cptvf)
200{
201 struct pci_dev *pdev = cptvf->pdev;
202 struct cpt_mbox mbx = {};
203
204 mbx.msg = CPT_MSG_VF_DOWN;
205 if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
206 dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n");
207 return -EBUSY;
208 }
209
210 return 0;
211}
diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
new file mode 100644
index 000000000000..7f57f30f8863
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
@@ -0,0 +1,593 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#include "cptvf.h"
10#include "request_manager.h"
11
12/**
13 * get_free_pending_entry - get free entry from pending queue
14 * @param pqinfo: pending_qinfo structure
15 * @param qno: queue number
16 */
17static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
18 int qlen)
19{
20 struct pending_entry *ent = NULL;
21
22 ent = &q->head[q->rear];
23 if (unlikely(ent->busy)) {
24 ent = NULL;
25 goto no_free_entry;
26 }
27
28 q->rear++;
29 if (unlikely(q->rear == qlen))
30 q->rear = 0;
31
32no_free_entry:
33 return ent;
34}
35
36static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
37 int qno)
38{
39 struct pending_queue *queue = &pqinfo->queue[qno];
40
41 queue->front++;
42 if (unlikely(queue->front == pqinfo->qlen))
43 queue->front = 0;
44}
45
46static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
47 int buf_count, u8 *buffer)
48{
49 int ret = 0, i, j;
50 int components;
51 struct sglist_component *sg_ptr = NULL;
52 struct pci_dev *pdev = cptvf->pdev;
53
54 if (unlikely(!list)) {
55 dev_err(&pdev->dev, "Input List pointer is NULL\n");
56 return -EFAULT;
57 }
58
59 for (i = 0; i < buf_count; i++) {
60 if (likely(list[i].vptr)) {
61 list[i].dma_addr = dma_map_single(&pdev->dev,
62 list[i].vptr,
63 list[i].size,
64 DMA_BIDIRECTIONAL);
65 if (unlikely(dma_mapping_error(&pdev->dev,
66 list[i].dma_addr))) {
67 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
68 i);
69 ret = -EIO;
70 goto sg_cleanup;
71 }
72 }
73 }
74
75 components = buf_count / 4;
76 sg_ptr = (struct sglist_component *)buffer;
77 for (i = 0; i < components; i++) {
78 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
79 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
80 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
81 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
82 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
83 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
84 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
85 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
86 sg_ptr++;
87 }
88
89 components = buf_count % 4;
90
91 switch (components) {
92 case 3:
93 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
94 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
95 /* Fall through */
96 case 2:
97 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
98 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
99 /* Fall through */
100 case 1:
101 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
102 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
103 break;
104 default:
105 break;
106 }
107
108 return ret;
109
110sg_cleanup:
111 for (j = 0; j < i; j++) {
112 if (list[j].dma_addr) {
113 dma_unmap_single(&pdev->dev, list[i].dma_addr,
114 list[i].size, DMA_BIDIRECTIONAL);
115 }
116
117 list[j].dma_addr = 0;
118 }
119
120 return ret;
121}
122
123static inline int setup_sgio_list(struct cpt_vf *cptvf,
124 struct cpt_info_buffer *info,
125 struct cpt_request_info *req)
126{
127 u16 g_sz_bytes = 0, s_sz_bytes = 0;
128 int ret = 0;
129 struct pci_dev *pdev = cptvf->pdev;
130
131 if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
132 dev_err(&pdev->dev, "Request SG components are higher than supported\n");
133 ret = -EINVAL;
134 goto scatter_gather_clean;
135 }
136
137 /* Setup gather (input) components */
138 g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
139 info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL);
140 if (!info->gather_components) {
141 ret = -ENOMEM;
142 goto scatter_gather_clean;
143 }
144
145 ret = setup_sgio_components(cptvf, req->in,
146 req->incnt,
147 info->gather_components);
148 if (ret) {
149 dev_err(&pdev->dev, "Failed to setup gather list\n");
150 ret = -EFAULT;
151 goto scatter_gather_clean;
152 }
153
154 /* Setup scatter (output) components */
155 s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
156 info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL);
157 if (!info->scatter_components) {
158 ret = -ENOMEM;
159 goto scatter_gather_clean;
160 }
161
162 ret = setup_sgio_components(cptvf, req->out,
163 req->outcnt,
164 info->scatter_components);
165 if (ret) {
166 dev_err(&pdev->dev, "Failed to setup gather list\n");
167 ret = -EFAULT;
168 goto scatter_gather_clean;
169 }
170
171 /* Create and initialize DPTR */
172 info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
173 info->in_buffer = kzalloc(info->dlen, GFP_KERNEL);
174 if (!info->in_buffer) {
175 ret = -ENOMEM;
176 goto scatter_gather_clean;
177 }
178
179 ((u16 *)info->in_buffer)[0] = req->outcnt;
180 ((u16 *)info->in_buffer)[1] = req->incnt;
181 ((u16 *)info->in_buffer)[2] = 0;
182 ((u16 *)info->in_buffer)[3] = 0;
183 *(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer);
184
185 memcpy(&info->in_buffer[8], info->gather_components,
186 g_sz_bytes);
187 memcpy(&info->in_buffer[8 + g_sz_bytes],
188 info->scatter_components, s_sz_bytes);
189
190 info->dptr_baddr = dma_map_single(&pdev->dev,
191 (void *)info->in_buffer,
192 info->dlen,
193 DMA_BIDIRECTIONAL);
194 if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
195 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
196 ret = -EIO;
197 goto scatter_gather_clean;
198 }
199
200 /* Create and initialize RPTR */
201 info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL);
202 if (!info->out_buffer) {
203 ret = -ENOMEM;
204 goto scatter_gather_clean;
205 }
206
207 *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
208 info->alternate_caddr = (u64 *)info->out_buffer;
209 info->rptr_baddr = dma_map_single(&pdev->dev,
210 (void *)info->out_buffer,
211 COMPLETION_CODE_SIZE,
212 DMA_BIDIRECTIONAL);
213 if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
214 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
215 COMPLETION_CODE_SIZE);
216 ret = -EIO;
217 goto scatter_gather_clean;
218 }
219
220 return 0;
221
222scatter_gather_clean:
223 return ret;
224}
225
226int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
227 u32 qno)
228{
229 struct pci_dev *pdev = cptvf->pdev;
230 struct command_qinfo *qinfo = NULL;
231 struct command_queue *queue;
232 struct command_chunk *chunk;
233 u8 *ent;
234 int ret = 0;
235
236 if (unlikely(qno >= cptvf->nr_queues)) {
237 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
238 qno, cptvf->nr_queues);
239 return -EINVAL;
240 }
241
242 qinfo = &cptvf->cqinfo;
243 queue = &qinfo->queue[qno];
244 /* lock commad queue */
245 spin_lock(&queue->lock);
246 ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
247 memcpy(ent, (void *)cmd, qinfo->cmd_size);
248
249 if (++queue->idx >= queue->qhead->size / 64) {
250 struct hlist_node *node;
251
252 hlist_for_each(node, &queue->chead) {
253 chunk = hlist_entry(node, struct command_chunk,
254 nextchunk);
255 if (chunk == queue->qhead) {
256 continue;
257 } else {
258 queue->qhead = chunk;
259 break;
260 }
261 }
262 queue->idx = 0;
263 }
264 /* make sure all memory stores are done before ringing doorbell */
265 smp_wmb();
266 cptvf_write_vq_doorbell(cptvf, 1);
267 /* unlock command queue */
268 spin_unlock(&queue->lock);
269
270 return ret;
271}
272
273void do_request_cleanup(struct cpt_vf *cptvf,
274 struct cpt_info_buffer *info)
275{
276 int i;
277 struct pci_dev *pdev = cptvf->pdev;
278 struct cpt_request_info *req;
279
280 if (info->dptr_baddr)
281 dma_unmap_single(&pdev->dev, info->dptr_baddr,
282 info->dlen, DMA_BIDIRECTIONAL);
283
284 if (info->rptr_baddr)
285 dma_unmap_single(&pdev->dev, info->rptr_baddr,
286 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
287
288 if (info->comp_baddr)
289 dma_unmap_single(&pdev->dev, info->comp_baddr,
290 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
291
292 if (info->req) {
293 req = info->req;
294 for (i = 0; i < req->outcnt; i++) {
295 if (req->out[i].dma_addr)
296 dma_unmap_single(&pdev->dev,
297 req->out[i].dma_addr,
298 req->out[i].size,
299 DMA_BIDIRECTIONAL);
300 }
301
302 for (i = 0; i < req->incnt; i++) {
303 if (req->in[i].dma_addr)
304 dma_unmap_single(&pdev->dev,
305 req->in[i].dma_addr,
306 req->in[i].size,
307 DMA_BIDIRECTIONAL);
308 }
309 }
310
311 if (info->scatter_components)
312 kzfree(info->scatter_components);
313
314 if (info->gather_components)
315 kzfree(info->gather_components);
316
317 if (info->out_buffer)
318 kzfree(info->out_buffer);
319
320 if (info->in_buffer)
321 kzfree(info->in_buffer);
322
323 if (info->completion_addr)
324 kzfree((void *)info->completion_addr);
325
326 kzfree(info);
327}
328
329void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
330{
331 struct pci_dev *pdev = cptvf->pdev;
332
333 if (!info || !cptvf) {
334 dev_err(&pdev->dev, "Input params are incorrect for post processing\n");
335 return;
336 }
337
338 do_request_cleanup(cptvf, info);
339}
340
341static inline void process_pending_queue(struct cpt_vf *cptvf,
342 struct pending_qinfo *pqinfo,
343 int qno)
344{
345 struct pci_dev *pdev = cptvf->pdev;
346 struct pending_queue *pqueue = &pqinfo->queue[qno];
347 struct pending_entry *pentry = NULL;
348 struct cpt_info_buffer *info = NULL;
349 union cpt_res_s *status = NULL;
350 unsigned char ccode;
351
352 while (1) {
353 spin_lock_bh(&pqueue->lock);
354 pentry = &pqueue->head[pqueue->front];
355 if (unlikely(!pentry->busy)) {
356 spin_unlock_bh(&pqueue->lock);
357 break;
358 }
359
360 info = (struct cpt_info_buffer *)pentry->post_arg;
361 if (unlikely(!info)) {
362 dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
363 pending_queue_inc_front(pqinfo, qno);
364 spin_unlock_bh(&pqueue->lock);
365 continue;
366 }
367
368 status = (union cpt_res_s *)pentry->completion_addr;
369 ccode = status->s.compcode;
370 if ((status->s.compcode == CPT_COMP_E_FAULT) ||
371 (status->s.compcode == CPT_COMP_E_SWERR)) {
372 dev_err(&pdev->dev, "Request failed with %s\n",
373 (status->s.compcode == CPT_COMP_E_FAULT) ?
374 "DMA Fault" : "Software error");
375 pentry->completion_addr = NULL;
376 pentry->busy = false;
377 atomic64_dec((&pqueue->pending_count));
378 pentry->post_arg = NULL;
379 pending_queue_inc_front(pqinfo, qno);
380 do_request_cleanup(cptvf, info);
381 spin_unlock_bh(&pqueue->lock);
382 break;
383 } else if (status->s.compcode == COMPLETION_CODE_INIT) {
384 /* check for timeout */
385 if (time_after_eq(jiffies,
386 (info->time_in +
387 (CPT_COMMAND_TIMEOUT * HZ)))) {
388 dev_err(&pdev->dev, "Request timed out");
389 pentry->completion_addr = NULL;
390 pentry->busy = false;
391 atomic64_dec((&pqueue->pending_count));
392 pentry->post_arg = NULL;
393 pending_queue_inc_front(pqinfo, qno);
394 do_request_cleanup(cptvf, info);
395 spin_unlock_bh(&pqueue->lock);
396 break;
397 } else if ((*info->alternate_caddr ==
398 (~COMPLETION_CODE_INIT)) &&
399 (info->extra_time < TIME_IN_RESET_COUNT)) {
400 info->time_in = jiffies;
401 info->extra_time++;
402 spin_unlock_bh(&pqueue->lock);
403 break;
404 }
405 }
406
407 pentry->completion_addr = NULL;
408 pentry->busy = false;
409 pentry->post_arg = NULL;
410 atomic64_dec((&pqueue->pending_count));
411 pending_queue_inc_front(pqinfo, qno);
412 spin_unlock_bh(&pqueue->lock);
413
414 do_post_process(info->cptvf, info);
415 /*
416 * Calling callback after we find
417 * that the request has been serviced
418 */
419 pentry->callback(ccode, pentry->callback_arg);
420 }
421}
422
423int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
424{
425 int ret = 0, clear = 0, queue = 0;
426 struct cpt_info_buffer *info = NULL;
427 struct cptvf_request *cpt_req = NULL;
428 union ctrl_info *ctrl = NULL;
429 union cpt_res_s *result = NULL;
430 struct pending_entry *pentry = NULL;
431 struct pending_queue *pqueue = NULL;
432 struct pci_dev *pdev = cptvf->pdev;
433 u8 group = 0;
434 struct cpt_vq_command vq_cmd;
435 union cpt_inst_s cptinst;
436
437 info = kzalloc(sizeof(*info), GFP_KERNEL);
438 if (unlikely(!info)) {
439 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
440 return -ENOMEM;
441 }
442
443 cpt_req = (struct cptvf_request *)&req->req;
444 ctrl = (union ctrl_info *)&req->ctrl;
445
446 info->cptvf = cptvf;
447 group = ctrl->s.grp;
448 ret = setup_sgio_list(cptvf, info, req);
449 if (ret) {
450 dev_err(&pdev->dev, "Setting up SG list failed");
451 goto request_cleanup;
452 }
453
454 cpt_req->dlen = info->dlen;
455 /*
456 * Get buffer for union cpt_res_s response
457 * structure and its physical address
458 */
459 info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL);
460 if (unlikely(!info->completion_addr)) {
461 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
462 return -ENOMEM;
463 }
464
465 result = (union cpt_res_s *)info->completion_addr;
466 result->s.compcode = COMPLETION_CODE_INIT;
467 info->comp_baddr = dma_map_single(&pdev->dev,
468 (void *)info->completion_addr,
469 sizeof(union cpt_res_s),
470 DMA_BIDIRECTIONAL);
471 if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
472 dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
473 sizeof(union cpt_res_s));
474 ret = -EFAULT;
475 goto request_cleanup;
476 }
477
478 /* Fill the VQ command */
479 vq_cmd.cmd.u64 = 0;
480 vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
481 vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
482 vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
483 vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
484
485 /* 64-bit swap for microcode data reads, not needed for addresses*/
486 vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64);
487 vq_cmd.dptr = info->dptr_baddr;
488 vq_cmd.rptr = info->rptr_baddr;
489 vq_cmd.cptr.u64 = 0;
490 vq_cmd.cptr.s.grp = group;
491 /* Get Pending Entry to submit command */
492 /* Always queue 0, because 1 queue per VF */
493 queue = 0;
494 pqueue = &cptvf->pqinfo.queue[queue];
495
496 if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
497 dev_err(&pdev->dev, "pending threshold reached\n");
498 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
499 }
500
501get_pending_entry:
502 spin_lock_bh(&pqueue->lock);
503 pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
504 if (unlikely(!pentry)) {
505 spin_unlock_bh(&pqueue->lock);
506 if (clear == 0) {
507 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
508 clear = 1;
509 goto get_pending_entry;
510 }
511 dev_err(&pdev->dev, "Get free entry failed\n");
512 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
513 queue, pqueue->rear, pqueue->front);
514 ret = -EFAULT;
515 goto request_cleanup;
516 }
517
518 pentry->completion_addr = info->completion_addr;
519 pentry->post_arg = (void *)info;
520 pentry->callback = req->callback;
521 pentry->callback_arg = req->callback_arg;
522 info->pentry = pentry;
523 pentry->busy = true;
524 atomic64_inc(&pqueue->pending_count);
525
526 /* Send CPT command */
527 info->pentry = pentry;
528 info->time_in = jiffies;
529 info->req = req;
530
531 /* Create the CPT_INST_S type command for HW intrepretation */
532 cptinst.s.doneint = true;
533 cptinst.s.res_addr = (u64)info->comp_baddr;
534 cptinst.s.tag = 0;
535 cptinst.s.grp = 0;
536 cptinst.s.wq_ptr = 0;
537 cptinst.s.ei0 = vq_cmd.cmd.u64;
538 cptinst.s.ei1 = vq_cmd.dptr;
539 cptinst.s.ei2 = vq_cmd.rptr;
540 cptinst.s.ei3 = vq_cmd.cptr.u64;
541
542 ret = send_cpt_command(cptvf, &cptinst, queue);
543 spin_unlock_bh(&pqueue->lock);
544 if (unlikely(ret)) {
545 dev_err(&pdev->dev, "Send command failed for AE\n");
546 ret = -EFAULT;
547 goto request_cleanup;
548 }
549
550 return 0;
551
552request_cleanup:
553 dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
554 do_request_cleanup(cptvf, info);
555
556 return ret;
557}
558
559void vq_post_process(struct cpt_vf *cptvf, u32 qno)
560{
561 struct pci_dev *pdev = cptvf->pdev;
562
563 if (unlikely(qno > cptvf->nr_queues)) {
564 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
565 qno);
566 return;
567 }
568
569 process_pending_queue(cptvf, &cptvf->pqinfo, qno);
570}
571
572int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
573{
574 struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
575 struct pci_dev *pdev = cptvf->pdev;
576
577 if (!cpt_device_ready(cptvf)) {
578 dev_err(&pdev->dev, "CPT Device is not ready");
579 return -ENODEV;
580 }
581
582 if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
583 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
584 cptvf->vfid);
585 return -EINVAL;
586 } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
587 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
588 cptvf->vfid);
589 return -EINVAL;
590 }
591
592 return process_request(cptvf, req);
593}
diff --git a/drivers/crypto/cavium/cpt/request_manager.h b/drivers/crypto/cavium/cpt/request_manager.h
new file mode 100644
index 000000000000..80ee074c6e0c
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/request_manager.h
@@ -0,0 +1,147 @@
1/*
2 * Copyright (C) 2016 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#ifndef __REQUEST_MANAGER_H
10#define __REQUEST_MANAGER_H
11
12#include "cpt_common.h"
13
14#define TIME_IN_RESET_COUNT 5
15#define COMPLETION_CODE_SIZE 8
16#define COMPLETION_CODE_INIT 0
17#define PENDING_THOLD 100
18#define MAX_SG_IN_CNT 12
19#define MAX_SG_OUT_CNT 13
20#define SG_LIST_HDR_SIZE 8
21#define MAX_BUF_CNT 16
22
23union ctrl_info {
24 u32 flags;
25 struct {
26#if defined(__BIG_ENDIAN_BITFIELD)
27 u32 reserved0:26;
28 u32 grp:3; /* Group bits */
29 u32 dma_mode:2; /* DMA mode */
30 u32 se_req:1;/* To SE core */
31#else
32 u32 se_req:1; /* To SE core */
33 u32 dma_mode:2; /* DMA mode */
34 u32 grp:3; /* Group bits */
35 u32 reserved0:26;
36#endif
37 } s;
38};
39
40union opcode_info {
41 u16 flags;
42 struct {
43 u8 major;
44 u8 minor;
45 } s;
46};
47
48struct cptvf_request {
49 union opcode_info opcode;
50 u16 param1;
51 u16 param2;
52 u16 dlen;
53};
54
55struct buf_ptr {
56 u8 *vptr;
57 dma_addr_t dma_addr;
58 u16 size;
59};
60
61struct cpt_request_info {
62 u8 incnt; /* Number of input buffers */
63 u8 outcnt; /* Number of output buffers */
64 u16 rlen; /* Output length */
65 union ctrl_info ctrl; /* User control information */
66 struct cptvf_request req; /* Request Information (Core specific) */
67
68 struct buf_ptr in[MAX_BUF_CNT];
69 struct buf_ptr out[MAX_BUF_CNT];
70
71 void (*callback)(int, void *); /* Kernel ASYNC request callabck */
72 void *callback_arg; /* Kernel ASYNC request callabck arg */
73};
74
75struct sglist_component {
76 union {
77 u64 len;
78 struct {
79 u16 len0;
80 u16 len1;
81 u16 len2;
82 u16 len3;
83 } s;
84 } u;
85 u64 ptr0;
86 u64 ptr1;
87 u64 ptr2;
88 u64 ptr3;
89};
90
91struct cpt_info_buffer {
92 struct cpt_vf *cptvf;
93 unsigned long time_in;
94 u8 extra_time;
95
96 struct cpt_request_info *req;
97 dma_addr_t dptr_baddr;
98 u32 dlen;
99 dma_addr_t rptr_baddr;
100 dma_addr_t comp_baddr;
101 u8 *in_buffer;
102 u8 *out_buffer;
103 u8 *gather_components;
104 u8 *scatter_components;
105
106 struct pending_entry *pentry;
107 volatile u64 *completion_addr;
108 volatile u64 *alternate_caddr;
109};
110
111/*
112 * CPT_INST_S software command definitions
113 * Words EI (0-3)
114 */
115union vq_cmd_word0 {
116 u64 u64;
117 struct {
118 u16 opcode;
119 u16 param1;
120 u16 param2;
121 u16 dlen;
122 } s;
123};
124
125union vq_cmd_word3 {
126 u64 u64;
127 struct {
128#if defined(__BIG_ENDIAN_BITFIELD)
129 u64 grp:3;
130 u64 cptr:61;
131#else
132 u64 cptr:61;
133 u64 grp:3;
134#endif
135 } s;
136};
137
138struct cpt_vq_command {
139 union vq_cmd_word0 cmd;
140 u64 dptr;
141 u64 rptr;
142 union vq_cmd_word3 cptr;
143};
144
145void vq_post_process(struct cpt_vf *cptvf, u32 qno);
146int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req);
147#endif /* __REQUEST_MANAGER_H */
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index 612898b4aaad..41cc853f8569 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -250,17 +250,20 @@ static int ccp5_do_cmd(struct ccp5_desc *desc,
250 ret = wait_event_interruptible(cmd_q->int_queue, 250 ret = wait_event_interruptible(cmd_q->int_queue,
251 cmd_q->int_rcvd); 251 cmd_q->int_rcvd);
252 if (ret || cmd_q->cmd_error) { 252 if (ret || cmd_q->cmd_error) {
253 /* Log the error and flush the queue by
254 * moving the head pointer
255 */
253 if (cmd_q->cmd_error) 256 if (cmd_q->cmd_error)
254 ccp_log_error(cmd_q->ccp, 257 ccp_log_error(cmd_q->ccp,
255 cmd_q->cmd_error); 258 cmd_q->cmd_error);
256 /* A version 5 device doesn't use Job IDs... */ 259 iowrite32(tail, cmd_q->reg_head_lo);
257 if (!ret) 260 if (!ret)
258 ret = -EIO; 261 ret = -EIO;
259 } 262 }
260 cmd_q->int_rcvd = 0; 263 cmd_q->int_rcvd = 0;
261 } 264 }
262 265
263 return 0; 266 return ret;
264} 267}
265 268
266static int ccp5_perform_aes(struct ccp_op *op) 269static int ccp5_perform_aes(struct ccp_op *op)
@@ -284,8 +287,7 @@ static int ccp5_perform_aes(struct ccp_op *op)
284 CCP_AES_ENCRYPT(&function) = op->u.aes.action; 287 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
285 CCP_AES_MODE(&function) = op->u.aes.mode; 288 CCP_AES_MODE(&function) = op->u.aes.mode;
286 CCP_AES_TYPE(&function) = op->u.aes.type; 289 CCP_AES_TYPE(&function) = op->u.aes.type;
287 if (op->u.aes.mode == CCP_AES_MODE_CFB) 290 CCP_AES_SIZE(&function) = op->u.aes.size;
288 CCP_AES_SIZE(&function) = 0x7f;
289 291
290 CCP5_CMD_FUNCTION(&desc) = function.raw; 292 CCP5_CMD_FUNCTION(&desc) = function.raw;
291 293
@@ -532,7 +534,7 @@ static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
532 status >>= LSB_REGION_WIDTH; 534 status >>= LSB_REGION_WIDTH;
533 } 535 }
534 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 536 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
535 dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", 537 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
536 cmd_q->id, queues); 538 cmd_q->id, queues);
537 539
538 return queues ? 0 : -EINVAL; 540 return queues ? 0 : -EINVAL;
@@ -574,7 +576,7 @@ static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
574 */ 576 */
575 cmd_q->lsb = bitno; 577 cmd_q->lsb = bitno;
576 bitmap_clear(lsb_pub, bitno, 1); 578 bitmap_clear(lsb_pub, bitno, 1);
577 dev_info(ccp->dev, 579 dev_dbg(ccp->dev,
578 "Queue %d gets LSB %d\n", 580 "Queue %d gets LSB %d\n",
579 i, bitno); 581 i, bitno);
580 break; 582 break;
@@ -732,7 +734,6 @@ static int ccp5_init(struct ccp_device *ccp)
732 ret = -EIO; 734 ret = -EIO;
733 goto e_pool; 735 goto e_pool;
734 } 736 }
735 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
736 737
737 /* Turn off the queues and disable interrupts until ready */ 738 /* Turn off the queues and disable interrupts until ready */
738 for (i = 0; i < ccp->cmd_q_count; i++) { 739 for (i = 0; i < ccp->cmd_q_count; i++) {
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index 649e5610a5ce..2b5c01fade05 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -467,6 +467,7 @@ struct ccp_aes_op {
467 enum ccp_aes_type type; 467 enum ccp_aes_type type;
468 enum ccp_aes_mode mode; 468 enum ccp_aes_mode mode;
469 enum ccp_aes_action action; 469 enum ccp_aes_action action;
470 unsigned int size;
470}; 471};
471 472
472struct ccp_xts_aes_op { 473struct ccp_xts_aes_op {
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index 50fae4442801..f1396c3aedac 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -184,62 +184,46 @@ static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
184} 184}
185 185
186static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 186static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
187 unsigned int wa_offset,
187 struct scatterlist *sg, 188 struct scatterlist *sg,
188 unsigned int len, unsigned int se_len, 189 unsigned int sg_offset,
189 bool sign_extend) 190 unsigned int len)
190{ 191{
191 unsigned int nbytes, sg_offset, dm_offset, sb_len, i; 192 u8 *p, *q;
192 u8 buffer[CCP_REVERSE_BUF_SIZE]; 193
193 194 ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
194 if (WARN_ON(se_len > sizeof(buffer))) 195
195 return -EINVAL; 196 p = wa->address + wa_offset;
196 197 q = p + len - 1;
197 sg_offset = len; 198 while (p < q) {
198 dm_offset = 0; 199 *p = *p ^ *q;
199 nbytes = len; 200 *q = *p ^ *q;
200 while (nbytes) { 201 *p = *p ^ *q;
201 sb_len = min_t(unsigned int, nbytes, se_len); 202 p++;
202 sg_offset -= sb_len; 203 q--;
203
204 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0);
205 for (i = 0; i < sb_len; i++)
206 wa->address[dm_offset + i] = buffer[sb_len - i - 1];
207
208 dm_offset += sb_len;
209 nbytes -= sb_len;
210
211 if ((sb_len != se_len) && sign_extend) {
212 /* Must sign-extend to nearest sign-extend length */
213 if (wa->address[dm_offset - 1] & 0x80)
214 memset(wa->address + dm_offset, 0xff,
215 se_len - sb_len);
216 }
217 } 204 }
218
219 return 0; 205 return 0;
220} 206}
221 207
222static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 208static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
209 unsigned int wa_offset,
223 struct scatterlist *sg, 210 struct scatterlist *sg,
211 unsigned int sg_offset,
224 unsigned int len) 212 unsigned int len)
225{ 213{
226 unsigned int nbytes, sg_offset, dm_offset, sb_len, i; 214 u8 *p, *q;
227 u8 buffer[CCP_REVERSE_BUF_SIZE]; 215
228 216 p = wa->address + wa_offset;
229 sg_offset = 0; 217 q = p + len - 1;
230 dm_offset = len; 218 while (p < q) {
231 nbytes = len; 219 *p = *p ^ *q;
232 while (nbytes) { 220 *q = *p ^ *q;
233 sb_len = min_t(unsigned int, nbytes, sizeof(buffer)); 221 *p = *p ^ *q;
234 dm_offset -= sb_len; 222 p++;
235 223 q--;
236 for (i = 0; i < sb_len; i++)
237 buffer[sb_len - i - 1] = wa->address[dm_offset + i];
238 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1);
239
240 sg_offset += sb_len;
241 nbytes -= sb_len;
242 } 224 }
225
226 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
243} 227}
244 228
245static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 229static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
@@ -692,6 +676,14 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
692 goto e_ctx; 676 goto e_ctx;
693 } 677 }
694 } 678 }
679 switch (aes->mode) {
680 case CCP_AES_MODE_CFB: /* CFB128 only */
681 case CCP_AES_MODE_CTR:
682 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
683 break;
684 default:
685 op.u.aes.size = 0;
686 }
695 687
696 /* Prepare the input and output data workareas. For in-place 688 /* Prepare the input and output data workareas. For in-place
697 * operations we need to set the dma direction to BIDIRECTIONAL 689 * operations we need to set the dma direction to BIDIRECTIONAL
@@ -1261,8 +1253,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1261 if (ret) 1253 if (ret)
1262 goto e_sb; 1254 goto e_sb;
1263 1255
1264 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, 1256 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1265 CCP_SB_BYTES, false);
1266 if (ret) 1257 if (ret)
1267 goto e_exp; 1258 goto e_exp;
1268 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1259 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
@@ -1280,16 +1271,12 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1280 if (ret) 1271 if (ret)
1281 goto e_exp; 1272 goto e_exp;
1282 1273
1283 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, 1274 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1284 CCP_SB_BYTES, false);
1285 if (ret) 1275 if (ret)
1286 goto e_src; 1276 goto e_src;
1287 src.address += o_len; /* Adjust the address for the copy operation */ 1277 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1288 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1289 CCP_SB_BYTES, false);
1290 if (ret) 1278 if (ret)
1291 goto e_src; 1279 goto e_src;
1292 src.address -= o_len; /* Reset the address to original value */
1293 1280
1294 /* Prepare the output area for the operation */ 1281 /* Prepare the output area for the operation */
1295 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1282 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
@@ -1314,7 +1301,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1314 goto e_dst; 1301 goto e_dst;
1315 } 1302 }
1316 1303
1317 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); 1304 ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len);
1318 1305
1319e_dst: 1306e_dst:
1320 ccp_free_data(&dst, cmd_q); 1307 ccp_free_data(&dst, cmd_q);
@@ -1566,25 +1553,22 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1566 save = src.address; 1553 save = src.address;
1567 1554
1568 /* Copy the ECC modulus */ 1555 /* Copy the ECC modulus */
1569 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1556 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
1570 CCP_ECC_OPERAND_SIZE, false);
1571 if (ret) 1557 if (ret)
1572 goto e_src; 1558 goto e_src;
1573 src.address += CCP_ECC_OPERAND_SIZE; 1559 src.address += CCP_ECC_OPERAND_SIZE;
1574 1560
1575 /* Copy the first operand */ 1561 /* Copy the first operand */
1576 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, 1562 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
1577 ecc->u.mm.operand_1_len, 1563 ecc->u.mm.operand_1_len);
1578 CCP_ECC_OPERAND_SIZE, false);
1579 if (ret) 1564 if (ret)
1580 goto e_src; 1565 goto e_src;
1581 src.address += CCP_ECC_OPERAND_SIZE; 1566 src.address += CCP_ECC_OPERAND_SIZE;
1582 1567
1583 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1568 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1584 /* Copy the second operand */ 1569 /* Copy the second operand */
1585 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, 1570 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
1586 ecc->u.mm.operand_2_len, 1571 ecc->u.mm.operand_2_len);
1587 CCP_ECC_OPERAND_SIZE, false);
1588 if (ret) 1572 if (ret)
1589 goto e_src; 1573 goto e_src;
1590 src.address += CCP_ECC_OPERAND_SIZE; 1574 src.address += CCP_ECC_OPERAND_SIZE;
@@ -1623,7 +1607,8 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1623 } 1607 }
1624 1608
1625 /* Save the ECC result */ 1609 /* Save the ECC result */
1626 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); 1610 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
1611 CCP_ECC_MODULUS_BYTES);
1627 1612
1628e_dst: 1613e_dst:
1629 ccp_dm_free(&dst); 1614 ccp_dm_free(&dst);
@@ -1691,22 +1676,19 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1691 save = src.address; 1676 save = src.address;
1692 1677
1693 /* Copy the ECC modulus */ 1678 /* Copy the ECC modulus */
1694 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1679 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
1695 CCP_ECC_OPERAND_SIZE, false);
1696 if (ret) 1680 if (ret)
1697 goto e_src; 1681 goto e_src;
1698 src.address += CCP_ECC_OPERAND_SIZE; 1682 src.address += CCP_ECC_OPERAND_SIZE;
1699 1683
1700 /* Copy the first point X and Y coordinate */ 1684 /* Copy the first point X and Y coordinate */
1701 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, 1685 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
1702 ecc->u.pm.point_1.x_len, 1686 ecc->u.pm.point_1.x_len);
1703 CCP_ECC_OPERAND_SIZE, false);
1704 if (ret) 1687 if (ret)
1705 goto e_src; 1688 goto e_src;
1706 src.address += CCP_ECC_OPERAND_SIZE; 1689 src.address += CCP_ECC_OPERAND_SIZE;
1707 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, 1690 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
1708 ecc->u.pm.point_1.y_len, 1691 ecc->u.pm.point_1.y_len);
1709 CCP_ECC_OPERAND_SIZE, false);
1710 if (ret) 1692 if (ret)
1711 goto e_src; 1693 goto e_src;
1712 src.address += CCP_ECC_OPERAND_SIZE; 1694 src.address += CCP_ECC_OPERAND_SIZE;
@@ -1717,15 +1699,13 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1717 1699
1718 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1700 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1719 /* Copy the second point X and Y coordinate */ 1701 /* Copy the second point X and Y coordinate */
1720 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, 1702 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
1721 ecc->u.pm.point_2.x_len, 1703 ecc->u.pm.point_2.x_len);
1722 CCP_ECC_OPERAND_SIZE, false);
1723 if (ret) 1704 if (ret)
1724 goto e_src; 1705 goto e_src;
1725 src.address += CCP_ECC_OPERAND_SIZE; 1706 src.address += CCP_ECC_OPERAND_SIZE;
1726 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, 1707 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
1727 ecc->u.pm.point_2.y_len, 1708 ecc->u.pm.point_2.y_len);
1728 CCP_ECC_OPERAND_SIZE, false);
1729 if (ret) 1709 if (ret)
1730 goto e_src; 1710 goto e_src;
1731 src.address += CCP_ECC_OPERAND_SIZE; 1711 src.address += CCP_ECC_OPERAND_SIZE;
@@ -1735,19 +1715,17 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1735 src.address += CCP_ECC_OPERAND_SIZE; 1715 src.address += CCP_ECC_OPERAND_SIZE;
1736 } else { 1716 } else {
1737 /* Copy the Domain "a" parameter */ 1717 /* Copy the Domain "a" parameter */
1738 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, 1718 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
1739 ecc->u.pm.domain_a_len, 1719 ecc->u.pm.domain_a_len);
1740 CCP_ECC_OPERAND_SIZE, false);
1741 if (ret) 1720 if (ret)
1742 goto e_src; 1721 goto e_src;
1743 src.address += CCP_ECC_OPERAND_SIZE; 1722 src.address += CCP_ECC_OPERAND_SIZE;
1744 1723
1745 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 1724 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1746 /* Copy the scalar value */ 1725 /* Copy the scalar value */
1747 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, 1726 ret = ccp_reverse_set_dm_area(&src, 0,
1748 ecc->u.pm.scalar_len, 1727 ecc->u.pm.scalar, 0,
1749 CCP_ECC_OPERAND_SIZE, 1728 ecc->u.pm.scalar_len);
1750 false);
1751 if (ret) 1729 if (ret)
1752 goto e_src; 1730 goto e_src;
1753 src.address += CCP_ECC_OPERAND_SIZE; 1731 src.address += CCP_ECC_OPERAND_SIZE;
@@ -1792,10 +1770,10 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1792 save = dst.address; 1770 save = dst.address;
1793 1771
1794 /* Save the ECC result X and Y coordinates */ 1772 /* Save the ECC result X and Y coordinates */
1795 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, 1773 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
1796 CCP_ECC_MODULUS_BYTES); 1774 CCP_ECC_MODULUS_BYTES);
1797 dst.address += CCP_ECC_OUTPUT_SIZE; 1775 dst.address += CCP_ECC_OUTPUT_SIZE;
1798 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, 1776 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
1799 CCP_ECC_MODULUS_BYTES); 1777 CCP_ECC_MODULUS_BYTES);
1800 dst.address += CCP_ECC_OUTPUT_SIZE; 1778 dst.address += CCP_ECC_OUTPUT_SIZE;
1801 1779
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index b4b78b37f8a6..41bc7f4f58cd 100644..100755
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -171,7 +171,7 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
171 } 171 }
172 break; 172 break;
173 173
174 case CRYPTO_ALG_TYPE_BLKCIPHER: 174 case CRYPTO_ALG_TYPE_ABLKCIPHER:
175 ctx_req.req.ablk_req = (struct ablkcipher_request *)req; 175 ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
176 ctx_req.ctx.ablk_ctx = 176 ctx_req.ctx.ablk_ctx =
177 ablkcipher_request_ctx(ctx_req.req.ablk_req); 177 ablkcipher_request_ctx(ctx_req.req.ablk_req);
@@ -542,10 +542,11 @@ static inline void create_wreq(struct chcr_context *ctx,
542 (calc_tx_flits_ofld(skb) * 8), 16))); 542 (calc_tx_flits_ofld(skb) * 8), 16)));
543 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 543 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
544 chcr_req->wreq.rx_chid_to_rx_q_id = 544 chcr_req->wreq.rx_chid_to_rx_q_id =
545 FILL_WR_RX_Q_ID(ctx->dev->tx_channel_id, qid, 545 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
546 is_iv ? iv_loc : IV_NOP); 546 is_iv ? iv_loc : IV_NOP, ctx->tx_channel_id);
547 547
548 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id); 548 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
549 qid);
549 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), 550 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
550 16) - ((sizeof(chcr_req->wreq)) >> 4))); 551 16) - ((sizeof(chcr_req->wreq)) >> 4)));
551 552
@@ -606,7 +607,7 @@ static struct sk_buff
606 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 607 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
607 memset(chcr_req, 0, transhdr_len); 608 memset(chcr_req, 0, transhdr_len);
608 chcr_req->sec_cpl.op_ivinsrtofst = 609 chcr_req->sec_cpl.op_ivinsrtofst =
609 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1); 610 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
610 611
611 chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes); 612 chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes);
612 chcr_req->sec_cpl.aadstart_cipherstop_hi = 613 chcr_req->sec_cpl.aadstart_cipherstop_hi =
@@ -782,6 +783,7 @@ static int chcr_device_init(struct chcr_context *ctx)
782 spin_lock(&ctx->dev->lock_chcr_dev); 783 spin_lock(&ctx->dev->lock_chcr_dev);
783 ctx->tx_channel_id = rxq_idx; 784 ctx->tx_channel_id = rxq_idx;
784 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 785 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
786 ctx->dev->rx_channel_id = 0;
785 spin_unlock(&ctx->dev->lock_chcr_dev); 787 spin_unlock(&ctx->dev->lock_chcr_dev);
786 } 788 }
787out: 789out:
@@ -874,7 +876,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
874 memset(chcr_req, 0, transhdr_len); 876 memset(chcr_req, 0, transhdr_len);
875 877
876 chcr_req->sec_cpl.op_ivinsrtofst = 878 chcr_req->sec_cpl.op_ivinsrtofst =
877 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0); 879 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
878 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 880 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
879 881
880 chcr_req->sec_cpl.aadstart_cipherstop_hi = 882 chcr_req->sec_cpl.aadstart_cipherstop_hi =
@@ -1425,7 +1427,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
1425 * to the hardware spec 1427 * to the hardware spec
1426 */ 1428 */
1427 chcr_req->sec_cpl.op_ivinsrtofst = 1429 chcr_req->sec_cpl.op_ivinsrtofst =
1428 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1430 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
1429 (ivsize ? (assoclen + 1) : 0)); 1431 (ivsize ? (assoclen + 1) : 0));
1430 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); 1432 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
1431 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1433 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
@@ -1601,7 +1603,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
1601 unsigned int ivsize = AES_BLOCK_SIZE; 1603 unsigned int ivsize = AES_BLOCK_SIZE;
1602 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 1604 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
1603 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 1605 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
1604 unsigned int c_id = chcrctx->dev->tx_channel_id; 1606 unsigned int c_id = chcrctx->dev->rx_channel_id;
1605 unsigned int ccm_xtra; 1607 unsigned int ccm_xtra;
1606 unsigned char tag_offset = 0, auth_offset = 0; 1608 unsigned char tag_offset = 0, auth_offset = 0;
1607 unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm)); 1609 unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm));
@@ -1877,7 +1879,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
1877 1879
1878 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 1880 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
1879 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 1881 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
1880 ctx->dev->tx_channel_id, 2, (ivsize ? 1882 ctx->dev->rx_channel_id, 2, (ivsize ?
1881 (req->assoclen + 1) : 0)); 1883 (req->assoclen + 1) : 0));
1882 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len); 1884 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len);
1883 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1885 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
@@ -2187,8 +2189,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2187 struct chcr_context *ctx = crypto_aead_ctx(aead); 2189 struct chcr_context *ctx = crypto_aead_ctx(aead);
2188 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2190 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2189 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 2191 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
2190 struct blkcipher_desc h_desc; 2192 struct crypto_cipher *cipher;
2191 struct scatterlist src[1];
2192 unsigned int ck_size; 2193 unsigned int ck_size;
2193 int ret = 0, key_ctx_size = 0; 2194 int ret = 0, key_ctx_size = 0;
2194 2195
@@ -2221,27 +2222,26 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2221 CHCR_KEYCTX_MAC_KEY_SIZE_128, 2222 CHCR_KEYCTX_MAC_KEY_SIZE_128,
2222 0, 0, 2223 0, 0,
2223 key_ctx_size >> 4); 2224 key_ctx_size >> 4);
2224 /* Calculate the H = CIPH(K, 0 repeated 16 times) using sync aes 2225 /* Calculate the H = CIPH(K, 0 repeated 16 times).
2225 * blkcipher It will go on key context 2226 * It will go in key context
2226 */ 2227 */
2227 h_desc.tfm = crypto_alloc_blkcipher("cbc(aes-generic)", 0, 0); 2228 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
2228 if (IS_ERR(h_desc.tfm)) { 2229 if (IS_ERR(cipher)) {
2229 aeadctx->enckey_len = 0; 2230 aeadctx->enckey_len = 0;
2230 ret = -ENOMEM; 2231 ret = -ENOMEM;
2231 goto out; 2232 goto out;
2232 } 2233 }
2233 h_desc.flags = 0; 2234
2234 ret = crypto_blkcipher_setkey(h_desc.tfm, key, keylen); 2235 ret = crypto_cipher_setkey(cipher, key, keylen);
2235 if (ret) { 2236 if (ret) {
2236 aeadctx->enckey_len = 0; 2237 aeadctx->enckey_len = 0;
2237 goto out1; 2238 goto out1;
2238 } 2239 }
2239 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 2240 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
2240 sg_init_one(&src[0], gctx->ghash_h, AEAD_H_SIZE); 2241 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2241 ret = crypto_blkcipher_encrypt(&h_desc, &src[0], &src[0], AEAD_H_SIZE);
2242 2242
2243out1: 2243out1:
2244 crypto_free_blkcipher(h_desc.tfm); 2244 crypto_free_cipher(cipher);
2245out: 2245out:
2246 return ret; 2246 return ret;
2247} 2247}
@@ -2456,13 +2456,14 @@ static int chcr_aead_op(struct aead_request *req,
2456{ 2456{
2457 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2457 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2458 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2458 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2459 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2459 struct uld_ctx *u_ctx;
2460 struct sk_buff *skb; 2460 struct sk_buff *skb;
2461 2461
2462 if (ctx && !ctx->dev) { 2462 if (!ctx->dev) {
2463 pr_err("chcr : %s : No crypto device.\n", __func__); 2463 pr_err("chcr : %s : No crypto device.\n", __func__);
2464 return -ENXIO; 2464 return -ENXIO;
2465 } 2465 }
2466 u_ctx = ULD_CTX(ctx);
2466 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 2467 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2467 ctx->tx_channel_id)) { 2468 ctx->tx_channel_id)) {
2468 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2469 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
@@ -2492,7 +2493,7 @@ static struct chcr_alg_template driver_algs[] = {
2492 .cra_name = "cbc(aes)", 2493 .cra_name = "cbc(aes)",
2493 .cra_driver_name = "cbc-aes-chcr", 2494 .cra_driver_name = "cbc-aes-chcr",
2494 .cra_priority = CHCR_CRA_PRIORITY, 2495 .cra_priority = CHCR_CRA_PRIORITY,
2495 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 2496 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2496 CRYPTO_ALG_ASYNC, 2497 CRYPTO_ALG_ASYNC,
2497 .cra_blocksize = AES_BLOCK_SIZE, 2498 .cra_blocksize = AES_BLOCK_SIZE,
2498 .cra_ctxsize = sizeof(struct chcr_context) 2499 .cra_ctxsize = sizeof(struct chcr_context)
@@ -2519,7 +2520,7 @@ static struct chcr_alg_template driver_algs[] = {
2519 .cra_name = "xts(aes)", 2520 .cra_name = "xts(aes)",
2520 .cra_driver_name = "xts-aes-chcr", 2521 .cra_driver_name = "xts-aes-chcr",
2521 .cra_priority = CHCR_CRA_PRIORITY, 2522 .cra_priority = CHCR_CRA_PRIORITY,
2522 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 2523 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2523 CRYPTO_ALG_ASYNC, 2524 CRYPTO_ALG_ASYNC,
2524 .cra_blocksize = AES_BLOCK_SIZE, 2525 .cra_blocksize = AES_BLOCK_SIZE,
2525 .cra_ctxsize = sizeof(struct chcr_context) + 2526 .cra_ctxsize = sizeof(struct chcr_context) +
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index 3c7c51f7bedf..ba38bae7ce80 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -185,20 +185,21 @@
185 FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ 185 FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \
186 FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) 186 FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len)))
187 187
188#define FILL_WR_RX_Q_ID(cid, qid, wr_iv) \ 188#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, fid) \
189 htonl( \ 189 htonl( \
190 FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ 190 FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \
191 FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ 191 FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \
192 FW_CRYPTO_LOOKASIDE_WR_LCB_V(0) | \ 192 FW_CRYPTO_LOOKASIDE_WR_LCB_V(0) | \
193 FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv))) 193 FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \
194 FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid))
194 195
195#define FILL_ULPTX_CMD_DEST(cid) \ 196#define FILL_ULPTX_CMD_DEST(cid, qid) \
196 htonl(ULPTX_CMD_V(ULP_TX_PKT) | \ 197 htonl(ULPTX_CMD_V(ULP_TX_PKT) | \
197 ULP_TXPKT_DEST_V(0) | \ 198 ULP_TXPKT_DEST_V(0) | \
198 ULP_TXPKT_DATAMODIFY_V(0) | \ 199 ULP_TXPKT_DATAMODIFY_V(0) | \
199 ULP_TXPKT_CHANNELID_V((cid)) | \ 200 ULP_TXPKT_CHANNELID_V((cid)) | \
200 ULP_TXPKT_RO_V(1) | \ 201 ULP_TXPKT_RO_V(1) | \
201 ULP_TXPKT_FID_V(0)) 202 ULP_TXPKT_FID_V(qid))
202 203
203#define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\ 204#define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\
204 _bs == SHA1_DIGEST_SIZE ? 12 : 0; }) 205 _bs == SHA1_DIGEST_SIZE ? 12 : 0; })
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index 1c65f07e1cc9..c28e018e0773 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -61,7 +61,7 @@ int assign_chcr_device(struct chcr_dev **dev)
61 */ 61 */
62 mutex_lock(&dev_mutex); /* TODO ? */ 62 mutex_lock(&dev_mutex); /* TODO ? */
63 list_for_each_entry(u_ctx, &uld_ctx_list, entry) 63 list_for_each_entry(u_ctx, &uld_ctx_list, entry)
64 if (u_ctx && u_ctx->dev) { 64 if (u_ctx->dev) {
65 *dev = u_ctx->dev; 65 *dev = u_ctx->dev;
66 ret = 0; 66 ret = 0;
67 break; 67 break;
@@ -151,18 +151,17 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
151{ 151{
152 struct uld_ctx *u_ctx = (struct uld_ctx *)handle; 152 struct uld_ctx *u_ctx = (struct uld_ctx *)handle;
153 struct chcr_dev *dev = u_ctx->dev; 153 struct chcr_dev *dev = u_ctx->dev;
154 const struct cpl_act_establish *rpl = (struct cpl_act_establish 154 const struct cpl_fw6_pld *rpl = (struct cpl_fw6_pld *)rsp;
155 *)rsp;
156 155
157 if (rpl->ot.opcode != CPL_FW6_PLD) { 156 if (rpl->opcode != CPL_FW6_PLD) {
158 pr_err("Unsupported opcode\n"); 157 pr_err("Unsupported opcode\n");
159 return 0; 158 return 0;
160 } 159 }
161 160
162 if (!pgl) 161 if (!pgl)
163 work_handlers[rpl->ot.opcode](dev, (unsigned char *)&rsp[1]); 162 work_handlers[rpl->opcode](dev, (unsigned char *)&rsp[1]);
164 else 163 else
165 work_handlers[rpl->ot.opcode](dev, pgl->va); 164 work_handlers[rpl->opcode](dev, pgl->va);
166 return 0; 165 return 0;
167} 166}
168 167
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index c7088a4e0a49..79da22b5cdc9 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -75,6 +75,7 @@ struct chcr_dev {
75 spinlock_t lock_chcr_dev; 75 spinlock_t lock_chcr_dev;
76 struct uld_ctx *u_ctx; 76 struct uld_ctx *u_ctx;
77 unsigned char tx_channel_id; 77 unsigned char tx_channel_id;
78 unsigned char rx_channel_id;
78}; 79};
79 80
80struct uld_ctx { 81struct uld_ctx {
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 7ec0a8f12475..81cfd0ba132e 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -48,7 +48,7 @@
48 * giving the processed data 48 * giving the processed data
49 */ 49 */
50 50
51#define CHCR_CRA_PRIORITY 300 51#define CHCR_CRA_PRIORITY 3000
52 52
53#define CHCR_AES_MAX_KEY_LEN (2 * (AES_MAX_KEY_SIZE)) /* consider xts */ 53#define CHCR_AES_MAX_KEY_LEN (2 * (AES_MAX_KEY_SIZE)) /* consider xts */
54#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */ 54#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index a2e77b87485b..9b07f3d88feb 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -226,7 +226,7 @@ static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
226 struct dma_async_tx_descriptor *desc; 226 struct dma_async_tx_descriptor *desc;
227 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 227 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
228 228
229 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV); 229 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
230 if (ctx->dma_ct == 0) { 230 if (ctx->dma_ct == 0) {
231 dev_err(hdev->dev, "Invalid DMA sg\n"); 231 dev_err(hdev->dev, "Invalid DMA sg\n");
232 hdev->err = -EINVAL; 232 hdev->err = -EINVAL;
@@ -241,7 +241,7 @@ static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
241 if (!desc) { 241 if (!desc) {
242 dev_err(hdev->dev, "Null DMA descriptor\n"); 242 dev_err(hdev->dev, "Null DMA descriptor\n");
243 hdev->err = -EINVAL; 243 hdev->err = -EINVAL;
244 dma_unmap_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV); 244 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
245 return -EINVAL; 245 return -EINVAL;
246 } 246 }
247 desc->callback = img_hash_dma_callback; 247 desc->callback = img_hash_dma_callback;
diff --git a/drivers/crypto/mediatek/Makefile b/drivers/crypto/mediatek/Makefile
new file mode 100644
index 000000000000..187be79c7f3e
--- /dev/null
+++ b/drivers/crypto/mediatek/Makefile
@@ -0,0 +1,2 @@
1obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mtk-crypto.o
2mtk-crypto-objs:= mtk-platform.o mtk-aes.o mtk-sha.o
diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
new file mode 100644
index 000000000000..3a47cdb8f0c8
--- /dev/null
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -0,0 +1,1299 @@
1/*
2 * Cryptographic API.
3 *
4 * Driver for EIP97 AES acceleration.
5 *
6 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * Some ideas are from atmel-aes.c drivers.
13 */
14
15#include <crypto/aes.h>
16#include "mtk-platform.h"
17
18#define AES_QUEUE_SIZE 512
19#define AES_BUF_ORDER 2
20#define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \
21 & ~(AES_BLOCK_SIZE - 1))
22
23/* AES command token size */
24#define AES_CT_SIZE_ECB 2
25#define AES_CT_SIZE_CBC 3
26#define AES_CT_SIZE_CTR 3
27#define AES_CT_SIZE_GCM_OUT 5
28#define AES_CT_SIZE_GCM_IN 6
29#define AES_CT_CTRL_HDR cpu_to_le32(0x00220000)
30
31/* AES-CBC/ECB/CTR command token */
32#define AES_CMD0 cpu_to_le32(0x05000000)
33#define AES_CMD1 cpu_to_le32(0x2d060000)
34#define AES_CMD2 cpu_to_le32(0xe4a63806)
35/* AES-GCM command token */
36#define AES_GCM_CMD0 cpu_to_le32(0x0b000000)
37#define AES_GCM_CMD1 cpu_to_le32(0xa0800000)
38#define AES_GCM_CMD2 cpu_to_le32(0x25000010)
39#define AES_GCM_CMD3 cpu_to_le32(0x0f020000)
40#define AES_GCM_CMD4 cpu_to_le32(0x21e60000)
41#define AES_GCM_CMD5 cpu_to_le32(0x40e60000)
42#define AES_GCM_CMD6 cpu_to_le32(0xd0070000)
43
44/* AES transform information word 0 fields */
45#define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0)
46#define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0)
47#define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0)
48#define AES_TFM_GCM_IN cpu_to_le32(0xf << 0)
49#define AES_TFM_SIZE(x) cpu_to_le32((x) << 8)
50#define AES_TFM_128BITS cpu_to_le32(0xb << 16)
51#define AES_TFM_192BITS cpu_to_le32(0xd << 16)
52#define AES_TFM_256BITS cpu_to_le32(0xf << 16)
53/* AES transform information word 1 fields */
54#define AES_TFM_ECB cpu_to_le32(0x0 << 0)
55#define AES_TFM_CBC cpu_to_le32(0x1 << 0)
56#define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */
57#define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */
58#define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */
59#define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */
60#define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10)
61#define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17)
62#define AES_TFM_GHASH_DIG cpu_to_le32(0x2 << 21)
63#define AES_TFM_GHASH cpu_to_le32(0x4 << 23)
64
65/* AES flags */
66#define AES_FLAGS_ECB BIT(0)
67#define AES_FLAGS_CBC BIT(1)
68#define AES_FLAGS_CTR BIT(2)
69#define AES_FLAGS_GCM BIT(3)
70#define AES_FLAGS_ENCRYPT BIT(4)
71#define AES_FLAGS_BUSY BIT(5)
72
73/**
74 * Command token(CT) is a set of hardware instructions that
75 * are used to control engine's processing flow of AES.
76 *
77 * Transform information(TFM) is used to define AES state and
78 * contains all keys and initial vectors.
79 *
80 * The engine requires CT and TFM to do:
81 * - Commands decoding and control of the engine's data path.
82 * - Coordinating hardware data fetch and store operations.
83 * - Result token construction and output.
84 *
85 * Memory map of GCM's TFM:
86 * /-----------\
87 * | AES KEY | 128/196/256 bits
88 * |-----------|
89 * | HASH KEY | a string 128 zero bits encrypted using the block cipher
90 * |-----------|
91 * | IVs | 4 * 4 bytes
92 * \-----------/
93 */
94struct mtk_aes_ct {
95 __le32 cmd[AES_CT_SIZE_GCM_IN];
96};
97
98struct mtk_aes_tfm {
99 __le32 ctrl[2];
100 __le32 state[SIZE_IN_WORDS(AES_KEYSIZE_256 + AES_BLOCK_SIZE * 2)];
101};
102
103struct mtk_aes_reqctx {
104 u64 mode;
105};
106
107struct mtk_aes_base_ctx {
108 struct mtk_cryp *cryp;
109 u32 keylen;
110 mtk_aes_fn start;
111
112 struct mtk_aes_ct ct;
113 dma_addr_t ct_dma;
114 struct mtk_aes_tfm tfm;
115 dma_addr_t tfm_dma;
116
117 __le32 ct_hdr;
118 u32 ct_size;
119};
120
121struct mtk_aes_ctx {
122 struct mtk_aes_base_ctx base;
123};
124
125struct mtk_aes_ctr_ctx {
126 struct mtk_aes_base_ctx base;
127
128 u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
129 size_t offset;
130 struct scatterlist src[2];
131 struct scatterlist dst[2];
132};
133
134struct mtk_aes_gcm_ctx {
135 struct mtk_aes_base_ctx base;
136
137 u32 authsize;
138 size_t textlen;
139
140 struct crypto_skcipher *ctr;
141};
142
143struct mtk_aes_gcm_setkey_result {
144 int err;
145 struct completion completion;
146};
147
148struct mtk_aes_drv {
149 struct list_head dev_list;
150 /* Device list lock */
151 spinlock_t lock;
152};
153
154static struct mtk_aes_drv mtk_aes = {
155 .dev_list = LIST_HEAD_INIT(mtk_aes.dev_list),
156 .lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock),
157};
158
159static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset)
160{
161 return readl_relaxed(cryp->base + offset);
162}
163
164static inline void mtk_aes_write(struct mtk_cryp *cryp,
165 u32 offset, u32 value)
166{
167 writel_relaxed(value, cryp->base + offset);
168}
169
170static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx)
171{
172 struct mtk_cryp *cryp = NULL;
173 struct mtk_cryp *tmp;
174
175 spin_lock_bh(&mtk_aes.lock);
176 if (!ctx->cryp) {
177 list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) {
178 cryp = tmp;
179 break;
180 }
181 ctx->cryp = cryp;
182 } else {
183 cryp = ctx->cryp;
184 }
185 spin_unlock_bh(&mtk_aes.lock);
186
187 return cryp;
188}
189
190static inline size_t mtk_aes_padlen(size_t len)
191{
192 len &= AES_BLOCK_SIZE - 1;
193 return len ? AES_BLOCK_SIZE - len : 0;
194}
195
196static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len,
197 struct mtk_aes_dma *dma)
198{
199 int nents;
200
201 if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
202 return false;
203
204 for (nents = 0; sg; sg = sg_next(sg), ++nents) {
205 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
206 return false;
207
208 if (len <= sg->length) {
209 if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
210 return false;
211
212 dma->nents = nents + 1;
213 dma->remainder = sg->length - len;
214 sg->length = len;
215 return true;
216 }
217
218 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
219 return false;
220
221 len -= sg->length;
222 }
223
224 return false;
225}
226
227static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,
228 const struct mtk_aes_reqctx *rctx)
229{
230 /* Clear all but persistent flags and set request flags. */
231 aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;
232}
233
234static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
235{
236 struct scatterlist *sg = dma->sg;
237 int nents = dma->nents;
238
239 if (!dma->remainder)
240 return;
241
242 while (--nents > 0 && sg)
243 sg = sg_next(sg);
244
245 if (!sg)
246 return;
247
248 sg->length += dma->remainder;
249}
250
251/*
252 * Write descriptors for processing. This will configure the engine, load
253 * the transform information and then start the packet processing.
254 */
255static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
256{
257 struct mtk_ring *ring = cryp->ring[aes->id];
258 struct mtk_desc *cmd = NULL, *res = NULL;
259 struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;
260 u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;
261 int nents;
262
263 /* Write command descriptors */
264 for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) {
265 cmd = ring->cmd_base + ring->cmd_pos;
266 cmd->hdr = MTK_DESC_BUF_LEN(ssg->length);
267 cmd->buf = cpu_to_le32(sg_dma_address(ssg));
268
269 if (nents == 0) {
270 cmd->hdr |= MTK_DESC_FIRST |
271 MTK_DESC_CT_LEN(aes->ctx->ct_size);
272 cmd->ct = cpu_to_le32(aes->ctx->ct_dma);
273 cmd->ct_hdr = aes->ctx->ct_hdr;
274 cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);
275 }
276
277 if (++ring->cmd_pos == MTK_DESC_NUM)
278 ring->cmd_pos = 0;
279 }
280 cmd->hdr |= MTK_DESC_LAST;
281
282 /* Prepare result descriptors */
283 for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) {
284 res = ring->res_base + ring->res_pos;
285 res->hdr = MTK_DESC_BUF_LEN(dsg->length);
286 res->buf = cpu_to_le32(sg_dma_address(dsg));
287
288 if (nents == 0)
289 res->hdr |= MTK_DESC_FIRST;
290
291 if (++ring->res_pos == MTK_DESC_NUM)
292 ring->res_pos = 0;
293 }
294 res->hdr |= MTK_DESC_LAST;
295
296 /* Prepare enough space for authenticated tag */
297 if (aes->flags & AES_FLAGS_GCM)
298 res->hdr += AES_BLOCK_SIZE;
299
300 /*
301 * Make sure that all changes to the DMA ring are done before we
302 * start engine.
303 */
304 wmb();
305 /* Start DMA transfer */
306 mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));
307 mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));
308
309 return -EINPROGRESS;
310}
311
312static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
313{
314 struct mtk_aes_base_ctx *ctx = aes->ctx;
315
316 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
317 DMA_TO_DEVICE);
318 dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
319 DMA_TO_DEVICE);
320
321 if (aes->src.sg == aes->dst.sg) {
322 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
323 DMA_BIDIRECTIONAL);
324
325 if (aes->src.sg != &aes->aligned_sg)
326 mtk_aes_restore_sg(&aes->src);
327 } else {
328 dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,
329 DMA_FROM_DEVICE);
330
331 if (aes->dst.sg != &aes->aligned_sg)
332 mtk_aes_restore_sg(&aes->dst);
333
334 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
335 DMA_TO_DEVICE);
336
337 if (aes->src.sg != &aes->aligned_sg)
338 mtk_aes_restore_sg(&aes->src);
339 }
340
341 if (aes->dst.sg == &aes->aligned_sg)
342 sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),
343 aes->buf, aes->total);
344}
345
346static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
347{
348 struct mtk_aes_base_ctx *ctx = aes->ctx;
349
350 ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct),
351 DMA_TO_DEVICE);
352 if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
353 return -EINVAL;
354
355 ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm),
356 DMA_TO_DEVICE);
357 if (unlikely(dma_mapping_error(cryp->dev, ctx->tfm_dma)))
358 goto tfm_map_err;
359
360 if (aes->src.sg == aes->dst.sg) {
361 aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
362 aes->src.nents,
363 DMA_BIDIRECTIONAL);
364 aes->dst.sg_len = aes->src.sg_len;
365 if (unlikely(!aes->src.sg_len))
366 goto sg_map_err;
367 } else {
368 aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
369 aes->src.nents, DMA_TO_DEVICE);
370 if (unlikely(!aes->src.sg_len))
371 goto sg_map_err;
372
373 aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
374 aes->dst.nents, DMA_FROM_DEVICE);
375 if (unlikely(!aes->dst.sg_len)) {
376 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
377 DMA_TO_DEVICE);
378 goto sg_map_err;
379 }
380 }
381
382 return mtk_aes_xmit(cryp, aes);
383
384sg_map_err:
385 dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm),
386 DMA_TO_DEVICE);
387tfm_map_err:
388 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct),
389 DMA_TO_DEVICE);
390
391 return -EINVAL;
392}
393
394/* Initialize transform information of CBC/ECB/CTR mode */
395static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
396 size_t len)
397{
398 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
399 struct mtk_aes_base_ctx *ctx = aes->ctx;
400
401 ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
402 ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len);
403 ctx->ct.cmd[1] = AES_CMD1;
404
405 if (aes->flags & AES_FLAGS_ENCRYPT)
406 ctx->tfm.ctrl[0] = AES_TFM_BASIC_OUT;
407 else
408 ctx->tfm.ctrl[0] = AES_TFM_BASIC_IN;
409
410 if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128))
411 ctx->tfm.ctrl[0] |= AES_TFM_128BITS;
412 else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256))
413 ctx->tfm.ctrl[0] |= AES_TFM_256BITS;
414 else
415 ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
416
417 if (aes->flags & AES_FLAGS_CBC) {
418 const u32 *iv = (const u32 *)req->info;
419 u32 *iv_state = ctx->tfm.state + ctx->keylen;
420 int i;
421
422 ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen +
423 SIZE_IN_WORDS(AES_BLOCK_SIZE));
424 ctx->tfm.ctrl[1] = AES_TFM_CBC | AES_TFM_FULL_IV;
425
426 for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
427 iv_state[i] = cpu_to_le32(iv[i]);
428
429 ctx->ct.cmd[2] = AES_CMD2;
430 ctx->ct_size = AES_CT_SIZE_CBC;
431 } else if (aes->flags & AES_FLAGS_ECB) {
432 ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen);
433 ctx->tfm.ctrl[1] = AES_TFM_ECB;
434
435 ctx->ct_size = AES_CT_SIZE_ECB;
436 } else if (aes->flags & AES_FLAGS_CTR) {
437 ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen +
438 SIZE_IN_WORDS(AES_BLOCK_SIZE));
439 ctx->tfm.ctrl[1] = AES_TFM_CTR_LOAD | AES_TFM_FULL_IV;
440
441 ctx->ct.cmd[2] = AES_CMD2;
442 ctx->ct_size = AES_CT_SIZE_CTR;
443 }
444}
445
446static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
447 struct scatterlist *src, struct scatterlist *dst,
448 size_t len)
449{
450 size_t padlen = 0;
451 bool src_aligned, dst_aligned;
452
453 aes->total = len;
454 aes->src.sg = src;
455 aes->dst.sg = dst;
456 aes->real_dst = dst;
457
458 src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
459 if (src == dst)
460 dst_aligned = src_aligned;
461 else
462 dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
463
464 if (!src_aligned || !dst_aligned) {
465 padlen = mtk_aes_padlen(len);
466
467 if (len + padlen > AES_BUF_SIZE)
468 return -ENOMEM;
469
470 if (!src_aligned) {
471 sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
472 aes->src.sg = &aes->aligned_sg;
473 aes->src.nents = 1;
474 aes->src.remainder = 0;
475 }
476
477 if (!dst_aligned) {
478 aes->dst.sg = &aes->aligned_sg;
479 aes->dst.nents = 1;
480 aes->dst.remainder = 0;
481 }
482
483 sg_init_table(&aes->aligned_sg, 1);
484 sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);
485 }
486
487 mtk_aes_info_init(cryp, aes, len + padlen);
488
489 return mtk_aes_map(cryp, aes);
490}
491
492static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
493 struct crypto_async_request *new_areq)
494{
495 struct mtk_aes_rec *aes = cryp->aes[id];
496 struct crypto_async_request *areq, *backlog;
497 struct mtk_aes_base_ctx *ctx;
498 unsigned long flags;
499 int ret = 0;
500
501 spin_lock_irqsave(&aes->lock, flags);
502 if (new_areq)
503 ret = crypto_enqueue_request(&aes->queue, new_areq);
504 if (aes->flags & AES_FLAGS_BUSY) {
505 spin_unlock_irqrestore(&aes->lock, flags);
506 return ret;
507 }
508 backlog = crypto_get_backlog(&aes->queue);
509 areq = crypto_dequeue_request(&aes->queue);
510 if (areq)
511 aes->flags |= AES_FLAGS_BUSY;
512 spin_unlock_irqrestore(&aes->lock, flags);
513
514 if (!areq)
515 return ret;
516
517 if (backlog)
518 backlog->complete(backlog, -EINPROGRESS);
519
520 ctx = crypto_tfm_ctx(areq->tfm);
521
522 aes->areq = areq;
523 aes->ctx = ctx;
524
525 return ctx->start(cryp, aes);
526}
527
528static int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
529{
530 aes->flags &= ~AES_FLAGS_BUSY;
531 aes->areq->complete(aes->areq, 0);
532
533 /* Handle new request */
534 return mtk_aes_handle_queue(cryp, aes->id, NULL);
535}
536
537static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
538{
539 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
540 struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
541
542 mtk_aes_set_mode(aes, rctx);
543 aes->resume = mtk_aes_complete;
544
545 return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);
546}
547
548static inline struct mtk_aes_ctr_ctx *
549mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx)
550{
551 return container_of(ctx, struct mtk_aes_ctr_ctx, base);
552}
553
554static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
555{
556 struct mtk_aes_base_ctx *ctx = aes->ctx;
557 struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx);
558 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
559 struct scatterlist *src, *dst;
560 int i;
561 u32 start, end, ctr, blocks, *iv_state;
562 size_t datalen;
563 bool fragmented = false;
564
565 /* Check for transfer completion. */
566 cctx->offset += aes->total;
567 if (cctx->offset >= req->nbytes)
568 return mtk_aes_complete(cryp, aes);
569
570 /* Compute data length. */
571 datalen = req->nbytes - cctx->offset;
572 blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
573 ctr = be32_to_cpu(cctx->iv[3]);
574
575 /* Check 32bit counter overflow. */
576 start = ctr;
577 end = start + blocks - 1;
578 if (end < start) {
579 ctr |= 0xffffffff;
580 datalen = AES_BLOCK_SIZE * -start;
581 fragmented = true;
582 }
583
584 /* Jump to offset. */
585 src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset);
586 dst = ((req->src == req->dst) ? src :
587 scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset));
588
589 /* Write IVs into transform state buffer. */
590 iv_state = ctx->tfm.state + ctx->keylen;
591 for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
592 iv_state[i] = cpu_to_le32(cctx->iv[i]);
593
594 if (unlikely(fragmented)) {
595 /*
596 * Increment the counter manually to cope with the hardware
597 * counter overflow.
598 */
599 cctx->iv[3] = cpu_to_be32(ctr);
600 crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE);
601 }
602 aes->resume = mtk_aes_ctr_transfer;
603
604 return mtk_aes_dma(cryp, aes, src, dst, datalen);
605}
606
607static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
608{
609 struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);
610 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
611 struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
612
613 mtk_aes_set_mode(aes, rctx);
614
615 memcpy(cctx->iv, req->info, AES_BLOCK_SIZE);
616 cctx->offset = 0;
617 aes->total = 0;
618
619 return mtk_aes_ctr_transfer(cryp, aes);
620}
621
622/* Check and set the AES key to transform state buffer */
623static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
624 const u8 *key, u32 keylen)
625{
626 struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm);
627 const u32 *aes_key = (const u32 *)key;
628 u32 *key_state = ctx->tfm.state;
629 int i;
630
631 if (keylen != AES_KEYSIZE_128 &&
632 keylen != AES_KEYSIZE_192 &&
633 keylen != AES_KEYSIZE_256) {
634 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
635 return -EINVAL;
636 }
637
638 ctx->keylen = SIZE_IN_WORDS(keylen);
639
640 for (i = 0; i < ctx->keylen; i++)
641 key_state[i] = cpu_to_le32(aes_key[i]);
642
643 return 0;
644}
645
646static int mtk_aes_crypt(struct ablkcipher_request *req, u64 mode)
647{
648 struct mtk_aes_base_ctx *ctx;
649 struct mtk_aes_reqctx *rctx;
650
651 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
652 rctx = ablkcipher_request_ctx(req);
653 rctx->mode = mode;
654
655 return mtk_aes_handle_queue(ctx->cryp, !(mode & AES_FLAGS_ENCRYPT),
656 &req->base);
657}
658
659static int mtk_aes_ecb_encrypt(struct ablkcipher_request *req)
660{
661 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB);
662}
663
664static int mtk_aes_ecb_decrypt(struct ablkcipher_request *req)
665{
666 return mtk_aes_crypt(req, AES_FLAGS_ECB);
667}
668
669static int mtk_aes_cbc_encrypt(struct ablkcipher_request *req)
670{
671 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC);
672}
673
674static int mtk_aes_cbc_decrypt(struct ablkcipher_request *req)
675{
676 return mtk_aes_crypt(req, AES_FLAGS_CBC);
677}
678
679static int mtk_aes_ctr_encrypt(struct ablkcipher_request *req)
680{
681 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR);
682}
683
684static int mtk_aes_ctr_decrypt(struct ablkcipher_request *req)
685{
686 return mtk_aes_crypt(req, AES_FLAGS_CTR);
687}
688
689static int mtk_aes_cra_init(struct crypto_tfm *tfm)
690{
691 struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
692 struct mtk_cryp *cryp = NULL;
693
694 cryp = mtk_aes_find_dev(&ctx->base);
695 if (!cryp) {
696 pr_err("can't find crypto device\n");
697 return -ENODEV;
698 }
699
700 tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
701 ctx->base.start = mtk_aes_start;
702 return 0;
703}
704
705static int mtk_aes_ctr_cra_init(struct crypto_tfm *tfm)
706{
707 struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
708 struct mtk_cryp *cryp = NULL;
709
710 cryp = mtk_aes_find_dev(&ctx->base);
711 if (!cryp) {
712 pr_err("can't find crypto device\n");
713 return -ENODEV;
714 }
715
716 tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
717 ctx->base.start = mtk_aes_ctr_start;
718 return 0;
719}
720
721static struct crypto_alg aes_algs[] = {
722{
723 .cra_name = "cbc(aes)",
724 .cra_driver_name = "cbc-aes-mtk",
725 .cra_priority = 400,
726 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
727 CRYPTO_ALG_ASYNC,
728 .cra_init = mtk_aes_cra_init,
729 .cra_blocksize = AES_BLOCK_SIZE,
730 .cra_ctxsize = sizeof(struct mtk_aes_ctx),
731 .cra_alignmask = 0xf,
732 .cra_type = &crypto_ablkcipher_type,
733 .cra_module = THIS_MODULE,
734 .cra_u.ablkcipher = {
735 .min_keysize = AES_MIN_KEY_SIZE,
736 .max_keysize = AES_MAX_KEY_SIZE,
737 .setkey = mtk_aes_setkey,
738 .encrypt = mtk_aes_cbc_encrypt,
739 .decrypt = mtk_aes_cbc_decrypt,
740 .ivsize = AES_BLOCK_SIZE,
741 }
742},
743{
744 .cra_name = "ecb(aes)",
745 .cra_driver_name = "ecb-aes-mtk",
746 .cra_priority = 400,
747 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
748 CRYPTO_ALG_ASYNC,
749 .cra_init = mtk_aes_cra_init,
750 .cra_blocksize = AES_BLOCK_SIZE,
751 .cra_ctxsize = sizeof(struct mtk_aes_ctx),
752 .cra_alignmask = 0xf,
753 .cra_type = &crypto_ablkcipher_type,
754 .cra_module = THIS_MODULE,
755 .cra_u.ablkcipher = {
756 .min_keysize = AES_MIN_KEY_SIZE,
757 .max_keysize = AES_MAX_KEY_SIZE,
758 .setkey = mtk_aes_setkey,
759 .encrypt = mtk_aes_ecb_encrypt,
760 .decrypt = mtk_aes_ecb_decrypt,
761 }
762},
763{
764 .cra_name = "ctr(aes)",
765 .cra_driver_name = "ctr-aes-mtk",
766 .cra_priority = 400,
767 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
768 CRYPTO_ALG_ASYNC,
769 .cra_init = mtk_aes_ctr_cra_init,
770 .cra_blocksize = 1,
771 .cra_ctxsize = sizeof(struct mtk_aes_ctr_ctx),
772 .cra_alignmask = 0xf,
773 .cra_type = &crypto_ablkcipher_type,
774 .cra_module = THIS_MODULE,
775 .cra_u.ablkcipher = {
776 .min_keysize = AES_MIN_KEY_SIZE,
777 .max_keysize = AES_MAX_KEY_SIZE,
778 .ivsize = AES_BLOCK_SIZE,
779 .setkey = mtk_aes_setkey,
780 .encrypt = mtk_aes_ctr_encrypt,
781 .decrypt = mtk_aes_ctr_decrypt,
782 }
783},
784};
785
786static inline struct mtk_aes_gcm_ctx *
787mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx)
788{
789 return container_of(ctx, struct mtk_aes_gcm_ctx, base);
790}
791
792/* Initialize transform information of GCM mode */
793static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp,
794 struct mtk_aes_rec *aes,
795 size_t len)
796{
797 struct aead_request *req = aead_request_cast(aes->areq);
798 struct mtk_aes_base_ctx *ctx = aes->ctx;
799 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
800 const u32 *iv = (const u32 *)req->iv;
801 u32 *iv_state = ctx->tfm.state + ctx->keylen +
802 SIZE_IN_WORDS(AES_BLOCK_SIZE);
803 u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
804 int i;
805
806 ctx->ct_hdr = AES_CT_CTRL_HDR | len;
807
808 ctx->ct.cmd[0] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen);
809 ctx->ct.cmd[1] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen);
810 ctx->ct.cmd[2] = AES_GCM_CMD2;
811 ctx->ct.cmd[3] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen);
812
813 if (aes->flags & AES_FLAGS_ENCRYPT) {
814 ctx->ct.cmd[4] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize);
815 ctx->ct_size = AES_CT_SIZE_GCM_OUT;
816 ctx->tfm.ctrl[0] = AES_TFM_GCM_OUT;
817 } else {
818 ctx->ct.cmd[4] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize);
819 ctx->ct.cmd[5] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize);
820 ctx->ct_size = AES_CT_SIZE_GCM_IN;
821 ctx->tfm.ctrl[0] = AES_TFM_GCM_IN;
822 }
823
824 if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128))
825 ctx->tfm.ctrl[0] |= AES_TFM_128BITS;
826 else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256))
827 ctx->tfm.ctrl[0] |= AES_TFM_256BITS;
828 else
829 ctx->tfm.ctrl[0] |= AES_TFM_192BITS;
830
831 ctx->tfm.ctrl[0] |= AES_TFM_GHASH_DIG | AES_TFM_GHASH |
832 AES_TFM_SIZE(ctx->keylen + SIZE_IN_WORDS(
833 AES_BLOCK_SIZE + ivsize));
834 ctx->tfm.ctrl[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE |
835 AES_TFM_3IV | AES_TFM_ENC_HASH;
836
837 for (i = 0; i < SIZE_IN_WORDS(ivsize); i++)
838 iv_state[i] = cpu_to_le32(iv[i]);
839}
840
841static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
842 struct scatterlist *src, struct scatterlist *dst,
843 size_t len)
844{
845 bool src_aligned, dst_aligned;
846
847 aes->src.sg = src;
848 aes->dst.sg = dst;
849 aes->real_dst = dst;
850
851 src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
852 if (src == dst)
853 dst_aligned = src_aligned;
854 else
855 dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
856
857 if (!src_aligned || !dst_aligned) {
858 if (aes->total > AES_BUF_SIZE)
859 return -ENOMEM;
860
861 if (!src_aligned) {
862 sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
863 aes->src.sg = &aes->aligned_sg;
864 aes->src.nents = 1;
865 aes->src.remainder = 0;
866 }
867
868 if (!dst_aligned) {
869 aes->dst.sg = &aes->aligned_sg;
870 aes->dst.nents = 1;
871 aes->dst.remainder = 0;
872 }
873
874 sg_init_table(&aes->aligned_sg, 1);
875 sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);
876 }
877
878 mtk_aes_gcm_info_init(cryp, aes, len);
879
880 return mtk_aes_map(cryp, aes);
881}
882
883/* Todo: GMAC */
884static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
885{
886 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);
887 struct aead_request *req = aead_request_cast(aes->areq);
888 struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
889 u32 len = req->assoclen + req->cryptlen;
890
891 mtk_aes_set_mode(aes, rctx);
892
893 if (aes->flags & AES_FLAGS_ENCRYPT) {
894 u32 tag[4];
895 /* Compute total process length. */
896 aes->total = len + gctx->authsize;
897 /* Compute text length. */
898 gctx->textlen = req->cryptlen;
899 /* Hardware will append authenticated tag to output buffer */
900 scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1);
901 } else {
902 aes->total = len;
903 gctx->textlen = req->cryptlen - gctx->authsize;
904 }
905 aes->resume = mtk_aes_complete;
906
907 return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);
908}
909
910static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
911{
912 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
913 struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
914
915 rctx->mode = AES_FLAGS_GCM | mode;
916
917 return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT),
918 &req->base);
919}
920
921static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err)
922{
923 struct mtk_aes_gcm_setkey_result *result = req->data;
924
925 if (err == -EINPROGRESS)
926 return;
927
928 result->err = err;
929 complete(&result->completion);
930}
931
932/*
933 * Because of the hardware limitation, we need to pre-calculate key(H)
934 * for the GHASH operation. The result of the encryption operation
935 * need to be stored in the transform state buffer.
936 */
937static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
938 u32 keylen)
939{
940 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
941 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
942 struct crypto_skcipher *ctr = gctx->ctr;
943 struct {
944 u32 hash[4];
945 u8 iv[8];
946
947 struct mtk_aes_gcm_setkey_result result;
948
949 struct scatterlist sg[1];
950 struct skcipher_request req;
951 } *data;
952 const u32 *aes_key;
953 u32 *key_state, *hash_state;
954 int err, i;
955
956 if (keylen != AES_KEYSIZE_256 &&
957 keylen != AES_KEYSIZE_192 &&
958 keylen != AES_KEYSIZE_128) {
959 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
960 return -EINVAL;
961 }
962
963 key_state = ctx->tfm.state;
964 aes_key = (u32 *)key;
965 ctx->keylen = SIZE_IN_WORDS(keylen);
966
967 for (i = 0; i < ctx->keylen; i++)
968 ctx->tfm.state[i] = cpu_to_le32(aes_key[i]);
969
970 /* Same as crypto_gcm_setkey() from crypto/gcm.c */
971 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
972 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
973 CRYPTO_TFM_REQ_MASK);
974 err = crypto_skcipher_setkey(ctr, key, keylen);
975 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
976 CRYPTO_TFM_RES_MASK);
977 if (err)
978 return err;
979
980 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
981 GFP_KERNEL);
982 if (!data)
983 return -ENOMEM;
984
985 init_completion(&data->result.completion);
986 sg_init_one(data->sg, &data->hash, AES_BLOCK_SIZE);
987 skcipher_request_set_tfm(&data->req, ctr);
988 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
989 CRYPTO_TFM_REQ_MAY_BACKLOG,
990 mtk_gcm_setkey_done, &data->result);
991 skcipher_request_set_crypt(&data->req, data->sg, data->sg,
992 AES_BLOCK_SIZE, data->iv);
993
994 err = crypto_skcipher_encrypt(&data->req);
995 if (err == -EINPROGRESS || err == -EBUSY) {
996 err = wait_for_completion_interruptible(
997 &data->result.completion);
998 if (!err)
999 err = data->result.err;
1000 }
1001 if (err)
1002 goto out;
1003
1004 hash_state = key_state + ctx->keylen;
1005
1006 for (i = 0; i < 4; i++)
1007 hash_state[i] = cpu_to_be32(data->hash[i]);
1008out:
1009 kzfree(data);
1010 return err;
1011}
1012
1013static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead,
1014 u32 authsize)
1015{
1016 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
1017 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
1018
1019 /* Same as crypto_gcm_authsize() from crypto/gcm.c */
1020 switch (authsize) {
1021 case 8:
1022 case 12:
1023 case 16:
1024 break;
1025 default:
1026 return -EINVAL;
1027 }
1028
1029 gctx->authsize = authsize;
1030 return 0;
1031}
1032
1033static int mtk_aes_gcm_encrypt(struct aead_request *req)
1034{
1035 return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
1036}
1037
1038static int mtk_aes_gcm_decrypt(struct aead_request *req)
1039{
1040 return mtk_aes_gcm_crypt(req, 0);
1041}
1042
1043static int mtk_aes_gcm_init(struct crypto_aead *aead)
1044{
1045 struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
1046 struct mtk_cryp *cryp = NULL;
1047
1048 cryp = mtk_aes_find_dev(&ctx->base);
1049 if (!cryp) {
1050 pr_err("can't find crypto device\n");
1051 return -ENODEV;
1052 }
1053
1054 ctx->ctr = crypto_alloc_skcipher("ctr(aes)", 0,
1055 CRYPTO_ALG_ASYNC);
1056 if (IS_ERR(ctx->ctr)) {
1057 pr_err("Error allocating ctr(aes)\n");
1058 return PTR_ERR(ctx->ctr);
1059 }
1060
1061 crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx));
1062 ctx->base.start = mtk_aes_gcm_start;
1063 return 0;
1064}
1065
1066static void mtk_aes_gcm_exit(struct crypto_aead *aead)
1067{
1068 struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
1069
1070 crypto_free_skcipher(ctx->ctr);
1071}
1072
1073static struct aead_alg aes_gcm_alg = {
1074 .setkey = mtk_aes_gcm_setkey,
1075 .setauthsize = mtk_aes_gcm_setauthsize,
1076 .encrypt = mtk_aes_gcm_encrypt,
1077 .decrypt = mtk_aes_gcm_decrypt,
1078 .init = mtk_aes_gcm_init,
1079 .exit = mtk_aes_gcm_exit,
1080 .ivsize = 12,
1081 .maxauthsize = AES_BLOCK_SIZE,
1082
1083 .base = {
1084 .cra_name = "gcm(aes)",
1085 .cra_driver_name = "gcm-aes-mtk",
1086 .cra_priority = 400,
1087 .cra_flags = CRYPTO_ALG_ASYNC,
1088 .cra_blocksize = 1,
1089 .cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx),
1090 .cra_alignmask = 0xf,
1091 .cra_module = THIS_MODULE,
1092 },
1093};
1094
1095static void mtk_aes_enc_task(unsigned long data)
1096{
1097 struct mtk_cryp *cryp = (struct mtk_cryp *)data;
1098 struct mtk_aes_rec *aes = cryp->aes[0];
1099
1100 mtk_aes_unmap(cryp, aes);
1101 aes->resume(cryp, aes);
1102}
1103
1104static void mtk_aes_dec_task(unsigned long data)
1105{
1106 struct mtk_cryp *cryp = (struct mtk_cryp *)data;
1107 struct mtk_aes_rec *aes = cryp->aes[1];
1108
1109 mtk_aes_unmap(cryp, aes);
1110 aes->resume(cryp, aes);
1111}
1112
1113static irqreturn_t mtk_aes_enc_irq(int irq, void *dev_id)
1114{
1115 struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id;
1116 struct mtk_aes_rec *aes = cryp->aes[0];
1117 u32 val = mtk_aes_read(cryp, RDR_STAT(RING0));
1118
1119 mtk_aes_write(cryp, RDR_STAT(RING0), val);
1120
1121 if (likely(AES_FLAGS_BUSY & aes->flags)) {
1122 mtk_aes_write(cryp, RDR_PROC_COUNT(RING0), MTK_CNT_RST);
1123 mtk_aes_write(cryp, RDR_THRESH(RING0),
1124 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1125
1126 tasklet_schedule(&aes->task);
1127 } else {
1128 dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
1129 }
1130 return IRQ_HANDLED;
1131}
1132
1133static irqreturn_t mtk_aes_dec_irq(int irq, void *dev_id)
1134{
1135 struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id;
1136 struct mtk_aes_rec *aes = cryp->aes[1];
1137 u32 val = mtk_aes_read(cryp, RDR_STAT(RING1));
1138
1139 mtk_aes_write(cryp, RDR_STAT(RING1), val);
1140
1141 if (likely(AES_FLAGS_BUSY & aes->flags)) {
1142 mtk_aes_write(cryp, RDR_PROC_COUNT(RING1), MTK_CNT_RST);
1143 mtk_aes_write(cryp, RDR_THRESH(RING1),
1144 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1145
1146 tasklet_schedule(&aes->task);
1147 } else {
1148 dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
1149 }
1150 return IRQ_HANDLED;
1151}
1152
1153/*
1154 * The purpose of creating encryption and decryption records is
1155 * to process outbound/inbound data in parallel, it can improve
1156 * performance in most use cases, such as IPSec VPN, especially
1157 * under heavy network traffic.
1158 */
1159static int mtk_aes_record_init(struct mtk_cryp *cryp)
1160{
1161 struct mtk_aes_rec **aes = cryp->aes;
1162 int i, err = -ENOMEM;
1163
1164 for (i = 0; i < MTK_REC_NUM; i++) {
1165 aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL);
1166 if (!aes[i])
1167 goto err_cleanup;
1168
1169 aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL,
1170 AES_BUF_ORDER);
1171 if (!aes[i]->buf)
1172 goto err_cleanup;
1173
1174 aes[i]->id = i;
1175
1176 spin_lock_init(&aes[i]->lock);
1177 crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE);
1178 }
1179
1180 tasklet_init(&aes[0]->task, mtk_aes_enc_task, (unsigned long)cryp);
1181 tasklet_init(&aes[1]->task, mtk_aes_dec_task, (unsigned long)cryp);
1182
1183 return 0;
1184
1185err_cleanup:
1186 for (; i--; ) {
1187 free_page((unsigned long)aes[i]->buf);
1188 kfree(aes[i]);
1189 }
1190
1191 return err;
1192}
1193
1194static void mtk_aes_record_free(struct mtk_cryp *cryp)
1195{
1196 int i;
1197
1198 for (i = 0; i < MTK_REC_NUM; i++) {
1199 tasklet_kill(&cryp->aes[i]->task);
1200 free_page((unsigned long)cryp->aes[i]->buf);
1201 kfree(cryp->aes[i]);
1202 }
1203}
1204
1205static void mtk_aes_unregister_algs(void)
1206{
1207 int i;
1208
1209 crypto_unregister_aead(&aes_gcm_alg);
1210
1211 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1212 crypto_unregister_alg(&aes_algs[i]);
1213}
1214
1215static int mtk_aes_register_algs(void)
1216{
1217 int err, i;
1218
1219 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1220 err = crypto_register_alg(&aes_algs[i]);
1221 if (err)
1222 goto err_aes_algs;
1223 }
1224
1225 err = crypto_register_aead(&aes_gcm_alg);
1226 if (err)
1227 goto err_aes_algs;
1228
1229 return 0;
1230
1231err_aes_algs:
1232 for (; i--; )
1233 crypto_unregister_alg(&aes_algs[i]);
1234
1235 return err;
1236}
1237
1238int mtk_cipher_alg_register(struct mtk_cryp *cryp)
1239{
1240 int ret;
1241
1242 INIT_LIST_HEAD(&cryp->aes_list);
1243
1244 /* Initialize two cipher records */
1245 ret = mtk_aes_record_init(cryp);
1246 if (ret)
1247 goto err_record;
1248
1249 /* Ring0 is use by encryption record */
1250 ret = devm_request_irq(cryp->dev, cryp->irq[RING0], mtk_aes_enc_irq,
1251 IRQF_TRIGGER_LOW, "mtk-aes", cryp);
1252 if (ret) {
1253 dev_err(cryp->dev, "unable to request AES encryption irq.\n");
1254 goto err_res;
1255 }
1256
1257 /* Ring1 is use by decryption record */
1258 ret = devm_request_irq(cryp->dev, cryp->irq[RING1], mtk_aes_dec_irq,
1259 IRQF_TRIGGER_LOW, "mtk-aes", cryp);
1260 if (ret) {
1261 dev_err(cryp->dev, "unable to request AES decryption irq.\n");
1262 goto err_res;
1263 }
1264
1265 /* Enable ring0 and ring1 interrupt */
1266 mtk_aes_write(cryp, AIC_ENABLE_SET(RING0), MTK_IRQ_RDR0);
1267 mtk_aes_write(cryp, AIC_ENABLE_SET(RING1), MTK_IRQ_RDR1);
1268
1269 spin_lock(&mtk_aes.lock);
1270 list_add_tail(&cryp->aes_list, &mtk_aes.dev_list);
1271 spin_unlock(&mtk_aes.lock);
1272
1273 ret = mtk_aes_register_algs();
1274 if (ret)
1275 goto err_algs;
1276
1277 return 0;
1278
1279err_algs:
1280 spin_lock(&mtk_aes.lock);
1281 list_del(&cryp->aes_list);
1282 spin_unlock(&mtk_aes.lock);
1283err_res:
1284 mtk_aes_record_free(cryp);
1285err_record:
1286
1287 dev_err(cryp->dev, "mtk-aes initialization failed.\n");
1288 return ret;
1289}
1290
1291void mtk_cipher_alg_release(struct mtk_cryp *cryp)
1292{
1293 spin_lock(&mtk_aes.lock);
1294 list_del(&cryp->aes_list);
1295 spin_unlock(&mtk_aes.lock);
1296
1297 mtk_aes_unregister_algs();
1298 mtk_aes_record_free(cryp);
1299}
diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c
new file mode 100644
index 000000000000..a9c713d4c733
--- /dev/null
+++ b/drivers/crypto/mediatek/mtk-platform.c
@@ -0,0 +1,604 @@
1/*
2 * Driver for EIP97 cryptographic accelerator.
3 *
4 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 */
11
12#include <linux/clk.h>
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/platform_device.h>
17#include <linux/pm_runtime.h>
18#include "mtk-platform.h"
19
20#define MTK_BURST_SIZE_MSK GENMASK(7, 4)
21#define MTK_BURST_SIZE(x) ((x) << 4)
22#define MTK_DESC_SIZE(x) ((x) << 0)
23#define MTK_DESC_OFFSET(x) ((x) << 16)
24#define MTK_DESC_FETCH_SIZE(x) ((x) << 0)
25#define MTK_DESC_FETCH_THRESH(x) ((x) << 16)
26#define MTK_DESC_OVL_IRQ_EN BIT(25)
27#define MTK_DESC_ATP_PRESENT BIT(30)
28
29#define MTK_DFSE_IDLE GENMASK(3, 0)
30#define MTK_DFSE_THR_CTRL_EN BIT(30)
31#define MTK_DFSE_THR_CTRL_RESET BIT(31)
32#define MTK_DFSE_RING_ID(x) (((x) >> 12) & GENMASK(3, 0))
33#define MTK_DFSE_MIN_DATA(x) ((x) << 0)
34#define MTK_DFSE_MAX_DATA(x) ((x) << 8)
35#define MTK_DFE_MIN_CTRL(x) ((x) << 16)
36#define MTK_DFE_MAX_CTRL(x) ((x) << 24)
37
38#define MTK_IN_BUF_MIN_THRESH(x) ((x) << 8)
39#define MTK_IN_BUF_MAX_THRESH(x) ((x) << 12)
40#define MTK_OUT_BUF_MIN_THRESH(x) ((x) << 0)
41#define MTK_OUT_BUF_MAX_THRESH(x) ((x) << 4)
42#define MTK_IN_TBUF_SIZE(x) (((x) >> 4) & GENMASK(3, 0))
43#define MTK_IN_DBUF_SIZE(x) (((x) >> 8) & GENMASK(3, 0))
44#define MTK_OUT_DBUF_SIZE(x) (((x) >> 16) & GENMASK(3, 0))
45#define MTK_CMD_FIFO_SIZE(x) (((x) >> 8) & GENMASK(3, 0))
46#define MTK_RES_FIFO_SIZE(x) (((x) >> 12) & GENMASK(3, 0))
47
48#define MTK_PE_TK_LOC_AVL BIT(2)
49#define MTK_PE_PROC_HELD BIT(14)
50#define MTK_PE_TK_TIMEOUT_EN BIT(22)
51#define MTK_PE_INPUT_DMA_ERR BIT(0)
52#define MTK_PE_OUTPUT_DMA_ERR BIT(1)
53#define MTK_PE_PKT_PORC_ERR BIT(2)
54#define MTK_PE_PKT_TIMEOUT BIT(3)
55#define MTK_PE_FATAL_ERR BIT(14)
56#define MTK_PE_INPUT_DMA_ERR_EN BIT(16)
57#define MTK_PE_OUTPUT_DMA_ERR_EN BIT(17)
58#define MTK_PE_PKT_PORC_ERR_EN BIT(18)
59#define MTK_PE_PKT_TIMEOUT_EN BIT(19)
60#define MTK_PE_FATAL_ERR_EN BIT(30)
61#define MTK_PE_INT_OUT_EN BIT(31)
62
63#define MTK_HIA_SIGNATURE ((u16)0x35ca)
64#define MTK_HIA_DATA_WIDTH(x) (((x) >> 25) & GENMASK(1, 0))
65#define MTK_HIA_DMA_LENGTH(x) (((x) >> 20) & GENMASK(4, 0))
66#define MTK_CDR_STAT_CLR GENMASK(4, 0)
67#define MTK_RDR_STAT_CLR GENMASK(7, 0)
68
69#define MTK_AIC_INT_MSK GENMASK(5, 0)
70#define MTK_AIC_VER_MSK (GENMASK(15, 0) | GENMASK(27, 20))
71#define MTK_AIC_VER11 0x011036c9
72#define MTK_AIC_VER12 0x012036c9
73#define MTK_AIC_G_CLR GENMASK(30, 20)
74
75/**
76 * EIP97 is an integrated security subsystem to accelerate cryptographic
77 * functions and protocols to offload the host processor.
78 * Some important hardware modules are briefly introduced below:
79 *
80 * Host Interface Adapter(HIA) - the main interface between the host
81 * system and the hardware subsystem. It is responsible for attaching
82 * processing engine to the specific host bus interface and provides a
83 * standardized software view for off loading tasks to the engine.
84 *
85 * Command Descriptor Ring Manager(CDR Manager) - keeps track of how many
86 * CD the host has prepared in the CDR. It monitors the fill level of its
87 * CD-FIFO and if there's sufficient space for the next block of descriptors,
88 * then it fires off a DMA request to fetch a block of CDs.
89 *
90 * Data fetch engine(DFE) - It is responsible for parsing the CD and
91 * setting up the required control and packet data DMA transfers from
92 * system memory to the processing engine.
93 *
94 * Result Descriptor Ring Manager(RDR Manager) - same as CDR Manager,
95 * but target is result descriptors, Moreover, it also handles the RD
96 * updates under control of the DSE. For each packet data segment
97 * processed, the DSE triggers the RDR Manager to write the updated RD.
98 * If triggered to update, the RDR Manager sets up a DMA operation to
99 * copy the RD from the DSE to the correct location in the RDR.
100 *
101 * Data Store Engine(DSE) - It is responsible for parsing the prepared RD
102 * and setting up the required control and packet data DMA transfers from
103 * the processing engine to system memory.
104 *
105 * Advanced Interrupt Controllers(AICs) - receive interrupt request signals
106 * from various sources and combine them into one interrupt output.
107 * The AICs are used by:
108 * - One for the HIA global and processing engine interrupts.
109 * - The others for the descriptor ring interrupts.
110 */
111
112/* Cryptographic engine capabilities */
113struct mtk_sys_cap {
114 /* host interface adapter */
115 u32 hia_ver;
116 u32 hia_opt;
117 /* packet engine */
118 u32 pkt_eng_opt;
119 /* global hardware */
120 u32 hw_opt;
121};
122
123static void mtk_desc_ring_link(struct mtk_cryp *cryp, u32 mask)
124{
125 /* Assign rings to DFE/DSE thread and enable it */
126 writel(MTK_DFSE_THR_CTRL_EN | mask, cryp->base + DFE_THR_CTRL);
127 writel(MTK_DFSE_THR_CTRL_EN | mask, cryp->base + DSE_THR_CTRL);
128}
129
130static void mtk_dfe_dse_buf_setup(struct mtk_cryp *cryp,
131 struct mtk_sys_cap *cap)
132{
133 u32 width = MTK_HIA_DATA_WIDTH(cap->hia_opt) + 2;
134 u32 len = MTK_HIA_DMA_LENGTH(cap->hia_opt) - 1;
135 u32 ipbuf = min((u32)MTK_IN_DBUF_SIZE(cap->hw_opt) + width, len);
136 u32 opbuf = min((u32)MTK_OUT_DBUF_SIZE(cap->hw_opt) + width, len);
137 u32 itbuf = min((u32)MTK_IN_TBUF_SIZE(cap->hw_opt) + width, len);
138
139 writel(MTK_DFSE_MIN_DATA(ipbuf - 1) |
140 MTK_DFSE_MAX_DATA(ipbuf) |
141 MTK_DFE_MIN_CTRL(itbuf - 1) |
142 MTK_DFE_MAX_CTRL(itbuf),
143 cryp->base + DFE_CFG);
144
145 writel(MTK_DFSE_MIN_DATA(opbuf - 1) |
146 MTK_DFSE_MAX_DATA(opbuf),
147 cryp->base + DSE_CFG);
148
149 writel(MTK_IN_BUF_MIN_THRESH(ipbuf - 1) |
150 MTK_IN_BUF_MAX_THRESH(ipbuf),
151 cryp->base + PE_IN_DBUF_THRESH);
152
153 writel(MTK_IN_BUF_MIN_THRESH(itbuf - 1) |
154 MTK_IN_BUF_MAX_THRESH(itbuf),
155 cryp->base + PE_IN_TBUF_THRESH);
156
157 writel(MTK_OUT_BUF_MIN_THRESH(opbuf - 1) |
158 MTK_OUT_BUF_MAX_THRESH(opbuf),
159 cryp->base + PE_OUT_DBUF_THRESH);
160
161 writel(0, cryp->base + PE_OUT_TBUF_THRESH);
162 writel(0, cryp->base + PE_OUT_BUF_CTRL);
163}
164
165static int mtk_dfe_dse_state_check(struct mtk_cryp *cryp)
166{
167 int ret = -EINVAL;
168 u32 val;
169
170 /* Check for completion of all DMA transfers */
171 val = readl(cryp->base + DFE_THR_STAT);
172 if (MTK_DFSE_RING_ID(val) == MTK_DFSE_IDLE) {
173 val = readl(cryp->base + DSE_THR_STAT);
174 if (MTK_DFSE_RING_ID(val) == MTK_DFSE_IDLE)
175 ret = 0;
176 }
177
178 if (!ret) {
179 /* Take DFE/DSE thread out of reset */
180 writel(0, cryp->base + DFE_THR_CTRL);
181 writel(0, cryp->base + DSE_THR_CTRL);
182 } else {
183 return -EBUSY;
184 }
185
186 return 0;
187}
188
189static int mtk_dfe_dse_reset(struct mtk_cryp *cryp)
190{
191 int err;
192
193 /* Reset DSE/DFE and correct system priorities for all rings. */
194 writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DFE_THR_CTRL);
195 writel(0, cryp->base + DFE_PRIO_0);
196 writel(0, cryp->base + DFE_PRIO_1);
197 writel(0, cryp->base + DFE_PRIO_2);
198 writel(0, cryp->base + DFE_PRIO_3);
199
200 writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DSE_THR_CTRL);
201 writel(0, cryp->base + DSE_PRIO_0);
202 writel(0, cryp->base + DSE_PRIO_1);
203 writel(0, cryp->base + DSE_PRIO_2);
204 writel(0, cryp->base + DSE_PRIO_3);
205
206 err = mtk_dfe_dse_state_check(cryp);
207 if (err)
208 return err;
209
210 return 0;
211}
212
213static void mtk_cmd_desc_ring_setup(struct mtk_cryp *cryp,
214 int i, struct mtk_sys_cap *cap)
215{
216 /* Full descriptor that fits FIFO minus one */
217 u32 count =
218 ((1 << MTK_CMD_FIFO_SIZE(cap->hia_opt)) / MTK_DESC_SZ) - 1;
219
220 /* Temporarily disable external triggering */
221 writel(0, cryp->base + CDR_CFG(i));
222
223 /* Clear CDR count */
224 writel(MTK_CNT_RST, cryp->base + CDR_PREP_COUNT(i));
225 writel(MTK_CNT_RST, cryp->base + CDR_PROC_COUNT(i));
226
227 writel(0, cryp->base + CDR_PREP_PNTR(i));
228 writel(0, cryp->base + CDR_PROC_PNTR(i));
229 writel(0, cryp->base + CDR_DMA_CFG(i));
230
231 /* Configure CDR host address space */
232 writel(0, cryp->base + CDR_BASE_ADDR_HI(i));
233 writel(cryp->ring[i]->cmd_dma, cryp->base + CDR_BASE_ADDR_LO(i));
234
235 writel(MTK_DESC_RING_SZ, cryp->base + CDR_RING_SIZE(i));
236
237 /* Clear and disable all CDR interrupts */
238 writel(MTK_CDR_STAT_CLR, cryp->base + CDR_STAT(i));
239
240 /*
241 * Set command descriptor offset and enable additional
242 * token present in descriptor.
243 */
244 writel(MTK_DESC_SIZE(MTK_DESC_SZ) |
245 MTK_DESC_OFFSET(MTK_DESC_OFF) |
246 MTK_DESC_ATP_PRESENT,
247 cryp->base + CDR_DESC_SIZE(i));
248
249 writel(MTK_DESC_FETCH_SIZE(count * MTK_DESC_OFF) |
250 MTK_DESC_FETCH_THRESH(count * MTK_DESC_SZ),
251 cryp->base + CDR_CFG(i));
252}
253
254static void mtk_res_desc_ring_setup(struct mtk_cryp *cryp,
255 int i, struct mtk_sys_cap *cap)
256{
257 u32 rndup = 2;
258 u32 count = ((1 << MTK_RES_FIFO_SIZE(cap->hia_opt)) / rndup) - 1;
259
260 /* Temporarily disable external triggering */
261 writel(0, cryp->base + RDR_CFG(i));
262
263 /* Clear RDR count */
264 writel(MTK_CNT_RST, cryp->base + RDR_PREP_COUNT(i));
265 writel(MTK_CNT_RST, cryp->base + RDR_PROC_COUNT(i));
266
267 writel(0, cryp->base + RDR_PREP_PNTR(i));
268 writel(0, cryp->base + RDR_PROC_PNTR(i));
269 writel(0, cryp->base + RDR_DMA_CFG(i));
270
271 /* Configure RDR host address space */
272 writel(0, cryp->base + RDR_BASE_ADDR_HI(i));
273 writel(cryp->ring[i]->res_dma, cryp->base + RDR_BASE_ADDR_LO(i));
274
275 writel(MTK_DESC_RING_SZ, cryp->base + RDR_RING_SIZE(i));
276 writel(MTK_RDR_STAT_CLR, cryp->base + RDR_STAT(i));
277
278 /*
279 * RDR manager generates update interrupts on a per-completed-packet,
280 * and the rd_proc_thresh_irq interrupt is fired when proc_pkt_count
281 * for the RDR exceeds the number of packets.
282 */
283 writel(MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE,
284 cryp->base + RDR_THRESH(i));
285
286 /*
287 * Configure a threshold and time-out value for the processed
288 * result descriptors (or complete packets) that are written to
289 * the RDR.
290 */
291 writel(MTK_DESC_SIZE(MTK_DESC_SZ) | MTK_DESC_OFFSET(MTK_DESC_OFF),
292 cryp->base + RDR_DESC_SIZE(i));
293
294 /*
295 * Configure HIA fetch size and fetch threshold that are used to
296 * fetch blocks of multiple descriptors.
297 */
298 writel(MTK_DESC_FETCH_SIZE(count * MTK_DESC_OFF) |
299 MTK_DESC_FETCH_THRESH(count * rndup) |
300 MTK_DESC_OVL_IRQ_EN,
301 cryp->base + RDR_CFG(i));
302}
303
304static int mtk_packet_engine_setup(struct mtk_cryp *cryp)
305{
306 struct mtk_sys_cap cap;
307 int i, err;
308 u32 val;
309
310 cap.hia_ver = readl(cryp->base + HIA_VERSION);
311 cap.hia_opt = readl(cryp->base + HIA_OPTIONS);
312 cap.hw_opt = readl(cryp->base + EIP97_OPTIONS);
313
314 if (!(((u16)cap.hia_ver) == MTK_HIA_SIGNATURE))
315 return -EINVAL;
316
317 /* Configure endianness conversion method for master (DMA) interface */
318 writel(0, cryp->base + EIP97_MST_CTRL);
319
320 /* Set HIA burst size */
321 val = readl(cryp->base + HIA_MST_CTRL);
322 val &= ~MTK_BURST_SIZE_MSK;
323 val |= MTK_BURST_SIZE(5);
324 writel(val, cryp->base + HIA_MST_CTRL);
325
326 err = mtk_dfe_dse_reset(cryp);
327 if (err) {
328 dev_err(cryp->dev, "Failed to reset DFE and DSE.\n");
329 return err;
330 }
331
332 mtk_dfe_dse_buf_setup(cryp, &cap);
333
334 /* Enable the 4 rings for the packet engines. */
335 mtk_desc_ring_link(cryp, 0xf);
336
337 for (i = 0; i < RING_MAX; i++) {
338 mtk_cmd_desc_ring_setup(cryp, i, &cap);
339 mtk_res_desc_ring_setup(cryp, i, &cap);
340 }
341
342 writel(MTK_PE_TK_LOC_AVL | MTK_PE_PROC_HELD | MTK_PE_TK_TIMEOUT_EN,
343 cryp->base + PE_TOKEN_CTRL_STAT);
344
345 /* Clear all pending interrupts */
346 writel(MTK_AIC_G_CLR, cryp->base + AIC_G_ACK);
347 writel(MTK_PE_INPUT_DMA_ERR | MTK_PE_OUTPUT_DMA_ERR |
348 MTK_PE_PKT_PORC_ERR | MTK_PE_PKT_TIMEOUT |
349 MTK_PE_FATAL_ERR | MTK_PE_INPUT_DMA_ERR_EN |
350 MTK_PE_OUTPUT_DMA_ERR_EN | MTK_PE_PKT_PORC_ERR_EN |
351 MTK_PE_PKT_TIMEOUT_EN | MTK_PE_FATAL_ERR_EN |
352 MTK_PE_INT_OUT_EN,
353 cryp->base + PE_INTERRUPT_CTRL_STAT);
354
355 return 0;
356}
357
358static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw)
359{
360 u32 val;
361
362 if (hw == RING_MAX)
363 val = readl(cryp->base + AIC_G_VERSION);
364 else
365 val = readl(cryp->base + AIC_VERSION(hw));
366
367 val &= MTK_AIC_VER_MSK;
368 if (val != MTK_AIC_VER11 && val != MTK_AIC_VER12)
369 return -ENXIO;
370
371 if (hw == RING_MAX)
372 val = readl(cryp->base + AIC_G_OPTIONS);
373 else
374 val = readl(cryp->base + AIC_OPTIONS(hw));
375
376 val &= MTK_AIC_INT_MSK;
377 if (!val || val > 32)
378 return -ENXIO;
379
380 return 0;
381}
382
383static int mtk_aic_init(struct mtk_cryp *cryp, int hw)
384{
385 int err;
386
387 err = mtk_aic_cap_check(cryp, hw);
388 if (err)
389 return err;
390
391 /* Disable all interrupts and set initial configuration */
392 if (hw == RING_MAX) {
393 writel(0, cryp->base + AIC_G_ENABLE_CTRL);
394 writel(0, cryp->base + AIC_G_POL_CTRL);
395 writel(0, cryp->base + AIC_G_TYPE_CTRL);
396 writel(0, cryp->base + AIC_G_ENABLE_SET);
397 } else {
398 writel(0, cryp->base + AIC_ENABLE_CTRL(hw));
399 writel(0, cryp->base + AIC_POL_CTRL(hw));
400 writel(0, cryp->base + AIC_TYPE_CTRL(hw));
401 writel(0, cryp->base + AIC_ENABLE_SET(hw));
402 }
403
404 return 0;
405}
406
407static int mtk_accelerator_init(struct mtk_cryp *cryp)
408{
409 int i, err;
410
411 /* Initialize advanced interrupt controller(AIC) */
412 for (i = 0; i < MTK_IRQ_NUM; i++) {
413 err = mtk_aic_init(cryp, i);
414 if (err) {
415 dev_err(cryp->dev, "Failed to initialize AIC.\n");
416 return err;
417 }
418 }
419
420 /* Initialize packet engine */
421 err = mtk_packet_engine_setup(cryp);
422 if (err) {
423 dev_err(cryp->dev, "Failed to configure packet engine.\n");
424 return err;
425 }
426
427 return 0;
428}
429
430static void mtk_desc_dma_free(struct mtk_cryp *cryp)
431{
432 int i;
433
434 for (i = 0; i < RING_MAX; i++) {
435 dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ,
436 cryp->ring[i]->res_base,
437 cryp->ring[i]->res_dma);
438 dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ,
439 cryp->ring[i]->cmd_base,
440 cryp->ring[i]->cmd_dma);
441 kfree(cryp->ring[i]);
442 }
443}
444
445static int mtk_desc_ring_alloc(struct mtk_cryp *cryp)
446{
447 struct mtk_ring **ring = cryp->ring;
448 int i, err = ENOMEM;
449
450 for (i = 0; i < RING_MAX; i++) {
451 ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL);
452 if (!ring[i])
453 goto err_cleanup;
454
455 ring[i]->cmd_base = dma_zalloc_coherent(cryp->dev,
456 MTK_DESC_RING_SZ,
457 &ring[i]->cmd_dma,
458 GFP_KERNEL);
459 if (!ring[i]->cmd_base)
460 goto err_cleanup;
461
462 ring[i]->res_base = dma_zalloc_coherent(cryp->dev,
463 MTK_DESC_RING_SZ,
464 &ring[i]->res_dma,
465 GFP_KERNEL);
466 if (!ring[i]->res_base)
467 goto err_cleanup;
468 }
469 return 0;
470
471err_cleanup:
472 for (; i--; ) {
473 dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ,
474 ring[i]->res_base, ring[i]->res_dma);
475 dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ,
476 ring[i]->cmd_base, ring[i]->cmd_dma);
477 kfree(ring[i]);
478 }
479 return err;
480}
481
482static int mtk_crypto_probe(struct platform_device *pdev)
483{
484 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
485 struct mtk_cryp *cryp;
486 int i, err;
487
488 cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL);
489 if (!cryp)
490 return -ENOMEM;
491
492 cryp->base = devm_ioremap_resource(&pdev->dev, res);
493 if (IS_ERR(cryp->base))
494 return PTR_ERR(cryp->base);
495
496 for (i = 0; i < MTK_IRQ_NUM; i++) {
497 cryp->irq[i] = platform_get_irq(pdev, i);
498 if (cryp->irq[i] < 0) {
499 dev_err(cryp->dev, "no IRQ:%d resource info\n", i);
500 return -ENXIO;
501 }
502 }
503
504 cryp->clk_ethif = devm_clk_get(&pdev->dev, "ethif");
505 cryp->clk_cryp = devm_clk_get(&pdev->dev, "cryp");
506 if (IS_ERR(cryp->clk_ethif) || IS_ERR(cryp->clk_cryp))
507 return -EPROBE_DEFER;
508
509 cryp->dev = &pdev->dev;
510 pm_runtime_enable(cryp->dev);
511 pm_runtime_get_sync(cryp->dev);
512
513 err = clk_prepare_enable(cryp->clk_ethif);
514 if (err)
515 goto err_clk_ethif;
516
517 err = clk_prepare_enable(cryp->clk_cryp);
518 if (err)
519 goto err_clk_cryp;
520
521 /* Allocate four command/result descriptor rings */
522 err = mtk_desc_ring_alloc(cryp);
523 if (err) {
524 dev_err(cryp->dev, "Unable to allocate descriptor rings.\n");
525 goto err_resource;
526 }
527
528 /* Initialize hardware modules */
529 err = mtk_accelerator_init(cryp);
530 if (err) {
531 dev_err(cryp->dev, "Failed to initialize cryptographic engine.\n");
532 goto err_engine;
533 }
534
535 err = mtk_cipher_alg_register(cryp);
536 if (err) {
537 dev_err(cryp->dev, "Unable to register cipher algorithm.\n");
538 goto err_cipher;
539 }
540
541 err = mtk_hash_alg_register(cryp);
542 if (err) {
543 dev_err(cryp->dev, "Unable to register hash algorithm.\n");
544 goto err_hash;
545 }
546
547 platform_set_drvdata(pdev, cryp);
548 return 0;
549
550err_hash:
551 mtk_cipher_alg_release(cryp);
552err_cipher:
553 mtk_dfe_dse_reset(cryp);
554err_engine:
555 mtk_desc_dma_free(cryp);
556err_resource:
557 clk_disable_unprepare(cryp->clk_cryp);
558err_clk_cryp:
559 clk_disable_unprepare(cryp->clk_ethif);
560err_clk_ethif:
561 pm_runtime_put_sync(cryp->dev);
562 pm_runtime_disable(cryp->dev);
563
564 return err;
565}
566
567static int mtk_crypto_remove(struct platform_device *pdev)
568{
569 struct mtk_cryp *cryp = platform_get_drvdata(pdev);
570
571 mtk_hash_alg_release(cryp);
572 mtk_cipher_alg_release(cryp);
573 mtk_desc_dma_free(cryp);
574
575 clk_disable_unprepare(cryp->clk_cryp);
576 clk_disable_unprepare(cryp->clk_ethif);
577
578 pm_runtime_put_sync(cryp->dev);
579 pm_runtime_disable(cryp->dev);
580 platform_set_drvdata(pdev, NULL);
581
582 return 0;
583}
584
585static const struct of_device_id of_crypto_id[] = {
586 { .compatible = "mediatek,eip97-crypto" },
587 {},
588};
589MODULE_DEVICE_TABLE(of, of_crypto_id);
590
591static struct platform_driver mtk_crypto_driver = {
592 .probe = mtk_crypto_probe,
593 .remove = mtk_crypto_remove,
594 .driver = {
595 .name = "mtk-crypto",
596 .owner = THIS_MODULE,
597 .of_match_table = of_crypto_id,
598 },
599};
600module_platform_driver(mtk_crypto_driver);
601
602MODULE_LICENSE("GPL");
603MODULE_AUTHOR("Ryder Lee <ryder.lee@mediatek.com>");
604MODULE_DESCRIPTION("Cryptographic accelerator driver for EIP97");
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
new file mode 100644
index 000000000000..ed6d8717f7f4
--- /dev/null
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -0,0 +1,231 @@
1/*
2 * Driver for EIP97 cryptographic accelerator.
3 *
4 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 */
11
12#ifndef __MTK_PLATFORM_H_
13#define __MTK_PLATFORM_H_
14
15#include <crypto/algapi.h>
16#include <crypto/internal/aead.h>
17#include <crypto/internal/hash.h>
18#include <crypto/scatterwalk.h>
19#include <crypto/skcipher.h>
20#include <linux/crypto.h>
21#include <linux/dma-mapping.h>
22#include <linux/interrupt.h>
23#include <linux/scatterlist.h>
24#include "mtk-regs.h"
25
26#define MTK_RDR_PROC_THRESH BIT(0)
27#define MTK_RDR_PROC_MODE BIT(23)
28#define MTK_CNT_RST BIT(31)
29#define MTK_IRQ_RDR0 BIT(1)
30#define MTK_IRQ_RDR1 BIT(3)
31#define MTK_IRQ_RDR2 BIT(5)
32#define MTK_IRQ_RDR3 BIT(7)
33
34#define SIZE_IN_WORDS(x) ((x) >> 2)
35
36/**
37 * Ring 0/1 are used by AES encrypt and decrypt.
38 * Ring 2/3 are used by SHA.
39 */
40enum {
41 RING0 = 0,
42 RING1,
43 RING2,
44 RING3,
45 RING_MAX,
46};
47
48#define MTK_REC_NUM (RING_MAX / 2)
49#define MTK_IRQ_NUM 5
50
51/**
52 * struct mtk_desc - DMA descriptor
53 * @hdr: the descriptor control header
54 * @buf: DMA address of input buffer segment
55 * @ct: DMA address of command token that control operation flow
56 * @ct_hdr: the command token control header
57 * @tag: the user-defined field
58 * @tfm: DMA address of transform state
59 * @bound: align descriptors offset boundary
60 *
61 * Structure passed to the crypto engine to describe where source
62 * data needs to be fetched and how it needs to be processed.
63 */
64struct mtk_desc {
65 __le32 hdr;
66 __le32 buf;
67 __le32 ct;
68 __le32 ct_hdr;
69 __le32 tag;
70 __le32 tfm;
71 __le32 bound[2];
72};
73
74#define MTK_DESC_NUM 512
75#define MTK_DESC_OFF SIZE_IN_WORDS(sizeof(struct mtk_desc))
76#define MTK_DESC_SZ (MTK_DESC_OFF - 2)
77#define MTK_DESC_RING_SZ ((sizeof(struct mtk_desc) * MTK_DESC_NUM))
78#define MTK_DESC_CNT(x) ((MTK_DESC_OFF * (x)) << 2)
79#define MTK_DESC_LAST cpu_to_le32(BIT(22))
80#define MTK_DESC_FIRST cpu_to_le32(BIT(23))
81#define MTK_DESC_BUF_LEN(x) cpu_to_le32(x)
82#define MTK_DESC_CT_LEN(x) cpu_to_le32((x) << 24)
83
84/**
85 * struct mtk_ring - Descriptor ring
86 * @cmd_base: pointer to command descriptor ring base
87 * @cmd_dma: DMA address of command descriptor ring
88 * @cmd_pos: current position in the command descriptor ring
89 * @res_base: pointer to result descriptor ring base
90 * @res_dma: DMA address of result descriptor ring
91 * @res_pos: current position in the result descriptor ring
92 *
93 * A descriptor ring is a circular buffer that is used to manage
94 * one or more descriptors. There are two type of descriptor rings;
95 * the command descriptor ring and result descriptor ring.
96 */
97struct mtk_ring {
98 struct mtk_desc *cmd_base;
99 dma_addr_t cmd_dma;
100 u32 cmd_pos;
101 struct mtk_desc *res_base;
102 dma_addr_t res_dma;
103 u32 res_pos;
104};
105
106/**
107 * struct mtk_aes_dma - Structure that holds sg list info
108 * @sg: pointer to scatter-gather list
109 * @nents: number of entries in the sg list
110 * @remainder: remainder of sg list
111 * @sg_len: number of entries in the sg mapped list
112 */
113struct mtk_aes_dma {
114 struct scatterlist *sg;
115 int nents;
116 u32 remainder;
117 u32 sg_len;
118};
119
120struct mtk_aes_base_ctx;
121struct mtk_aes_rec;
122struct mtk_cryp;
123
124typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes);
125
126/**
127 * struct mtk_aes_rec - AES operation record
128 * @queue: crypto request queue
129 * @areq: pointer to async request
130 * @task: the tasklet is use in AES interrupt
131 * @ctx: pointer to current context
132 * @src: the structure that holds source sg list info
133 * @dst: the structure that holds destination sg list info
134 * @aligned_sg: the scatter list is use to alignment
135 * @real_dst: pointer to the destination sg list
136 * @resume: pointer to resume function
137 * @total: request buffer length
138 * @buf: pointer to page buffer
139 * @id: record identification
140 * @flags: it's describing AES operation state
141 * @lock: the async queue lock
142 *
143 * Structure used to record AES execution state.
144 */
145struct mtk_aes_rec {
146 struct crypto_queue queue;
147 struct crypto_async_request *areq;
148 struct tasklet_struct task;
149 struct mtk_aes_base_ctx *ctx;
150 struct mtk_aes_dma src;
151 struct mtk_aes_dma dst;
152
153 struct scatterlist aligned_sg;
154 struct scatterlist *real_dst;
155
156 mtk_aes_fn resume;
157
158 size_t total;
159 void *buf;
160
161 u8 id;
162 unsigned long flags;
163 /* queue lock */
164 spinlock_t lock;
165};
166
167/**
168 * struct mtk_sha_rec - SHA operation record
169 * @queue: crypto request queue
170 * @req: pointer to ahash request
171 * @task: the tasklet is use in SHA interrupt
172 * @id: record identification
173 * @flags: it's describing SHA operation state
174 * @lock: the ablkcipher queue lock
175 *
176 * Structure used to record SHA execution state.
177 */
178struct mtk_sha_rec {
179 struct crypto_queue queue;
180 struct ahash_request *req;
181 struct tasklet_struct task;
182
183 u8 id;
184 unsigned long flags;
185 /* queue lock */
186 spinlock_t lock;
187};
188
189/**
190 * struct mtk_cryp - Cryptographic device
191 * @base: pointer to mapped register I/O base
192 * @dev: pointer to device
193 * @clk_ethif: pointer to ethif clock
194 * @clk_cryp: pointer to crypto clock
195 * @irq: global system and rings IRQ
196 * @ring: pointer to execution state of AES
197 * @aes: pointer to execution state of SHA
198 * @sha: each execution record map to a ring
199 * @aes_list: device list of AES
200 * @sha_list: device list of SHA
201 * @tmp: pointer to temporary buffer for internal use
202 * @tmp_dma: DMA address of temporary buffer
203 * @rec: it's used to select SHA record for tfm
204 *
205 * Structure storing cryptographic device information.
206 */
207struct mtk_cryp {
208 void __iomem *base;
209 struct device *dev;
210 struct clk *clk_ethif;
211 struct clk *clk_cryp;
212 int irq[MTK_IRQ_NUM];
213
214 struct mtk_ring *ring[RING_MAX];
215 struct mtk_aes_rec *aes[MTK_REC_NUM];
216 struct mtk_sha_rec *sha[MTK_REC_NUM];
217
218 struct list_head aes_list;
219 struct list_head sha_list;
220
221 void *tmp;
222 dma_addr_t tmp_dma;
223 bool rec;
224};
225
226int mtk_cipher_alg_register(struct mtk_cryp *cryp);
227void mtk_cipher_alg_release(struct mtk_cryp *cryp);
228int mtk_hash_alg_register(struct mtk_cryp *cryp);
229void mtk_hash_alg_release(struct mtk_cryp *cryp);
230
231#endif /* __MTK_PLATFORM_H_ */
diff --git a/drivers/crypto/mediatek/mtk-regs.h b/drivers/crypto/mediatek/mtk-regs.h
new file mode 100644
index 000000000000..94f4eb85be3f
--- /dev/null
+++ b/drivers/crypto/mediatek/mtk-regs.h
@@ -0,0 +1,194 @@
1/*
2 * Support for MediaTek cryptographic accelerator.
3 *
4 * Copyright (c) 2016 MediaTek Inc.
5 * Author: Ryder Lee <ryder.lee@mediatek.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License.
10 *
11 */
12
13#ifndef __MTK_REGS_H__
14#define __MTK_REGS_H__
15
16/* HIA, Command Descriptor Ring Manager */
17#define CDR_BASE_ADDR_LO(x) (0x0 + ((x) << 12))
18#define CDR_BASE_ADDR_HI(x) (0x4 + ((x) << 12))
19#define CDR_DATA_BASE_ADDR_LO(x) (0x8 + ((x) << 12))
20#define CDR_DATA_BASE_ADDR_HI(x) (0xC + ((x) << 12))
21#define CDR_ACD_BASE_ADDR_LO(x) (0x10 + ((x) << 12))
22#define CDR_ACD_BASE_ADDR_HI(x) (0x14 + ((x) << 12))
23#define CDR_RING_SIZE(x) (0x18 + ((x) << 12))
24#define CDR_DESC_SIZE(x) (0x1C + ((x) << 12))
25#define CDR_CFG(x) (0x20 + ((x) << 12))
26#define CDR_DMA_CFG(x) (0x24 + ((x) << 12))
27#define CDR_THRESH(x) (0x28 + ((x) << 12))
28#define CDR_PREP_COUNT(x) (0x2C + ((x) << 12))
29#define CDR_PROC_COUNT(x) (0x30 + ((x) << 12))
30#define CDR_PREP_PNTR(x) (0x34 + ((x) << 12))
31#define CDR_PROC_PNTR(x) (0x38 + ((x) << 12))
32#define CDR_STAT(x) (0x3C + ((x) << 12))
33
34/* HIA, Result Descriptor Ring Manager */
35#define RDR_BASE_ADDR_LO(x) (0x800 + ((x) << 12))
36#define RDR_BASE_ADDR_HI(x) (0x804 + ((x) << 12))
37#define RDR_DATA_BASE_ADDR_LO(x) (0x808 + ((x) << 12))
38#define RDR_DATA_BASE_ADDR_HI(x) (0x80C + ((x) << 12))
39#define RDR_ACD_BASE_ADDR_LO(x) (0x810 + ((x) << 12))
40#define RDR_ACD_BASE_ADDR_HI(x) (0x814 + ((x) << 12))
41#define RDR_RING_SIZE(x) (0x818 + ((x) << 12))
42#define RDR_DESC_SIZE(x) (0x81C + ((x) << 12))
43#define RDR_CFG(x) (0x820 + ((x) << 12))
44#define RDR_DMA_CFG(x) (0x824 + ((x) << 12))
45#define RDR_THRESH(x) (0x828 + ((x) << 12))
46#define RDR_PREP_COUNT(x) (0x82C + ((x) << 12))
47#define RDR_PROC_COUNT(x) (0x830 + ((x) << 12))
48#define RDR_PREP_PNTR(x) (0x834 + ((x) << 12))
49#define RDR_PROC_PNTR(x) (0x838 + ((x) << 12))
50#define RDR_STAT(x) (0x83C + ((x) << 12))
51
52/* HIA, Ring AIC */
53#define AIC_POL_CTRL(x) (0xE000 - ((x) << 12))
54#define AIC_TYPE_CTRL(x) (0xE004 - ((x) << 12))
55#define AIC_ENABLE_CTRL(x) (0xE008 - ((x) << 12))
56#define AIC_RAW_STAL(x) (0xE00C - ((x) << 12))
57#define AIC_ENABLE_SET(x) (0xE00C - ((x) << 12))
58#define AIC_ENABLED_STAT(x) (0xE010 - ((x) << 12))
59#define AIC_ACK(x) (0xE010 - ((x) << 12))
60#define AIC_ENABLE_CLR(x) (0xE014 - ((x) << 12))
61#define AIC_OPTIONS(x) (0xE018 - ((x) << 12))
62#define AIC_VERSION(x) (0xE01C - ((x) << 12))
63
64/* HIA, Global AIC */
65#define AIC_G_POL_CTRL 0xF800
66#define AIC_G_TYPE_CTRL 0xF804
67#define AIC_G_ENABLE_CTRL 0xF808
68#define AIC_G_RAW_STAT 0xF80C
69#define AIC_G_ENABLE_SET 0xF80C
70#define AIC_G_ENABLED_STAT 0xF810
71#define AIC_G_ACK 0xF810
72#define AIC_G_ENABLE_CLR 0xF814
73#define AIC_G_OPTIONS 0xF818
74#define AIC_G_VERSION 0xF81C
75
76/* HIA, Data Fetch Engine */
77#define DFE_CFG 0xF000
78#define DFE_PRIO_0 0xF010
79#define DFE_PRIO_1 0xF014
80#define DFE_PRIO_2 0xF018
81#define DFE_PRIO_3 0xF01C
82
83/* HIA, Data Fetch Engine access monitoring for CDR */
84#define DFE_RING_REGION_LO(x) (0xF080 + ((x) << 3))
85#define DFE_RING_REGION_HI(x) (0xF084 + ((x) << 3))
86
87/* HIA, Data Fetch Engine thread control and status for thread */
88#define DFE_THR_CTRL 0xF200
89#define DFE_THR_STAT 0xF204
90#define DFE_THR_DESC_CTRL 0xF208
91#define DFE_THR_DESC_DPTR_LO 0xF210
92#define DFE_THR_DESC_DPTR_HI 0xF214
93#define DFE_THR_DESC_ACDPTR_LO 0xF218
94#define DFE_THR_DESC_ACDPTR_HI 0xF21C
95
96/* HIA, Data Store Engine */
97#define DSE_CFG 0xF400
98#define DSE_PRIO_0 0xF410
99#define DSE_PRIO_1 0xF414
100#define DSE_PRIO_2 0xF418
101#define DSE_PRIO_3 0xF41C
102
103/* HIA, Data Store Engine access monitoring for RDR */
104#define DSE_RING_REGION_LO(x) (0xF480 + ((x) << 3))
105#define DSE_RING_REGION_HI(x) (0xF484 + ((x) << 3))
106
107/* HIA, Data Store Engine thread control and status for thread */
108#define DSE_THR_CTRL 0xF600
109#define DSE_THR_STAT 0xF604
110#define DSE_THR_DESC_CTRL 0xF608
111#define DSE_THR_DESC_DPTR_LO 0xF610
112#define DSE_THR_DESC_DPTR_HI 0xF614
113#define DSE_THR_DESC_S_DPTR_LO 0xF618
114#define DSE_THR_DESC_S_DPTR_HI 0xF61C
115#define DSE_THR_ERROR_STAT 0xF620
116
117/* HIA Global */
118#define HIA_MST_CTRL 0xFFF4
119#define HIA_OPTIONS 0xFFF8
120#define HIA_VERSION 0xFFFC
121
122/* Processing Engine Input Side, Processing Engine */
123#define PE_IN_DBUF_THRESH 0x10000
124#define PE_IN_TBUF_THRESH 0x10100
125
126/* Packet Engine Configuration / Status Registers */
127#define PE_TOKEN_CTRL_STAT 0x11000
128#define PE_FUNCTION_EN 0x11004
129#define PE_CONTEXT_CTRL 0x11008
130#define PE_INTERRUPT_CTRL_STAT 0x11010
131#define PE_CONTEXT_STAT 0x1100C
132#define PE_OUT_TRANS_CTRL_STAT 0x11018
133#define PE_OUT_BUF_CTRL 0x1101C
134
135/* Packet Engine PRNG Registers */
136#define PE_PRNG_STAT 0x11040
137#define PE_PRNG_CTRL 0x11044
138#define PE_PRNG_SEED_L 0x11048
139#define PE_PRNG_SEED_H 0x1104C
140#define PE_PRNG_KEY_0_L 0x11050
141#define PE_PRNG_KEY_0_H 0x11054
142#define PE_PRNG_KEY_1_L 0x11058
143#define PE_PRNG_KEY_1_H 0x1105C
144#define PE_PRNG_RES_0 0x11060
145#define PE_PRNG_RES_1 0x11064
146#define PE_PRNG_RES_2 0x11068
147#define PE_PRNG_RES_3 0x1106C
148#define PE_PRNG_LFSR_L 0x11070
149#define PE_PRNG_LFSR_H 0x11074
150
151/* Packet Engine AIC */
152#define PE_EIP96_AIC_POL_CTRL 0x113C0
153#define PE_EIP96_AIC_TYPE_CTRL 0x113C4
154#define PE_EIP96_AIC_ENABLE_CTRL 0x113C8
155#define PE_EIP96_AIC_RAW_STAT 0x113CC
156#define PE_EIP96_AIC_ENABLE_SET 0x113CC
157#define PE_EIP96_AIC_ENABLED_STAT 0x113D0
158#define PE_EIP96_AIC_ACK 0x113D0
159#define PE_EIP96_AIC_ENABLE_CLR 0x113D4
160#define PE_EIP96_AIC_OPTIONS 0x113D8
161#define PE_EIP96_AIC_VERSION 0x113DC
162
163/* Packet Engine Options & Version Registers */
164#define PE_EIP96_OPTIONS 0x113F8
165#define PE_EIP96_VERSION 0x113FC
166
167/* Processing Engine Output Side */
168#define PE_OUT_DBUF_THRESH 0x11C00
169#define PE_OUT_TBUF_THRESH 0x11D00
170
171/* Processing Engine Local AIC */
172#define PE_AIC_POL_CTRL 0x11F00
173#define PE_AIC_TYPE_CTRL 0x11F04
174#define PE_AIC_ENABLE_CTRL 0x11F08
175#define PE_AIC_RAW_STAT 0x11F0C
176#define PE_AIC_ENABLE_SET 0x11F0C
177#define PE_AIC_ENABLED_STAT 0x11F10
178#define PE_AIC_ENABLE_CLR 0x11F14
179#define PE_AIC_OPTIONS 0x11F18
180#define PE_AIC_VERSION 0x11F1C
181
182/* Processing Engine General Configuration and Version */
183#define PE_IN_FLIGHT 0x11FF0
184#define PE_OPTIONS 0x11FF8
185#define PE_VERSION 0x11FFC
186
187/* EIP-97 - Global */
188#define EIP97_CLOCK_STATE 0x1FFE4
189#define EIP97_FORCE_CLOCK_ON 0x1FFE8
190#define EIP97_FORCE_CLOCK_OFF 0x1FFEC
191#define EIP97_MST_CTRL 0x1FFF4
192#define EIP97_OPTIONS 0x1FFF8
193#define EIP97_VERSION 0x1FFFC
194#endif /* __MTK_REGS_H__ */
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
new file mode 100644
index 000000000000..55e3805fba07
--- /dev/null
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -0,0 +1,1435 @@
1/*
2 * Cryptographic API.
3 *
4 * Driver for EIP97 SHA1/SHA2(HMAC) acceleration.
5 *
6 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * Some ideas are from atmel-sha.c and omap-sham.c drivers.
13 */
14
15#include <crypto/sha.h>
16#include "mtk-platform.h"
17
18#define SHA_ALIGN_MSK (sizeof(u32) - 1)
19#define SHA_QUEUE_SIZE 512
20#define SHA_TMP_BUF_SIZE 512
21#define SHA_BUF_SIZE ((u32)PAGE_SIZE)
22
23#define SHA_OP_UPDATE 1
24#define SHA_OP_FINAL 2
25
26#define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0))
27
28/* SHA command token */
29#define SHA_CT_SIZE 5
30#define SHA_CT_CTRL_HDR cpu_to_le32(0x02220000)
31#define SHA_CMD0 cpu_to_le32(0x03020000)
32#define SHA_CMD1 cpu_to_le32(0x21060000)
33#define SHA_CMD2 cpu_to_le32(0xe0e63802)
34
35/* SHA transform information */
36#define SHA_TFM_HASH cpu_to_le32(0x2 << 0)
37#define SHA_TFM_INNER_DIG cpu_to_le32(0x1 << 21)
38#define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8)
39#define SHA_TFM_START cpu_to_le32(0x1 << 4)
40#define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5)
41#define SHA_TFM_HASH_STORE cpu_to_le32(0x1 << 19)
42#define SHA_TFM_SHA1 cpu_to_le32(0x2 << 23)
43#define SHA_TFM_SHA256 cpu_to_le32(0x3 << 23)
44#define SHA_TFM_SHA224 cpu_to_le32(0x4 << 23)
45#define SHA_TFM_SHA512 cpu_to_le32(0x5 << 23)
46#define SHA_TFM_SHA384 cpu_to_le32(0x6 << 23)
47#define SHA_TFM_DIGEST(x) cpu_to_le32(((x) & GENMASK(3, 0)) << 24)
48
49/* SHA flags */
50#define SHA_FLAGS_BUSY BIT(0)
51#define SHA_FLAGS_FINAL BIT(1)
52#define SHA_FLAGS_FINUP BIT(2)
53#define SHA_FLAGS_SG BIT(3)
54#define SHA_FLAGS_ALGO_MSK GENMASK(8, 4)
55#define SHA_FLAGS_SHA1 BIT(4)
56#define SHA_FLAGS_SHA224 BIT(5)
57#define SHA_FLAGS_SHA256 BIT(6)
58#define SHA_FLAGS_SHA384 BIT(7)
59#define SHA_FLAGS_SHA512 BIT(8)
60#define SHA_FLAGS_HMAC BIT(9)
61#define SHA_FLAGS_PAD BIT(10)
62
63/**
64 * mtk_sha_ct is a set of hardware instructions(command token)
65 * that are used to control engine's processing flow of SHA,
66 * and it contains the first two words of transform state.
67 */
68struct mtk_sha_ct {
69 __le32 ctrl[2];
70 __le32 cmd[3];
71};
72
73/**
74 * mtk_sha_tfm is used to define SHA transform state
75 * and store result digest that produced by engine.
76 */
77struct mtk_sha_tfm {
78 __le32 ctrl[2];
79 __le32 digest[SIZE_IN_WORDS(SHA512_DIGEST_SIZE)];
80};
81
82/**
83 * mtk_sha_info consists of command token and transform state
84 * of SHA, its role is similar to mtk_aes_info.
85 */
86struct mtk_sha_info {
87 struct mtk_sha_ct ct;
88 struct mtk_sha_tfm tfm;
89};
90
91struct mtk_sha_reqctx {
92 struct mtk_sha_info info;
93 unsigned long flags;
94 unsigned long op;
95
96 u64 digcnt;
97 bool start;
98 size_t bufcnt;
99 dma_addr_t dma_addr;
100
101 __le32 ct_hdr;
102 u32 ct_size;
103 dma_addr_t ct_dma;
104 dma_addr_t tfm_dma;
105
106 /* Walk state */
107 struct scatterlist *sg;
108 u32 offset; /* Offset in current sg */
109 u32 total; /* Total request */
110 size_t ds;
111 size_t bs;
112
113 u8 *buffer;
114};
115
116struct mtk_sha_hmac_ctx {
117 struct crypto_shash *shash;
118 u8 ipad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
119 u8 opad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
120};
121
122struct mtk_sha_ctx {
123 struct mtk_cryp *cryp;
124 unsigned long flags;
125 u8 id;
126 u8 buf[SHA_BUF_SIZE] __aligned(sizeof(u32));
127
128 struct mtk_sha_hmac_ctx base[0];
129};
130
131struct mtk_sha_drv {
132 struct list_head dev_list;
133 /* Device list lock */
134 spinlock_t lock;
135};
136
137static struct mtk_sha_drv mtk_sha = {
138 .dev_list = LIST_HEAD_INIT(mtk_sha.dev_list),
139 .lock = __SPIN_LOCK_UNLOCKED(mtk_sha.lock),
140};
141
142static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id,
143 struct ahash_request *req);
144
145static inline u32 mtk_sha_read(struct mtk_cryp *cryp, u32 offset)
146{
147 return readl_relaxed(cryp->base + offset);
148}
149
150static inline void mtk_sha_write(struct mtk_cryp *cryp,
151 u32 offset, u32 value)
152{
153 writel_relaxed(value, cryp->base + offset);
154}
155
156static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx)
157{
158 struct mtk_cryp *cryp = NULL;
159 struct mtk_cryp *tmp;
160
161 spin_lock_bh(&mtk_sha.lock);
162 if (!tctx->cryp) {
163 list_for_each_entry(tmp, &mtk_sha.dev_list, sha_list) {
164 cryp = tmp;
165 break;
166 }
167 tctx->cryp = cryp;
168 } else {
169 cryp = tctx->cryp;
170 }
171
172 /*
173 * Assign record id to tfm in round-robin fashion, and this
174 * will help tfm to bind to corresponding descriptor rings.
175 */
176 tctx->id = cryp->rec;
177 cryp->rec = !cryp->rec;
178
179 spin_unlock_bh(&mtk_sha.lock);
180
181 return cryp;
182}
183
184static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx)
185{
186 size_t count;
187
188 while ((ctx->bufcnt < SHA_BUF_SIZE) && ctx->total) {
189 count = min(ctx->sg->length - ctx->offset, ctx->total);
190 count = min(count, SHA_BUF_SIZE - ctx->bufcnt);
191
192 if (count <= 0) {
193 /*
194 * Check if count <= 0 because the buffer is full or
195 * because the sg length is 0. In the latest case,
196 * check if there is another sg in the list, a 0 length
197 * sg doesn't necessarily mean the end of the sg list.
198 */
199 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {
200 ctx->sg = sg_next(ctx->sg);
201 continue;
202 } else {
203 break;
204 }
205 }
206
207 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
208 ctx->offset, count, 0);
209
210 ctx->bufcnt += count;
211 ctx->offset += count;
212 ctx->total -= count;
213
214 if (ctx->offset == ctx->sg->length) {
215 ctx->sg = sg_next(ctx->sg);
216 if (ctx->sg)
217 ctx->offset = 0;
218 else
219 ctx->total = 0;
220 }
221 }
222
223 return 0;
224}
225
226/*
227 * The purpose of this padding is to ensure that the padded message is a
228 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).
229 * The bit "1" is appended at the end of the message followed by
230 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or
231 * 128 bits block (SHA384/SHA512) equals to the message length in bits
232 * is appended.
233 *
234 * For SHA1/SHA224/SHA256, padlen is calculated as followed:
235 * - if message length < 56 bytes then padlen = 56 - message length
236 * - else padlen = 64 + 56 - message length
237 *
238 * For SHA384/SHA512, padlen is calculated as followed:
239 * - if message length < 112 bytes then padlen = 112 - message length
240 * - else padlen = 128 + 112 - message length
241 */
242static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len)
243{
244 u32 index, padlen;
245 u64 bits[2];
246 u64 size = ctx->digcnt;
247
248 size += ctx->bufcnt;
249 size += len;
250
251 bits[1] = cpu_to_be64(size << 3);
252 bits[0] = cpu_to_be64(size >> 61);
253
254 if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) {
255 index = ctx->bufcnt & 0x7f;
256 padlen = (index < 112) ? (112 - index) : ((128 + 112) - index);
257 *(ctx->buffer + ctx->bufcnt) = 0x80;
258 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1);
259 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
260 ctx->bufcnt += padlen + 16;
261 ctx->flags |= SHA_FLAGS_PAD;
262 } else {
263 index = ctx->bufcnt & 0x3f;
264 padlen = (index < 56) ? (56 - index) : ((64 + 56) - index);
265 *(ctx->buffer + ctx->bufcnt) = 0x80;
266 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1);
267 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
268 ctx->bufcnt += padlen + 8;
269 ctx->flags |= SHA_FLAGS_PAD;
270 }
271}
272
273/* Initialize basic transform information of SHA */
274static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx)
275{
276 struct mtk_sha_ct *ct = &ctx->info.ct;
277 struct mtk_sha_tfm *tfm = &ctx->info.tfm;
278
279 ctx->ct_hdr = SHA_CT_CTRL_HDR;
280 ctx->ct_size = SHA_CT_SIZE;
281
282 tfm->ctrl[0] = SHA_TFM_HASH | SHA_TFM_INNER_DIG |
283 SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds));
284
285 switch (ctx->flags & SHA_FLAGS_ALGO_MSK) {
286 case SHA_FLAGS_SHA1:
287 tfm->ctrl[0] |= SHA_TFM_SHA1;
288 break;
289 case SHA_FLAGS_SHA224:
290 tfm->ctrl[0] |= SHA_TFM_SHA224;
291 break;
292 case SHA_FLAGS_SHA256:
293 tfm->ctrl[0] |= SHA_TFM_SHA256;
294 break;
295 case SHA_FLAGS_SHA384:
296 tfm->ctrl[0] |= SHA_TFM_SHA384;
297 break;
298 case SHA_FLAGS_SHA512:
299 tfm->ctrl[0] |= SHA_TFM_SHA512;
300 break;
301
302 default:
303 /* Should not happen... */
304 return;
305 }
306
307 tfm->ctrl[1] = SHA_TFM_HASH_STORE;
308 ct->ctrl[0] = tfm->ctrl[0] | SHA_TFM_CONTINUE | SHA_TFM_START;
309 ct->ctrl[1] = tfm->ctrl[1];
310
311 ct->cmd[0] = SHA_CMD0;
312 ct->cmd[1] = SHA_CMD1;
313 ct->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds));
314}
315
316/*
317 * Update input data length field of transform information and
318 * map it to DMA region.
319 */
320static int mtk_sha_info_update(struct mtk_cryp *cryp,
321 struct mtk_sha_rec *sha,
322 size_t len)
323{
324 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
325 struct mtk_sha_info *info = &ctx->info;
326 struct mtk_sha_ct *ct = &info->ct;
327
328 if (ctx->start)
329 ctx->start = false;
330 else
331 ct->ctrl[0] &= ~SHA_TFM_START;
332
333 ctx->ct_hdr &= ~SHA_DATA_LEN_MSK;
334 ctx->ct_hdr |= cpu_to_le32(len);
335 ct->cmd[0] &= ~SHA_DATA_LEN_MSK;
336 ct->cmd[0] |= cpu_to_le32(len);
337
338 ctx->digcnt += len;
339
340 ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
341 DMA_BIDIRECTIONAL);
342 if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) {
343 dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info));
344 return -EINVAL;
345 }
346 ctx->tfm_dma = ctx->ct_dma + sizeof(*ct);
347
348 return 0;
349}
350
351/*
352 * Because of hardware limitation, we must pre-calculate the inner
353 * and outer digest that need to be processed firstly by engine, then
354 * apply the result digest to the input message. These complex hashing
355 * procedures limits HMAC performance, so we use fallback SW encoding.
356 */
357static int mtk_sha_finish_hmac(struct ahash_request *req)
358{
359 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
360 struct mtk_sha_hmac_ctx *bctx = tctx->base;
361 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
362
363 SHASH_DESC_ON_STACK(shash, bctx->shash);
364
365 shash->tfm = bctx->shash;
366 shash->flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
367
368 return crypto_shash_init(shash) ?:
369 crypto_shash_update(shash, bctx->opad, ctx->bs) ?:
370 crypto_shash_finup(shash, req->result, ctx->ds, req->result);
371}
372
373/* Initialize request context */
374static int mtk_sha_init(struct ahash_request *req)
375{
376 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
377 struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm);
378 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
379
380 ctx->flags = 0;
381 ctx->ds = crypto_ahash_digestsize(tfm);
382
383 switch (ctx->ds) {
384 case SHA1_DIGEST_SIZE:
385 ctx->flags |= SHA_FLAGS_SHA1;
386 ctx->bs = SHA1_BLOCK_SIZE;
387 break;
388 case SHA224_DIGEST_SIZE:
389 ctx->flags |= SHA_FLAGS_SHA224;
390 ctx->bs = SHA224_BLOCK_SIZE;
391 break;
392 case SHA256_DIGEST_SIZE:
393 ctx->flags |= SHA_FLAGS_SHA256;
394 ctx->bs = SHA256_BLOCK_SIZE;
395 break;
396 case SHA384_DIGEST_SIZE:
397 ctx->flags |= SHA_FLAGS_SHA384;
398 ctx->bs = SHA384_BLOCK_SIZE;
399 break;
400 case SHA512_DIGEST_SIZE:
401 ctx->flags |= SHA_FLAGS_SHA512;
402 ctx->bs = SHA512_BLOCK_SIZE;
403 break;
404 default:
405 return -EINVAL;
406 }
407
408 ctx->bufcnt = 0;
409 ctx->digcnt = 0;
410 ctx->buffer = tctx->buf;
411 ctx->start = true;
412
413 if (tctx->flags & SHA_FLAGS_HMAC) {
414 struct mtk_sha_hmac_ctx *bctx = tctx->base;
415
416 memcpy(ctx->buffer, bctx->ipad, ctx->bs);
417 ctx->bufcnt = ctx->bs;
418 ctx->flags |= SHA_FLAGS_HMAC;
419 }
420
421 return 0;
422}
423
424static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
425 dma_addr_t addr, size_t len)
426{
427 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
428 struct mtk_ring *ring = cryp->ring[sha->id];
429 struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos;
430 struct mtk_desc *res = ring->res_base + ring->res_pos;
431 int err;
432
433 err = mtk_sha_info_update(cryp, sha, len);
434 if (err)
435 return err;
436
437 /* Fill in the command/result descriptors */
438 res->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len);
439 res->buf = cpu_to_le32(cryp->tmp_dma);
440
441 cmd->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len) |
442 MTK_DESC_CT_LEN(ctx->ct_size);
443
444 cmd->buf = cpu_to_le32(addr);
445 cmd->ct = cpu_to_le32(ctx->ct_dma);
446 cmd->ct_hdr = ctx->ct_hdr;
447 cmd->tfm = cpu_to_le32(ctx->tfm_dma);
448
449 if (++ring->cmd_pos == MTK_DESC_NUM)
450 ring->cmd_pos = 0;
451
452 ring->res_pos = ring->cmd_pos;
453 /*
454 * Make sure that all changes to the DMA ring are done before we
455 * start engine.
456 */
457 wmb();
458 /* Start DMA transfer */
459 mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(1));
460 mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(1));
461
462 return -EINPROGRESS;
463}
464
465static int mtk_sha_xmit2(struct mtk_cryp *cryp,
466 struct mtk_sha_rec *sha,
467 struct mtk_sha_reqctx *ctx,
468 size_t len1, size_t len2)
469{
470 struct mtk_ring *ring = cryp->ring[sha->id];
471 struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos;
472 struct mtk_desc *res = ring->res_base + ring->res_pos;
473 int err;
474
475 err = mtk_sha_info_update(cryp, sha, len1 + len2);
476 if (err)
477 return err;
478
479 /* Fill in the command/result descriptors */
480 res->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST;
481 res->buf = cpu_to_le32(cryp->tmp_dma);
482
483 cmd->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST |
484 MTK_DESC_CT_LEN(ctx->ct_size);
485 cmd->buf = cpu_to_le32(sg_dma_address(ctx->sg));
486 cmd->ct = cpu_to_le32(ctx->ct_dma);
487 cmd->ct_hdr = ctx->ct_hdr;
488 cmd->tfm = cpu_to_le32(ctx->tfm_dma);
489
490 if (++ring->cmd_pos == MTK_DESC_NUM)
491 ring->cmd_pos = 0;
492
493 ring->res_pos = ring->cmd_pos;
494
495 cmd = ring->cmd_base + ring->cmd_pos;
496 res = ring->res_base + ring->res_pos;
497
498 res->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST;
499 res->buf = cpu_to_le32(cryp->tmp_dma);
500
501 cmd->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST;
502 cmd->buf = cpu_to_le32(ctx->dma_addr);
503
504 if (++ring->cmd_pos == MTK_DESC_NUM)
505 ring->cmd_pos = 0;
506
507 ring->res_pos = ring->cmd_pos;
508
509 /*
510 * Make sure that all changes to the DMA ring are done before we
511 * start engine.
512 */
513 wmb();
514 /* Start DMA transfer */
515 mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(2));
516 mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(2));
517
518 return -EINPROGRESS;
519}
520
521static int mtk_sha_dma_map(struct mtk_cryp *cryp,
522 struct mtk_sha_rec *sha,
523 struct mtk_sha_reqctx *ctx,
524 size_t count)
525{
526 ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer,
527 SHA_BUF_SIZE, DMA_TO_DEVICE);
528 if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) {
529 dev_err(cryp->dev, "dma map error\n");
530 return -EINVAL;
531 }
532
533 ctx->flags &= ~SHA_FLAGS_SG;
534
535 return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count);
536}
537
538static int mtk_sha_update_slow(struct mtk_cryp *cryp,
539 struct mtk_sha_rec *sha)
540{
541 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
542 size_t count;
543 u32 final;
544
545 mtk_sha_append_sg(ctx);
546
547 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
548
549 dev_dbg(cryp->dev, "slow: bufcnt: %zu\n", ctx->bufcnt);
550
551 if (final) {
552 sha->flags |= SHA_FLAGS_FINAL;
553 mtk_sha_fill_padding(ctx, 0);
554 }
555
556 if (final || (ctx->bufcnt == SHA_BUF_SIZE && ctx->total)) {
557 count = ctx->bufcnt;
558 ctx->bufcnt = 0;
559
560 return mtk_sha_dma_map(cryp, sha, ctx, count);
561 }
562 return 0;
563}
564
565static int mtk_sha_update_start(struct mtk_cryp *cryp,
566 struct mtk_sha_rec *sha)
567{
568 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
569 u32 len, final, tail;
570 struct scatterlist *sg;
571
572 if (!ctx->total)
573 return 0;
574
575 if (ctx->bufcnt || ctx->offset)
576 return mtk_sha_update_slow(cryp, sha);
577
578 sg = ctx->sg;
579
580 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
581 return mtk_sha_update_slow(cryp, sha);
582
583 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->bs))
584 /* size is not ctx->bs aligned */
585 return mtk_sha_update_slow(cryp, sha);
586
587 len = min(ctx->total, sg->length);
588
589 if (sg_is_last(sg)) {
590 if (!(ctx->flags & SHA_FLAGS_FINUP)) {
591 /* not last sg must be ctx->bs aligned */
592 tail = len & (ctx->bs - 1);
593 len -= tail;
594 }
595 }
596
597 ctx->total -= len;
598 ctx->offset = len; /* offset where to start slow */
599
600 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
601
602 /* Add padding */
603 if (final) {
604 size_t count;
605
606 tail = len & (ctx->bs - 1);
607 len -= tail;
608 ctx->total += tail;
609 ctx->offset = len; /* offset where to start slow */
610
611 sg = ctx->sg;
612 mtk_sha_append_sg(ctx);
613 mtk_sha_fill_padding(ctx, len);
614
615 ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer,
616 SHA_BUF_SIZE, DMA_TO_DEVICE);
617 if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) {
618 dev_err(cryp->dev, "dma map bytes error\n");
619 return -EINVAL;
620 }
621
622 sha->flags |= SHA_FLAGS_FINAL;
623 count = ctx->bufcnt;
624 ctx->bufcnt = 0;
625
626 if (len == 0) {
627 ctx->flags &= ~SHA_FLAGS_SG;
628 return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count);
629
630 } else {
631 ctx->sg = sg;
632 if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
633 dev_err(cryp->dev, "dma_map_sg error\n");
634 return -EINVAL;
635 }
636
637 ctx->flags |= SHA_FLAGS_SG;
638 return mtk_sha_xmit2(cryp, sha, ctx, len, count);
639 }
640 }
641
642 if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
643 dev_err(cryp->dev, "dma_map_sg error\n");
644 return -EINVAL;
645 }
646
647 ctx->flags |= SHA_FLAGS_SG;
648
649 return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), len);
650}
651
652static int mtk_sha_final_req(struct mtk_cryp *cryp,
653 struct mtk_sha_rec *sha)
654{
655 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
656 size_t count;
657
658 mtk_sha_fill_padding(ctx, 0);
659
660 sha->flags |= SHA_FLAGS_FINAL;
661 count = ctx->bufcnt;
662 ctx->bufcnt = 0;
663
664 return mtk_sha_dma_map(cryp, sha, ctx, count);
665}
666
667/* Copy ready hash (+ finalize hmac) */
668static int mtk_sha_finish(struct ahash_request *req)
669{
670 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
671 u32 *digest = ctx->info.tfm.digest;
672 u32 *result = (u32 *)req->result;
673 int i;
674
675 /* Get the hash from the digest buffer */
676 for (i = 0; i < SIZE_IN_WORDS(ctx->ds); i++)
677 result[i] = le32_to_cpu(digest[i]);
678
679 if (ctx->flags & SHA_FLAGS_HMAC)
680 return mtk_sha_finish_hmac(req);
681
682 return 0;
683}
684
685static void mtk_sha_finish_req(struct mtk_cryp *cryp,
686 struct mtk_sha_rec *sha,
687 int err)
688{
689 if (likely(!err && (SHA_FLAGS_FINAL & sha->flags)))
690 err = mtk_sha_finish(sha->req);
691
692 sha->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL);
693
694 sha->req->base.complete(&sha->req->base, err);
695
696 /* Handle new request */
697 mtk_sha_handle_queue(cryp, sha->id - RING2, NULL);
698}
699
700static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id,
701 struct ahash_request *req)
702{
703 struct mtk_sha_rec *sha = cryp->sha[id];
704 struct crypto_async_request *async_req, *backlog;
705 struct mtk_sha_reqctx *ctx;
706 unsigned long flags;
707 int err = 0, ret = 0;
708
709 spin_lock_irqsave(&sha->lock, flags);
710 if (req)
711 ret = ahash_enqueue_request(&sha->queue, req);
712
713 if (SHA_FLAGS_BUSY & sha->flags) {
714 spin_unlock_irqrestore(&sha->lock, flags);
715 return ret;
716 }
717
718 backlog = crypto_get_backlog(&sha->queue);
719 async_req = crypto_dequeue_request(&sha->queue);
720 if (async_req)
721 sha->flags |= SHA_FLAGS_BUSY;
722 spin_unlock_irqrestore(&sha->lock, flags);
723
724 if (!async_req)
725 return ret;
726
727 if (backlog)
728 backlog->complete(backlog, -EINPROGRESS);
729
730 req = ahash_request_cast(async_req);
731 ctx = ahash_request_ctx(req);
732
733 sha->req = req;
734
735 mtk_sha_info_init(ctx);
736
737 if (ctx->op == SHA_OP_UPDATE) {
738 err = mtk_sha_update_start(cryp, sha);
739 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
740 /* No final() after finup() */
741 err = mtk_sha_final_req(cryp, sha);
742 } else if (ctx->op == SHA_OP_FINAL) {
743 err = mtk_sha_final_req(cryp, sha);
744 }
745
746 if (unlikely(err != -EINPROGRESS))
747 /* Task will not finish it, so do it here */
748 mtk_sha_finish_req(cryp, sha, err);
749
750 return ret;
751}
752
753static int mtk_sha_enqueue(struct ahash_request *req, u32 op)
754{
755 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
756 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
757
758 ctx->op = op;
759
760 return mtk_sha_handle_queue(tctx->cryp, tctx->id, req);
761}
762
763static void mtk_sha_unmap(struct mtk_cryp *cryp, struct mtk_sha_rec *sha)
764{
765 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
766
767 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),
768 DMA_BIDIRECTIONAL);
769
770 if (ctx->flags & SHA_FLAGS_SG) {
771 dma_unmap_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE);
772 if (ctx->sg->length == ctx->offset) {
773 ctx->sg = sg_next(ctx->sg);
774 if (ctx->sg)
775 ctx->offset = 0;
776 }
777 if (ctx->flags & SHA_FLAGS_PAD) {
778 dma_unmap_single(cryp->dev, ctx->dma_addr,
779 SHA_BUF_SIZE, DMA_TO_DEVICE);
780 }
781 } else
782 dma_unmap_single(cryp->dev, ctx->dma_addr,
783 SHA_BUF_SIZE, DMA_TO_DEVICE);
784}
785
786static void mtk_sha_complete(struct mtk_cryp *cryp,
787 struct mtk_sha_rec *sha)
788{
789 int err = 0;
790
791 err = mtk_sha_update_start(cryp, sha);
792 if (err != -EINPROGRESS)
793 mtk_sha_finish_req(cryp, sha, err);
794}
795
796static int mtk_sha_update(struct ahash_request *req)
797{
798 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
799
800 ctx->total = req->nbytes;
801 ctx->sg = req->src;
802 ctx->offset = 0;
803
804 if ((ctx->bufcnt + ctx->total < SHA_BUF_SIZE) &&
805 !(ctx->flags & SHA_FLAGS_FINUP))
806 return mtk_sha_append_sg(ctx);
807
808 return mtk_sha_enqueue(req, SHA_OP_UPDATE);
809}
810
811static int mtk_sha_final(struct ahash_request *req)
812{
813 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
814
815 ctx->flags |= SHA_FLAGS_FINUP;
816
817 if (ctx->flags & SHA_FLAGS_PAD)
818 return mtk_sha_finish(req);
819
820 return mtk_sha_enqueue(req, SHA_OP_FINAL);
821}
822
823static int mtk_sha_finup(struct ahash_request *req)
824{
825 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
826 int err1, err2;
827
828 ctx->flags |= SHA_FLAGS_FINUP;
829
830 err1 = mtk_sha_update(req);
831 if (err1 == -EINPROGRESS || err1 == -EBUSY)
832 return err1;
833 /*
834 * final() has to be always called to cleanup resources
835 * even if update() failed
836 */
837 err2 = mtk_sha_final(req);
838
839 return err1 ?: err2;
840}
841
842static int mtk_sha_digest(struct ahash_request *req)
843{
844 return mtk_sha_init(req) ?: mtk_sha_finup(req);
845}
846
847static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
848 u32 keylen)
849{
850 struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm);
851 struct mtk_sha_hmac_ctx *bctx = tctx->base;
852 size_t bs = crypto_shash_blocksize(bctx->shash);
853 size_t ds = crypto_shash_digestsize(bctx->shash);
854 int err, i;
855
856 SHASH_DESC_ON_STACK(shash, bctx->shash);
857
858 shash->tfm = bctx->shash;
859 shash->flags = crypto_shash_get_flags(bctx->shash) &
860 CRYPTO_TFM_REQ_MAY_SLEEP;
861
862 if (keylen > bs) {
863 err = crypto_shash_digest(shash, key, keylen, bctx->ipad);
864 if (err)
865 return err;
866 keylen = ds;
867 } else {
868 memcpy(bctx->ipad, key, keylen);
869 }
870
871 memset(bctx->ipad + keylen, 0, bs - keylen);
872 memcpy(bctx->opad, bctx->ipad, bs);
873
874 for (i = 0; i < bs; i++) {
875 bctx->ipad[i] ^= 0x36;
876 bctx->opad[i] ^= 0x5c;
877 }
878
879 return 0;
880}
881
882static int mtk_sha_export(struct ahash_request *req, void *out)
883{
884 const struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
885
886 memcpy(out, ctx, sizeof(*ctx));
887 return 0;
888}
889
890static int mtk_sha_import(struct ahash_request *req, const void *in)
891{
892 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
893
894 memcpy(ctx, in, sizeof(*ctx));
895 return 0;
896}
897
898static int mtk_sha_cra_init_alg(struct crypto_tfm *tfm,
899 const char *alg_base)
900{
901 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm);
902 struct mtk_cryp *cryp = NULL;
903
904 cryp = mtk_sha_find_dev(tctx);
905 if (!cryp)
906 return -ENODEV;
907
908 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
909 sizeof(struct mtk_sha_reqctx));
910
911 if (alg_base) {
912 struct mtk_sha_hmac_ctx *bctx = tctx->base;
913
914 tctx->flags |= SHA_FLAGS_HMAC;
915 bctx->shash = crypto_alloc_shash(alg_base, 0,
916 CRYPTO_ALG_NEED_FALLBACK);
917 if (IS_ERR(bctx->shash)) {
918 pr_err("base driver %s could not be loaded.\n",
919 alg_base);
920
921 return PTR_ERR(bctx->shash);
922 }
923 }
924 return 0;
925}
926
927static int mtk_sha_cra_init(struct crypto_tfm *tfm)
928{
929 return mtk_sha_cra_init_alg(tfm, NULL);
930}
931
932static int mtk_sha_cra_sha1_init(struct crypto_tfm *tfm)
933{
934 return mtk_sha_cra_init_alg(tfm, "sha1");
935}
936
937static int mtk_sha_cra_sha224_init(struct crypto_tfm *tfm)
938{
939 return mtk_sha_cra_init_alg(tfm, "sha224");
940}
941
942static int mtk_sha_cra_sha256_init(struct crypto_tfm *tfm)
943{
944 return mtk_sha_cra_init_alg(tfm, "sha256");
945}
946
947static int mtk_sha_cra_sha384_init(struct crypto_tfm *tfm)
948{
949 return mtk_sha_cra_init_alg(tfm, "sha384");
950}
951
952static int mtk_sha_cra_sha512_init(struct crypto_tfm *tfm)
953{
954 return mtk_sha_cra_init_alg(tfm, "sha512");
955}
956
957static void mtk_sha_cra_exit(struct crypto_tfm *tfm)
958{
959 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm);
960
961 if (tctx->flags & SHA_FLAGS_HMAC) {
962 struct mtk_sha_hmac_ctx *bctx = tctx->base;
963
964 crypto_free_shash(bctx->shash);
965 }
966}
967
968static struct ahash_alg algs_sha1_sha224_sha256[] = {
969{
970 .init = mtk_sha_init,
971 .update = mtk_sha_update,
972 .final = mtk_sha_final,
973 .finup = mtk_sha_finup,
974 .digest = mtk_sha_digest,
975 .export = mtk_sha_export,
976 .import = mtk_sha_import,
977 .halg.digestsize = SHA1_DIGEST_SIZE,
978 .halg.statesize = sizeof(struct mtk_sha_reqctx),
979 .halg.base = {
980 .cra_name = "sha1",
981 .cra_driver_name = "mtk-sha1",
982 .cra_priority = 400,
983 .cra_flags = CRYPTO_ALG_ASYNC,
984 .cra_blocksize = SHA1_BLOCK_SIZE,
985 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
986 .cra_alignmask = SHA_ALIGN_MSK,
987 .cra_module = THIS_MODULE,
988 .cra_init = mtk_sha_cra_init,
989 .cra_exit = mtk_sha_cra_exit,
990 }
991},
992{
993 .init = mtk_sha_init,
994 .update = mtk_sha_update,
995 .final = mtk_sha_final,
996 .finup = mtk_sha_finup,
997 .digest = mtk_sha_digest,
998 .export = mtk_sha_export,
999 .import = mtk_sha_import,
1000 .halg.digestsize = SHA224_DIGEST_SIZE,
1001 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1002 .halg.base = {
1003 .cra_name = "sha224",
1004 .cra_driver_name = "mtk-sha224",
1005 .cra_priority = 400,
1006 .cra_flags = CRYPTO_ALG_ASYNC,
1007 .cra_blocksize = SHA224_BLOCK_SIZE,
1008 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
1009 .cra_alignmask = SHA_ALIGN_MSK,
1010 .cra_module = THIS_MODULE,
1011 .cra_init = mtk_sha_cra_init,
1012 .cra_exit = mtk_sha_cra_exit,
1013 }
1014},
1015{
1016 .init = mtk_sha_init,
1017 .update = mtk_sha_update,
1018 .final = mtk_sha_final,
1019 .finup = mtk_sha_finup,
1020 .digest = mtk_sha_digest,
1021 .export = mtk_sha_export,
1022 .import = mtk_sha_import,
1023 .halg.digestsize = SHA256_DIGEST_SIZE,
1024 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1025 .halg.base = {
1026 .cra_name = "sha256",
1027 .cra_driver_name = "mtk-sha256",
1028 .cra_priority = 400,
1029 .cra_flags = CRYPTO_ALG_ASYNC,
1030 .cra_blocksize = SHA256_BLOCK_SIZE,
1031 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
1032 .cra_alignmask = SHA_ALIGN_MSK,
1033 .cra_module = THIS_MODULE,
1034 .cra_init = mtk_sha_cra_init,
1035 .cra_exit = mtk_sha_cra_exit,
1036 }
1037},
1038{
1039 .init = mtk_sha_init,
1040 .update = mtk_sha_update,
1041 .final = mtk_sha_final,
1042 .finup = mtk_sha_finup,
1043 .digest = mtk_sha_digest,
1044 .export = mtk_sha_export,
1045 .import = mtk_sha_import,
1046 .setkey = mtk_sha_setkey,
1047 .halg.digestsize = SHA1_DIGEST_SIZE,
1048 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1049 .halg.base = {
1050 .cra_name = "hmac(sha1)",
1051 .cra_driver_name = "mtk-hmac-sha1",
1052 .cra_priority = 400,
1053 .cra_flags = CRYPTO_ALG_ASYNC |
1054 CRYPTO_ALG_NEED_FALLBACK,
1055 .cra_blocksize = SHA1_BLOCK_SIZE,
1056 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1057 sizeof(struct mtk_sha_hmac_ctx),
1058 .cra_alignmask = SHA_ALIGN_MSK,
1059 .cra_module = THIS_MODULE,
1060 .cra_init = mtk_sha_cra_sha1_init,
1061 .cra_exit = mtk_sha_cra_exit,
1062 }
1063},
1064{
1065 .init = mtk_sha_init,
1066 .update = mtk_sha_update,
1067 .final = mtk_sha_final,
1068 .finup = mtk_sha_finup,
1069 .digest = mtk_sha_digest,
1070 .export = mtk_sha_export,
1071 .import = mtk_sha_import,
1072 .setkey = mtk_sha_setkey,
1073 .halg.digestsize = SHA224_DIGEST_SIZE,
1074 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1075 .halg.base = {
1076 .cra_name = "hmac(sha224)",
1077 .cra_driver_name = "mtk-hmac-sha224",
1078 .cra_priority = 400,
1079 .cra_flags = CRYPTO_ALG_ASYNC |
1080 CRYPTO_ALG_NEED_FALLBACK,
1081 .cra_blocksize = SHA224_BLOCK_SIZE,
1082 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1083 sizeof(struct mtk_sha_hmac_ctx),
1084 .cra_alignmask = SHA_ALIGN_MSK,
1085 .cra_module = THIS_MODULE,
1086 .cra_init = mtk_sha_cra_sha224_init,
1087 .cra_exit = mtk_sha_cra_exit,
1088 }
1089},
1090{
1091 .init = mtk_sha_init,
1092 .update = mtk_sha_update,
1093 .final = mtk_sha_final,
1094 .finup = mtk_sha_finup,
1095 .digest = mtk_sha_digest,
1096 .export = mtk_sha_export,
1097 .import = mtk_sha_import,
1098 .setkey = mtk_sha_setkey,
1099 .halg.digestsize = SHA256_DIGEST_SIZE,
1100 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1101 .halg.base = {
1102 .cra_name = "hmac(sha256)",
1103 .cra_driver_name = "mtk-hmac-sha256",
1104 .cra_priority = 400,
1105 .cra_flags = CRYPTO_ALG_ASYNC |
1106 CRYPTO_ALG_NEED_FALLBACK,
1107 .cra_blocksize = SHA256_BLOCK_SIZE,
1108 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1109 sizeof(struct mtk_sha_hmac_ctx),
1110 .cra_alignmask = SHA_ALIGN_MSK,
1111 .cra_module = THIS_MODULE,
1112 .cra_init = mtk_sha_cra_sha256_init,
1113 .cra_exit = mtk_sha_cra_exit,
1114 }
1115},
1116};
1117
1118static struct ahash_alg algs_sha384_sha512[] = {
1119{
1120 .init = mtk_sha_init,
1121 .update = mtk_sha_update,
1122 .final = mtk_sha_final,
1123 .finup = mtk_sha_finup,
1124 .digest = mtk_sha_digest,
1125 .export = mtk_sha_export,
1126 .import = mtk_sha_import,
1127 .halg.digestsize = SHA384_DIGEST_SIZE,
1128 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1129 .halg.base = {
1130 .cra_name = "sha384",
1131 .cra_driver_name = "mtk-sha384",
1132 .cra_priority = 400,
1133 .cra_flags = CRYPTO_ALG_ASYNC,
1134 .cra_blocksize = SHA384_BLOCK_SIZE,
1135 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
1136 .cra_alignmask = SHA_ALIGN_MSK,
1137 .cra_module = THIS_MODULE,
1138 .cra_init = mtk_sha_cra_init,
1139 .cra_exit = mtk_sha_cra_exit,
1140 }
1141},
1142{
1143 .init = mtk_sha_init,
1144 .update = mtk_sha_update,
1145 .final = mtk_sha_final,
1146 .finup = mtk_sha_finup,
1147 .digest = mtk_sha_digest,
1148 .export = mtk_sha_export,
1149 .import = mtk_sha_import,
1150 .halg.digestsize = SHA512_DIGEST_SIZE,
1151 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1152 .halg.base = {
1153 .cra_name = "sha512",
1154 .cra_driver_name = "mtk-sha512",
1155 .cra_priority = 400,
1156 .cra_flags = CRYPTO_ALG_ASYNC,
1157 .cra_blocksize = SHA512_BLOCK_SIZE,
1158 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
1159 .cra_alignmask = SHA_ALIGN_MSK,
1160 .cra_module = THIS_MODULE,
1161 .cra_init = mtk_sha_cra_init,
1162 .cra_exit = mtk_sha_cra_exit,
1163 }
1164},
1165{
1166 .init = mtk_sha_init,
1167 .update = mtk_sha_update,
1168 .final = mtk_sha_final,
1169 .finup = mtk_sha_finup,
1170 .digest = mtk_sha_digest,
1171 .export = mtk_sha_export,
1172 .import = mtk_sha_import,
1173 .setkey = mtk_sha_setkey,
1174 .halg.digestsize = SHA384_DIGEST_SIZE,
1175 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1176 .halg.base = {
1177 .cra_name = "hmac(sha384)",
1178 .cra_driver_name = "mtk-hmac-sha384",
1179 .cra_priority = 400,
1180 .cra_flags = CRYPTO_ALG_ASYNC |
1181 CRYPTO_ALG_NEED_FALLBACK,
1182 .cra_blocksize = SHA384_BLOCK_SIZE,
1183 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1184 sizeof(struct mtk_sha_hmac_ctx),
1185 .cra_alignmask = SHA_ALIGN_MSK,
1186 .cra_module = THIS_MODULE,
1187 .cra_init = mtk_sha_cra_sha384_init,
1188 .cra_exit = mtk_sha_cra_exit,
1189 }
1190},
1191{
1192 .init = mtk_sha_init,
1193 .update = mtk_sha_update,
1194 .final = mtk_sha_final,
1195 .finup = mtk_sha_finup,
1196 .digest = mtk_sha_digest,
1197 .export = mtk_sha_export,
1198 .import = mtk_sha_import,
1199 .setkey = mtk_sha_setkey,
1200 .halg.digestsize = SHA512_DIGEST_SIZE,
1201 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1202 .halg.base = {
1203 .cra_name = "hmac(sha512)",
1204 .cra_driver_name = "mtk-hmac-sha512",
1205 .cra_priority = 400,
1206 .cra_flags = CRYPTO_ALG_ASYNC |
1207 CRYPTO_ALG_NEED_FALLBACK,
1208 .cra_blocksize = SHA512_BLOCK_SIZE,
1209 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1210 sizeof(struct mtk_sha_hmac_ctx),
1211 .cra_alignmask = SHA_ALIGN_MSK,
1212 .cra_module = THIS_MODULE,
1213 .cra_init = mtk_sha_cra_sha512_init,
1214 .cra_exit = mtk_sha_cra_exit,
1215 }
1216},
1217};
1218
1219static void mtk_sha_task0(unsigned long data)
1220{
1221 struct mtk_cryp *cryp = (struct mtk_cryp *)data;
1222 struct mtk_sha_rec *sha = cryp->sha[0];
1223
1224 mtk_sha_unmap(cryp, sha);
1225 mtk_sha_complete(cryp, sha);
1226}
1227
1228static void mtk_sha_task1(unsigned long data)
1229{
1230 struct mtk_cryp *cryp = (struct mtk_cryp *)data;
1231 struct mtk_sha_rec *sha = cryp->sha[1];
1232
1233 mtk_sha_unmap(cryp, sha);
1234 mtk_sha_complete(cryp, sha);
1235}
1236
1237static irqreturn_t mtk_sha_ring2_irq(int irq, void *dev_id)
1238{
1239 struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id;
1240 struct mtk_sha_rec *sha = cryp->sha[0];
1241 u32 val = mtk_sha_read(cryp, RDR_STAT(RING2));
1242
1243 mtk_sha_write(cryp, RDR_STAT(RING2), val);
1244
1245 if (likely((SHA_FLAGS_BUSY & sha->flags))) {
1246 mtk_sha_write(cryp, RDR_PROC_COUNT(RING2), MTK_CNT_RST);
1247 mtk_sha_write(cryp, RDR_THRESH(RING2),
1248 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1249
1250 tasklet_schedule(&sha->task);
1251 } else {
1252 dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
1253 }
1254 return IRQ_HANDLED;
1255}
1256
1257static irqreturn_t mtk_sha_ring3_irq(int irq, void *dev_id)
1258{
1259 struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id;
1260 struct mtk_sha_rec *sha = cryp->sha[1];
1261 u32 val = mtk_sha_read(cryp, RDR_STAT(RING3));
1262
1263 mtk_sha_write(cryp, RDR_STAT(RING3), val);
1264
1265 if (likely((SHA_FLAGS_BUSY & sha->flags))) {
1266 mtk_sha_write(cryp, RDR_PROC_COUNT(RING3), MTK_CNT_RST);
1267 mtk_sha_write(cryp, RDR_THRESH(RING3),
1268 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1269
1270 tasklet_schedule(&sha->task);
1271 } else {
1272 dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
1273 }
1274 return IRQ_HANDLED;
1275}
1276
1277/*
1278 * The purpose of two SHA records is used to get extra performance.
1279 * It is similar to mtk_aes_record_init().
1280 */
1281static int mtk_sha_record_init(struct mtk_cryp *cryp)
1282{
1283 struct mtk_sha_rec **sha = cryp->sha;
1284 int i, err = -ENOMEM;
1285
1286 for (i = 0; i < MTK_REC_NUM; i++) {
1287 sha[i] = kzalloc(sizeof(**sha), GFP_KERNEL);
1288 if (!sha[i])
1289 goto err_cleanup;
1290
1291 sha[i]->id = i + RING2;
1292
1293 spin_lock_init(&sha[i]->lock);
1294 crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE);
1295 }
1296
1297 tasklet_init(&sha[0]->task, mtk_sha_task0, (unsigned long)cryp);
1298 tasklet_init(&sha[1]->task, mtk_sha_task1, (unsigned long)cryp);
1299
1300 cryp->rec = 1;
1301
1302 return 0;
1303
1304err_cleanup:
1305 for (; i--; )
1306 kfree(sha[i]);
1307 return err;
1308}
1309
1310static void mtk_sha_record_free(struct mtk_cryp *cryp)
1311{
1312 int i;
1313
1314 for (i = 0; i < MTK_REC_NUM; i++) {
1315 tasklet_kill(&cryp->sha[i]->task);
1316 kfree(cryp->sha[i]);
1317 }
1318}
1319
1320static void mtk_sha_unregister_algs(void)
1321{
1322 int i;
1323
1324 for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++)
1325 crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]);
1326
1327 for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++)
1328 crypto_unregister_ahash(&algs_sha384_sha512[i]);
1329}
1330
1331static int mtk_sha_register_algs(void)
1332{
1333 int err, i;
1334
1335 for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) {
1336 err = crypto_register_ahash(&algs_sha1_sha224_sha256[i]);
1337 if (err)
1338 goto err_sha_224_256_algs;
1339 }
1340
1341 for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) {
1342 err = crypto_register_ahash(&algs_sha384_sha512[i]);
1343 if (err)
1344 goto err_sha_384_512_algs;
1345 }
1346
1347 return 0;
1348
1349err_sha_384_512_algs:
1350 for (; i--; )
1351 crypto_unregister_ahash(&algs_sha384_sha512[i]);
1352 i = ARRAY_SIZE(algs_sha1_sha224_sha256);
1353err_sha_224_256_algs:
1354 for (; i--; )
1355 crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]);
1356
1357 return err;
1358}
1359
1360int mtk_hash_alg_register(struct mtk_cryp *cryp)
1361{
1362 int err;
1363
1364 INIT_LIST_HEAD(&cryp->sha_list);
1365
1366 /* Initialize two hash records */
1367 err = mtk_sha_record_init(cryp);
1368 if (err)
1369 goto err_record;
1370
1371 /* Ring2 is use by SHA record0 */
1372 err = devm_request_irq(cryp->dev, cryp->irq[RING2],
1373 mtk_sha_ring2_irq, IRQF_TRIGGER_LOW,
1374 "mtk-sha", cryp);
1375 if (err) {
1376 dev_err(cryp->dev, "unable to request sha irq0.\n");
1377 goto err_res;
1378 }
1379
1380 /* Ring3 is use by SHA record1 */
1381 err = devm_request_irq(cryp->dev, cryp->irq[RING3],
1382 mtk_sha_ring3_irq, IRQF_TRIGGER_LOW,
1383 "mtk-sha", cryp);
1384 if (err) {
1385 dev_err(cryp->dev, "unable to request sha irq1.\n");
1386 goto err_res;
1387 }
1388
1389 /* Enable ring2 and ring3 interrupt for hash */
1390 mtk_sha_write(cryp, AIC_ENABLE_SET(RING2), MTK_IRQ_RDR2);
1391 mtk_sha_write(cryp, AIC_ENABLE_SET(RING3), MTK_IRQ_RDR3);
1392
1393 cryp->tmp = dma_alloc_coherent(cryp->dev, SHA_TMP_BUF_SIZE,
1394 &cryp->tmp_dma, GFP_KERNEL);
1395 if (!cryp->tmp) {
1396 dev_err(cryp->dev, "unable to allocate tmp buffer.\n");
1397 err = -EINVAL;
1398 goto err_res;
1399 }
1400
1401 spin_lock(&mtk_sha.lock);
1402 list_add_tail(&cryp->sha_list, &mtk_sha.dev_list);
1403 spin_unlock(&mtk_sha.lock);
1404
1405 err = mtk_sha_register_algs();
1406 if (err)
1407 goto err_algs;
1408
1409 return 0;
1410
1411err_algs:
1412 spin_lock(&mtk_sha.lock);
1413 list_del(&cryp->sha_list);
1414 spin_unlock(&mtk_sha.lock);
1415 dma_free_coherent(cryp->dev, SHA_TMP_BUF_SIZE,
1416 cryp->tmp, cryp->tmp_dma);
1417err_res:
1418 mtk_sha_record_free(cryp);
1419err_record:
1420
1421 dev_err(cryp->dev, "mtk-sha initialization failed.\n");
1422 return err;
1423}
1424
1425void mtk_hash_alg_release(struct mtk_cryp *cryp)
1426{
1427 spin_lock(&mtk_sha.lock);
1428 list_del(&cryp->sha_list);
1429 spin_unlock(&mtk_sha.lock);
1430
1431 mtk_sha_unregister_algs();
1432 dma_free_coherent(cryp->dev, SHA_TMP_BUF_SIZE,
1433 cryp->tmp, cryp->tmp_dma);
1434 mtk_sha_record_free(cryp);
1435}
diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c
index 47576098831f..b6f14844702e 100644
--- a/drivers/crypto/picoxcell_crypto.c
+++ b/drivers/crypto/picoxcell_crypto.c
@@ -1616,32 +1616,17 @@ static const struct of_device_id spacc_of_id_table[] = {
1616MODULE_DEVICE_TABLE(of, spacc_of_id_table); 1616MODULE_DEVICE_TABLE(of, spacc_of_id_table);
1617#endif /* CONFIG_OF */ 1617#endif /* CONFIG_OF */
1618 1618
1619static bool spacc_is_compatible(struct platform_device *pdev,
1620 const char *spacc_type)
1621{
1622 const struct platform_device_id *platid = platform_get_device_id(pdev);
1623
1624 if (platid && !strcmp(platid->name, spacc_type))
1625 return true;
1626
1627#ifdef CONFIG_OF
1628 if (of_device_is_compatible(pdev->dev.of_node, spacc_type))
1629 return true;
1630#endif /* CONFIG_OF */
1631
1632 return false;
1633}
1634
1635static int spacc_probe(struct platform_device *pdev) 1619static int spacc_probe(struct platform_device *pdev)
1636{ 1620{
1637 int i, err, ret = -EINVAL; 1621 int i, err, ret = -EINVAL;
1638 struct resource *mem, *irq; 1622 struct resource *mem, *irq;
1623 struct device_node *np = pdev->dev.of_node;
1639 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), 1624 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
1640 GFP_KERNEL); 1625 GFP_KERNEL);
1641 if (!engine) 1626 if (!engine)
1642 return -ENOMEM; 1627 return -ENOMEM;
1643 1628
1644 if (spacc_is_compatible(pdev, "picochip,spacc-ipsec")) { 1629 if (of_device_is_compatible(np, "picochip,spacc-ipsec")) {
1645 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; 1630 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS;
1646 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; 1631 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;
1647 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; 1632 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ;
@@ -1650,7 +1635,7 @@ static int spacc_probe(struct platform_device *pdev)
1650 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); 1635 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs);
1651 engine->aeads = ipsec_engine_aeads; 1636 engine->aeads = ipsec_engine_aeads;
1652 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); 1637 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads);
1653 } else if (spacc_is_compatible(pdev, "picochip,spacc-l2")) { 1638 } else if (of_device_is_compatible(np, "picochip,spacc-l2")) {
1654 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; 1639 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS;
1655 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; 1640 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ;
1656 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; 1641 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ;
@@ -1803,12 +1788,6 @@ static int spacc_remove(struct platform_device *pdev)
1803 return 0; 1788 return 0;
1804} 1789}
1805 1790
1806static const struct platform_device_id spacc_id_table[] = {
1807 { "picochip,spacc-ipsec", },
1808 { "picochip,spacc-l2", },
1809 { }
1810};
1811
1812static struct platform_driver spacc_driver = { 1791static struct platform_driver spacc_driver = {
1813 .probe = spacc_probe, 1792 .probe = spacc_probe,
1814 .remove = spacc_remove, 1793 .remove = spacc_remove,
@@ -1819,7 +1798,6 @@ static struct platform_driver spacc_driver = {
1819#endif /* CONFIG_PM */ 1798#endif /* CONFIG_PM */
1820 .of_match_table = of_match_ptr(spacc_of_id_table), 1799 .of_match_table = of_match_ptr(spacc_of_id_table),
1821 }, 1800 },
1822 .id_table = spacc_id_table,
1823}; 1801};
1824 1802
1825module_platform_driver(spacc_driver); 1803module_platform_driver(spacc_driver);
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c
index 640c3fc870fd..f172171668ee 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c
+++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c
@@ -186,7 +186,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
186 } 186 }
187 187
188 /* Create dev top level debugfs entry */ 188 /* Create dev top level debugfs entry */
189 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 189 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
190 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 190 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
191 pdev->bus->number, PCI_SLOT(pdev->devfn), 191 pdev->bus->number, PCI_SLOT(pdev->devfn),
192 PCI_FUNC(pdev->devfn)); 192 PCI_FUNC(pdev->devfn));
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
index 949d77b79fbe..24ec908eb26c 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
+++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
@@ -170,7 +170,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
170 accel_pci_dev->sku = hw_data->get_sku(hw_data); 170 accel_pci_dev->sku = hw_data->get_sku(hw_data);
171 171
172 /* Create dev top level debugfs entry */ 172 /* Create dev top level debugfs entry */
173 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 173 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
174 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 174 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
175 pdev->bus->number, PCI_SLOT(pdev->devfn), 175 pdev->bus->number, PCI_SLOT(pdev->devfn),
176 PCI_FUNC(pdev->devfn)); 176 PCI_FUNC(pdev->devfn));
diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c
index 5b2d78a5b5aa..58a984c9c3ec 100644
--- a/drivers/crypto/qat/qat_c62x/adf_drv.c
+++ b/drivers/crypto/qat/qat_c62x/adf_drv.c
@@ -186,7 +186,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
186 } 186 }
187 187
188 /* Create dev top level debugfs entry */ 188 /* Create dev top level debugfs entry */
189 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 189 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
190 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 190 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
191 pdev->bus->number, PCI_SLOT(pdev->devfn), 191 pdev->bus->number, PCI_SLOT(pdev->devfn),
192 PCI_FUNC(pdev->devfn)); 192 PCI_FUNC(pdev->devfn));
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c
index 7540ce13b0d0..b9f3e0e4fde9 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c
+++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c
@@ -170,7 +170,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
170 accel_pci_dev->sku = hw_data->get_sku(hw_data); 170 accel_pci_dev->sku = hw_data->get_sku(hw_data);
171 171
172 /* Create dev top level debugfs entry */ 172 /* Create dev top level debugfs entry */
173 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 173 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
174 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 174 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
175 pdev->bus->number, PCI_SLOT(pdev->devfn), 175 pdev->bus->number, PCI_SLOT(pdev->devfn),
176 PCI_FUNC(pdev->devfn)); 176 PCI_FUNC(pdev->devfn));
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h
index 8c4f6573ce59..1211261de7c2 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_common.h
+++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h
@@ -61,6 +61,7 @@
61#define ADF_CFG_AFFINITY_WHATEVER 0xFF 61#define ADF_CFG_AFFINITY_WHATEVER 0xFF
62#define MAX_DEVICE_NAME_SIZE 32 62#define MAX_DEVICE_NAME_SIZE 32
63#define ADF_MAX_DEVICES (32 * 32) 63#define ADF_MAX_DEVICES (32 * 32)
64#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
64 65
65enum adf_cfg_val_type { 66enum adf_cfg_val_type {
66 ADF_DEC, 67 ADF_DEC,
diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h
index 980e07475012..5c4c0a253129 100644
--- a/drivers/crypto/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/qat/qat_common/adf_common_drv.h
@@ -87,8 +87,8 @@ enum adf_event {
87struct service_hndl { 87struct service_hndl {
88 int (*event_hld)(struct adf_accel_dev *accel_dev, 88 int (*event_hld)(struct adf_accel_dev *accel_dev,
89 enum adf_event event); 89 enum adf_event event);
90 unsigned long init_status; 90 unsigned long init_status[ADF_DEVS_ARRAY_SIZE];
91 unsigned long start_status; 91 unsigned long start_status[ADF_DEVS_ARRAY_SIZE];
92 char *name; 92 char *name;
93 struct list_head list; 93 struct list_head list;
94}; 94};
diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c
index b3ebb25f9ca7..8afac52677a6 100644
--- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c
+++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c
@@ -152,7 +152,7 @@ void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data)
152 ptr->hw_device->instance_id = i++; 152 ptr->hw_device->instance_id = i++;
153 153
154 if (i == class->instances) 154 if (i == class->instances)
155 break; 155 break;
156 } 156 }
157} 157}
158EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index); 158EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index);
diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c
index 888c6675e7e5..26556c713049 100644
--- a/drivers/crypto/qat/qat_common/adf_init.c
+++ b/drivers/crypto/qat/qat_common/adf_init.c
@@ -64,8 +64,8 @@ static void adf_service_add(struct service_hndl *service)
64 64
65int adf_service_register(struct service_hndl *service) 65int adf_service_register(struct service_hndl *service)
66{ 66{
67 service->init_status = 0; 67 memset(service->init_status, 0, sizeof(service->init_status));
68 service->start_status = 0; 68 memset(service->start_status, 0, sizeof(service->start_status));
69 adf_service_add(service); 69 adf_service_add(service);
70 return 0; 70 return 0;
71} 71}
@@ -79,9 +79,13 @@ static void adf_service_remove(struct service_hndl *service)
79 79
80int adf_service_unregister(struct service_hndl *service) 80int adf_service_unregister(struct service_hndl *service)
81{ 81{
82 if (service->init_status || service->start_status) { 82 int i;
83 pr_err("QAT: Could not remove active service\n"); 83
84 return -EFAULT; 84 for (i = 0; i < ARRAY_SIZE(service->init_status); i++) {
85 if (service->init_status[i] || service->start_status[i]) {
86 pr_err("QAT: Could not remove active service\n");
87 return -EFAULT;
88 }
85 } 89 }
86 adf_service_remove(service); 90 adf_service_remove(service);
87 return 0; 91 return 0;
@@ -163,7 +167,7 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
163 service->name); 167 service->name);
164 return -EFAULT; 168 return -EFAULT;
165 } 169 }
166 set_bit(accel_dev->accel_id, &service->init_status); 170 set_bit(accel_dev->accel_id, service->init_status);
167 } 171 }
168 172
169 hw_data->enable_error_correction(accel_dev); 173 hw_data->enable_error_correction(accel_dev);
@@ -210,7 +214,7 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
210 service->name); 214 service->name);
211 return -EFAULT; 215 return -EFAULT;
212 } 216 }
213 set_bit(accel_dev->accel_id, &service->start_status); 217 set_bit(accel_dev->accel_id, service->start_status);
214 } 218 }
215 219
216 clear_bit(ADF_STATUS_STARTING, &accel_dev->status); 220 clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
@@ -259,14 +263,14 @@ void adf_dev_stop(struct adf_accel_dev *accel_dev)
259 263
260 list_for_each(list_itr, &service_table) { 264 list_for_each(list_itr, &service_table) {
261 service = list_entry(list_itr, struct service_hndl, list); 265 service = list_entry(list_itr, struct service_hndl, list);
262 if (!test_bit(accel_dev->accel_id, &service->start_status)) 266 if (!test_bit(accel_dev->accel_id, service->start_status))
263 continue; 267 continue;
264 ret = service->event_hld(accel_dev, ADF_EVENT_STOP); 268 ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
265 if (!ret) { 269 if (!ret) {
266 clear_bit(accel_dev->accel_id, &service->start_status); 270 clear_bit(accel_dev->accel_id, service->start_status);
267 } else if (ret == -EAGAIN) { 271 } else if (ret == -EAGAIN) {
268 wait = true; 272 wait = true;
269 clear_bit(accel_dev->accel_id, &service->start_status); 273 clear_bit(accel_dev->accel_id, service->start_status);
270 } 274 }
271 } 275 }
272 276
@@ -317,14 +321,14 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
317 321
318 list_for_each(list_itr, &service_table) { 322 list_for_each(list_itr, &service_table) {
319 service = list_entry(list_itr, struct service_hndl, list); 323 service = list_entry(list_itr, struct service_hndl, list);
320 if (!test_bit(accel_dev->accel_id, &service->init_status)) 324 if (!test_bit(accel_dev->accel_id, service->init_status))
321 continue; 325 continue;
322 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) 326 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
323 dev_err(&GET_DEV(accel_dev), 327 dev_err(&GET_DEV(accel_dev),
324 "Failed to shutdown service %s\n", 328 "Failed to shutdown service %s\n",
325 service->name); 329 service->name);
326 else 330 else
327 clear_bit(accel_dev->accel_id, &service->init_status); 331 clear_bit(accel_dev->accel_id, service->init_status);
328 } 332 }
329 333
330 hw_data->disable_iov(accel_dev); 334 hw_data->disable_iov(accel_dev);
diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c
index 9320ae1d005b..b36d8653b1ba 100644
--- a/drivers/crypto/qat/qat_common/adf_sriov.c
+++ b/drivers/crypto/qat/qat_common/adf_sriov.c
@@ -162,9 +162,9 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev)
162 162
163/** 163/**
164 * adf_disable_sriov() - Disable SRIOV for the device 164 * adf_disable_sriov() - Disable SRIOV for the device
165 * @pdev: Pointer to pci device. 165 * @accel_dev: Pointer to accel device.
166 * 166 *
167 * Function disables SRIOV for the pci device. 167 * Function disables SRIOV for the accel device.
168 * 168 *
169 * Return: 0 on success, error code otherwise. 169 * Return: 0 on success, error code otherwise.
170 */ 170 */
diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c
index bf99e11a3403..4a73fc70f7a9 100644
--- a/drivers/crypto/qat/qat_common/adf_vf_isr.c
+++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c
@@ -148,7 +148,7 @@ static void adf_pf2vf_bh_handler(void *data)
148 INIT_WORK(&stop_data->work, adf_dev_stop_async); 148 INIT_WORK(&stop_data->work, adf_dev_stop_async);
149 queue_work(adf_vf_stop_wq, &stop_data->work); 149 queue_work(adf_vf_stop_wq, &stop_data->work);
150 /* To ack, clear the PF2VFINT bit */ 150 /* To ack, clear the PF2VFINT bit */
151 msg &= ~BIT(0); 151 msg &= ~ADF_PF2VF_INT;
152 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg); 152 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
153 return; 153 return;
154 } 154 }
@@ -168,7 +168,7 @@ static void adf_pf2vf_bh_handler(void *data)
168 } 168 }
169 169
170 /* To ack, clear the PF2VFINT bit */ 170 /* To ack, clear the PF2VFINT bit */
171 msg &= ~BIT(0); 171 msg &= ~ADF_PF2VF_INT;
172 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg); 172 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
173 173
174 /* Re-enable PF2VF interrupts */ 174 /* Re-enable PF2VF interrupts */
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
index 4d2de2838451..2ce01f010c74 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
@@ -186,7 +186,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
186 } 186 }
187 187
188 /* Create dev top level debugfs entry */ 188 /* Create dev top level debugfs entry */
189 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 189 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
190 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 190 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
191 pdev->bus->number, PCI_SLOT(pdev->devfn), 191 pdev->bus->number, PCI_SLOT(pdev->devfn),
192 PCI_FUNC(pdev->devfn)); 192 PCI_FUNC(pdev->devfn));
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
index 60df98632fa2..26ab17bfc6da 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
+++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
@@ -170,7 +170,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
170 accel_pci_dev->sku = hw_data->get_sku(hw_data); 170 accel_pci_dev->sku = hw_data->get_sku(hw_data);
171 171
172 /* Create dev top level debugfs entry */ 172 /* Create dev top level debugfs entry */
173 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 173 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
174 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 174 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
175 pdev->bus->number, PCI_SLOT(pdev->devfn), 175 pdev->bus->number, PCI_SLOT(pdev->devfn),
176 PCI_FUNC(pdev->devfn)); 176 PCI_FUNC(pdev->devfn));
diff --git a/drivers/crypto/virtio/Kconfig b/drivers/crypto/virtio/Kconfig
index d80f73366ae2..5db07495ddc5 100644
--- a/drivers/crypto/virtio/Kconfig
+++ b/drivers/crypto/virtio/Kconfig
@@ -4,6 +4,7 @@ config CRYPTO_DEV_VIRTIO
4 select CRYPTO_AEAD 4 select CRYPTO_AEAD
5 select CRYPTO_AUTHENC 5 select CRYPTO_AUTHENC
6 select CRYPTO_BLKCIPHER 6 select CRYPTO_BLKCIPHER
7 select CRYPTO_ENGINE
7 default m 8 default m
8 help 9 help
9 This driver provides support for virtio crypto device. If you 10 This driver provides support for virtio crypto device. If you
diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c
index c2374df9abae..49defda4e03d 100644
--- a/drivers/crypto/virtio/virtio_crypto_algs.c
+++ b/drivers/crypto/virtio/virtio_crypto_algs.c
@@ -288,8 +288,7 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
288static int 288static int
289__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req, 289__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
290 struct ablkcipher_request *req, 290 struct ablkcipher_request *req,
291 struct data_queue *data_vq, 291 struct data_queue *data_vq)
292 __u8 op)
293{ 292{
294 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 293 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
295 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 294 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
@@ -329,7 +328,7 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
329 vc_req->req_data = req_data; 328 vc_req->req_data = req_data;
330 vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER; 329 vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
331 /* Head of operation */ 330 /* Head of operation */
332 if (op) { 331 if (vc_req->encrypt) {
333 req_data->header.session_id = 332 req_data->header.session_id =
334 cpu_to_le64(ctx->enc_sess_info.session_id); 333 cpu_to_le64(ctx->enc_sess_info.session_id);
335 req_data->header.opcode = 334 req_data->header.opcode =
@@ -424,19 +423,15 @@ static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
424 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm); 423 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
425 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); 424 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
426 struct virtio_crypto *vcrypto = ctx->vcrypto; 425 struct virtio_crypto *vcrypto = ctx->vcrypto;
427 int ret;
428 /* Use the first data virtqueue as default */ 426 /* Use the first data virtqueue as default */
429 struct data_queue *data_vq = &vcrypto->data_vq[0]; 427 struct data_queue *data_vq = &vcrypto->data_vq[0];
430 428
431 vc_req->ablkcipher_ctx = ctx; 429 vc_req->ablkcipher_ctx = ctx;
432 vc_req->ablkcipher_req = req; 430 vc_req->ablkcipher_req = req;
433 ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq, 1); 431 vc_req->encrypt = true;
434 if (ret < 0) { 432 vc_req->dataq = data_vq;
435 pr_err("virtio_crypto: Encryption failed!\n");
436 return ret;
437 }
438 433
439 return -EINPROGRESS; 434 return crypto_transfer_cipher_request_to_engine(data_vq->engine, req);
440} 435}
441 436
442static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req) 437static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
@@ -445,20 +440,16 @@ static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
445 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm); 440 struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
446 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); 441 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
447 struct virtio_crypto *vcrypto = ctx->vcrypto; 442 struct virtio_crypto *vcrypto = ctx->vcrypto;
448 int ret;
449 /* Use the first data virtqueue as default */ 443 /* Use the first data virtqueue as default */
450 struct data_queue *data_vq = &vcrypto->data_vq[0]; 444 struct data_queue *data_vq = &vcrypto->data_vq[0];
451 445
452 vc_req->ablkcipher_ctx = ctx; 446 vc_req->ablkcipher_ctx = ctx;
453 vc_req->ablkcipher_req = req; 447 vc_req->ablkcipher_req = req;
454 448
455 ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq, 0); 449 vc_req->encrypt = false;
456 if (ret < 0) { 450 vc_req->dataq = data_vq;
457 pr_err("virtio_crypto: Decryption failed!\n");
458 return ret;
459 }
460 451
461 return -EINPROGRESS; 452 return crypto_transfer_cipher_request_to_engine(data_vq->engine, req);
462} 453}
463 454
464static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm) 455static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm)
@@ -484,10 +475,37 @@ static void virtio_crypto_ablkcipher_exit(struct crypto_tfm *tfm)
484 ctx->vcrypto = NULL; 475 ctx->vcrypto = NULL;
485} 476}
486 477
478int virtio_crypto_ablkcipher_crypt_req(
479 struct crypto_engine *engine,
480 struct ablkcipher_request *req)
481{
482 struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
483 struct data_queue *data_vq = vc_req->dataq;
484 int ret;
485
486 ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq);
487 if (ret < 0)
488 return ret;
489
490 virtqueue_kick(data_vq->vq);
491
492 return 0;
493}
494
495void virtio_crypto_ablkcipher_finalize_req(
496 struct virtio_crypto_request *vc_req,
497 struct ablkcipher_request *req,
498 int err)
499{
500 crypto_finalize_cipher_request(vc_req->dataq->engine, req, err);
501
502 virtcrypto_clear_request(vc_req);
503}
504
487static struct crypto_alg virtio_crypto_algs[] = { { 505static struct crypto_alg virtio_crypto_algs[] = { {
488 .cra_name = "cbc(aes)", 506 .cra_name = "cbc(aes)",
489 .cra_driver_name = "virtio_crypto_aes_cbc", 507 .cra_driver_name = "virtio_crypto_aes_cbc",
490 .cra_priority = 501, 508 .cra_priority = 150,
491 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 509 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
492 .cra_blocksize = AES_BLOCK_SIZE, 510 .cra_blocksize = AES_BLOCK_SIZE,
493 .cra_ctxsize = sizeof(struct virtio_crypto_ablkcipher_ctx), 511 .cra_ctxsize = sizeof(struct virtio_crypto_ablkcipher_ctx),
diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h
index 3d6566b02876..da6d8c0ea407 100644
--- a/drivers/crypto/virtio/virtio_crypto_common.h
+++ b/drivers/crypto/virtio/virtio_crypto_common.h
@@ -25,6 +25,7 @@
25#include <crypto/aead.h> 25#include <crypto/aead.h>
26#include <crypto/aes.h> 26#include <crypto/aes.h>
27#include <crypto/authenc.h> 27#include <crypto/authenc.h>
28#include <crypto/engine.h>
28 29
29 30
30/* Internal representation of a data virtqueue */ 31/* Internal representation of a data virtqueue */
@@ -37,6 +38,8 @@ struct data_queue {
37 38
38 /* Name of the tx queue: dataq.$index */ 39 /* Name of the tx queue: dataq.$index */
39 char name[32]; 40 char name[32];
41
42 struct crypto_engine *engine;
40}; 43};
41 44
42struct virtio_crypto { 45struct virtio_crypto {
@@ -97,6 +100,9 @@ struct virtio_crypto_request {
97 struct virtio_crypto_op_data_req *req_data; 100 struct virtio_crypto_op_data_req *req_data;
98 struct scatterlist **sgs; 101 struct scatterlist **sgs;
99 uint8_t *iv; 102 uint8_t *iv;
103 /* Encryption? */
104 bool encrypt;
105 struct data_queue *dataq;
100}; 106};
101 107
102int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev); 108int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev);
@@ -110,6 +116,16 @@ int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev);
110struct virtio_crypto *virtcrypto_get_dev_node(int node); 116struct virtio_crypto *virtcrypto_get_dev_node(int node);
111int virtcrypto_dev_start(struct virtio_crypto *vcrypto); 117int virtcrypto_dev_start(struct virtio_crypto *vcrypto);
112void virtcrypto_dev_stop(struct virtio_crypto *vcrypto); 118void virtcrypto_dev_stop(struct virtio_crypto *vcrypto);
119int virtio_crypto_ablkcipher_crypt_req(
120 struct crypto_engine *engine,
121 struct ablkcipher_request *req);
122void virtio_crypto_ablkcipher_finalize_req(
123 struct virtio_crypto_request *vc_req,
124 struct ablkcipher_request *req,
125 int err);
126
127void
128virtcrypto_clear_request(struct virtio_crypto_request *vc_req);
113 129
114static inline int virtio_crypto_get_current_node(void) 130static inline int virtio_crypto_get_current_node(void)
115{ 131{
diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c
index fe70ec823b27..b5b153317376 100644
--- a/drivers/crypto/virtio/virtio_crypto_core.c
+++ b/drivers/crypto/virtio/virtio_crypto_core.c
@@ -25,7 +25,7 @@
25#include "virtio_crypto_common.h" 25#include "virtio_crypto_common.h"
26 26
27 27
28static void 28void
29virtcrypto_clear_request(struct virtio_crypto_request *vc_req) 29virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
30{ 30{
31 if (vc_req) { 31 if (vc_req) {
@@ -66,12 +66,12 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq)
66 break; 66 break;
67 } 67 }
68 ablk_req = vc_req->ablkcipher_req; 68 ablk_req = vc_req->ablkcipher_req;
69 virtcrypto_clear_request(vc_req);
70 69
71 spin_unlock_irqrestore( 70 spin_unlock_irqrestore(
72 &vcrypto->data_vq[qid].lock, flags); 71 &vcrypto->data_vq[qid].lock, flags);
73 /* Finish the encrypt or decrypt process */ 72 /* Finish the encrypt or decrypt process */
74 ablk_req->base.complete(&ablk_req->base, error); 73 virtio_crypto_ablkcipher_finalize_req(vc_req,
74 ablk_req, error);
75 spin_lock_irqsave( 75 spin_lock_irqsave(
76 &vcrypto->data_vq[qid].lock, flags); 76 &vcrypto->data_vq[qid].lock, flags);
77 } 77 }
@@ -87,6 +87,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
87 int ret = -ENOMEM; 87 int ret = -ENOMEM;
88 int i, total_vqs; 88 int i, total_vqs;
89 const char **names; 89 const char **names;
90 struct device *dev = &vi->vdev->dev;
90 91
91 /* 92 /*
92 * We expect 1 data virtqueue, followed by 93 * We expect 1 data virtqueue, followed by
@@ -128,6 +129,15 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
128 for (i = 0; i < vi->max_data_queues; i++) { 129 for (i = 0; i < vi->max_data_queues; i++) {
129 spin_lock_init(&vi->data_vq[i].lock); 130 spin_lock_init(&vi->data_vq[i].lock);
130 vi->data_vq[i].vq = vqs[i]; 131 vi->data_vq[i].vq = vqs[i];
132 /* Initialize crypto engine */
133 vi->data_vq[i].engine = crypto_engine_alloc_init(dev, 1);
134 if (!vi->data_vq[i].engine) {
135 ret = -ENOMEM;
136 goto err_engine;
137 }
138
139 vi->data_vq[i].engine->cipher_one_request =
140 virtio_crypto_ablkcipher_crypt_req;
131 } 141 }
132 142
133 kfree(names); 143 kfree(names);
@@ -136,6 +146,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
136 146
137 return 0; 147 return 0;
138 148
149err_engine:
139err_find: 150err_find:
140 kfree(names); 151 kfree(names);
141err_names: 152err_names:
@@ -269,6 +280,38 @@ static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
269 return 0; 280 return 0;
270} 281}
271 282
283static int virtcrypto_start_crypto_engines(struct virtio_crypto *vcrypto)
284{
285 int32_t i;
286 int ret;
287
288 for (i = 0; i < vcrypto->max_data_queues; i++) {
289 if (vcrypto->data_vq[i].engine) {
290 ret = crypto_engine_start(vcrypto->data_vq[i].engine);
291 if (ret)
292 goto err;
293 }
294 }
295
296 return 0;
297
298err:
299 while (--i >= 0)
300 if (vcrypto->data_vq[i].engine)
301 crypto_engine_exit(vcrypto->data_vq[i].engine);
302
303 return ret;
304}
305
306static void virtcrypto_clear_crypto_engines(struct virtio_crypto *vcrypto)
307{
308 u32 i;
309
310 for (i = 0; i < vcrypto->max_data_queues; i++)
311 if (vcrypto->data_vq[i].engine)
312 crypto_engine_exit(vcrypto->data_vq[i].engine);
313}
314
272static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto) 315static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto)
273{ 316{
274 struct virtio_device *vdev = vcrypto->vdev; 317 struct virtio_device *vdev = vcrypto->vdev;
@@ -355,14 +398,21 @@ static int virtcrypto_probe(struct virtio_device *vdev)
355 dev_err(&vdev->dev, "Failed to initialize vqs.\n"); 398 dev_err(&vdev->dev, "Failed to initialize vqs.\n");
356 goto free_dev; 399 goto free_dev;
357 } 400 }
401
402 err = virtcrypto_start_crypto_engines(vcrypto);
403 if (err)
404 goto free_vqs;
405
358 virtio_device_ready(vdev); 406 virtio_device_ready(vdev);
359 407
360 err = virtcrypto_update_status(vcrypto); 408 err = virtcrypto_update_status(vcrypto);
361 if (err) 409 if (err)
362 goto free_vqs; 410 goto free_engines;
363 411
364 return 0; 412 return 0;
365 413
414free_engines:
415 virtcrypto_clear_crypto_engines(vcrypto);
366free_vqs: 416free_vqs:
367 vcrypto->vdev->config->reset(vdev); 417 vcrypto->vdev->config->reset(vdev);
368 virtcrypto_del_vqs(vcrypto); 418 virtcrypto_del_vqs(vcrypto);
@@ -398,6 +448,7 @@ static void virtcrypto_remove(struct virtio_device *vdev)
398 virtcrypto_dev_stop(vcrypto); 448 virtcrypto_dev_stop(vcrypto);
399 vdev->config->reset(vdev); 449 vdev->config->reset(vdev);
400 virtcrypto_free_unused_reqs(vcrypto); 450 virtcrypto_free_unused_reqs(vcrypto);
451 virtcrypto_clear_crypto_engines(vcrypto);
401 virtcrypto_del_vqs(vcrypto); 452 virtcrypto_del_vqs(vcrypto);
402 virtcrypto_devmgr_rm_dev(vcrypto); 453 virtcrypto_devmgr_rm_dev(vcrypto);
403 kfree(vcrypto); 454 kfree(vcrypto);
@@ -420,6 +471,7 @@ static int virtcrypto_freeze(struct virtio_device *vdev)
420 if (virtcrypto_dev_started(vcrypto)) 471 if (virtcrypto_dev_started(vcrypto))
421 virtcrypto_dev_stop(vcrypto); 472 virtcrypto_dev_stop(vcrypto);
422 473
474 virtcrypto_clear_crypto_engines(vcrypto);
423 virtcrypto_del_vqs(vcrypto); 475 virtcrypto_del_vqs(vcrypto);
424 return 0; 476 return 0;
425} 477}
@@ -433,14 +485,26 @@ static int virtcrypto_restore(struct virtio_device *vdev)
433 if (err) 485 if (err)
434 return err; 486 return err;
435 487
488 err = virtcrypto_start_crypto_engines(vcrypto);
489 if (err)
490 goto free_vqs;
491
436 virtio_device_ready(vdev); 492 virtio_device_ready(vdev);
493
437 err = virtcrypto_dev_start(vcrypto); 494 err = virtcrypto_dev_start(vcrypto);
438 if (err) { 495 if (err) {
439 dev_err(&vdev->dev, "Failed to start virtio crypto device.\n"); 496 dev_err(&vdev->dev, "Failed to start virtio crypto device.\n");
440 return -EFAULT; 497 goto free_engines;
441 } 498 }
442 499
443 return 0; 500 return 0;
501
502free_engines:
503 virtcrypto_clear_crypto_engines(vcrypto);
504free_vqs:
505 vcrypto->vdev->config->reset(vdev);
506 virtcrypto_del_vqs(vcrypto);
507 return err;
444} 508}
445#endif 509#endif
446 510
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 38ed10d761d0..7cf6d31c1123 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -80,11 +80,13 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
80 int ret; 80 int ret;
81 struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); 81 struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
82 82
83 preempt_disable();
83 pagefault_disable(); 84 pagefault_disable();
84 enable_kernel_vsx(); 85 enable_kernel_vsx();
85 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); 86 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
86 disable_kernel_vsx(); 87 disable_kernel_vsx();
87 pagefault_enable(); 88 pagefault_enable();
89 preempt_enable();
88 90
89 ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); 91 ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
90 return ret; 92 return ret;
@@ -99,11 +101,13 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
99 u8 *dst = walk->dst.virt.addr; 101 u8 *dst = walk->dst.virt.addr;
100 unsigned int nbytes = walk->nbytes; 102 unsigned int nbytes = walk->nbytes;
101 103
104 preempt_disable();
102 pagefault_disable(); 105 pagefault_disable();
103 enable_kernel_vsx(); 106 enable_kernel_vsx();
104 aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key); 107 aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key);
105 disable_kernel_vsx(); 108 disable_kernel_vsx();
106 pagefault_enable(); 109 pagefault_enable();
110 preempt_enable();
107 111
108 crypto_xor(keystream, src, nbytes); 112 crypto_xor(keystream, src, nbytes);
109 memcpy(dst, keystream, nbytes); 113 memcpy(dst, keystream, nbytes);
@@ -132,6 +136,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
132 blkcipher_walk_init(&walk, dst, src, nbytes); 136 blkcipher_walk_init(&walk, dst, src, nbytes);
133 ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 137 ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
134 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 138 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
139 preempt_disable();
135 pagefault_disable(); 140 pagefault_disable();
136 enable_kernel_vsx(); 141 enable_kernel_vsx();
137 aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, 142 aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
@@ -143,6 +148,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
143 walk.iv); 148 walk.iv);
144 disable_kernel_vsx(); 149 disable_kernel_vsx();
145 pagefault_enable(); 150 pagefault_enable();
151 preempt_enable();
146 152
147 /* We need to update IV mostly for last bytes/round */ 153 /* We need to update IV mostly for last bytes/round */
148 inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE; 154 inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
index 8d9e4b7a8e84..ccc05f874419 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
@@ -3385,6 +3385,14 @@ struct fw_crypto_lookaside_wr {
3385#define FW_CRYPTO_LOOKASIDE_WR_IV_G(x) \ 3385#define FW_CRYPTO_LOOKASIDE_WR_IV_G(x) \
3386 (((x) >> FW_CRYPTO_LOOKASIDE_WR_IV_S) & FW_CRYPTO_LOOKASIDE_WR_IV_M) 3386 (((x) >> FW_CRYPTO_LOOKASIDE_WR_IV_S) & FW_CRYPTO_LOOKASIDE_WR_IV_M)
3387 3387
3388#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_S 15
3389#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_M 0xff
3390#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(x) \
3391 ((x) << FW_CRYPTO_LOOKASIDE_WR_FQIDX_S)
3392#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_G(x) \
3393 (((x) >> FW_CRYPTO_LOOKASIDE_WR_FQIDX_S) & \
3394 FW_CRYPTO_LOOKASIDE_WR_FQIDX_M)
3395
3388#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_S 10 3396#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_S 10
3389#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_M 0x3 3397#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_M 0x3
3390#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_V(x) \ 3398#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_V(x) \
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 404e9558e879..ebe4ded0c55d 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -191,9 +191,25 @@ static inline unsigned int crypto_queue_len(struct crypto_queue *queue)
191 return queue->qlen; 191 return queue->qlen;
192} 192}
193 193
194/* These functions require the input/output to be aligned as u32. */
195void crypto_inc(u8 *a, unsigned int size); 194void crypto_inc(u8 *a, unsigned int size);
196void crypto_xor(u8 *dst, const u8 *src, unsigned int size); 195void __crypto_xor(u8 *dst, const u8 *src, unsigned int size);
196
197static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
198{
199 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
200 __builtin_constant_p(size) &&
201 (size % sizeof(unsigned long)) == 0) {
202 unsigned long *d = (unsigned long *)dst;
203 unsigned long *s = (unsigned long *)src;
204
205 while (size > 0) {
206 *d++ ^= *s++;
207 size -= sizeof(unsigned long);
208 }
209 } else {
210 __crypto_xor(dst, src, size);
211 }
212}
197 213
198int blkcipher_walk_done(struct blkcipher_desc *desc, 214int blkcipher_walk_done(struct blkcipher_desc *desc,
199 struct blkcipher_walk *walk, int err); 215 struct blkcipher_walk *walk, int err);
diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h
index 20d20f681a72..445fc45f4b5b 100644
--- a/include/crypto/chacha20.h
+++ b/include/crypto/chacha20.h
@@ -5,6 +5,7 @@
5#ifndef _CRYPTO_CHACHA20_H 5#ifndef _CRYPTO_CHACHA20_H
6#define _CRYPTO_CHACHA20_H 6#define _CRYPTO_CHACHA20_H
7 7
8#include <crypto/skcipher.h>
8#include <linux/types.h> 9#include <linux/types.h>
9#include <linux/crypto.h> 10#include <linux/crypto.h>
10 11
@@ -18,9 +19,8 @@ struct chacha20_ctx {
18 19
19void chacha20_block(u32 *state, void *stream); 20void chacha20_block(u32 *state, void *stream);
20void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv); 21void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv);
21int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key, 22int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
22 unsigned int keysize); 23 unsigned int keysize);
23int crypto_chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 24int crypto_chacha20_crypt(struct skcipher_request *req);
24 struct scatterlist *src, unsigned int nbytes);
25 25
26#endif 26#endif
diff --git a/include/crypto/hash.h b/include/crypto/hash.h
index 216a2b876147..b5727bcd2336 100644
--- a/include/crypto/hash.h
+++ b/include/crypto/hash.h
@@ -329,6 +329,16 @@ static inline unsigned int crypto_ahash_digestsize(struct crypto_ahash *tfm)
329 return crypto_hash_alg_common(tfm)->digestsize; 329 return crypto_hash_alg_common(tfm)->digestsize;
330} 330}
331 331
332/**
333 * crypto_ahash_statesize() - obtain size of the ahash state
334 * @tfm: cipher handle
335 *
336 * Return the size of the ahash state. With the crypto_ahash_export()
337 * function, the caller can export the state into a buffer whose size is
338 * defined with this function.
339 *
340 * Return: size of the ahash state
341 */
332static inline unsigned int crypto_ahash_statesize(struct crypto_ahash *tfm) 342static inline unsigned int crypto_ahash_statesize(struct crypto_ahash *tfm)
333{ 343{
334 return crypto_hash_alg_common(tfm)->statesize; 344 return crypto_hash_alg_common(tfm)->statesize;
@@ -369,11 +379,7 @@ static inline struct crypto_ahash *crypto_ahash_reqtfm(
369 * crypto_ahash_reqsize() - obtain size of the request data structure 379 * crypto_ahash_reqsize() - obtain size of the request data structure
370 * @tfm: cipher handle 380 * @tfm: cipher handle
371 * 381 *
372 * Return the size of the ahash state size. With the crypto_ahash_export 382 * Return: size of the request data
373 * function, the caller can export the state into a buffer whose size is
374 * defined with this function.
375 *
376 * Return: size of the ahash state
377 */ 383 */
378static inline unsigned int crypto_ahash_reqsize(struct crypto_ahash *tfm) 384static inline unsigned int crypto_ahash_reqsize(struct crypto_ahash *tfm)
379{ 385{
@@ -453,7 +459,7 @@ int crypto_ahash_digest(struct ahash_request *req);
453 * 459 *
454 * This function exports the hash state of the ahash_request handle into the 460 * This function exports the hash state of the ahash_request handle into the
455 * caller-allocated output buffer out which must have sufficient size (e.g. by 461 * caller-allocated output buffer out which must have sufficient size (e.g. by
456 * calling crypto_ahash_reqsize). 462 * calling crypto_ahash_statesize()).
457 * 463 *
458 * Return: 0 if the export was successful; < 0 if an error occurred 464 * Return: 0 if the export was successful; < 0 if an error occurred
459 */ 465 */
diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h
index 8735979ed341..e42f7063f245 100644
--- a/include/crypto/internal/skcipher.h
+++ b/include/crypto/internal/skcipher.h
@@ -66,7 +66,7 @@ struct skcipher_walk {
66 66
67 int flags; 67 int flags;
68 unsigned int blocksize; 68 unsigned int blocksize;
69 unsigned int chunksize; 69 unsigned int stride;
70 unsigned int alignmask; 70 unsigned int alignmask;
71}; 71};
72 72
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index 750b14f1ada4..562001cb412b 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -115,6 +115,9 @@ struct crypto_skcipher {
115 * IV of exactly that size to perform the encrypt or decrypt operation. 115 * IV of exactly that size to perform the encrypt or decrypt operation.
116 * @chunksize: Equal to the block size except for stream ciphers such as 116 * @chunksize: Equal to the block size except for stream ciphers such as
117 * CTR where it is set to the underlying block size. 117 * CTR where it is set to the underlying block size.
118 * @walksize: Equal to the chunk size except in cases where the algorithm is
119 * considerably more efficient if it can operate on multiple chunks
120 * in parallel. Should be a multiple of chunksize.
118 * @base: Definition of a generic crypto algorithm. 121 * @base: Definition of a generic crypto algorithm.
119 * 122 *
120 * All fields except @ivsize are mandatory and must be filled. 123 * All fields except @ivsize are mandatory and must be filled.
@@ -131,6 +134,7 @@ struct skcipher_alg {
131 unsigned int max_keysize; 134 unsigned int max_keysize;
132 unsigned int ivsize; 135 unsigned int ivsize;
133 unsigned int chunksize; 136 unsigned int chunksize;
137 unsigned int walksize;
134 138
135 struct crypto_alg base; 139 struct crypto_alg base;
136}; 140};
@@ -289,6 +293,19 @@ static inline unsigned int crypto_skcipher_alg_chunksize(
289 return alg->chunksize; 293 return alg->chunksize;
290} 294}
291 295
296static inline unsigned int crypto_skcipher_alg_walksize(
297 struct skcipher_alg *alg)
298{
299 if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
300 CRYPTO_ALG_TYPE_BLKCIPHER)
301 return alg->base.cra_blocksize;
302
303 if (alg->base.cra_ablkcipher.encrypt)
304 return alg->base.cra_blocksize;
305
306 return alg->walksize;
307}
308
292/** 309/**
293 * crypto_skcipher_chunksize() - obtain chunk size 310 * crypto_skcipher_chunksize() - obtain chunk size
294 * @tfm: cipher handle 311 * @tfm: cipher handle
@@ -307,6 +324,23 @@ static inline unsigned int crypto_skcipher_chunksize(
307} 324}
308 325
309/** 326/**
327 * crypto_skcipher_walksize() - obtain walk size
328 * @tfm: cipher handle
329 *
330 * In some cases, algorithms can only perform optimally when operating on
331 * multiple blocks in parallel. This is reflected by the walksize, which
332 * must be a multiple of the chunksize (or equal if the concern does not
333 * apply)
334 *
335 * Return: walk size in bytes
336 */
337static inline unsigned int crypto_skcipher_walksize(
338 struct crypto_skcipher *tfm)
339{
340 return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm));
341}
342
343/**
310 * crypto_skcipher_blocksize() - obtain block size of cipher 344 * crypto_skcipher_blocksize() - obtain block size of cipher
311 * @tfm: cipher handle 345 * @tfm: cipher handle
312 * 346 *
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index 0444b1336268..fddd1a5eb322 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -116,6 +116,7 @@
116 */ 116 */
117#define __pure __attribute__((pure)) 117#define __pure __attribute__((pure))
118#define __aligned(x) __attribute__((aligned(x))) 118#define __aligned(x) __attribute__((aligned(x)))
119#define __aligned_largest __attribute__((aligned))
119#define __printf(a, b) __attribute__((format(printf, a, b))) 120#define __printf(a, b) __attribute__((format(printf, a, b)))
120#define __scanf(a, b) __attribute__((format(scanf, a, b))) 121#define __scanf(a, b) __attribute__((format(scanf, a, b)))
121#define __attribute_const__ __attribute__((__const__)) 122#define __attribute_const__ __attribute__((__const__))
diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h
index 0590263c462c..762b5fec3383 100644
--- a/include/linux/miscdevice.h
+++ b/include/linux/miscdevice.h
@@ -32,6 +32,7 @@
32#define SGI_MMTIMER 153 32#define SGI_MMTIMER 153
33#define STORE_QUEUE_MINOR 155 /* unused */ 33#define STORE_QUEUE_MINOR 155 /* unused */
34#define I2O_MINOR 166 34#define I2O_MINOR 166
35#define HWRNG_MINOR 183
35#define MICROCODE_MINOR 184 36#define MICROCODE_MINOR 184
36#define IRNET_MINOR 187 37#define IRNET_MINOR 187
37#define VFIO_MINOR 196 38#define VFIO_MINOR 196
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-22 22:18:32 -0400