Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto updates from Herbert Xu: "API: - Remove VLA usage - Add cryptostat user-space interface - Add notifier for new crypto algorithms Algorithms: - Add OFB mode - Remove speck Drivers: - Remove x86/sha*-mb as they are buggy - Remove pcbc(aes) from x86/aesni - Improve performance of arm/ghash-ce by up to 85% - Implement CTS-CBC in arm64/aes-blk, faster by up to 50% - Remove PMULL based arm64/crc32 driver - Use PMULL in arm64/crct10dif - Add aes-ctr support in s5p-sss - Add caam/qi2 driver Others: - Pick better transform if one becomes available in crc-t10dif" * 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (124 commits) crypto: chelsio - Update ntx queue received from cxgb4 crypto: ccree - avoid implicit enum conversion crypto: caam - add SPDX license identifier to all files crypto: caam/qi - simplify CGR allocation, freeing crypto: mxs-dcp - make symbols 'sha1_null_hash' and 'sha256_null_hash' static crypto: arm64/aes-blk - ensure XTS mask is always loaded crypto: testmgr - fix sizeof() on COMP_BUF_SIZE crypto: chtls - remove set but not used variable 'csk' crypto: axis - fix platform_no_drv_owner.cocci warnings crypto: x86/aes-ni - fix build error following fpu template removal crypto: arm64/aes - fix handling sub-block CTS-CBC inputs crypto: caam/qi2 - avoid double export crypto: mxs-dcp - Fix AES issues crypto: mxs-dcp - Fix SHA null hashes and output length crypto: mxs-dcp - Implement sha import/export crypto: aegis/generic - fix for big endian systems crypto: morus/generic - fix for big endian systems crypto: lrw - fix rebase error after out of bounds fix crypto: cavium/nitrox - use pci_alloc_irq_vectors() while enabling MSI-X. crypto: cavium/nitrox - NITROX command queue changes. ...
author: Linus Torvalds <torvalds@linux-foundation.org> 2018-10-25 19:43:35 -0400
committer: Linus Torvalds <torvalds@linux-foundation.org> 2018-10-25 19:43:35 -0400
commit: 62606c224d72a98c35d21a849f95cccf95b0a252 (patch)
tree: 6f6f3466451edf9baa2ea8b5f9fc558aa555c69a
parent: 24ed334f33666f2ae929ccc08f72e7e72e353c64 (diff)
parent: a1c6fd4308d37f072e939a2782f24214115fc7e8 (diff)
234 files changed, 11921 insertions, 15829 deletions
diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst
index 48b424de85bb..cfbc18f0d9c9 100644
--- a/Documentation/filesystems/fscrypt.rst
+++ b/Documentation/filesystems/fscrypt.rst
@@ -191,21 +191,11 @@ Currently, the following pairs of encryption modes are supported:
 - AES-256-XTS for contents and AES-256-CTS-CBC for filenames
 - AES-128-CBC for contents and AES-128-CTS-CBC for filenames
- Speck128/256-XTS for contents and Speck128/256-CTS-CBC for filenames
 It is strongly recommended to use AES-256-XTS for contents encryption.
 AES-128-CBC was added only for low-powered embedded devices with
 crypto accelerators such as CAAM or CESA that do not support XTS.
-Similarly, Speck128/256 support was only added for older or low-end
-CPUs which cannot do AES fast enough -- especially ARM CPUs which have
-NEON instructions but not the Cryptography Extensions -- and for which
-it would not otherwise be feasible to use encryption at all.  It is
-not recommended to use Speck on CPUs that have AES instructions.
-Speck support is only available if it has been enabled in the crypto
-API via CONFIG_CRYPTO_SPECK.  Also, on ARM platforms, to get
-acceptable performance CONFIG_CRYPTO_SPECK_NEON must be enabled.
 New encryption modes can be added relatively easily, without changes
 to individual filesystems.  However, authenticated encryption (AE)
 modes are not currently supported because of the difficulty of dealing
diff --git a/MAINTAINERS b/MAINTAINERS
index 99540a5a4e19..8f22f6af3782 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7578,14 +7578,6 @@ S:	Supported
 F:      drivers/infiniband/hw/i40iw/
 F:      include/uapi/rdma/i40iw-abi.h
-INTEL SHA MULTIBUFFER DRIVER
-M:      Megha Dey <megha.dey@linux.intel.com>
-R:      Tim Chen <tim.c.chen@linux.intel.com>
-L:      linux-crypto@vger.kernel.org
-S:      Supported
-F:      arch/x86/crypto/sha*-mb/
-F:      crypto/mcryptd.c
 INTEL TELEMETRY DRIVER
 M:      Souvik Kumar Chakravarty <souvik.k.chakravarty@intel.com>
 L:      platform-driver-x86@vger.kernel.org
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 925d1364727a..ef0c7feea6e2 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -99,6 +99,7 @@ config CRYPTO_GHASH_ARM_CE
        depends on KERNEL_MODE_NEON
        select CRYPTO_HASH
        select CRYPTO_CRYPTD
+        select CRYPTO_GF128MUL
        help
          Use an implementation of GHASH (used by the GCM AEAD chaining mode)
          that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64)
@@ -121,10 +122,4 @@ config CRYPTO_CHACHA20_NEON
        select CRYPTO_BLKCIPHER
        select CRYPTO_CHACHA20
-config CRYPTO_SPECK_NEON
-        tristate "NEON accelerated Speck cipher algorithms"
-        depends on KERNEL_MODE_NEON
-        select CRYPTO_BLKCIPHER
-        select CRYPTO_SPECK
 endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index 8de542c48ade..bd5bceef0605 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -10,7 +10,6 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
 obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
 obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o
 obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
-obj-$(CONFIG_CRYPTO_SPECK_NEON) += speck-neon.o
 ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
 ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
@@ -54,7 +53,6 @@ ghash-arm-ce-y	:= ghash-ce-core.o ghash-ce-glue.o
 crct10dif-arm-ce-y      := crct10dif-ce-core.o crct10dif-ce-glue.o
 crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
 chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
-speck-neon-y := speck-neon-core.o speck-neon-glue.o
 ifdef REGENERATE_ARM_CRYPTO
 quiet_cmd_perl = PERL    $@
diff --git a/arch/arm/crypto/chacha20-neon-core.S b/arch/arm/crypto/chacha20-neon-core.S
index 451a849ad518..50e7b9896818 100644
--- a/arch/arm/crypto/chacha20-neon-core.S
+++ b/arch/arm/crypto/chacha20-neon-core.S
@@ -18,6 +18,34 @@
 * (at your option) any later version.
 */
+ /*
+  * NEON doesn't have a rotate instruction.  The alternatives are, more or less:
+  *
+  * (a)  vshl.u32 + vsri.u32            (needs temporary register)
+  * (b)  vshl.u32 + vshr.u32 + vorr     (needs temporary register)
+  * (c)  vrev32.16                      (16-bit rotations only)
+  * (d)  vtbl.8 + vtbl.8                (multiple of 8 bits rotations only,
+  *                                      needs index vector)
+  *
+  * ChaCha20 has 16, 12, 8, and 7-bit rotations.  For the 12 and 7-bit
+  * rotations, the only choices are (a) and (b).  We use (a) since it takes
+  * two-thirds the cycles of (b) on both Cortex-A7 and Cortex-A53.
+  *
+  * For the 16-bit rotation, we use vrev32.16 since it's consistently fastest
+  * and doesn't need a temporary register.
+  *
+  * For the 8-bit rotation, we use vtbl.8 + vtbl.8.  On Cortex-A7, this sequence
+  * is twice as fast as (a), even when doing (a) on multiple registers
+  * simultaneously to eliminate the stall between vshl and vsri.  Also, it
+  * parallelizes better when temporary registers are scarce.
+  *
+  * A disadvantage is that on Cortex-A53, the vtbl sequence is the same speed as
+  * (a), so the need to load the rotation table actually makes the vtbl method
+  * slightly slower overall on that CPU (~1.3% slower ChaCha20).  Still, it
+  * seems to be a good compromise to get a more significant speed boost on some
+  * CPUs, e.g. ~4.8% faster ChaCha20 on Cortex-A7.
+  */
 #include <linux/linkage.h>
        .text
@@ -46,7 +74,9 @@ ENTRY(chacha20_block_xor_neon)
        vmov            q10, q2
        vmov            q11, q3
+        adr             ip, .Lrol8_table
        mov             r3, #10
+        vld1.8          {d10}, [ip, :64]
 .Ldoubleround:
        // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
@@ -62,9 +92,9 @@ ENTRY(chacha20_block_xor_neon)
        // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
        vadd.i32        q0, q0, q1
-        veor            q4, q3, q0
+        veor            q3, q3, q0
-        vshl.u32        q3, q4, #8
+        vtbl.8          d6, {d6}, d10
-        vsri.u32        q3, q4, #24
+        vtbl.8          d7, {d7}, d10
        // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
        vadd.i32        q2, q2, q3
@@ -92,9 +122,9 @@ ENTRY(chacha20_block_xor_neon)
        // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
        vadd.i32        q0, q0, q1
-        veor            q4, q3, q0
+        veor            q3, q3, q0
-        vshl.u32        q3, q4, #8
+        vtbl.8          d6, {d6}, d10
-        vsri.u32        q3, q4, #24
+        vtbl.8          d7, {d7}, d10
        // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
        vadd.i32        q2, q2, q3
@@ -139,13 +169,17 @@ ENTRY(chacha20_block_xor_neon)
        bx              lr
 ENDPROC(chacha20_block_xor_neon)
+        .align          4
+.Lctrinc:       .word   0, 1, 2, 3
+.Lrol8_table:   .byte   3, 0, 1, 2, 7, 4, 5, 6
        .align          5
 ENTRY(chacha20_4block_xor_neon)
-        push            {r4-r6, lr}
+        push            {r4-r5}
-        mov             ip, sp                  // preserve the stack pointer
+        mov             r4, sp                  // preserve the stack pointer
-        sub             r3, sp, #0x20           // allocate a 32 byte buffer
+        sub             ip, sp, #0x20           // allocate a 32 byte buffer
-        bic             r3, r3, #0x1f           // aligned to 32 bytes
+        bic             ip, ip, #0x1f           // aligned to 32 bytes
-        mov             sp, r3
+        mov             sp, ip
        // r0: Input state matrix, s
        // r1: 4 data blocks output, o
@@ -155,25 +189,24 @@ ENTRY(chacha20_4block_xor_neon)
        // This function encrypts four consecutive ChaCha20 blocks by loading
        // the state matrix in NEON registers four times. The algorithm performs
        // each operation on the corresponding word of each state matrix, hence
-        // requires no word shuffling. For final XORing step we transpose the
+        // requires no word shuffling. The words are re-interleaved before the
-        // matrix by interleaving 32- and then 64-bit words, which allows us to
+        // final addition of the original state and the XORing step.
-        // do XOR in NEON registers.
        //
-        // x0..15[0-3] = s0..3[0..3]
+        // x0..15[0-3] = s0..15[0-3]
-        add             r3, r0, #0x20
+        add             ip, r0, #0x20
        vld1.32         {q0-q1}, [r0]
-        vld1.32         {q2-q3}, [r3]
+        vld1.32         {q2-q3}, [ip]
-        adr             r3, CTRINC
+        adr             r5, .Lctrinc
        vdup.32         q15, d7[1]
        vdup.32         q14, d7[0]
-        vld1.32         {q11}, [r3, :128]
+        vld1.32         {q4}, [r5, :128]
        vdup.32         q13, d6[1]
        vdup.32         q12, d6[0]
-        vadd.i32        q12, q12, q11           // x12 += counter values 0-3
        vdup.32         q11, d5[1]
        vdup.32         q10, d5[0]
+        vadd.u32        q12, q12, q4            // x12 += counter values 0-3
        vdup.32         q9, d4[1]
        vdup.32         q8, d4[0]
        vdup.32         q7, d3[1]
@@ -185,9 +218,13 @@ ENTRY(chacha20_4block_xor_neon)
        vdup.32         q1, d0[1]
        vdup.32         q0, d0[0]
+        adr             ip, .Lrol8_table
        mov             r3, #10
+        b               1f
 .Ldoubleround4:
+        vld1.32         {q8-q9}, [sp, :256]
+1:
        // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
        // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
        // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
@@ -236,24 +273,25 @@ ENTRY(chacha20_4block_xor_neon)
        // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
        // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
        // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
+        vld1.8          {d16}, [ip, :64]
        vadd.i32        q0, q0, q4
        vadd.i32        q1, q1, q5
        vadd.i32        q2, q2, q6
        vadd.i32        q3, q3, q7
-        veor            q8, q12, q0
+        veor            q12, q12, q0
-        veor            q9, q13, q1
+        veor            q13, q13, q1
-        vshl.u32        q12, q8, #8
+        veor            q14, q14, q2
-        vshl.u32        q13, q9, #8
+        veor            q15, q15, q3
-        vsri.u32        q12, q8, #24
-        vsri.u32        q13, q9, #24
-        veor            q8, q14, q2
+        vtbl.8          d24, {d24}, d16
-        veor            q9, q15, q3
+        vtbl.8          d25, {d25}, d16
-        vshl.u32        q14, q8, #8
+        vtbl.8          d26, {d26}, d16
-        vshl.u32        q15, q9, #8
+        vtbl.8          d27, {d27}, d16
-        vsri.u32        q14, q8, #24
+        vtbl.8          d28, {d28}, d16
-        vsri.u32        q15, q9, #24
+        vtbl.8          d29, {d29}, d16
+        vtbl.8          d30, {d30}, d16
+        vtbl.8          d31, {d31}, d16
        vld1.32         {q8-q9}, [sp, :256]
@@ -332,24 +370,25 @@ ENTRY(chacha20_4block_xor_neon)
        // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
        // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
        // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
+        vld1.8          {d16}, [ip, :64]
        vadd.i32        q0, q0, q5
        vadd.i32        q1, q1, q6
        vadd.i32        q2, q2, q7
        vadd.i32        q3, q3, q4
-        veor            q8, q15, q0
+        veor            q15, q15, q0
-        veor            q9, q12, q1
+        veor            q12, q12, q1
-        vshl.u32        q15, q8, #8
+        veor            q13, q13, q2
-        vshl.u32        q12, q9, #8
+        veor            q14, q14, q3
-        vsri.u32        q15, q8, #24
-        vsri.u32        q12, q9, #24
-        veor            q8, q13, q2
+        vtbl.8          d30, {d30}, d16
-        veor            q9, q14, q3
+        vtbl.8          d31, {d31}, d16
-        vshl.u32        q13, q8, #8
+        vtbl.8          d24, {d24}, d16
-        vshl.u32        q14, q9, #8
+        vtbl.8          d25, {d25}, d16
-        vsri.u32        q13, q8, #24
+        vtbl.8          d26, {d26}, d16
-        vsri.u32        q14, q9, #24
+        vtbl.8          d27, {d27}, d16
+        vtbl.8          d28, {d28}, d16
+        vtbl.8          d29, {d29}, d16
        vld1.32         {q8-q9}, [sp, :256]
@@ -379,104 +418,76 @@ ENTRY(chacha20_4block_xor_neon)
        vsri.u32        q6, q9, #25
        subs            r3, r3, #1
-        beq             0f
+        bne             .Ldoubleround4
-        vld1.32         {q8-q9}, [sp, :256]
+        // x0..7[0-3] are in q0-q7, x10..15[0-3] are in q10-q15.
-        b               .Ldoubleround4
+        // x8..9[0-3] are on the stack.
-        // x0[0-3] += s0[0]
+        // Re-interleave the words in the first two rows of each block (x0..7).
-        // x1[0-3] += s0[1]
+        // Also add the counter values 0-3 to x12[0-3].
-        // x2[0-3] += s0[2]
+          vld1.32       {q8}, [r5, :128]        // load counter values 0-3
-        // x3[0-3] += s0[3]
+        vzip.32         q0, q1                  // => (0 1 0 1) (0 1 0 1)
-0:      ldmia           r0!, {r3-r6}
+        vzip.32         q2, q3                  // => (2 3 2 3) (2 3 2 3)
-        vdup.32         q8, r3
+        vzip.32         q4, q5                  // => (4 5 4 5) (4 5 4 5)
-        vdup.32         q9, r4
+        vzip.32         q6, q7                  // => (6 7 6 7) (6 7 6 7)
-        vadd.i32        q0, q0, q8
+          vadd.u32      q12, q8                 // x12 += counter values 0-3
-        vadd.i32        q1, q1, q9
-        vdup.32         q8, r5
-        vdup.32         q9, r6
-        vadd.i32        q2, q2, q8
-        vadd.i32        q3, q3, q9
-        // x4[0-3] += s1[0]
-        // x5[0-3] += s1[1]
-        // x6[0-3] += s1[2]
-        // x7[0-3] += s1[3]
-        ldmia           r0!, {r3-r6}
-        vdup.32         q8, r3
-        vdup.32         q9, r4
-        vadd.i32        q4, q4, q8
-        vadd.i32        q5, q5, q9
-        vdup.32         q8, r5
-        vdup.32         q9, r6
-        vadd.i32        q6, q6, q8
-        vadd.i32        q7, q7, q9
-        // interleave 32-bit words in state n, n+1
-        vzip.32         q0, q1
-        vzip.32         q2, q3
-        vzip.32         q4, q5
-        vzip.32         q6, q7
-        // interleave 64-bit words in state n, n+2
        vswp            d1, d4
        vswp            d3, d6
+          vld1.32       {q8-q9}, [r0]!          // load s0..7
        vswp            d9, d12
        vswp            d11, d14
-        // xor with corresponding input, write to output
+        // Swap q1 and q4 so that we'll free up consecutive registers (q0-q1)
+        // after XORing the first 32 bytes.
+        vswp            q1, q4
+        // First two rows of each block are (q0 q1) (q2 q6) (q4 q5) (q3 q7)
+        // x0..3[0-3] += s0..3[0-3]     (add orig state to 1st row of each block)
+        vadd.u32        q0, q0, q8
+        vadd.u32        q2, q2, q8
+        vadd.u32        q4, q4, q8
+        vadd.u32        q3, q3, q8
+        // x4..7[0-3] += s4..7[0-3]     (add orig state to 2nd row of each block)
+        vadd.u32        q1, q1, q9
+        vadd.u32        q6, q6, q9
+        vadd.u32        q5, q5, q9
+        vadd.u32        q7, q7, q9
+        // XOR first 32 bytes using keystream from first two rows of first block
        vld1.8          {q8-q9}, [r2]!
        veor            q8, q8, q0
-        veor            q9, q9, q4
+        veor            q9, q9, q1
        vst1.8          {q8-q9}, [r1]!
+        // Re-interleave the words in the last two rows of each block (x8..15).
        vld1.32         {q8-q9}, [sp, :256]
+        vzip.32         q12, q13        // => (12 13 12 13) (12 13 12 13)
-        // x8[0-3] += s2[0]
+        vzip.32         q14, q15        // => (14 15 14 15) (14 15 14 15)
-        // x9[0-3] += s2[1]
+        vzip.32         q8, q9          // => (8 9 8 9) (8 9 8 9)
-        // x10[0-3] += s2[2]
+        vzip.32         q10, q11        // => (10 11 10 11) (10 11 10 11)
-        // x11[0-3] += s2[3]
+          vld1.32       {q0-q1}, [r0]   // load s8..15
-        ldmia           r0!, {r3-r6}
-        vdup.32         q0, r3
-        vdup.32         q4, r4
-        vadd.i32        q8, q8, q0
-        vadd.i32        q9, q9, q4
-        vdup.32         q0, r5
-        vdup.32         q4, r6
-        vadd.i32        q10, q10, q0
-        vadd.i32        q11, q11, q4
-        // x12[0-3] += s3[0]
-        // x13[0-3] += s3[1]
-        // x14[0-3] += s3[2]
-        // x15[0-3] += s3[3]
-        ldmia           r0!, {r3-r6}
-        vdup.32         q0, r3
-        vdup.32         q4, r4
-        adr             r3, CTRINC
-        vadd.i32        q12, q12, q0
-        vld1.32         {q0}, [r3, :128]
-        vadd.i32        q13, q13, q4
-        vadd.i32        q12, q12, q0            // x12 += counter values 0-3
-        vdup.32         q0, r5
-        vdup.32         q4, r6
-        vadd.i32        q14, q14, q0
-        vadd.i32        q15, q15, q4
-        // interleave 32-bit words in state n, n+1
-        vzip.32         q8, q9
-        vzip.32         q10, q11
-        vzip.32         q12, q13
-        vzip.32         q14, q15
-        // interleave 64-bit words in state n, n+2
-        vswp            d17, d20
-        vswp            d19, d22
        vswp            d25, d28
        vswp            d27, d30
+        vswp            d17, d20
+        vswp            d19, d22
+        // Last two rows of each block are (q8 q12) (q10 q14) (q9 q13) (q11 q15)
+        // x8..11[0-3] += s8..11[0-3]   (add orig state to 3rd row of each block)
+        vadd.u32        q8,  q8,  q0
+        vadd.u32        q10, q10, q0
+        vadd.u32        q9,  q9,  q0
+        vadd.u32        q11, q11, q0
+        // x12..15[0-3] += s12..15[0-3] (add orig state to 4th row of each block)
+        vadd.u32        q12, q12, q1
+        vadd.u32        q14, q14, q1
+        vadd.u32        q13, q13, q1
+        vadd.u32        q15, q15, q1
-        vmov            q4, q1
+        // XOR the rest of the data with the keystream
        vld1.8          {q0-q1}, [r2]!
        veor            q0, q0, q8
@@ -509,13 +520,11 @@ ENTRY(chacha20_4block_xor_neon)
        vst1.8          {q0-q1}, [r1]!
        vld1.8          {q0-q1}, [r2]
+          mov           sp, r4          // restore original stack pointer
        veor            q0, q0, q11
        veor            q1, q1, q15
        vst1.8          {q0-q1}, [r1]
-        mov             sp, ip
+        pop             {r4-r5}
-        pop             {r4-r6, pc}
+        bx              lr
 ENDPROC(chacha20_4block_xor_neon)
-        .align          4
-CTRINC: .word           0, 1, 2, 3
diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c
index 96e62ec105d0..cd9e93b46c2d 100644
--- a/arch/arm/crypto/crc32-ce-glue.c
+++ b/arch/arm/crypto/crc32-ce-glue.c
@@ -236,7 +236,7 @@ static void __exit crc32_pmull_mod_exit(void)
                                  ARRAY_SIZE(crc32_pmull_algs));
 }
-static const struct cpu_feature crc32_cpu_feature[] = {
+static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
        { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
 };
 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
diff --git a/arch/arm/crypto/ghash-ce-core.S b/arch/arm/crypto/ghash-ce-core.S
index 2f78c10b1881..406009afa9cf 100644
--- a/arch/arm/crypto/ghash-ce-core.S
+++ b/arch/arm/crypto/ghash-ce-core.S
@@ -63,6 +63,33 @@
        k48             .req    d31
        SHASH2_p64      .req    d31
+        HH              .req    q10
+        HH3             .req    q11
+        HH4             .req    q12
+        HH34            .req    q13
+        HH_L            .req    d20
+        HH_H            .req    d21
+        HH3_L           .req    d22
+        HH3_H           .req    d23
+        HH4_L           .req    d24
+        HH4_H           .req    d25
+        HH34_L          .req    d26
+        HH34_H          .req    d27
+        SHASH2_H        .req    d29
+        XL2             .req    q5
+        XM2             .req    q6
+        XH2             .req    q7
+        T3              .req    q8
+        XL2_L           .req    d10
+        XL2_H           .req    d11
+        XM2_L           .req    d12
+        XM2_H           .req    d13
+        T3_L            .req    d16
+        T3_H            .req    d17
        .text
        .fpu            crypto-neon-fp-armv8
@@ -175,12 +202,77 @@
        beq             0f
        vld1.64         {T1}, [ip]
        teq             r0, #0
-        b               1f
+        b               3f
+0:      .ifc            \pn, p64
+        tst             r0, #3                  // skip until #blocks is a
+        bne             2f                      // round multiple of 4
+        vld1.8          {XL2-XM2}, [r2]!
+1:      vld1.8          {T3-T2}, [r2]!
+        vrev64.8        XL2, XL2
+        vrev64.8        XM2, XM2
+        subs            r0, r0, #4
+        vext.8          T1, XL2, XL2, #8
+        veor            XL2_H, XL2_H, XL_L
+        veor            XL, XL, T1
+        vrev64.8        T3, T3
+        vrev64.8        T1, T2
+        vmull.p64       XH, HH4_H, XL_H                 // a1 * b1
+        veor            XL2_H, XL2_H, XL_H
+        vmull.p64       XL, HH4_L, XL_L                 // a0 * b0
+        vmull.p64       XM, HH34_H, XL2_H               // (a1 + a0)(b1 + b0)
+        vmull.p64       XH2, HH3_H, XM2_L               // a1 * b1
+        veor            XM2_L, XM2_L, XM2_H
+        vmull.p64       XL2, HH3_L, XM2_H               // a0 * b0
+        vmull.p64       XM2, HH34_L, XM2_L              // (a1 + a0)(b1 + b0)
+        veor            XH, XH, XH2
+        veor            XL, XL, XL2
+        veor            XM, XM, XM2
+        vmull.p64       XH2, HH_H, T3_L                 // a1 * b1
+        veor            T3_L, T3_L, T3_H
+        vmull.p64       XL2, HH_L, T3_H                 // a0 * b0
+        vmull.p64       XM2, SHASH2_H, T3_L             // (a1 + a0)(b1 + b0)
+        veor            XH, XH, XH2
+        veor            XL, XL, XL2
+        veor            XM, XM, XM2
+        vmull.p64       XH2, SHASH_H, T1_L              // a1 * b1
+        veor            T1_L, T1_L, T1_H
+        vmull.p64       XL2, SHASH_L, T1_H              // a0 * b0
+        vmull.p64       XM2, SHASH2_p64, T1_L           // (a1 + a0)(b1 + b0)
+        veor            XH, XH, XH2
+        veor            XL, XL, XL2
+        veor            XM, XM, XM2
-0:      vld1.64         {T1}, [r2]!
+        beq             4f
+        vld1.8          {XL2-XM2}, [r2]!
+        veor            T1, XL, XH
+        veor            XM, XM, T1
+        __pmull_reduce_p64
+        veor            T1, T1, XH
+        veor            XL, XL, T1
+        b               1b
+        .endif
+2:      vld1.64         {T1}, [r2]!
        subs            r0, r0, #1
-1:      /* multiply XL by SHASH in GF(2^128) */
+3:      /* multiply XL by SHASH in GF(2^128) */
 #ifndef CONFIG_CPU_BIG_ENDIAN
        vrev64.8        T1, T1
 #endif
@@ -193,7 +285,7 @@
        __pmull_\pn     XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l   @ a0 * b0
        __pmull_\pn     XM, T1_L, SHASH2_\pn                    @ (a1+a0)(b1+b0)
-        veor            T1, XL, XH
+4:      veor            T1, XL, XH
        veor            XM, XM, T1
        __pmull_reduce_\pn
@@ -212,8 +304,14 @@
         *                         struct ghash_key const *k, const char *head)
         */
 ENTRY(pmull_ghash_update_p64)
-        vld1.64         {SHASH}, [r3]
+        vld1.64         {SHASH}, [r3]!
+        vld1.64         {HH}, [r3]!
+        vld1.64         {HH3-HH4}, [r3]
        veor            SHASH2_p64, SHASH_L, SHASH_H
+        veor            SHASH2_H, HH_L, HH_H
+        veor            HH34_L, HH3_L, HH3_H
+        veor            HH34_H, HH4_L, HH4_H
        vmov.i8         MASK, #0xe1
        vshl.u64        MASK, MASK, #57
diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index 8930fc4e7c22..b7d30b6cf49c 100644
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -1,7 +1,7 @@
 /*
 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
 *
- * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ * Copyright (C) 2015 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
@@ -28,8 +28,10 @@ MODULE_ALIAS_CRYPTO("ghash");
 #define GHASH_DIGEST_SIZE       16
 struct ghash_key {
-        u64     a;
+        u64     h[2];
-        u64     b;
+        u64     h2[2];
+        u64     h3[2];
+        u64     h4[2];
 };
 struct ghash_desc_ctx {
@@ -117,26 +119,40 @@ static int ghash_final(struct shash_desc *desc, u8 *dst)
        return 0;
 }
+static void ghash_reflect(u64 h[], const be128 *k)
+{
+        u64 carry = be64_to_cpu(k->a) >> 63;
+        h[0] = (be64_to_cpu(k->b) << 1) | carry;
+        h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
+        if (carry)
+                h[1] ^= 0xc200000000000000UL;
+}
 static int ghash_setkey(struct crypto_shash *tfm,
                        const u8 *inkey, unsigned int keylen)
 {
        struct ghash_key *key = crypto_shash_ctx(tfm);
-        u64 a, b;
+        be128 h, k;
        if (keylen != GHASH_BLOCK_SIZE) {
                crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
                return -EINVAL;
        }
-        /* perform multiplication by 'x' in GF(2^128) */
+        memcpy(&k, inkey, GHASH_BLOCK_SIZE);
-        b = get_unaligned_be64(inkey);
+        ghash_reflect(key->h, &k);
-        a = get_unaligned_be64(inkey + 8);
+        h = k;
+        gf128mul_lle(&h, &k);
+        ghash_reflect(key->h2, &h);
-        key->a = (a << 1) | (b >> 63);
+        gf128mul_lle(&h, &k);
-        key->b = (b << 1) | (a >> 63);
+        ghash_reflect(key->h3, &h);
-        if (b >> 63)
+        gf128mul_lle(&h, &k);
-                key->b ^= 0xc200000000000000UL;
+        ghash_reflect(key->h4, &h);
        return 0;
 }
diff --git a/arch/arm/crypto/speck-neon-core.S b/arch/arm/crypto/speck-neon-core.S
deleted file mode 100644
index 57caa742016e..000000000000
--- a/arch/arm/crypto/speck-neon-core.S
+++ /dev/null
@@ -1,434 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
- *
- * Copyright (c) 2018 Google, Inc
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-#include <linux/linkage.h>
-        .text
-        .fpu            neon
-        // arguments
-        ROUND_KEYS      .req    r0      // const {u64,u32} *round_keys
-        NROUNDS         .req    r1      // int nrounds
-        DST             .req    r2      // void *dst
-        SRC             .req    r3      // const void *src
-        NBYTES          .req    r4      // unsigned int nbytes
-        TWEAK           .req    r5      // void *tweak
-        // registers which hold the data being encrypted/decrypted
-        X0              .req    q0
-        X0_L            .req    d0
-        X0_H            .req    d1
-        Y0              .req    q1
-        Y0_H            .req    d3
-        X1              .req    q2
-        X1_L            .req    d4
-        X1_H            .req    d5
-        Y1              .req    q3
-        Y1_H            .req    d7
-        X2              .req    q4
-        X2_L            .req    d8
-        X2_H            .req    d9
-        Y2              .req    q5
-        Y2_H            .req    d11
-        X3              .req    q6
-        X3_L            .req    d12
-        X3_H            .req    d13
-        Y3              .req    q7
-        Y3_H            .req    d15
-        // the round key, duplicated in all lanes
-        ROUND_KEY       .req    q8
-        ROUND_KEY_L     .req    d16
-        ROUND_KEY_H     .req    d17
-        // index vector for vtbl-based 8-bit rotates
-        ROTATE_TABLE    .req    d18
-        // multiplication table for updating XTS tweaks
-        GF128MUL_TABLE  .req    d19
-        GF64MUL_TABLE   .req    d19
-        // current XTS tweak value(s)
-        TWEAKV          .req    q10
-        TWEAKV_L        .req    d20
-        TWEAKV_H        .req    d21
-        TMP0            .req    q12
-        TMP0_L          .req    d24
-        TMP0_H          .req    d25
-        TMP1            .req    q13
-        TMP2            .req    q14
-        TMP3            .req    q15
-        .align          4
-.Lror64_8_table:
-        .byte           1, 2, 3, 4, 5, 6, 7, 0
-.Lror32_8_table:
-        .byte           1, 2, 3, 0, 5, 6, 7, 4
-.Lrol64_8_table:
-        .byte           7, 0, 1, 2, 3, 4, 5, 6
-.Lrol32_8_table:
-        .byte           3, 0, 1, 2, 7, 4, 5, 6
-.Lgf128mul_table:
-        .byte           0, 0x87
-        .fill           14
-.Lgf64mul_table:
-        .byte           0, 0x1b, (0x1b << 1), (0x1b << 1) ^ 0x1b
-        .fill           12
-/*
- * _speck_round_128bytes() - Speck encryption round on 128 bytes at a time
- *
- * Do one Speck encryption round on the 128 bytes (8 blocks for Speck128, 16 for
- * Speck64) stored in X0-X3 and Y0-Y3, using the round key stored in all lanes
- * of ROUND_KEY.  'n' is the lane size: 64 for Speck128, or 32 for Speck64.
- *
- * The 8-bit rotates are implemented using vtbl instead of vshr + vsli because
- * the vtbl approach is faster on some processors and the same speed on others.
- */
-.macro _speck_round_128bytes    n
-        // x = ror(x, 8)
-        vtbl.8          X0_L, {X0_L}, ROTATE_TABLE
-        vtbl.8          X0_H, {X0_H}, ROTATE_TABLE
-        vtbl.8          X1_L, {X1_L}, ROTATE_TABLE
-        vtbl.8          X1_H, {X1_H}, ROTATE_TABLE
-        vtbl.8          X2_L, {X2_L}, ROTATE_TABLE
-        vtbl.8          X2_H, {X2_H}, ROTATE_TABLE
-        vtbl.8          X3_L, {X3_L}, ROTATE_TABLE
-        vtbl.8          X3_H, {X3_H}, ROTATE_TABLE
-        // x += y
-        vadd.u\n        X0, Y0
-        vadd.u\n        X1, Y1
-        vadd.u\n        X2, Y2
-        vadd.u\n        X3, Y3
-        // x ^= k
-        veor            X0, ROUND_KEY
-        veor            X1, ROUND_KEY
-        veor            X2, ROUND_KEY
-        veor            X3, ROUND_KEY
-        // y = rol(y, 3)
-        vshl.u\n        TMP0, Y0, #3
-        vshl.u\n        TMP1, Y1, #3
-        vshl.u\n        TMP2, Y2, #3
-        vshl.u\n        TMP3, Y3, #3
-        vsri.u\n        TMP0, Y0, #(\n - 3)
-        vsri.u\n        TMP1, Y1, #(\n - 3)
-        vsri.u\n        TMP2, Y2, #(\n - 3)
-        vsri.u\n        TMP3, Y3, #(\n - 3)
-        // y ^= x
-        veor            Y0, TMP0, X0
-        veor            Y1, TMP1, X1
-        veor            Y2, TMP2, X2
-        veor            Y3, TMP3, X3
-.endm
-/*
- * _speck_unround_128bytes() - Speck decryption round on 128 bytes at a time
- *
- * This is the inverse of _speck_round_128bytes().
- */
-.macro _speck_unround_128bytes  n
-        // y ^= x
-        veor            TMP0, Y0, X0
-        veor            TMP1, Y1, X1
-        veor            TMP2, Y2, X2
-        veor            TMP3, Y3, X3
-        // y = ror(y, 3)
-        vshr.u\n        Y0, TMP0, #3
-        vshr.u\n        Y1, TMP1, #3
-        vshr.u\n        Y2, TMP2, #3
-        vshr.u\n        Y3, TMP3, #3
-        vsli.u\n        Y0, TMP0, #(\n - 3)
-        vsli.u\n        Y1, TMP1, #(\n - 3)
-        vsli.u\n        Y2, TMP2, #(\n - 3)
-        vsli.u\n        Y3, TMP3, #(\n - 3)
-        // x ^= k
-        veor            X0, ROUND_KEY
-        veor            X1, ROUND_KEY
-        veor            X2, ROUND_KEY
-        veor            X3, ROUND_KEY
-        // x -= y
-        vsub.u\n        X0, Y0
-        vsub.u\n        X1, Y1
-        vsub.u\n        X2, Y2
-        vsub.u\n        X3, Y3
-        // x = rol(x, 8);
-        vtbl.8          X0_L, {X0_L}, ROTATE_TABLE
-        vtbl.8          X0_H, {X0_H}, ROTATE_TABLE
-        vtbl.8          X1_L, {X1_L}, ROTATE_TABLE
-        vtbl.8          X1_H, {X1_H}, ROTATE_TABLE
-        vtbl.8          X2_L, {X2_L}, ROTATE_TABLE
-        vtbl.8          X2_H, {X2_H}, ROTATE_TABLE
-        vtbl.8          X3_L, {X3_L}, ROTATE_TABLE
-        vtbl.8          X3_H, {X3_H}, ROTATE_TABLE
-.endm
-.macro _xts128_precrypt_one     dst_reg, tweak_buf, tmp
-        // Load the next source block
-        vld1.8          {\dst_reg}, [SRC]!
-        // Save the current tweak in the tweak buffer
-        vst1.8          {TWEAKV}, [\tweak_buf:128]!
-        // XOR the next source block with the current tweak
-        veor            \dst_reg, TWEAKV
-        /*
-         * Calculate the next tweak by multiplying the current one by x,
-         * modulo p(x) = x^128 + x^7 + x^2 + x + 1.
-         */
-        vshr.u64        \tmp, TWEAKV, #63
-        vshl.u64        TWEAKV, #1
-        veor            TWEAKV_H, \tmp\()_L
-        vtbl.8          \tmp\()_H, {GF128MUL_TABLE}, \tmp\()_H
-        veor            TWEAKV_L, \tmp\()_H
-.endm
-.macro _xts64_precrypt_two      dst_reg, tweak_buf, tmp
-        // Load the next two source blocks
-        vld1.8          {\dst_reg}, [SRC]!
-        // Save the current two tweaks in the tweak buffer
-        vst1.8          {TWEAKV}, [\tweak_buf:128]!
-        // XOR the next two source blocks with the current two tweaks
-        veor            \dst_reg, TWEAKV
-        /*
-         * Calculate the next two tweaks by multiplying the current ones by x^2,
-         * modulo p(x) = x^64 + x^4 + x^3 + x + 1.
-         */
-        vshr.u64        \tmp, TWEAKV, #62
-        vshl.u64        TWEAKV, #2
-        vtbl.8          \tmp\()_L, {GF64MUL_TABLE}, \tmp\()_L
-        vtbl.8          \tmp\()_H, {GF64MUL_TABLE}, \tmp\()_H
-        veor            TWEAKV, \tmp
-.endm
-/*
- * _speck_xts_crypt() - Speck-XTS encryption/decryption
- *
- * Encrypt or decrypt NBYTES bytes of data from the SRC buffer to the DST buffer
- * using Speck-XTS, specifically the variant with a block size of '2n' and round
- * count given by NROUNDS.  The expanded round keys are given in ROUND_KEYS, and
- * the current XTS tweak value is given in TWEAK.  It's assumed that NBYTES is a
- * nonzero multiple of 128.
- */
-.macro _speck_xts_crypt n, decrypting
-        push            {r4-r7}
-        mov             r7, sp
-        /*
-         * The first four parameters were passed in registers r0-r3.  Load the
-         * additional parameters, which were passed on the stack.
-         */
-        ldr             NBYTES, [sp, #16]
-        ldr             TWEAK, [sp, #20]
-        /*
-         * If decrypting, modify the ROUND_KEYS parameter to point to the last
-         * round key rather than the first, since for decryption the round keys
-         * are used in reverse order.
-         */
-.if \decrypting
-.if \n == 64
-        add             ROUND_KEYS, ROUND_KEYS, NROUNDS, lsl #3
-        sub             ROUND_KEYS, #8
-.else
-        add             ROUND_KEYS, ROUND_KEYS, NROUNDS, lsl #2
-        sub             ROUND_KEYS, #4
-.endif
-.endif
-        // Load the index vector for vtbl-based 8-bit rotates
-.if \decrypting
-        ldr             r12, =.Lrol\n\()_8_table
-.else
-        ldr             r12, =.Lror\n\()_8_table
-.endif
-        vld1.8          {ROTATE_TABLE}, [r12:64]
-        // One-time XTS preparation
-        /*
-         * Allocate stack space to store 128 bytes worth of tweaks.  For
-         * performance, this space is aligned to a 16-byte boundary so that we
-         * can use the load/store instructions that declare 16-byte alignment.
-         * For Thumb2 compatibility, don't do the 'bic' directly on 'sp'.
-         */
-        sub             r12, sp, #128
-        bic             r12, #0xf
-        mov             sp, r12
-.if \n == 64
-        // Load first tweak
-        vld1.8          {TWEAKV}, [TWEAK]
-        // Load GF(2^128) multiplication table
-        ldr             r12, =.Lgf128mul_table
-        vld1.8          {GF128MUL_TABLE}, [r12:64]
-.else
-        // Load first tweak
-        vld1.8          {TWEAKV_L}, [TWEAK]
-        // Load GF(2^64) multiplication table
-        ldr             r12, =.Lgf64mul_table
-        vld1.8          {GF64MUL_TABLE}, [r12:64]
-        // Calculate second tweak, packing it together with the first
-        vshr.u64        TMP0_L, TWEAKV_L, #63
-        vtbl.u8         TMP0_L, {GF64MUL_TABLE}, TMP0_L
-        vshl.u64        TWEAKV_H, TWEAKV_L, #1
-        veor            TWEAKV_H, TMP0_L
-.endif
-.Lnext_128bytes_\@:
-        /*
-         * Load the source blocks into {X,Y}[0-3], XOR them with their XTS tweak
-         * values, and save the tweaks on the stack for later.  Then
-         * de-interleave the 'x' and 'y' elements of each block, i.e. make it so
-         * that the X[0-3] registers contain only the second halves of blocks,
-         * and the Y[0-3] registers contain only the first halves of blocks.
-         * (Speck uses the order (y, x) rather than the more intuitive (x, y).)
-         */
-        mov             r12, sp
-.if \n == 64
-        _xts128_precrypt_one    X0, r12, TMP0
-        _xts128_precrypt_one    Y0, r12, TMP0
-        _xts128_precrypt_one    X1, r12, TMP0
-        _xts128_precrypt_one    Y1, r12, TMP0
-        _xts128_precrypt_one    X2, r12, TMP0
-        _xts128_precrypt_one    Y2, r12, TMP0
-        _xts128_precrypt_one    X3, r12, TMP0
-        _xts128_precrypt_one    Y3, r12, TMP0
-        vswp            X0_L, Y0_H
-        vswp            X1_L, Y1_H
-        vswp            X2_L, Y2_H
-        vswp            X3_L, Y3_H
-.else
-        _xts64_precrypt_two     X0, r12, TMP0
-        _xts64_precrypt_two     Y0, r12, TMP0
-        _xts64_precrypt_two     X1, r12, TMP0
-        _xts64_precrypt_two     Y1, r12, TMP0
-        _xts64_precrypt_two     X2, r12, TMP0
-        _xts64_precrypt_two     Y2, r12, TMP0
-        _xts64_precrypt_two     X3, r12, TMP0
-        _xts64_precrypt_two     Y3, r12, TMP0
-        vuzp.32         Y0, X0
-        vuzp.32         Y1, X1
-        vuzp.32         Y2, X2
-        vuzp.32         Y3, X3
-.endif
-        // Do the cipher rounds
-        mov             r12, ROUND_KEYS
-        mov             r6, NROUNDS
-.Lnext_round_\@:
-.if \decrypting
-.if \n == 64
-        vld1.64         ROUND_KEY_L, [r12]
-        sub             r12, #8
-        vmov            ROUND_KEY_H, ROUND_KEY_L
-.else
-        vld1.32         {ROUND_KEY_L[],ROUND_KEY_H[]}, [r12]
-        sub             r12, #4
-.endif
-        _speck_unround_128bytes \n
-.else
-.if \n == 64
-        vld1.64         ROUND_KEY_L, [r12]!
-        vmov            ROUND_KEY_H, ROUND_KEY_L
-.else
-        vld1.32         {ROUND_KEY_L[],ROUND_KEY_H[]}, [r12]!
-.endif
-        _speck_round_128bytes   \n
-.endif
-        subs            r6, r6, #1
-        bne             .Lnext_round_\@
-        // Re-interleave the 'x' and 'y' elements of each block
-.if \n == 64
-        vswp            X0_L, Y0_H
-        vswp            X1_L, Y1_H
-        vswp            X2_L, Y2_H
-        vswp            X3_L, Y3_H
-.else
-        vzip.32         Y0, X0
-        vzip.32         Y1, X1
-        vzip.32         Y2, X2
-        vzip.32         Y3, X3
-.endif
-        // XOR the encrypted/decrypted blocks with the tweaks we saved earlier
-        mov             r12, sp
-        vld1.8          {TMP0, TMP1}, [r12:128]!
-        vld1.8          {TMP2, TMP3}, [r12:128]!
-        veor            X0, TMP0
-        veor            Y0, TMP1
-        veor            X1, TMP2
-        veor            Y1, TMP3
-        vld1.8          {TMP0, TMP1}, [r12:128]!
-        vld1.8          {TMP2, TMP3}, [r12:128]!
-        veor            X2, TMP0
-        veor            Y2, TMP1
-        veor            X3, TMP2
-        veor            Y3, TMP3
-        // Store the ciphertext in the destination buffer
-        vst1.8          {X0, Y0}, [DST]!
-        vst1.8          {X1, Y1}, [DST]!
-        vst1.8          {X2, Y2}, [DST]!
-        vst1.8          {X3, Y3}, [DST]!
-        // Continue if there are more 128-byte chunks remaining, else return
-        subs            NBYTES, #128
-        bne             .Lnext_128bytes_\@
-        // Store the next tweak
-.if \n == 64
-        vst1.8          {TWEAKV}, [TWEAK]
-.else
-        vst1.8          {TWEAKV_L}, [TWEAK]
-.endif
-        mov             sp, r7
-        pop             {r4-r7}
-        bx              lr
-.endm
-ENTRY(speck128_xts_encrypt_neon)
-        _speck_xts_crypt        n=64, decrypting=0
-ENDPROC(speck128_xts_encrypt_neon)
-ENTRY(speck128_xts_decrypt_neon)
-        _speck_xts_crypt        n=64, decrypting=1
-ENDPROC(speck128_xts_decrypt_neon)
-ENTRY(speck64_xts_encrypt_neon)
-        _speck_xts_crypt        n=32, decrypting=0
-ENDPROC(speck64_xts_encrypt_neon)
-ENTRY(speck64_xts_decrypt_neon)
-        _speck_xts_crypt        n=32, decrypting=1
-ENDPROC(speck64_xts_decrypt_neon)
diff --git a/arch/arm/crypto/speck-neon-glue.c b/arch/arm/crypto/speck-neon-glue.c
deleted file mode 100644
index f012c3ea998f..000000000000
--- a/arch/arm/crypto/speck-neon-glue.c
+++ /dev/null
@@ -1,288 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
- *
- * Copyright (c) 2018 Google, Inc
- *
- * Note: the NIST recommendation for XTS only specifies a 128-bit block size,
- * but a 64-bit version (needed for Speck64) is fairly straightforward; the math
- * is just done in GF(2^64) instead of GF(2^128), with the reducing polynomial
- * x^64 + x^4 + x^3 + x + 1 from the original XEX paper (Rogaway, 2004:
- * "Efficient Instantiations of Tweakable Blockciphers and Refinements to Modes
- * OCB and PMAC"), represented as 0x1B.
- */
-#include <asm/hwcap.h>
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/algapi.h>
-#include <crypto/gf128mul.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/speck.h>
-#include <crypto/xts.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-/* The assembly functions only handle multiples of 128 bytes */
-#define SPECK_NEON_CHUNK_SIZE   128
-/* Speck128 */
-struct speck128_xts_tfm_ctx {
-        struct speck128_tfm_ctx main_key;
-        struct speck128_tfm_ctx tweak_key;
-};
-asmlinkage void speck128_xts_encrypt_neon(const u64 *round_keys, int nrounds,
-                                          void *dst, const void *src,
-                                          unsigned int nbytes, void *tweak);
-asmlinkage void speck128_xts_decrypt_neon(const u64 *round_keys, int nrounds,
-                                          void *dst, const void *src,
-                                          unsigned int nbytes, void *tweak);
-typedef void (*speck128_crypt_one_t)(const struct speck128_tfm_ctx *,
-                                     u8 *, const u8 *);
-typedef void (*speck128_xts_crypt_many_t)(const u64 *, int, void *,
-                                          const void *, unsigned int, void *);
-static __always_inline int
-__speck128_xts_crypt(struct skcipher_request *req,
-                     speck128_crypt_one_t crypt_one,
-                     speck128_xts_crypt_many_t crypt_many)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        const struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct skcipher_walk walk;
-        le128 tweak;
-        int err;
-        err = skcipher_walk_virt(&walk, req, true);
-        crypto_speck128_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
-        while (walk.nbytes > 0) {
-                unsigned int nbytes = walk.nbytes;
-                u8 *dst = walk.dst.virt.addr;
-                const u8 *src = walk.src.virt.addr;
-                if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
-                        unsigned int count;
-                        count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
-                        kernel_neon_begin();
-                        (*crypt_many)(ctx->main_key.round_keys,
-                                      ctx->main_key.nrounds,
-                                      dst, src, count, &tweak);
-                        kernel_neon_end();
-                        dst += count;
-                        src += count;
-                        nbytes -= count;
-                }
-                /* Handle any remainder with generic code */
-                while (nbytes >= sizeof(tweak)) {
-                        le128_xor((le128 *)dst, (const le128 *)src, &tweak);
-                        (*crypt_one)(&ctx->main_key, dst, dst);
-                        le128_xor((le128 *)dst, (const le128 *)dst, &tweak);
-                        gf128mul_x_ble(&tweak, &tweak);
-                        dst += sizeof(tweak);
-                        src += sizeof(tweak);
-                        nbytes -= sizeof(tweak);
-                }
-                err = skcipher_walk_done(&walk, nbytes);
-        }
-        return err;
-}
-static int speck128_xts_encrypt(struct skcipher_request *req)
-{
-        return __speck128_xts_crypt(req, crypto_speck128_encrypt,
-                                    speck128_xts_encrypt_neon);
-}
-static int speck128_xts_decrypt(struct skcipher_request *req)
-{
-        return __speck128_xts_crypt(req, crypto_speck128_decrypt,
-                                    speck128_xts_decrypt_neon);
-}
-static int speck128_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
-                               unsigned int keylen)
-{
-        struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        int err;
-        err = xts_verify_key(tfm, key, keylen);
-        if (err)
-                return err;
-        keylen /= 2;
-        err = crypto_speck128_setkey(&ctx->main_key, key, keylen);
-        if (err)
-                return err;
-        return crypto_speck128_setkey(&ctx->tweak_key, key + keylen, keylen);
-}
-/* Speck64 */
-struct speck64_xts_tfm_ctx {
-        struct speck64_tfm_ctx main_key;
-        struct speck64_tfm_ctx tweak_key;
-};
-asmlinkage void speck64_xts_encrypt_neon(const u32 *round_keys, int nrounds,
-                                         void *dst, const void *src,
-                                         unsigned int nbytes, void *tweak);
-asmlinkage void speck64_xts_decrypt_neon(const u32 *round_keys, int nrounds,
-                                         void *dst, const void *src,
-                                         unsigned int nbytes, void *tweak);
-typedef void (*speck64_crypt_one_t)(const struct speck64_tfm_ctx *,
-                                    u8 *, const u8 *);
-typedef void (*speck64_xts_crypt_many_t)(const u32 *, int, void *,
-                                         const void *, unsigned int, void *);
-static __always_inline int
-__speck64_xts_crypt(struct skcipher_request *req, speck64_crypt_one_t crypt_one,
-                    speck64_xts_crypt_many_t crypt_many)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        const struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct skcipher_walk walk;
-        __le64 tweak;
-        int err;
-        err = skcipher_walk_virt(&walk, req, true);
-        crypto_speck64_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
-        while (walk.nbytes > 0) {
-                unsigned int nbytes = walk.nbytes;
-                u8 *dst = walk.dst.virt.addr;
-                const u8 *src = walk.src.virt.addr;
-                if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
-                        unsigned int count;
-                        count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
-                        kernel_neon_begin();
-                        (*crypt_many)(ctx->main_key.round_keys,
-                                      ctx->main_key.nrounds,
-                                      dst, src, count, &tweak);
-                        kernel_neon_end();
-                        dst += count;
-                        src += count;
-                        nbytes -= count;
-                }
-                /* Handle any remainder with generic code */
-                while (nbytes >= sizeof(tweak)) {
-                        *(__le64 *)dst = *(__le64 *)src ^ tweak;
-                        (*crypt_one)(&ctx->main_key, dst, dst);
-                        *(__le64 *)dst ^= tweak;
-                        tweak = cpu_to_le64((le64_to_cpu(tweak) << 1) ^
-                                            ((tweak & cpu_to_le64(1ULL << 63)) ?
-                                             0x1B : 0));
-                        dst += sizeof(tweak);
-                        src += sizeof(tweak);
-                        nbytes -= sizeof(tweak);
-                }
-                err = skcipher_walk_done(&walk, nbytes);
-        }
-        return err;
-}
-static int speck64_xts_encrypt(struct skcipher_request *req)
-{
-        return __speck64_xts_crypt(req, crypto_speck64_encrypt,
-                                   speck64_xts_encrypt_neon);
-}
-static int speck64_xts_decrypt(struct skcipher_request *req)
-{
-        return __speck64_xts_crypt(req, crypto_speck64_decrypt,
-                                   speck64_xts_decrypt_neon);
-}
-static int speck64_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
-                              unsigned int keylen)
-{
-        struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        int err;
-        err = xts_verify_key(tfm, key, keylen);
-        if (err)
-                return err;
-        keylen /= 2;
-        err = crypto_speck64_setkey(&ctx->main_key, key, keylen);
-        if (err)
-                return err;
-        return crypto_speck64_setkey(&ctx->tweak_key, key + keylen, keylen);
-}
-static struct skcipher_alg speck_algs[] = {
-        {
-                .base.cra_name          = "xts(speck128)",
-                .base.cra_driver_name   = "xts-speck128-neon",
-                .base.cra_priority      = 300,
-                .base.cra_blocksize     = SPECK128_BLOCK_SIZE,
-                .base.cra_ctxsize       = sizeof(struct speck128_xts_tfm_ctx),
-                .base.cra_alignmask     = 7,
-                .base.cra_module        = THIS_MODULE,
-                .min_keysize            = 2 * SPECK128_128_KEY_SIZE,
-                .max_keysize            = 2 * SPECK128_256_KEY_SIZE,
-                .ivsize                 = SPECK128_BLOCK_SIZE,
-                .walksize               = SPECK_NEON_CHUNK_SIZE,
-                .setkey                 = speck128_xts_setkey,
-                .encrypt                = speck128_xts_encrypt,
-                .decrypt                = speck128_xts_decrypt,
-        }, {
-                .base.cra_name          = "xts(speck64)",
-                .base.cra_driver_name   = "xts-speck64-neon",
-                .base.cra_priority      = 300,
-                .base.cra_blocksize     = SPECK64_BLOCK_SIZE,
-                .base.cra_ctxsize       = sizeof(struct speck64_xts_tfm_ctx),
-                .base.cra_alignmask     = 7,
-                .base.cra_module        = THIS_MODULE,
-                .min_keysize            = 2 * SPECK64_96_KEY_SIZE,
-                .max_keysize            = 2 * SPECK64_128_KEY_SIZE,
-                .ivsize                 = SPECK64_BLOCK_SIZE,
-                .walksize               = SPECK_NEON_CHUNK_SIZE,
-                .setkey                 = speck64_xts_setkey,
-                .encrypt                = speck64_xts_encrypt,
-                .decrypt                = speck64_xts_decrypt,
-        }
-};
-static int __init speck_neon_module_init(void)
-{
-        if (!(elf_hwcap & HWCAP_NEON))
-                return -ENODEV;
-        return crypto_register_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
-}
-static void __exit speck_neon_module_exit(void)
-{
-        crypto_unregister_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
-}
-module_init(speck_neon_module_init);
-module_exit(speck_neon_module_exit);
-MODULE_DESCRIPTION("Speck block cipher (NEON-accelerated)");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("xts(speck128)");
-MODULE_ALIAS_CRYPTO("xts-speck128-neon");
-MODULE_ALIAS_CRYPTO("xts(speck64)");
-MODULE_ALIAS_CRYPTO("xts-speck64-neon");
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index db8d364f8476..3d165b4cdd2a 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -698,6 +698,7 @@ CONFIG_MEMTEST=y
 CONFIG_SECURITY=y
 CONFIG_CRYPTO_ECHAINIV=y
 CONFIG_CRYPTO_ANSI_CPRNG=y
+CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=y
 CONFIG_ARM64_CRYPTO=y
 CONFIG_CRYPTO_SHA1_ARM64_CE=y
 CONFIG_CRYPTO_SHA2_ARM64_CE=y
@@ -706,7 +707,6 @@ CONFIG_CRYPTO_SHA3_ARM64=m
 CONFIG_CRYPTO_SM3_ARM64_CE=m
 CONFIG_CRYPTO_GHASH_ARM64_CE=y
 CONFIG_CRYPTO_CRCT10DIF_ARM64_CE=m
-CONFIG_CRYPTO_CRC32_ARM64_CE=m
 CONFIG_CRYPTO_AES_ARM64_CE_CCM=y
 CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
 CONFIG_CRYPTO_CHACHA20_NEON=m
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index e3fdb0fd6f70..a5606823ed4d 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -66,11 +66,6 @@ config CRYPTO_CRCT10DIF_ARM64_CE
        depends on KERNEL_MODE_NEON && CRC_T10DIF
        select CRYPTO_HASH
-config CRYPTO_CRC32_ARM64_CE
-        tristate "CRC32 and CRC32C digest algorithms using ARMv8 extensions"
-        depends on CRC32
-        select CRYPTO_HASH
 config CRYPTO_AES_ARM64
        tristate "AES core cipher using scalar instructions"
        select CRYPTO_AES
@@ -119,10 +114,4 @@ config CRYPTO_AES_ARM64_BS
        select CRYPTO_AES_ARM64
        select CRYPTO_SIMD
-config CRYPTO_SPECK_NEON
-        tristate "NEON accelerated Speck cipher algorithms"
-        depends on KERNEL_MODE_NEON
-        select CRYPTO_BLKCIPHER
-        select CRYPTO_SPECK
 endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index bcafd016618e..f476fede09ba 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -32,9 +32,6 @@ ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
 obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM64_CE) += crct10dif-ce.o
 crct10dif-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
-obj-$(CONFIG_CRYPTO_CRC32_ARM64_CE) += crc32-ce.o
-crc32-ce-y:= crc32-ce-core.o crc32-ce-glue.o
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
 aes-ce-cipher-y := aes-ce-core.o aes-ce-glue.o
@@ -56,9 +53,6 @@ sha512-arm64-y := sha512-glue.o sha512-core.o
 obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
 chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
-obj-$(CONFIG_CRYPTO_SPECK_NEON) += speck-neon.o
-speck-neon-y := speck-neon-core.o speck-neon-glue.o
 obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
 aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S
index 623e74ed1c67..143070510809 100644
--- a/arch/arm64/crypto/aes-ce.S
+++ b/arch/arm64/crypto/aes-ce.S
@@ -17,6 +17,11 @@
        .arch           armv8-a+crypto
+        xtsmask         .req    v16
+        .macro          xts_reload_mask, tmp
+        .endm
        /* preload all round keys */
        .macro          load_round_keys, rounds, rk
        cmp             \rounds, #12
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index adcb83eb683c..1e676625ef33 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -15,6 +15,7 @@
 #include <crypto/internal/hash.h>
 #include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
 #include <linux/module.h>
 #include <linux/cpufeature.h>
 #include <crypto/xts.h>
@@ -31,6 +32,8 @@
 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
+#define aes_cbc_cts_encrypt     ce_aes_cbc_cts_encrypt
+#define aes_cbc_cts_decrypt     ce_aes_cbc_cts_decrypt
 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
 #define aes_xts_encrypt         ce_aes_xts_encrypt
 #define aes_xts_decrypt         ce_aes_xts_decrypt
@@ -45,6 +48,8 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
+#define aes_cbc_cts_encrypt     neon_aes_cbc_cts_encrypt
+#define aes_cbc_cts_decrypt     neon_aes_cbc_cts_decrypt
 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
 #define aes_xts_encrypt         neon_aes_xts_encrypt
 #define aes_xts_decrypt         neon_aes_xts_decrypt
@@ -63,30 +68,41 @@ MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_LICENSE("GPL v2");
 /* defined in aes-modes.S */
-asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
+asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
                                int rounds, int blocks);
-asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
+asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
                                int rounds, int blocks);
-asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
+asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
                                int rounds, int blocks, u8 iv[]);
-asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
+asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
                                int rounds, int blocks, u8 iv[]);
-asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
+                                int rounds, int bytes, u8 const iv[]);
+asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
+                                int rounds, int bytes, u8 const iv[]);
+asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
                                int rounds, int blocks, u8 ctr[]);
-asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
+asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
-                                int rounds, int blocks, u8 const rk2[], u8 iv[],
+                                int rounds, int blocks, u32 const rk2[], u8 iv[],
                                int first);
-asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
+asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
-                                int rounds, int blocks, u8 const rk2[], u8 iv[],
+                                int rounds, int blocks, u32 const rk2[], u8 iv[],
                                int first);
 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
                               int blocks, u8 dg[], int enc_before,
                               int enc_after);
+struct cts_cbc_req_ctx {
+        struct scatterlist sg_src[2];
+        struct scatterlist sg_dst[2];
+        struct skcipher_request subreq;
+};
 struct crypto_aes_xts_ctx {
        struct crypto_aes_ctx key1;
        struct crypto_aes_ctx __aligned(8) key2;
@@ -142,7 +158,7 @@ static int ecb_encrypt(struct skcipher_request *req)
        while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
                kernel_neon_begin();
                aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key_enc, rounds, blocks);
+                                ctx->key_enc, rounds, blocks);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
@@ -162,7 +178,7 @@ static int ecb_decrypt(struct skcipher_request *req)
        while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
                kernel_neon_begin();
                aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key_dec, rounds, blocks);
+                                ctx->key_dec, rounds, blocks);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
@@ -182,7 +198,7 @@ static int cbc_encrypt(struct skcipher_request *req)
        while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
                kernel_neon_begin();
                aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key_enc, rounds, blocks, walk.iv);
+                                ctx->key_enc, rounds, blocks, walk.iv);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
@@ -202,13 +218,149 @@ static int cbc_decrypt(struct skcipher_request *req)
        while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
                kernel_neon_begin();
                aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key_dec, rounds, blocks, walk.iv);
+                                ctx->key_dec, rounds, blocks, walk.iv);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
        return err;
 }
+static int cts_cbc_init_tfm(struct crypto_skcipher *tfm)
+{
+        crypto_skcipher_set_reqsize(tfm, sizeof(struct cts_cbc_req_ctx));
+        return 0;
+}
+static int cts_cbc_encrypt(struct skcipher_request *req)
+{
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req);
+        int err, rounds = 6 + ctx->key_length / 4;
+        int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
+        struct scatterlist *src = req->src, *dst = req->dst;
+        struct skcipher_walk walk;
+        skcipher_request_set_tfm(&rctx->subreq, tfm);
+        if (req->cryptlen <= AES_BLOCK_SIZE) {
+                if (req->cryptlen < AES_BLOCK_SIZE)
+                        return -EINVAL;
+                cbc_blocks = 1;
+        }
+        if (cbc_blocks > 0) {
+                unsigned int blocks;
+                skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst,
+                                           cbc_blocks * AES_BLOCK_SIZE,
+                                           req->iv);
+                err = skcipher_walk_virt(&walk, &rctx->subreq, false);
+                while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
+                        kernel_neon_begin();
+                        aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                                        ctx->key_enc, rounds, blocks, walk.iv);
+                        kernel_neon_end();
+                        err = skcipher_walk_done(&walk,
+                                                 walk.nbytes % AES_BLOCK_SIZE);
+                }
+                if (err)
+                        return err;
+                if (req->cryptlen == AES_BLOCK_SIZE)
+                        return 0;
+                dst = src = scatterwalk_ffwd(rctx->sg_src, req->src,
+                                             rctx->subreq.cryptlen);
+                if (req->dst != req->src)
+                        dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
+                                               rctx->subreq.cryptlen);
+        }
+        /* handle ciphertext stealing */
+        skcipher_request_set_crypt(&rctx->subreq, src, dst,
+                                   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
+                                   req->iv);
+        err = skcipher_walk_virt(&walk, &rctx->subreq, false);
+        if (err)
+                return err;
+        kernel_neon_begin();
+        aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                            ctx->key_enc, rounds, walk.nbytes, walk.iv);
+        kernel_neon_end();
+        return skcipher_walk_done(&walk, 0);
+}
+static int cts_cbc_decrypt(struct skcipher_request *req)
+{
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req);
+        int err, rounds = 6 + ctx->key_length / 4;
+        int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
+        struct scatterlist *src = req->src, *dst = req->dst;
+        struct skcipher_walk walk;
+        skcipher_request_set_tfm(&rctx->subreq, tfm);
+        if (req->cryptlen <= AES_BLOCK_SIZE) {
+                if (req->cryptlen < AES_BLOCK_SIZE)
+                        return -EINVAL;
+                cbc_blocks = 1;
+        }
+        if (cbc_blocks > 0) {
+                unsigned int blocks;
+                skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst,
+                                           cbc_blocks * AES_BLOCK_SIZE,
+                                           req->iv);
+                err = skcipher_walk_virt(&walk, &rctx->subreq, false);
+                while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
+                        kernel_neon_begin();
+                        aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                                        ctx->key_dec, rounds, blocks, walk.iv);
+                        kernel_neon_end();
+                        err = skcipher_walk_done(&walk,
+                                                 walk.nbytes % AES_BLOCK_SIZE);
+                }
+                if (err)
+                        return err;
+                if (req->cryptlen == AES_BLOCK_SIZE)
+                        return 0;
+                dst = src = scatterwalk_ffwd(rctx->sg_src, req->src,
+                                             rctx->subreq.cryptlen);
+                if (req->dst != req->src)
+                        dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
+                                               rctx->subreq.cryptlen);
+        }
+        /* handle ciphertext stealing */
+        skcipher_request_set_crypt(&rctx->subreq, src, dst,
+                                   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
+                                   req->iv);
+        err = skcipher_walk_virt(&walk, &rctx->subreq, false);
+        if (err)
+                return err;
+        kernel_neon_begin();
+        aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                            ctx->key_dec, rounds, walk.nbytes, walk.iv);
+        kernel_neon_end();
+        return skcipher_walk_done(&walk, 0);
+}
 static int ctr_encrypt(struct skcipher_request *req)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -222,7 +374,7 @@ static int ctr_encrypt(struct skcipher_request *req)
        while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
                kernel_neon_begin();
                aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key_enc, rounds, blocks, walk.iv);
+                                ctx->key_enc, rounds, blocks, walk.iv);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
@@ -238,7 +390,7 @@ static int ctr_encrypt(struct skcipher_request *req)
                blocks = -1;
                kernel_neon_begin();
-                aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
+                aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds,
                                blocks, walk.iv);
                kernel_neon_end();
                crypto_xor_cpy(tdst, tsrc, tail, nbytes);
@@ -272,8 +424,8 @@ static int xts_encrypt(struct skcipher_request *req)
        for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
                kernel_neon_begin();
                aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key1.key_enc, rounds, blocks,
+                                ctx->key1.key_enc, rounds, blocks,
-                                (u8 *)ctx->key2.key_enc, walk.iv, first);
+                                ctx->key2.key_enc, walk.iv, first);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
@@ -294,8 +446,8 @@ static int xts_decrypt(struct skcipher_request *req)
        for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
                kernel_neon_begin();
                aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
-                                (u8 *)ctx->key1.key_dec, rounds, blocks,
+                                ctx->key1.key_dec, rounds, blocks,
-                                (u8 *)ctx->key2.key_enc, walk.iv, first);
+                                ctx->key2.key_enc, walk.iv, first);
                kernel_neon_end();
                err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
        }
@@ -336,6 +488,24 @@ static struct skcipher_alg aes_algs[] = { {
        .decrypt        = cbc_decrypt,
 }, {
        .base = {
+                .cra_name               = "__cts(cbc(aes))",
+                .cra_driver_name        = "__cts-cbc-aes-" MODE,
+                .cra_priority           = PRIO,
+                .cra_flags              = CRYPTO_ALG_INTERNAL,
+                .cra_blocksize          = AES_BLOCK_SIZE,
+                .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
+                .cra_module             = THIS_MODULE,
+        },
+        .min_keysize    = AES_MIN_KEY_SIZE,
+        .max_keysize    = AES_MAX_KEY_SIZE,
+        .ivsize         = AES_BLOCK_SIZE,
+        .walksize       = 2 * AES_BLOCK_SIZE,
+        .setkey         = skcipher_aes_setkey,
+        .encrypt        = cts_cbc_encrypt,
+        .decrypt        = cts_cbc_decrypt,
+        .init           = cts_cbc_init_tfm,
+}, {
+        .base = {
                .cra_name               = "__ctr(aes)",
                .cra_driver_name        = "__ctr-aes-" MODE,
                .cra_priority           = PRIO,
@@ -412,7 +582,6 @@ static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
 {
        struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
        be128 *consts = (be128 *)ctx->consts;
-        u8 *rk = (u8 *)ctx->key.key_enc;
        int rounds = 6 + key_len / 4;
        int err;
@@ -422,7 +591,8 @@ static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
        /* encrypt the zero vector */
        kernel_neon_begin();
-        aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1);
+        aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc,
+                        rounds, 1);
        kernel_neon_end();
        cmac_gf128_mul_by_x(consts, consts);
@@ -441,7 +611,6 @@ static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
        };
        struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
-        u8 *rk = (u8 *)ctx->key.key_enc;
        int rounds = 6 + key_len / 4;
        u8 key[AES_BLOCK_SIZE];
        int err;
@@ -451,8 +620,8 @@ static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
                return err;
        kernel_neon_begin();
-        aes_ecb_encrypt(key, ks[0], rk, rounds, 1);
+        aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
-        aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2);
+        aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
        kernel_neon_end();
        return cbcmac_setkey(tfm, key, sizeof(key));
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index 483a7130cf0e..67700045a0e0 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -14,12 +14,12 @@
        .align          4
 aes_encrypt_block4x:
-        encrypt_block4x v0, v1, v2, v3, w22, x21, x8, w7
+        encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
        ret
 ENDPROC(aes_encrypt_block4x)
 aes_decrypt_block4x:
-        decrypt_block4x v0, v1, v2, v3, w22, x21, x8, w7
+        decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
        ret
 ENDPROC(aes_decrypt_block4x)
@@ -31,71 +31,57 @@ ENDPROC(aes_decrypt_block4x)
         */
 AES_ENTRY(aes_ecb_encrypt)
-        frame_push      5
+        stp             x29, x30, [sp, #-16]!
+        mov             x29, sp
-        mov             x19, x0
+        enc_prepare     w3, x2, x5
-        mov             x20, x1
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-.Lecbencrestart:
-        enc_prepare     w22, x21, x5
 .LecbencloopNx:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lecbenc1x
-        ld1             {v0.16b-v3.16b}, [x20], #64     /* get 4 pt blocks */
+        ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 pt blocks */
        bl              aes_encrypt_block4x
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
-        cond_yield_neon .Lecbencrestart
        b               .LecbencloopNx
 .Lecbenc1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lecbencout
 .Lecbencloop:
-        ld1             {v0.16b}, [x20], #16            /* get next pt block */
+        ld1             {v0.16b}, [x1], #16             /* get next pt block */
-        encrypt_block   v0, w22, x21, x5, w6
+        encrypt_block   v0, w3, x2, x5, w6
-        st1             {v0.16b}, [x19], #16
+        st1             {v0.16b}, [x0], #16
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        bne             .Lecbencloop
 .Lecbencout:
-        frame_pop
+        ldp             x29, x30, [sp], #16
        ret
 AES_ENDPROC(aes_ecb_encrypt)
 AES_ENTRY(aes_ecb_decrypt)
-        frame_push      5
+        stp             x29, x30, [sp, #-16]!
+        mov             x29, sp
-        mov             x19, x0
+        dec_prepare     w3, x2, x5
-        mov             x20, x1
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-.Lecbdecrestart:
-        dec_prepare     w22, x21, x5
 .LecbdecloopNx:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lecbdec1x
-        ld1             {v0.16b-v3.16b}, [x20], #64     /* get 4 ct blocks */
+        ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 ct blocks */
        bl              aes_decrypt_block4x
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
-        cond_yield_neon .Lecbdecrestart
        b               .LecbdecloopNx
 .Lecbdec1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lecbdecout
 .Lecbdecloop:
-        ld1             {v0.16b}, [x20], #16            /* get next ct block */
+        ld1             {v0.16b}, [x1], #16             /* get next ct block */
-        decrypt_block   v0, w22, x21, x5, w6
+        decrypt_block   v0, w3, x2, x5, w6
-        st1             {v0.16b}, [x19], #16
+        st1             {v0.16b}, [x0], #16
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        bne             .Lecbdecloop
 .Lecbdecout:
-        frame_pop
+        ldp             x29, x30, [sp], #16
        ret
 AES_ENDPROC(aes_ecb_decrypt)
@@ -108,162 +94,211 @@ AES_ENDPROC(aes_ecb_decrypt)
         */
 AES_ENTRY(aes_cbc_encrypt)
-        frame_push      6
+        ld1             {v4.16b}, [x5]                  /* get iv */
+        enc_prepare     w3, x2, x6
-        mov             x19, x0
-        mov             x20, x1
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-        mov             x24, x5
-.Lcbcencrestart:
-        ld1             {v4.16b}, [x24]                 /* get iv */
-        enc_prepare     w22, x21, x6
 .Lcbcencloop4x:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lcbcenc1x
-        ld1             {v0.16b-v3.16b}, [x20], #64     /* get 4 pt blocks */
+        ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 pt blocks */
        eor             v0.16b, v0.16b, v4.16b          /* ..and xor with iv */
-        encrypt_block   v0, w22, x21, x6, w7
+        encrypt_block   v0, w3, x2, x6, w7
        eor             v1.16b, v1.16b, v0.16b
-        encrypt_block   v1, w22, x21, x6, w7
+        encrypt_block   v1, w3, x2, x6, w7
        eor             v2.16b, v2.16b, v1.16b
-        encrypt_block   v2, w22, x21, x6, w7
+        encrypt_block   v2, w3, x2, x6, w7
        eor             v3.16b, v3.16b, v2.16b
-        encrypt_block   v3, w22, x21, x6, w7
+        encrypt_block   v3, w3, x2, x6, w7
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
        mov             v4.16b, v3.16b
-        st1             {v4.16b}, [x24]                 /* return iv */
-        cond_yield_neon .Lcbcencrestart
        b               .Lcbcencloop4x
 .Lcbcenc1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lcbcencout
 .Lcbcencloop:
-        ld1             {v0.16b}, [x20], #16            /* get next pt block */
+        ld1             {v0.16b}, [x1], #16             /* get next pt block */
        eor             v4.16b, v4.16b, v0.16b          /* ..and xor with iv */
-        encrypt_block   v4, w22, x21, x6, w7
+        encrypt_block   v4, w3, x2, x6, w7
-        st1             {v4.16b}, [x19], #16
+        st1             {v4.16b}, [x0], #16
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        bne             .Lcbcencloop
 .Lcbcencout:
-        st1             {v4.16b}, [x24]                 /* return iv */
+        st1             {v4.16b}, [x5]                  /* return iv */
-        frame_pop
        ret
 AES_ENDPROC(aes_cbc_encrypt)
 AES_ENTRY(aes_cbc_decrypt)
-        frame_push      6
+        stp             x29, x30, [sp, #-16]!
+        mov             x29, sp
-        mov             x19, x0
-        mov             x20, x1
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-        mov             x24, x5
-.Lcbcdecrestart:
+        ld1             {v7.16b}, [x5]                  /* get iv */
-        ld1             {v7.16b}, [x24]                 /* get iv */
+        dec_prepare     w3, x2, x6
-        dec_prepare     w22, x21, x6
 .LcbcdecloopNx:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lcbcdec1x
-        ld1             {v0.16b-v3.16b}, [x20], #64     /* get 4 ct blocks */
+        ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 ct blocks */
        mov             v4.16b, v0.16b
        mov             v5.16b, v1.16b
        mov             v6.16b, v2.16b
        bl              aes_decrypt_block4x
-        sub             x20, x20, #16
+        sub             x1, x1, #16
        eor             v0.16b, v0.16b, v7.16b
        eor             v1.16b, v1.16b, v4.16b
-        ld1             {v7.16b}, [x20], #16            /* reload 1 ct block */
+        ld1             {v7.16b}, [x1], #16             /* reload 1 ct block */
        eor             v2.16b, v2.16b, v5.16b
        eor             v3.16b, v3.16b, v6.16b
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
-        st1             {v7.16b}, [x24]                 /* return iv */
-        cond_yield_neon .Lcbcdecrestart
        b               .LcbcdecloopNx
 .Lcbcdec1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lcbcdecout
 .Lcbcdecloop:
-        ld1             {v1.16b}, [x20], #16            /* get next ct block */
+        ld1             {v1.16b}, [x1], #16             /* get next ct block */
        mov             v0.16b, v1.16b                  /* ...and copy to v0 */
-        decrypt_block   v0, w22, x21, x6, w7
+        decrypt_block   v0, w3, x2, x6, w7
        eor             v0.16b, v0.16b, v7.16b          /* xor with iv => pt */
        mov             v7.16b, v1.16b                  /* ct is next iv */
-        st1             {v0.16b}, [x19], #16
+        st1             {v0.16b}, [x0], #16
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        bne             .Lcbcdecloop
 .Lcbcdecout:
-        st1             {v7.16b}, [x24]                 /* return iv */
+        st1             {v7.16b}, [x5]                  /* return iv */
-        frame_pop
+        ldp             x29, x30, [sp], #16
        ret
 AES_ENDPROC(aes_cbc_decrypt)
        /*
+         * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
+         *                     int rounds, int bytes, u8 const iv[])
+         * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
+         *                     int rounds, int bytes, u8 const iv[])
+         */
+AES_ENTRY(aes_cbc_cts_encrypt)
+        adr_l           x8, .Lcts_permute_table
+        sub             x4, x4, #16
+        add             x9, x8, #32
+        add             x8, x8, x4
+        sub             x9, x9, x4
+        ld1             {v3.16b}, [x8]
+        ld1             {v4.16b}, [x9]
+        ld1             {v0.16b}, [x1], x4              /* overlapping loads */
+        ld1             {v1.16b}, [x1]
+        ld1             {v5.16b}, [x5]                  /* get iv */
+        enc_prepare     w3, x2, x6
+        eor             v0.16b, v0.16b, v5.16b          /* xor with iv */
+        tbl             v1.16b, {v1.16b}, v4.16b
+        encrypt_block   v0, w3, x2, x6, w7
+        eor             v1.16b, v1.16b, v0.16b
+        tbl             v0.16b, {v0.16b}, v3.16b
+        encrypt_block   v1, w3, x2, x6, w7
+        add             x4, x0, x4
+        st1             {v0.16b}, [x4]                  /* overlapping stores */
+        st1             {v1.16b}, [x0]
+        ret
+AES_ENDPROC(aes_cbc_cts_encrypt)
+AES_ENTRY(aes_cbc_cts_decrypt)
+        adr_l           x8, .Lcts_permute_table
+        sub             x4, x4, #16
+        add             x9, x8, #32
+        add             x8, x8, x4
+        sub             x9, x9, x4
+        ld1             {v3.16b}, [x8]
+        ld1             {v4.16b}, [x9]
+        ld1             {v0.16b}, [x1], x4              /* overlapping loads */
+        ld1             {v1.16b}, [x1]
+        ld1             {v5.16b}, [x5]                  /* get iv */
+        dec_prepare     w3, x2, x6
+        tbl             v2.16b, {v1.16b}, v4.16b
+        decrypt_block   v0, w3, x2, x6, w7
+        eor             v2.16b, v2.16b, v0.16b
+        tbx             v0.16b, {v1.16b}, v4.16b
+        tbl             v2.16b, {v2.16b}, v3.16b
+        decrypt_block   v0, w3, x2, x6, w7
+        eor             v0.16b, v0.16b, v5.16b          /* xor with iv */
+        add             x4, x0, x4
+        st1             {v2.16b}, [x4]                  /* overlapping stores */
+        st1             {v0.16b}, [x0]
+        ret
+AES_ENDPROC(aes_cbc_cts_decrypt)
+        .section        ".rodata", "a"
+        .align          6
+.Lcts_permute_table:
+        .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+        .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+        .byte            0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
+        .byte            0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
+        .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+        .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+        .previous
+        /*
         * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
         *                 int blocks, u8 ctr[])
         */
 AES_ENTRY(aes_ctr_encrypt)
-        frame_push      6
+        stp             x29, x30, [sp, #-16]!
+        mov             x29, sp
-        mov             x19, x0
+        enc_prepare     w3, x2, x6
-        mov             x20, x1
+        ld1             {v4.16b}, [x5]
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-        mov             x24, x5
-.Lctrrestart:
-        enc_prepare     w22, x21, x6
-        ld1             {v4.16b}, [x24]
        umov            x6, v4.d[1]             /* keep swabbed ctr in reg */
        rev             x6, x6
+        cmn             w6, w4                  /* 32 bit overflow? */
+        bcs             .Lctrloop
 .LctrloopNx:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lctr1x
-        cmn             w6, #4                  /* 32 bit overflow? */
+        add             w7, w6, #1
-        bcs             .Lctr1x
-        ldr             q8, =0x30000000200000001        /* addends 1,2,3[,0] */
-        dup             v7.4s, w6
        mov             v0.16b, v4.16b
-        add             v7.4s, v7.4s, v8.4s
+        add             w8, w6, #2
        mov             v1.16b, v4.16b
-        rev32           v8.16b, v7.16b
+        add             w9, w6, #3
        mov             v2.16b, v4.16b
+        rev             w7, w7
        mov             v3.16b, v4.16b
-        mov             v1.s[3], v8.s[0]
+        rev             w8, w8
-        mov             v2.s[3], v8.s[1]
+        mov             v1.s[3], w7
-        mov             v3.s[3], v8.s[2]
+        rev             w9, w9
-        ld1             {v5.16b-v7.16b}, [x20], #48     /* get 3 input blocks */
+        mov             v2.s[3], w8
+        mov             v3.s[3], w9
+        ld1             {v5.16b-v7.16b}, [x1], #48      /* get 3 input blocks */
        bl              aes_encrypt_block4x
        eor             v0.16b, v5.16b, v0.16b
-        ld1             {v5.16b}, [x20], #16            /* get 1 input block  */
+        ld1             {v5.16b}, [x1], #16             /* get 1 input block  */
        eor             v1.16b, v6.16b, v1.16b
        eor             v2.16b, v7.16b, v2.16b
        eor             v3.16b, v5.16b, v3.16b
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
        add             x6, x6, #4
        rev             x7, x6
        ins             v4.d[1], x7
-        cbz             w23, .Lctrout
+        cbz             w4, .Lctrout
-        st1             {v4.16b}, [x24]         /* return next CTR value */
-        cond_yield_neon .Lctrrestart
        b               .LctrloopNx
 .Lctr1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lctrout
 .Lctrloop:
        mov             v0.16b, v4.16b
-        encrypt_block   v0, w22, x21, x8, w7
+        encrypt_block   v0, w3, x2, x8, w7
        adds            x6, x6, #1              /* increment BE ctr */
        rev             x7, x6
@@ -271,22 +306,22 @@ AES_ENTRY(aes_ctr_encrypt)
        bcs             .Lctrcarry              /* overflow? */
 .Lctrcarrydone:
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        bmi             .Lctrtailblock          /* blocks <0 means tail block */
-        ld1             {v3.16b}, [x20], #16
+        ld1             {v3.16b}, [x1], #16
        eor             v3.16b, v0.16b, v3.16b
-        st1             {v3.16b}, [x19], #16
+        st1             {v3.16b}, [x0], #16
        bne             .Lctrloop
 .Lctrout:
-        st1             {v4.16b}, [x24]         /* return next CTR value */
+        st1             {v4.16b}, [x5]          /* return next CTR value */
-.Lctrret:
+        ldp             x29, x30, [sp], #16
-        frame_pop
        ret
 .Lctrtailblock:
-        st1             {v0.16b}, [x19]
+        st1             {v0.16b}, [x0]
-        b               .Lctrret
+        ldp             x29, x30, [sp], #16
+        ret
 .Lctrcarry:
        umov            x7, v4.d[0]             /* load upper word of ctr  */
@@ -296,7 +331,6 @@ AES_ENTRY(aes_ctr_encrypt)
        ins             v4.d[0], x7
        b               .Lctrcarrydone
 AES_ENDPROC(aes_ctr_encrypt)
-        .ltorg
        /*
@@ -306,150 +340,132 @@ AES_ENDPROC(aes_ctr_encrypt)
         *                 int blocks, u8 const rk2[], u8 iv[], int first)
         */
-        .macro          next_tweak, out, in, const, tmp
+        .macro          next_tweak, out, in, tmp
        sshr            \tmp\().2d,  \in\().2d,   #63
-        and             \tmp\().16b, \tmp\().16b, \const\().16b
+        and             \tmp\().16b, \tmp\().16b, xtsmask.16b
        add             \out\().2d,  \in\().2d,   \in\().2d
        ext             \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
        eor             \out\().16b, \out\().16b, \tmp\().16b
        .endm
-.Lxts_mul_x:
+        .macro          xts_load_mask, tmp
-CPU_LE( .quad           1, 0x87         )
+        movi            xtsmask.2s, #0x1
-CPU_BE( .quad           0x87, 1         )
+        movi            \tmp\().2s, #0x87
+        uzp1            xtsmask.4s, xtsmask.4s, \tmp\().4s
+        .endm
 AES_ENTRY(aes_xts_encrypt)
-        frame_push      6
+        stp             x29, x30, [sp, #-16]!
+        mov             x29, sp
-        mov             x19, x0
+        ld1             {v4.16b}, [x6]
-        mov             x20, x1
+        xts_load_mask   v8
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-        mov             x24, x6
-        ld1             {v4.16b}, [x24]
        cbz             w7, .Lxtsencnotfirst
        enc_prepare     w3, x5, x8
        encrypt_block   v4, w3, x5, x8, w7              /* first tweak */
        enc_switch_key  w3, x2, x8
-        ldr             q7, .Lxts_mul_x
        b               .LxtsencNx
-.Lxtsencrestart:
-        ld1             {v4.16b}, [x24]
 .Lxtsencnotfirst:
-        enc_prepare     w22, x21, x8
+        enc_prepare     w3, x2, x8
 .LxtsencloopNx:
-        ldr             q7, .Lxts_mul_x
+        next_tweak      v4, v4, v8
-        next_tweak      v4, v4, v7, v8
 .LxtsencNx:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lxtsenc1x
-        ld1             {v0.16b-v3.16b}, [x20], #64     /* get 4 pt blocks */
+        ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 pt blocks */
-        next_tweak      v5, v4, v7, v8
+        next_tweak      v5, v4, v8
        eor             v0.16b, v0.16b, v4.16b
-        next_tweak      v6, v5, v7, v8
+        next_tweak      v6, v5, v8
        eor             v1.16b, v1.16b, v5.16b
        eor             v2.16b, v2.16b, v6.16b
-        next_tweak      v7, v6, v7, v8
+        next_tweak      v7, v6, v8
        eor             v3.16b, v3.16b, v7.16b
        bl              aes_encrypt_block4x
        eor             v3.16b, v3.16b, v7.16b
        eor             v0.16b, v0.16b, v4.16b
        eor             v1.16b, v1.16b, v5.16b
        eor             v2.16b, v2.16b, v6.16b
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
        mov             v4.16b, v7.16b
-        cbz             w23, .Lxtsencout
+        cbz             w4, .Lxtsencout
-        st1             {v4.16b}, [x24]
+        xts_reload_mask v8
-        cond_yield_neon .Lxtsencrestart
        b               .LxtsencloopNx
 .Lxtsenc1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lxtsencout
 .Lxtsencloop:
-        ld1             {v1.16b}, [x20], #16
+        ld1             {v1.16b}, [x1], #16
        eor             v0.16b, v1.16b, v4.16b
-        encrypt_block   v0, w22, x21, x8, w7
+        encrypt_block   v0, w3, x2, x8, w7
        eor             v0.16b, v0.16b, v4.16b
-        st1             {v0.16b}, [x19], #16
+        st1             {v0.16b}, [x0], #16
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        beq             .Lxtsencout
-        next_tweak      v4, v4, v7, v8
+        next_tweak      v4, v4, v8
        b               .Lxtsencloop
 .Lxtsencout:
-        st1             {v4.16b}, [x24]
+        st1             {v4.16b}, [x6]
-        frame_pop
+        ldp             x29, x30, [sp], #16
        ret
 AES_ENDPROC(aes_xts_encrypt)
 AES_ENTRY(aes_xts_decrypt)
-        frame_push      6
+        stp             x29, x30, [sp, #-16]!
+        mov             x29, sp
-        mov             x19, x0
+        ld1             {v4.16b}, [x6]
-        mov             x20, x1
+        xts_load_mask   v8
-        mov             x21, x2
-        mov             x22, x3
-        mov             x23, x4
-        mov             x24, x6
-        ld1             {v4.16b}, [x24]
        cbz             w7, .Lxtsdecnotfirst
        enc_prepare     w3, x5, x8
        encrypt_block   v4, w3, x5, x8, w7              /* first tweak */
        dec_prepare     w3, x2, x8
-        ldr             q7, .Lxts_mul_x
        b               .LxtsdecNx
-.Lxtsdecrestart:
-        ld1             {v4.16b}, [x24]
 .Lxtsdecnotfirst:
-        dec_prepare     w22, x21, x8
+        dec_prepare     w3, x2, x8
 .LxtsdecloopNx:
-        ldr             q7, .Lxts_mul_x
+        next_tweak      v4, v4, v8
-        next_tweak      v4, v4, v7, v8
 .LxtsdecNx:
-        subs            w23, w23, #4
+        subs            w4, w4, #4
        bmi             .Lxtsdec1x
-        ld1             {v0.16b-v3.16b}, [x20], #64     /* get 4 ct blocks */
+        ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 ct blocks */
-        next_tweak      v5, v4, v7, v8
+        next_tweak      v5, v4, v8
        eor             v0.16b, v0.16b, v4.16b
-        next_tweak      v6, v5, v7, v8
+        next_tweak      v6, v5, v8
        eor             v1.16b, v1.16b, v5.16b
        eor             v2.16b, v2.16b, v6.16b
-        next_tweak      v7, v6, v7, v8
+        next_tweak      v7, v6, v8
        eor             v3.16b, v3.16b, v7.16b
        bl              aes_decrypt_block4x
        eor             v3.16b, v3.16b, v7.16b
        eor             v0.16b, v0.16b, v4.16b
        eor             v1.16b, v1.16b, v5.16b
        eor             v2.16b, v2.16b, v6.16b
-        st1             {v0.16b-v3.16b}, [x19], #64
+        st1             {v0.16b-v3.16b}, [x0], #64
        mov             v4.16b, v7.16b
-        cbz             w23, .Lxtsdecout
+        cbz             w4, .Lxtsdecout
-        st1             {v4.16b}, [x24]
+        xts_reload_mask v8
-        cond_yield_neon .Lxtsdecrestart
        b               .LxtsdecloopNx
 .Lxtsdec1x:
-        adds            w23, w23, #4
+        adds            w4, w4, #4
        beq             .Lxtsdecout
 .Lxtsdecloop:
-        ld1             {v1.16b}, [x20], #16
+        ld1             {v1.16b}, [x1], #16
        eor             v0.16b, v1.16b, v4.16b
-        decrypt_block   v0, w22, x21, x8, w7
+        decrypt_block   v0, w3, x2, x8, w7
        eor             v0.16b, v0.16b, v4.16b
-        st1             {v0.16b}, [x19], #16
+        st1             {v0.16b}, [x0], #16
-        subs            w23, w23, #1
+        subs            w4, w4, #1
        beq             .Lxtsdecout
-        next_tweak      v4, v4, v7, v8
+        next_tweak      v4, v4, v8
        b               .Lxtsdecloop
 .Lxtsdecout:
-        st1             {v4.16b}, [x24]
+        st1             {v4.16b}, [x6]
-        frame_pop
+        ldp             x29, x30, [sp], #16
        ret
 AES_ENDPROC(aes_xts_decrypt)
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
index 1c7b45b7268e..29100f692e8a 100644
--- a/arch/arm64/crypto/aes-neon.S
+++ b/arch/arm64/crypto/aes-neon.S
@@ -14,6 +14,12 @@
 #define AES_ENTRY(func)         ENTRY(neon_ ## func)
 #define AES_ENDPROC(func)       ENDPROC(neon_ ## func)
+        xtsmask         .req    v7
+        .macro          xts_reload_mask, tmp
+        xts_load_mask   \tmp
+        .endm
        /* multiply by polynomial 'x' in GF(2^8) */
        .macro          mul_by_x, out, in, temp, const
        sshr            \temp, \in, #7
diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S
deleted file mode 100644
index 8061bf0f9c66..000000000000
--- a/arch/arm64/crypto/crc32-ce-core.S
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Accelerated CRC32(C) using arm64 CRC, NEON and Crypto Extensions instructions
- *
- * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-/* GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see http://www.gnu.org/licenses
- *
- * Please  visit http://www.xyratex.com/contact if you need additional
- * information or have any questions.
- *
- * GPL HEADER END
- */
-/*
- * Copyright 2012 Xyratex Technology Limited
- *
- * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
- * calculation.
- * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
- * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
- * at:
- * http://www.intel.com/products/processor/manuals/
- * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
- * Volume 2B: Instruction Set Reference, N-Z
- *
- * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
- *            Alexander Boyko <Alexander_Boyko@xyratex.com>
- */
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-        .section        ".rodata", "a"
-        .align          6
-        .cpu            generic+crypto+crc
-.Lcrc32_constants:
-        /*
-         * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
-         * #define CONSTANT_R1  0x154442bd4LL
-         *
-         * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
-         * #define CONSTANT_R2  0x1c6e41596LL
-         */
-        .octa           0x00000001c6e415960000000154442bd4
-        /*
-         * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
-         * #define CONSTANT_R3  0x1751997d0LL
-         *
-         * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
-         * #define CONSTANT_R4  0x0ccaa009eLL
-         */
-        .octa           0x00000000ccaa009e00000001751997d0
-        /*
-         * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
-         * #define CONSTANT_R5  0x163cd6124LL
-         */
-        .quad           0x0000000163cd6124
-        .quad           0x00000000FFFFFFFF
-        /*
-         * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
-         *
-         * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))`
-         *                                                      = 0x1F7011641LL
-         * #define CONSTANT_RU  0x1F7011641LL
-         */
-        .octa           0x00000001F701164100000001DB710641
-.Lcrc32c_constants:
-        .octa           0x000000009e4addf800000000740eef02
-        .octa           0x000000014cd00bd600000000f20c0dfe
-        .quad           0x00000000dd45aab8
-        .quad           0x00000000FFFFFFFF
-        .octa           0x00000000dea713f10000000105ec76f0
-        vCONSTANT       .req    v0
-        dCONSTANT       .req    d0
-        qCONSTANT       .req    q0
-        BUF             .req    x19
-        LEN             .req    x20
-        CRC             .req    x21
-        CONST           .req    x22
-        vzr             .req    v9
-        /**
-         * Calculate crc32
-         * BUF - buffer
-         * LEN - sizeof buffer (multiple of 16 bytes), LEN should be > 63
-         * CRC - initial crc32
-         * return %eax crc32
-         * uint crc32_pmull_le(unsigned char const *buffer,
-         *                     size_t len, uint crc32)
-         */
-        .text
-ENTRY(crc32_pmull_le)
-        adr_l           x3, .Lcrc32_constants
-        b               0f
-ENTRY(crc32c_pmull_le)
-        adr_l           x3, .Lcrc32c_constants
-0:      frame_push      4, 64
-        mov             BUF, x0
-        mov             LEN, x1
-        mov             CRC, x2
-        mov             CONST, x3
-        bic             LEN, LEN, #15
-        ld1             {v1.16b-v4.16b}, [BUF], #0x40
-        movi            vzr.16b, #0
-        fmov            dCONSTANT, CRC
-        eor             v1.16b, v1.16b, vCONSTANT.16b
-        sub             LEN, LEN, #0x40
-        cmp             LEN, #0x40
-        b.lt            less_64
-        ldr             qCONSTANT, [CONST]
-loop_64:                /* 64 bytes Full cache line folding */
-        sub             LEN, LEN, #0x40
-        pmull2          v5.1q, v1.2d, vCONSTANT.2d
-        pmull2          v6.1q, v2.2d, vCONSTANT.2d
-        pmull2          v7.1q, v3.2d, vCONSTANT.2d
-        pmull2          v8.1q, v4.2d, vCONSTANT.2d
-        pmull           v1.1q, v1.1d, vCONSTANT.1d
-        pmull           v2.1q, v2.1d, vCONSTANT.1d
-        pmull           v3.1q, v3.1d, vCONSTANT.1d
-        pmull           v4.1q, v4.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v5.16b
-        ld1             {v5.16b}, [BUF], #0x10
-        eor             v2.16b, v2.16b, v6.16b
-        ld1             {v6.16b}, [BUF], #0x10
-        eor             v3.16b, v3.16b, v7.16b
-        ld1             {v7.16b}, [BUF], #0x10
-        eor             v4.16b, v4.16b, v8.16b
-        ld1             {v8.16b}, [BUF], #0x10
-        eor             v1.16b, v1.16b, v5.16b
-        eor             v2.16b, v2.16b, v6.16b
-        eor             v3.16b, v3.16b, v7.16b
-        eor             v4.16b, v4.16b, v8.16b
-        cmp             LEN, #0x40
-        b.lt            less_64
-        if_will_cond_yield_neon
-        stp             q1, q2, [sp, #.Lframe_local_offset]
-        stp             q3, q4, [sp, #.Lframe_local_offset + 32]
-        do_cond_yield_neon
-        ldp             q1, q2, [sp, #.Lframe_local_offset]
-        ldp             q3, q4, [sp, #.Lframe_local_offset + 32]
-        ldr             qCONSTANT, [CONST]
-        movi            vzr.16b, #0
-        endif_yield_neon
-        b               loop_64
-less_64:                /* Folding cache line into 128bit */
-        ldr             qCONSTANT, [CONST, #16]
-        pmull2          v5.1q, v1.2d, vCONSTANT.2d
-        pmull           v1.1q, v1.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v5.16b
-        eor             v1.16b, v1.16b, v2.16b
-        pmull2          v5.1q, v1.2d, vCONSTANT.2d
-        pmull           v1.1q, v1.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v5.16b
-        eor             v1.16b, v1.16b, v3.16b
-        pmull2          v5.1q, v1.2d, vCONSTANT.2d
-        pmull           v1.1q, v1.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v5.16b
-        eor             v1.16b, v1.16b, v4.16b
-        cbz             LEN, fold_64
-loop_16:                /* Folding rest buffer into 128bit */
-        subs            LEN, LEN, #0x10
-        ld1             {v2.16b}, [BUF], #0x10
-        pmull2          v5.1q, v1.2d, vCONSTANT.2d
-        pmull           v1.1q, v1.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v5.16b
-        eor             v1.16b, v1.16b, v2.16b
-        b.ne            loop_16
-fold_64:
-        /* perform the last 64 bit fold, also adds 32 zeroes
-         * to the input stream */
-        ext             v2.16b, v1.16b, v1.16b, #8
-        pmull2          v2.1q, v2.2d, vCONSTANT.2d
-        ext             v1.16b, v1.16b, vzr.16b, #8
-        eor             v1.16b, v1.16b, v2.16b
-        /* final 32-bit fold */
-        ldr             dCONSTANT, [CONST, #32]
-        ldr             d3, [CONST, #40]
-        ext             v2.16b, v1.16b, vzr.16b, #4
-        and             v1.16b, v1.16b, v3.16b
-        pmull           v1.1q, v1.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v2.16b
-        /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
-        ldr             qCONSTANT, [CONST, #48]
-        and             v2.16b, v1.16b, v3.16b
-        ext             v2.16b, vzr.16b, v2.16b, #8
-        pmull2          v2.1q, v2.2d, vCONSTANT.2d
-        and             v2.16b, v2.16b, v3.16b
-        pmull           v2.1q, v2.1d, vCONSTANT.1d
-        eor             v1.16b, v1.16b, v2.16b
-        mov             w0, v1.s[1]
-        frame_pop
-        ret
-ENDPROC(crc32_pmull_le)
-ENDPROC(crc32c_pmull_le)
-        .macro          __crc32, c
-0:      subs            x2, x2, #16
-        b.mi            8f
-        ldp             x3, x4, [x1], #16
-CPU_BE( rev             x3, x3          )
-CPU_BE( rev             x4, x4          )
-        crc32\c\()x     w0, w0, x3
-        crc32\c\()x     w0, w0, x4
-        b.ne            0b
-        ret
-8:      tbz             x2, #3, 4f
-        ldr             x3, [x1], #8
-CPU_BE( rev             x3, x3          )
-        crc32\c\()x     w0, w0, x3
-4:      tbz             x2, #2, 2f
-        ldr             w3, [x1], #4
-CPU_BE( rev             w3, w3          )
-        crc32\c\()w     w0, w0, w3
-2:      tbz             x2, #1, 1f
-        ldrh            w3, [x1], #2
-CPU_BE( rev16           w3, w3          )
-        crc32\c\()h     w0, w0, w3
-1:      tbz             x2, #0, 0f
-        ldrb            w3, [x1]
-        crc32\c\()b     w0, w0, w3
-0:      ret
-        .endm
-        .align          5
-ENTRY(crc32_armv8_le)
-        __crc32
-ENDPROC(crc32_armv8_le)
-        .align          5
-ENTRY(crc32c_armv8_le)
-        __crc32         c
-ENDPROC(crc32c_armv8_le)
diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
deleted file mode 100644
index 34b4e3d46aab..000000000000
--- a/arch/arm64/crypto/crc32-ce-glue.c
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
- *
- * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <linux/cpufeature.h>
-#include <linux/crc32.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/string.h>
-#include <crypto/internal/hash.h>
-#include <asm/hwcap.h>
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <asm/unaligned.h>
-#define PMULL_MIN_LEN           64L     /* minimum size of buffer
-                                         * for crc32_pmull_le_16 */
-#define SCALE_F                 16L     /* size of NEON register */
-asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
-asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);
-asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
-asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);
-static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
-static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);
-static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
-{
-        u32 *key = crypto_tfm_ctx(tfm);
-        *key = 0;
-        return 0;
-}
-static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
-{
-        u32 *key = crypto_tfm_ctx(tfm);
-        *key = ~0;
-        return 0;
-}
-static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
-                              unsigned int keylen)
-{
-        u32 *mctx = crypto_shash_ctx(hash);
-        if (keylen != sizeof(u32)) {
-                crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
-                return -EINVAL;
-        }
-        *mctx = le32_to_cpup((__le32 *)key);
-        return 0;
-}
-static int crc32_pmull_init(struct shash_desc *desc)
-{
-        u32 *mctx = crypto_shash_ctx(desc->tfm);
-        u32 *crc = shash_desc_ctx(desc);
-        *crc = *mctx;
-        return 0;
-}
-static int crc32_update(struct shash_desc *desc, const u8 *data,
-                        unsigned int length)
-{
-        u32 *crc = shash_desc_ctx(desc);
-        *crc = crc32_armv8_le(*crc, data, length);
-        return 0;
-}
-static int crc32c_update(struct shash_desc *desc, const u8 *data,
-                         unsigned int length)
-{
-        u32 *crc = shash_desc_ctx(desc);
-        *crc = crc32c_armv8_le(*crc, data, length);
-        return 0;
-}
-static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
-                         unsigned int length)
-{
-        u32 *crc = shash_desc_ctx(desc);
-        unsigned int l;
-        if ((u64)data % SCALE_F) {
-                l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
-                *crc = fallback_crc32(*crc, data, l);
-                data += l;
-                length -= l;
-        }
-        if (length >= PMULL_MIN_LEN && may_use_simd()) {
-                l = round_down(length, SCALE_F);
-                kernel_neon_begin();
-                *crc = crc32_pmull_le(data, l, *crc);
-                kernel_neon_end();
-                data += l;
-                length -= l;
-        }
-        if (length > 0)
-                *crc = fallback_crc32(*crc, data, length);
-        return 0;
-}
-static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
-                         unsigned int length)
-{
-        u32 *crc = shash_desc_ctx(desc);
-        unsigned int l;
-        if ((u64)data % SCALE_F) {
-                l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
-                *crc = fallback_crc32c(*crc, data, l);
-                data += l;
-                length -= l;
-        }
-        if (length >= PMULL_MIN_LEN && may_use_simd()) {
-                l = round_down(length, SCALE_F);
-                kernel_neon_begin();
-                *crc = crc32c_pmull_le(data, l, *crc);
-                kernel_neon_end();
-                data += l;
-                length -= l;
-        }
-        if (length > 0) {
-                *crc = fallback_crc32c(*crc, data, length);
-        }
-        return 0;
-}
-static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
-{
-        u32 *crc = shash_desc_ctx(desc);
-        put_unaligned_le32(*crc, out);
-        return 0;
-}
-static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
-{
-        u32 *crc = shash_desc_ctx(desc);
-        put_unaligned_le32(~*crc, out);
-        return 0;
-}
-static struct shash_alg crc32_pmull_algs[] = { {
-        .setkey                 = crc32_pmull_setkey,
-        .init                   = crc32_pmull_init,
-        .update                 = crc32_update,
-        .final                  = crc32_pmull_final,
-        .descsize               = sizeof(u32),
-        .digestsize             = sizeof(u32),
-        .base.cra_ctxsize       = sizeof(u32),
-        .base.cra_init          = crc32_pmull_cra_init,
-        .base.cra_name          = "crc32",
-        .base.cra_driver_name   = "crc32-arm64-ce",
-        .base.cra_priority      = 200,
-        .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
-        .base.cra_blocksize     = 1,
-        .base.cra_module        = THIS_MODULE,
-}, {
-        .setkey                 = crc32_pmull_setkey,
-        .init                   = crc32_pmull_init,
-        .update                 = crc32c_update,
-        .final                  = crc32c_pmull_final,
-        .descsize               = sizeof(u32),
-        .digestsize             = sizeof(u32),
-        .base.cra_ctxsize       = sizeof(u32),
-        .base.cra_init          = crc32c_pmull_cra_init,
-        .base.cra_name          = "crc32c",
-        .base.cra_driver_name   = "crc32c-arm64-ce",
-        .base.cra_priority      = 200,
-        .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
-        .base.cra_blocksize     = 1,
-        .base.cra_module        = THIS_MODULE,
-} };
-static int __init crc32_pmull_mod_init(void)
-{
-        if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
-                crc32_pmull_algs[0].update = crc32_pmull_update;
-                crc32_pmull_algs[1].update = crc32c_pmull_update;
-                if (elf_hwcap & HWCAP_CRC32) {
-                        fallback_crc32 = crc32_armv8_le;
-                        fallback_crc32c = crc32c_armv8_le;
-                } else {
-                        fallback_crc32 = crc32_le;
-                        fallback_crc32c = __crc32c_le;
-                }
-        } else if (!(elf_hwcap & HWCAP_CRC32)) {
-                return -ENODEV;
-        }
-        return crypto_register_shashes(crc32_pmull_algs,
-                                       ARRAY_SIZE(crc32_pmull_algs));
-}
-static void __exit crc32_pmull_mod_exit(void)
-{
-        crypto_unregister_shashes(crc32_pmull_algs,
-                                  ARRAY_SIZE(crc32_pmull_algs));
-}
-static const struct cpu_feature crc32_cpu_feature[] = {
-        { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
-};
-MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
-module_init(crc32_pmull_mod_init);
-module_exit(crc32_pmull_mod_exit);
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S
index 663ea71cdb38..9e82e8e8ed05 100644
--- a/arch/arm64/crypto/crct10dif-ce-core.S
+++ b/arch/arm64/crypto/crct10dif-ce-core.S
@@ -80,7 +80,186 @@
        vzr             .req    v13
-ENTRY(crc_t10dif_pmull)
+        ad              .req    v14
+        bd              .req    v10
+        k00_16          .req    v15
+        k32_48          .req    v16
+        t3              .req    v17
+        t4              .req    v18
+        t5              .req    v19
+        t6              .req    v20
+        t7              .req    v21
+        t8              .req    v22
+        t9              .req    v23
+        perm1           .req    v24
+        perm2           .req    v25
+        perm3           .req    v26
+        perm4           .req    v27
+        bd1             .req    v28
+        bd2             .req    v29
+        bd3             .req    v30
+        bd4             .req    v31
+        .macro          __pmull_init_p64
+        .endm
+        .macro          __pmull_pre_p64, bd
+        .endm
+        .macro          __pmull_init_p8
+        // k00_16 := 0x0000000000000000_000000000000ffff
+        // k32_48 := 0x00000000ffffffff_0000ffffffffffff
+        movi            k32_48.2d, #0xffffffff
+        mov             k32_48.h[2], k32_48.h[0]
+        ushr            k00_16.2d, k32_48.2d, #32
+        // prepare the permutation vectors
+        mov_q           x5, 0x080f0e0d0c0b0a09
+        movi            perm4.8b, #8
+        dup             perm1.2d, x5
+        eor             perm1.16b, perm1.16b, perm4.16b
+        ushr            perm2.2d, perm1.2d, #8
+        ushr            perm3.2d, perm1.2d, #16
+        ushr            perm4.2d, perm1.2d, #24
+        sli             perm2.2d, perm1.2d, #56
+        sli             perm3.2d, perm1.2d, #48
+        sli             perm4.2d, perm1.2d, #40
+        .endm
+        .macro          __pmull_pre_p8, bd
+        tbl             bd1.16b, {\bd\().16b}, perm1.16b
+        tbl             bd2.16b, {\bd\().16b}, perm2.16b
+        tbl             bd3.16b, {\bd\().16b}, perm3.16b
+        tbl             bd4.16b, {\bd\().16b}, perm4.16b
+        .endm
+__pmull_p8_core:
+.L__pmull_p8_core:
+        ext             t4.8b, ad.8b, ad.8b, #1                 // A1
+        ext             t5.8b, ad.8b, ad.8b, #2                 // A2
+        ext             t6.8b, ad.8b, ad.8b, #3                 // A3
+        pmull           t4.8h, t4.8b, bd.8b                     // F = A1*B
+        pmull           t8.8h, ad.8b, bd1.8b                    // E = A*B1
+        pmull           t5.8h, t5.8b, bd.8b                     // H = A2*B
+        pmull           t7.8h, ad.8b, bd2.8b                    // G = A*B2
+        pmull           t6.8h, t6.8b, bd.8b                     // J = A3*B
+        pmull           t9.8h, ad.8b, bd3.8b                    // I = A*B3
+        pmull           t3.8h, ad.8b, bd4.8b                    // K = A*B4
+        b               0f
+.L__pmull_p8_core2:
+        tbl             t4.16b, {ad.16b}, perm1.16b             // A1
+        tbl             t5.16b, {ad.16b}, perm2.16b             // A2
+        tbl             t6.16b, {ad.16b}, perm3.16b             // A3
+        pmull2          t4.8h, t4.16b, bd.16b                   // F = A1*B
+        pmull2          t8.8h, ad.16b, bd1.16b                  // E = A*B1
+        pmull2          t5.8h, t5.16b, bd.16b                   // H = A2*B
+        pmull2          t7.8h, ad.16b, bd2.16b                  // G = A*B2
+        pmull2          t6.8h, t6.16b, bd.16b                   // J = A3*B
+        pmull2          t9.8h, ad.16b, bd3.16b                  // I = A*B3
+        pmull2          t3.8h, ad.16b, bd4.16b                  // K = A*B4
+0:      eor             t4.16b, t4.16b, t8.16b                  // L = E + F
+        eor             t5.16b, t5.16b, t7.16b                  // M = G + H
+        eor             t6.16b, t6.16b, t9.16b                  // N = I + J
+        uzp1            t8.2d, t4.2d, t5.2d
+        uzp2            t4.2d, t4.2d, t5.2d
+        uzp1            t7.2d, t6.2d, t3.2d
+        uzp2            t6.2d, t6.2d, t3.2d
+        // t4 = (L) (P0 + P1) << 8
+        // t5 = (M) (P2 + P3) << 16
+        eor             t8.16b, t8.16b, t4.16b
+        and             t4.16b, t4.16b, k32_48.16b
+        // t6 = (N) (P4 + P5) << 24
+        // t7 = (K) (P6 + P7) << 32
+        eor             t7.16b, t7.16b, t6.16b
+        and             t6.16b, t6.16b, k00_16.16b
+        eor             t8.16b, t8.16b, t4.16b
+        eor             t7.16b, t7.16b, t6.16b
+        zip2            t5.2d, t8.2d, t4.2d
+        zip1            t4.2d, t8.2d, t4.2d
+        zip2            t3.2d, t7.2d, t6.2d
+        zip1            t6.2d, t7.2d, t6.2d
+        ext             t4.16b, t4.16b, t4.16b, #15
+        ext             t5.16b, t5.16b, t5.16b, #14
+        ext             t6.16b, t6.16b, t6.16b, #13
+        ext             t3.16b, t3.16b, t3.16b, #12
+        eor             t4.16b, t4.16b, t5.16b
+        eor             t6.16b, t6.16b, t3.16b
+        ret
+ENDPROC(__pmull_p8_core)
+        .macro          __pmull_p8, rq, ad, bd, i
+        .ifnc           \bd, v10
+        .err
+        .endif
+        mov             ad.16b, \ad\().16b
+        .ifb            \i
+        pmull           \rq\().8h, \ad\().8b, bd.8b             // D = A*B
+        .else
+        pmull2          \rq\().8h, \ad\().16b, bd.16b           // D = A*B
+        .endif
+        bl              .L__pmull_p8_core\i
+        eor             \rq\().16b, \rq\().16b, t4.16b
+        eor             \rq\().16b, \rq\().16b, t6.16b
+        .endm
+        .macro          fold64, p, reg1, reg2
+        ldp             q11, q12, [arg2], #0x20
+        __pmull_\p      v8, \reg1, v10, 2
+        __pmull_\p      \reg1, \reg1, v10
+CPU_LE( rev64           v11.16b, v11.16b                )
+CPU_LE( rev64           v12.16b, v12.16b                )
+        __pmull_\p      v9, \reg2, v10, 2
+        __pmull_\p      \reg2, \reg2, v10
+CPU_LE( ext             v11.16b, v11.16b, v11.16b, #8   )
+CPU_LE( ext             v12.16b, v12.16b, v12.16b, #8   )
+        eor             \reg1\().16b, \reg1\().16b, v8.16b
+        eor             \reg2\().16b, \reg2\().16b, v9.16b
+        eor             \reg1\().16b, \reg1\().16b, v11.16b
+        eor             \reg2\().16b, \reg2\().16b, v12.16b
+        .endm
+        .macro          fold16, p, reg, rk
+        __pmull_\p      v8, \reg, v10
+        __pmull_\p      \reg, \reg, v10, 2
+        .ifnb           \rk
+        ldr_l           q10, \rk, x8
+        __pmull_pre_\p  v10
+        .endif
+        eor             v7.16b, v7.16b, v8.16b
+        eor             v7.16b, v7.16b, \reg\().16b
+        .endm
+        .macro          __pmull_p64, rd, rn, rm, n
+        .ifb            \n
+        pmull           \rd\().1q, \rn\().1d, \rm\().1d
+        .else
+        pmull2          \rd\().1q, \rn\().2d, \rm\().2d
+        .endif
+        .endm
+        .macro          crc_t10dif_pmull, p
        frame_push      3, 128
        mov             arg1_low32, w0
@@ -89,6 +268,8 @@ ENTRY(crc_t10dif_pmull)
        movi            vzr.16b, #0             // init zero register
+        __pmull_init_\p
        // adjust the 16-bit initial_crc value, scale it to 32 bits
        lsl             arg1_low32, arg1_low32, #16
@@ -96,7 +277,7 @@ ENTRY(crc_t10dif_pmull)
        cmp             arg3, #256
        // for sizes less than 128, we can't fold 64B at a time...
-        b.lt            _less_than_128
+        b.lt            .L_less_than_128_\@
        // load the initial crc value
        // crc value does not need to be byte-reflected, but it needs
@@ -137,6 +318,7 @@ CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
        ldr_l           q10, rk3, x8    // xmm10 has rk3 and rk4
                                        // type of pmull instruction
                                        // will determine which constant to use
+        __pmull_pre_\p  v10
        //
        // we subtract 256 instead of 128 to save one instruction from the loop
@@ -147,41 +329,19 @@ CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
        // buffer. The _fold_64_B_loop will fold 64B at a time
        // until we have 64+y Bytes of buffer
        // fold 64B at a time. This section of the code folds 4 vector
        // registers in parallel
-_fold_64_B_loop:
+.L_fold_64_B_loop_\@:
-        .macro          fold64, reg1, reg2
+        fold64          \p, v0, v1
-        ldp             q11, q12, [arg2], #0x20
+        fold64          \p, v2, v3
+        fold64          \p, v4, v5
-        pmull2          v8.1q, \reg1\().2d, v10.2d
+        fold64          \p, v6, v7
-        pmull           \reg1\().1q, \reg1\().1d, v10.1d
-CPU_LE( rev64           v11.16b, v11.16b                )
-CPU_LE( rev64           v12.16b, v12.16b                )
-        pmull2          v9.1q, \reg2\().2d, v10.2d
-        pmull           \reg2\().1q, \reg2\().1d, v10.1d
-CPU_LE( ext             v11.16b, v11.16b, v11.16b, #8   )
-CPU_LE( ext             v12.16b, v12.16b, v12.16b, #8   )
-        eor             \reg1\().16b, \reg1\().16b, v8.16b
-        eor             \reg2\().16b, \reg2\().16b, v9.16b
-        eor             \reg1\().16b, \reg1\().16b, v11.16b
-        eor             \reg2\().16b, \reg2\().16b, v12.16b
-        .endm
-        fold64          v0, v1
-        fold64          v2, v3
-        fold64          v4, v5
-        fold64          v6, v7
        subs            arg3, arg3, #128
        // check if there is another 64B in the buffer to be able to fold
-        b.lt            _fold_64_B_end
+        b.lt            .L_fold_64_B_end_\@
        if_will_cond_yield_neon
        stp             q0, q1, [sp, #.Lframe_local_offset]
@@ -195,11 +355,13 @@ CPU_LE(	ext		v12.16b, v12.16b, v12.16b, #8	)
        ldp             q6, q7, [sp, #.Lframe_local_offset + 96]
        ldr_l           q10, rk3, x8
        movi            vzr.16b, #0             // init zero register
+        __pmull_init_\p
+        __pmull_pre_\p  v10
        endif_yield_neon
-        b               _fold_64_B_loop
+        b               .L_fold_64_B_loop_\@
-_fold_64_B_end:
+.L_fold_64_B_end_\@:
        // at this point, the buffer pointer is pointing at the last y Bytes
        // of the buffer the 64B of folded data is in 4 of the vector
        // registers: v0, v1, v2, v3
@@ -208,38 +370,29 @@ _fold_64_B_end:
        // constants
        ldr_l           q10, rk9, x8
+        __pmull_pre_\p  v10
-        .macro          fold16, reg, rk
+        fold16          \p, v0, rk11
-        pmull           v8.1q, \reg\().1d, v10.1d
+        fold16          \p, v1, rk13
-        pmull2          \reg\().1q, \reg\().2d, v10.2d
+        fold16          \p, v2, rk15
-        .ifnb           \rk
+        fold16          \p, v3, rk17
-        ldr_l           q10, \rk, x8
+        fold16          \p, v4, rk19
-        .endif
+        fold16          \p, v5, rk1
-        eor             v7.16b, v7.16b, v8.16b
+        fold16          \p, v6
-        eor             v7.16b, v7.16b, \reg\().16b
-        .endm
-        fold16          v0, rk11
-        fold16          v1, rk13
-        fold16          v2, rk15
-        fold16          v3, rk17
-        fold16          v4, rk19
-        fold16          v5, rk1
-        fold16          v6
        // instead of 64, we add 48 to the loop counter to save 1 instruction
        // from the loop instead of a cmp instruction, we use the negative
        // flag with the jl instruction
        adds            arg3, arg3, #(128-16)
-        b.lt            _final_reduction_for_128
+        b.lt            .L_final_reduction_for_128_\@
        // now we have 16+y bytes left to reduce. 16 Bytes is in register v7
        // and the rest is in memory. We can fold 16 bytes at a time if y>=16
        // continue folding 16B at a time
-_16B_reduction_loop:
+.L_16B_reduction_loop_\@:
-        pmull           v8.1q, v7.1d, v10.1d
+        __pmull_\p      v8, v7, v10
-        pmull2          v7.1q, v7.2d, v10.2d
+        __pmull_\p      v7, v7, v10, 2
        eor             v7.16b, v7.16b, v8.16b
        ldr             q0, [arg2], #16
@@ -251,22 +404,22 @@ CPU_LE(	ext		v0.16b, v0.16b, v0.16b, #8	)
        // instead of a cmp instruction, we utilize the flags with the
        // jge instruction equivalent of: cmp arg3, 16-16
        // check if there is any more 16B in the buffer to be able to fold
-        b.ge            _16B_reduction_loop
+        b.ge            .L_16B_reduction_loop_\@
        // now we have 16+z bytes left to reduce, where 0<= z < 16.
        // first, we reduce the data in the xmm7 register
-_final_reduction_for_128:
+.L_final_reduction_for_128_\@:
        // check if any more data to fold. If not, compute the CRC of
        // the final 128 bits
        adds            arg3, arg3, #16
-        b.eq            _128_done
+        b.eq            .L_128_done_\@
        // here we are getting data that is less than 16 bytes.
        // since we know that there was data before the pointer, we can
        // offset the input pointer before the actual point, to receive
        // exactly 16 bytes. after that the registers need to be adjusted.
-_get_last_two_regs:
+.L_get_last_two_regs_\@:
        add             arg2, arg2, arg3
        ldr             q1, [arg2, #-16]
 CPU_LE( rev64           v1.16b, v1.16b                  )
@@ -291,47 +444,48 @@ CPU_LE(	ext		v1.16b, v1.16b, v1.16b, #8	)
        bsl             v0.16b, v2.16b, v1.16b
        // fold 16 Bytes
-        pmull           v8.1q, v7.1d, v10.1d
+        __pmull_\p      v8, v7, v10
-        pmull2          v7.1q, v7.2d, v10.2d
+        __pmull_\p      v7, v7, v10, 2
        eor             v7.16b, v7.16b, v8.16b
        eor             v7.16b, v7.16b, v0.16b
-_128_done:
+.L_128_done_\@:
        // compute crc of a 128-bit value
        ldr_l           q10, rk5, x8            // rk5 and rk6 in xmm10
+        __pmull_pre_\p  v10
        // 64b fold
        ext             v0.16b, vzr.16b, v7.16b, #8
        mov             v7.d[0], v7.d[1]
-        pmull           v7.1q, v7.1d, v10.1d
+        __pmull_\p      v7, v7, v10
        eor             v7.16b, v7.16b, v0.16b
        // 32b fold
        ext             v0.16b, v7.16b, vzr.16b, #4
        mov             v7.s[3], vzr.s[0]
-        pmull2          v0.1q, v0.2d, v10.2d
+        __pmull_\p      v0, v0, v10, 2
        eor             v7.16b, v7.16b, v0.16b
        // barrett reduction
-_barrett:
        ldr_l           q10, rk7, x8
+        __pmull_pre_\p  v10
        mov             v0.d[0], v7.d[1]
-        pmull           v0.1q, v0.1d, v10.1d
+        __pmull_\p      v0, v0, v10
        ext             v0.16b, vzr.16b, v0.16b, #12
-        pmull2          v0.1q, v0.2d, v10.2d
+        __pmull_\p      v0, v0, v10, 2
        ext             v0.16b, vzr.16b, v0.16b, #12
        eor             v7.16b, v7.16b, v0.16b
        mov             w0, v7.s[1]
-_cleanup:
+.L_cleanup_\@:
        // scale the result back to 16 bits
        lsr             x0, x0, #16
        frame_pop
        ret
-_less_than_128:
+.L_less_than_128_\@:
-        cbz             arg3, _cleanup
+        cbz             arg3, .L_cleanup_\@
        movi            v0.16b, #0
        mov             v0.s[3], arg1_low32     // get the initial crc value
@@ -342,20 +496,21 @@ CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
        eor             v7.16b, v7.16b, v0.16b  // xor the initial crc value
        cmp             arg3, #16
-        b.eq            _128_done               // exactly 16 left
+        b.eq            .L_128_done_\@          // exactly 16 left
-        b.lt            _less_than_16_left
+        b.lt            .L_less_than_16_left_\@
        ldr_l           q10, rk1, x8            // rk1 and rk2 in xmm10
+        __pmull_pre_\p  v10
        // update the counter. subtract 32 instead of 16 to save one
        // instruction from the loop
        subs            arg3, arg3, #32
-        b.ge            _16B_reduction_loop
+        b.ge            .L_16B_reduction_loop_\@
        add             arg3, arg3, #16
-        b               _get_last_two_regs
+        b               .L_get_last_two_regs_\@
-_less_than_16_left:
+.L_less_than_16_left_\@:
        // shl r9, 4
        adr_l           x0, tbl_shf_table + 16
        sub             x0, x0, arg3
@@ -363,8 +518,17 @@ _less_than_16_left:
        movi            v9.16b, #0x80
        eor             v0.16b, v0.16b, v9.16b
        tbl             v7.16b, {v7.16b}, v0.16b
-        b               _128_done
+        b               .L_128_done_\@
-ENDPROC(crc_t10dif_pmull)
+        .endm
+ENTRY(crc_t10dif_pmull_p8)
+        crc_t10dif_pmull        p8
+ENDPROC(crc_t10dif_pmull_p8)
+        .align          5
+ENTRY(crc_t10dif_pmull_p64)
+        crc_t10dif_pmull        p64
+ENDPROC(crc_t10dif_pmull_p64)
 // precomputed constants
 // these constants are precomputed from the poly:
diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c
index 96f0cae4a022..b461d62023f2 100644
--- a/arch/arm64/crypto/crct10dif-ce-glue.c
+++ b/arch/arm64/crypto/crct10dif-ce-glue.c
@@ -22,7 +22,10 @@
 #define CRC_T10DIF_PMULL_CHUNK_SIZE     16U
-asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u64 len);
+asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 buf[], u64 len);
+asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 buf[], u64 len);
+static u16 (*crc_t10dif_pmull)(u16 init_crc, const u8 buf[], u64 len);
 static int crct10dif_init(struct shash_desc *desc)
 {
@@ -85,6 +88,11 @@ static struct shash_alg crc_t10dif_alg = {
 static int __init crc_t10dif_mod_init(void)
 {
+        if (elf_hwcap & HWCAP_PMULL)
+                crc_t10dif_pmull = crc_t10dif_pmull_p64;
+        else
+                crc_t10dif_pmull = crc_t10dif_pmull_p8;
        return crypto_register_shash(&crc_t10dif_alg);
 }
@@ -93,8 +101,10 @@ static void __exit crc_t10dif_mod_exit(void)
        crypto_unregister_shash(&crc_t10dif_alg);
 }
-module_cpu_feature_match(PMULL, crc_t10dif_mod_init);
+module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
 module_exit(crc_t10dif_mod_exit);
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("crct10dif");
+MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");
diff --git a/arch/arm64/crypto/speck-neon-core.S b/arch/arm64/crypto/speck-neon-core.S
deleted file mode 100644
index b14463438b09..000000000000
--- a/arch/arm64/crypto/speck-neon-core.S
+++ /dev/null
@@ -1,352 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * ARM64 NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
- *
- * Copyright (c) 2018 Google, Inc
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-#include <linux/linkage.h>
-        .text
-        // arguments
-        ROUND_KEYS      .req    x0      // const {u64,u32} *round_keys
-        NROUNDS         .req    w1      // int nrounds
-        NROUNDS_X       .req    x1
-        DST             .req    x2      // void *dst
-        SRC             .req    x3      // const void *src
-        NBYTES          .req    w4      // unsigned int nbytes
-        TWEAK           .req    x5      // void *tweak
-        // registers which hold the data being encrypted/decrypted
-        // (underscores avoid a naming collision with ARM64 registers x0-x3)
-        X_0             .req    v0
-        Y_0             .req    v1
-        X_1             .req    v2
-        Y_1             .req    v3
-        X_2             .req    v4
-        Y_2             .req    v5
-        X_3             .req    v6
-        Y_3             .req    v7
-        // the round key, duplicated in all lanes
-        ROUND_KEY       .req    v8
-        // index vector for tbl-based 8-bit rotates
-        ROTATE_TABLE    .req    v9
-        ROTATE_TABLE_Q  .req    q9
-        // temporary registers
-        TMP0            .req    v10
-        TMP1            .req    v11
-        TMP2            .req    v12
-        TMP3            .req    v13
-        // multiplication table for updating XTS tweaks
-        GFMUL_TABLE     .req    v14
-        GFMUL_TABLE_Q   .req    q14
-        // next XTS tweak value(s)
-        TWEAKV_NEXT     .req    v15
-        // XTS tweaks for the blocks currently being encrypted/decrypted
-        TWEAKV0         .req    v16
-        TWEAKV1         .req    v17
-        TWEAKV2         .req    v18
-        TWEAKV3         .req    v19
-        TWEAKV4         .req    v20
-        TWEAKV5         .req    v21
-        TWEAKV6         .req    v22
-        TWEAKV7         .req    v23
-        .align          4
-.Lror64_8_table:
-        .octa           0x080f0e0d0c0b0a090007060504030201
-.Lror32_8_table:
-        .octa           0x0c0f0e0d080b0a090407060500030201
-.Lrol64_8_table:
-        .octa           0x0e0d0c0b0a09080f0605040302010007
-.Lrol32_8_table:
-        .octa           0x0e0d0c0f0a09080b0605040702010003
-.Lgf128mul_table:
-        .octa           0x00000000000000870000000000000001
-.Lgf64mul_table:
-        .octa           0x0000000000000000000000002d361b00
-/*
- * _speck_round_128bytes() - Speck encryption round on 128 bytes at a time
- *
- * Do one Speck encryption round on the 128 bytes (8 blocks for Speck128, 16 for
- * Speck64) stored in X0-X3 and Y0-Y3, using the round key stored in all lanes
- * of ROUND_KEY.  'n' is the lane size: 64 for Speck128, or 32 for Speck64.
- * 'lanes' is the lane specifier: "2d" for Speck128 or "4s" for Speck64.
- */
-.macro _speck_round_128bytes    n, lanes
-        // x = ror(x, 8)
-        tbl             X_0.16b, {X_0.16b}, ROTATE_TABLE.16b
-        tbl             X_1.16b, {X_1.16b}, ROTATE_TABLE.16b
-        tbl             X_2.16b, {X_2.16b}, ROTATE_TABLE.16b
-        tbl             X_3.16b, {X_3.16b}, ROTATE_TABLE.16b
-        // x += y
-        add             X_0.\lanes, X_0.\lanes, Y_0.\lanes
-        add             X_1.\lanes, X_1.\lanes, Y_1.\lanes
-        add             X_2.\lanes, X_2.\lanes, Y_2.\lanes
-        add             X_3.\lanes, X_3.\lanes, Y_3.\lanes
-        // x ^= k
-        eor             X_0.16b, X_0.16b, ROUND_KEY.16b
-        eor             X_1.16b, X_1.16b, ROUND_KEY.16b
-        eor             X_2.16b, X_2.16b, ROUND_KEY.16b
-        eor             X_3.16b, X_3.16b, ROUND_KEY.16b
-        // y = rol(y, 3)
-        shl             TMP0.\lanes, Y_0.\lanes, #3
-        shl             TMP1.\lanes, Y_1.\lanes, #3
-        shl             TMP2.\lanes, Y_2.\lanes, #3
-        shl             TMP3.\lanes, Y_3.\lanes, #3
-        sri             TMP0.\lanes, Y_0.\lanes, #(\n - 3)
-        sri             TMP1.\lanes, Y_1.\lanes, #(\n - 3)
-        sri             TMP2.\lanes, Y_2.\lanes, #(\n - 3)
-        sri             TMP3.\lanes, Y_3.\lanes, #(\n - 3)
-        // y ^= x
-        eor             Y_0.16b, TMP0.16b, X_0.16b
-        eor             Y_1.16b, TMP1.16b, X_1.16b
-        eor             Y_2.16b, TMP2.16b, X_2.16b
-        eor             Y_3.16b, TMP3.16b, X_3.16b
-.endm
-/*
- * _speck_unround_128bytes() - Speck decryption round on 128 bytes at a time
- *
- * This is the inverse of _speck_round_128bytes().
- */
-.macro _speck_unround_128bytes  n, lanes
-        // y ^= x
-        eor             TMP0.16b, Y_0.16b, X_0.16b
-        eor             TMP1.16b, Y_1.16b, X_1.16b
-        eor             TMP2.16b, Y_2.16b, X_2.16b
-        eor             TMP3.16b, Y_3.16b, X_3.16b
-        // y = ror(y, 3)
-        ushr            Y_0.\lanes, TMP0.\lanes, #3
-        ushr            Y_1.\lanes, TMP1.\lanes, #3
-        ushr            Y_2.\lanes, TMP2.\lanes, #3
-        ushr            Y_3.\lanes, TMP3.\lanes, #3
-        sli             Y_0.\lanes, TMP0.\lanes, #(\n - 3)
-        sli             Y_1.\lanes, TMP1.\lanes, #(\n - 3)
-        sli             Y_2.\lanes, TMP2.\lanes, #(\n - 3)
-        sli             Y_3.\lanes, TMP3.\lanes, #(\n - 3)
-        // x ^= k
-        eor             X_0.16b, X_0.16b, ROUND_KEY.16b
-        eor             X_1.16b, X_1.16b, ROUND_KEY.16b
-        eor             X_2.16b, X_2.16b, ROUND_KEY.16b
-        eor             X_3.16b, X_3.16b, ROUND_KEY.16b
-        // x -= y
-        sub             X_0.\lanes, X_0.\lanes, Y_0.\lanes
-        sub             X_1.\lanes, X_1.\lanes, Y_1.\lanes
-        sub             X_2.\lanes, X_2.\lanes, Y_2.\lanes
-        sub             X_3.\lanes, X_3.\lanes, Y_3.\lanes
-        // x = rol(x, 8)
-        tbl             X_0.16b, {X_0.16b}, ROTATE_TABLE.16b
-        tbl             X_1.16b, {X_1.16b}, ROTATE_TABLE.16b
-        tbl             X_2.16b, {X_2.16b}, ROTATE_TABLE.16b
-        tbl             X_3.16b, {X_3.16b}, ROTATE_TABLE.16b
-.endm
-.macro _next_xts_tweak  next, cur, tmp, n
-.if \n == 64
-        /*
-         * Calculate the next tweak by multiplying the current one by x,
-         * modulo p(x) = x^128 + x^7 + x^2 + x + 1.
-         */
-        sshr            \tmp\().2d, \cur\().2d, #63
-        and             \tmp\().16b, \tmp\().16b, GFMUL_TABLE.16b
-        shl             \next\().2d, \cur\().2d, #1
-        ext             \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
-        eor             \next\().16b, \next\().16b, \tmp\().16b
-.else
-        /*
-         * Calculate the next two tweaks by multiplying the current ones by x^2,
-         * modulo p(x) = x^64 + x^4 + x^3 + x + 1.
-         */
-        ushr            \tmp\().2d, \cur\().2d, #62
-        shl             \next\().2d, \cur\().2d, #2
-        tbl             \tmp\().16b, {GFMUL_TABLE.16b}, \tmp\().16b
-        eor             \next\().16b, \next\().16b, \tmp\().16b
-.endif
-.endm
-/*
- * _speck_xts_crypt() - Speck-XTS encryption/decryption
- *
- * Encrypt or decrypt NBYTES bytes of data from the SRC buffer to the DST buffer
- * using Speck-XTS, specifically the variant with a block size of '2n' and round
- * count given by NROUNDS.  The expanded round keys are given in ROUND_KEYS, and
- * the current XTS tweak value is given in TWEAK.  It's assumed that NBYTES is a
- * nonzero multiple of 128.
- */
-.macro _speck_xts_crypt n, lanes, decrypting
-        /*
-         * If decrypting, modify the ROUND_KEYS parameter to point to the last
-         * round key rather than the first, since for decryption the round keys
-         * are used in reverse order.
-         */
-.if \decrypting
-        mov             NROUNDS, NROUNDS        /* zero the high 32 bits */
-.if \n == 64
-        add             ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #3
-        sub             ROUND_KEYS, ROUND_KEYS, #8
-.else
-        add             ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #2
-        sub             ROUND_KEYS, ROUND_KEYS, #4
-.endif
-.endif
-        // Load the index vector for tbl-based 8-bit rotates
-.if \decrypting
-        ldr             ROTATE_TABLE_Q, .Lrol\n\()_8_table
-.else
-        ldr             ROTATE_TABLE_Q, .Lror\n\()_8_table
-.endif
-        // One-time XTS preparation
-.if \n == 64
-        // Load first tweak
-        ld1             {TWEAKV0.16b}, [TWEAK]
-        // Load GF(2^128) multiplication table
-        ldr             GFMUL_TABLE_Q, .Lgf128mul_table
-.else
-        // Load first tweak
-        ld1             {TWEAKV0.8b}, [TWEAK]
-        // Load GF(2^64) multiplication table
-        ldr             GFMUL_TABLE_Q, .Lgf64mul_table
-        // Calculate second tweak, packing it together with the first
-        ushr            TMP0.2d, TWEAKV0.2d, #63
-        shl             TMP1.2d, TWEAKV0.2d, #1
-        tbl             TMP0.8b, {GFMUL_TABLE.16b}, TMP0.8b
-        eor             TMP0.8b, TMP0.8b, TMP1.8b
-        mov             TWEAKV0.d[1], TMP0.d[0]
-.endif
-.Lnext_128bytes_\@:
-        // Calculate XTS tweaks for next 128 bytes
-        _next_xts_tweak TWEAKV1, TWEAKV0, TMP0, \n
-        _next_xts_tweak TWEAKV2, TWEAKV1, TMP0, \n
-        _next_xts_tweak TWEAKV3, TWEAKV2, TMP0, \n
-        _next_xts_tweak TWEAKV4, TWEAKV3, TMP0, \n
-        _next_xts_tweak TWEAKV5, TWEAKV4, TMP0, \n
-        _next_xts_tweak TWEAKV6, TWEAKV5, TMP0, \n
-        _next_xts_tweak TWEAKV7, TWEAKV6, TMP0, \n
-        _next_xts_tweak TWEAKV_NEXT, TWEAKV7, TMP0, \n
-        // Load the next source blocks into {X,Y}[0-3]
-        ld1             {X_0.16b-Y_1.16b}, [SRC], #64
-        ld1             {X_2.16b-Y_3.16b}, [SRC], #64
-        // XOR the source blocks with their XTS tweaks
-        eor             TMP0.16b, X_0.16b, TWEAKV0.16b
-        eor             Y_0.16b,  Y_0.16b, TWEAKV1.16b
-        eor             TMP1.16b, X_1.16b, TWEAKV2.16b
-        eor             Y_1.16b,  Y_1.16b, TWEAKV3.16b
-        eor             TMP2.16b, X_2.16b, TWEAKV4.16b
-        eor             Y_2.16b,  Y_2.16b, TWEAKV5.16b
-        eor             TMP3.16b, X_3.16b, TWEAKV6.16b
-        eor             Y_3.16b,  Y_3.16b, TWEAKV7.16b
-        /*
-         * De-interleave the 'x' and 'y' elements of each block, i.e. make it so
-         * that the X[0-3] registers contain only the second halves of blocks,
-         * and the Y[0-3] registers contain only the first halves of blocks.
-         * (Speck uses the order (y, x) rather than the more intuitive (x, y).)
-         */
-        uzp2            X_0.\lanes, TMP0.\lanes, Y_0.\lanes
-        uzp1            Y_0.\lanes, TMP0.\lanes, Y_0.\lanes
-        uzp2            X_1.\lanes, TMP1.\lanes, Y_1.\lanes
-        uzp1            Y_1.\lanes, TMP1.\lanes, Y_1.\lanes
-        uzp2            X_2.\lanes, TMP2.\lanes, Y_2.\lanes
-        uzp1            Y_2.\lanes, TMP2.\lanes, Y_2.\lanes
-        uzp2            X_3.\lanes, TMP3.\lanes, Y_3.\lanes
-        uzp1            Y_3.\lanes, TMP3.\lanes, Y_3.\lanes
-        // Do the cipher rounds
-        mov             x6, ROUND_KEYS
-        mov             w7, NROUNDS
-.Lnext_round_\@:
-.if \decrypting
-        ld1r            {ROUND_KEY.\lanes}, [x6]
-        sub             x6, x6, #( \n / 8 )
-        _speck_unround_128bytes \n, \lanes
-.else
-        ld1r            {ROUND_KEY.\lanes}, [x6], #( \n / 8 )
-        _speck_round_128bytes   \n, \lanes
-.endif
-        subs            w7, w7, #1
-        bne             .Lnext_round_\@
-        // Re-interleave the 'x' and 'y' elements of each block
-        zip1            TMP0.\lanes, Y_0.\lanes, X_0.\lanes
-        zip2            Y_0.\lanes,  Y_0.\lanes, X_0.\lanes
-        zip1            TMP1.\lanes, Y_1.\lanes, X_1.\lanes
-        zip2            Y_1.\lanes,  Y_1.\lanes, X_1.\lanes
-        zip1            TMP2.\lanes, Y_2.\lanes, X_2.\lanes
-        zip2            Y_2.\lanes,  Y_2.\lanes, X_2.\lanes
-        zip1            TMP3.\lanes, Y_3.\lanes, X_3.\lanes
-        zip2            Y_3.\lanes,  Y_3.\lanes, X_3.\lanes
-        // XOR the encrypted/decrypted blocks with the tweaks calculated earlier
-        eor             X_0.16b, TMP0.16b, TWEAKV0.16b
-        eor             Y_0.16b, Y_0.16b,  TWEAKV1.16b
-        eor             X_1.16b, TMP1.16b, TWEAKV2.16b
-        eor             Y_1.16b, Y_1.16b,  TWEAKV3.16b
-        eor             X_2.16b, TMP2.16b, TWEAKV4.16b
-        eor             Y_2.16b, Y_2.16b,  TWEAKV5.16b
-        eor             X_3.16b, TMP3.16b, TWEAKV6.16b
-        eor             Y_3.16b, Y_3.16b,  TWEAKV7.16b
-        mov             TWEAKV0.16b, TWEAKV_NEXT.16b
-        // Store the ciphertext in the destination buffer
-        st1             {X_0.16b-Y_1.16b}, [DST], #64
-        st1             {X_2.16b-Y_3.16b}, [DST], #64
-        // Continue if there are more 128-byte chunks remaining
-        subs            NBYTES, NBYTES, #128
-        bne             .Lnext_128bytes_\@
-        // Store the next tweak and return
-.if \n == 64
-        st1             {TWEAKV_NEXT.16b}, [TWEAK]
-.else
-        st1             {TWEAKV_NEXT.8b}, [TWEAK]
-.endif
-        ret
-.endm
-ENTRY(speck128_xts_encrypt_neon)
-        _speck_xts_crypt        n=64, lanes=2d, decrypting=0
-ENDPROC(speck128_xts_encrypt_neon)
-ENTRY(speck128_xts_decrypt_neon)
-        _speck_xts_crypt        n=64, lanes=2d, decrypting=1
-ENDPROC(speck128_xts_decrypt_neon)
-ENTRY(speck64_xts_encrypt_neon)
-        _speck_xts_crypt        n=32, lanes=4s, decrypting=0
-ENDPROC(speck64_xts_encrypt_neon)
-ENTRY(speck64_xts_decrypt_neon)
-        _speck_xts_crypt        n=32, lanes=4s, decrypting=1
-ENDPROC(speck64_xts_decrypt_neon)
diff --git a/arch/arm64/crypto/speck-neon-glue.c b/arch/arm64/crypto/speck-neon-glue.c
deleted file mode 100644
index 6e233aeb4ff4..000000000000
--- a/arch/arm64/crypto/speck-neon-glue.c
+++ /dev/null
@@ -1,282 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
- * (64-bit version; based on the 32-bit version)
- *
- * Copyright (c) 2018 Google, Inc
- */
-#include <asm/hwcap.h>
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/algapi.h>
-#include <crypto/gf128mul.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/speck.h>
-#include <crypto/xts.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-/* The assembly functions only handle multiples of 128 bytes */
-#define SPECK_NEON_CHUNK_SIZE   128
-/* Speck128 */
-struct speck128_xts_tfm_ctx {
-        struct speck128_tfm_ctx main_key;
-        struct speck128_tfm_ctx tweak_key;
-};
-asmlinkage void speck128_xts_encrypt_neon(const u64 *round_keys, int nrounds,
-                                          void *dst, const void *src,
-                                          unsigned int nbytes, void *tweak);
-asmlinkage void speck128_xts_decrypt_neon(const u64 *round_keys, int nrounds,
-                                          void *dst, const void *src,
-                                          unsigned int nbytes, void *tweak);
-typedef void (*speck128_crypt_one_t)(const struct speck128_tfm_ctx *,
-                                     u8 *, const u8 *);
-typedef void (*speck128_xts_crypt_many_t)(const u64 *, int, void *,
-                                          const void *, unsigned int, void *);
-static __always_inline int
-__speck128_xts_crypt(struct skcipher_request *req,
-                     speck128_crypt_one_t crypt_one,
-                     speck128_xts_crypt_many_t crypt_many)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        const struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct skcipher_walk walk;
-        le128 tweak;
-        int err;
-        err = skcipher_walk_virt(&walk, req, true);
-        crypto_speck128_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
-        while (walk.nbytes > 0) {
-                unsigned int nbytes = walk.nbytes;
-                u8 *dst = walk.dst.virt.addr;
-                const u8 *src = walk.src.virt.addr;
-                if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
-                        unsigned int count;
-                        count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
-                        kernel_neon_begin();
-                        (*crypt_many)(ctx->main_key.round_keys,
-                                      ctx->main_key.nrounds,
-                                      dst, src, count, &tweak);
-                        kernel_neon_end();
-                        dst += count;
-                        src += count;
-                        nbytes -= count;
-                }
-                /* Handle any remainder with generic code */
-                while (nbytes >= sizeof(tweak)) {
-                        le128_xor((le128 *)dst, (const le128 *)src, &tweak);
-                        (*crypt_one)(&ctx->main_key, dst, dst);
-                        le128_xor((le128 *)dst, (const le128 *)dst, &tweak);
-                        gf128mul_x_ble(&tweak, &tweak);
-                        dst += sizeof(tweak);
-                        src += sizeof(tweak);
-                        nbytes -= sizeof(tweak);
-                }
-                err = skcipher_walk_done(&walk, nbytes);
-        }
-        return err;
-}
-static int speck128_xts_encrypt(struct skcipher_request *req)
-{
-        return __speck128_xts_crypt(req, crypto_speck128_encrypt,
-                                    speck128_xts_encrypt_neon);
-}
-static int speck128_xts_decrypt(struct skcipher_request *req)
-{
-        return __speck128_xts_crypt(req, crypto_speck128_decrypt,
-                                    speck128_xts_decrypt_neon);
-}
-static int speck128_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
-                               unsigned int keylen)
-{
-        struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        int err;
-        err = xts_verify_key(tfm, key, keylen);
-        if (err)
-                return err;
-        keylen /= 2;
-        err = crypto_speck128_setkey(&ctx->main_key, key, keylen);
-        if (err)
-                return err;
-        return crypto_speck128_setkey(&ctx->tweak_key, key + keylen, keylen);
-}
-/* Speck64 */
-struct speck64_xts_tfm_ctx {
-        struct speck64_tfm_ctx main_key;
-        struct speck64_tfm_ctx tweak_key;
-};
-asmlinkage void speck64_xts_encrypt_neon(const u32 *round_keys, int nrounds,
-                                         void *dst, const void *src,
-                                         unsigned int nbytes, void *tweak);
-asmlinkage void speck64_xts_decrypt_neon(const u32 *round_keys, int nrounds,
-                                         void *dst, const void *src,
-                                         unsigned int nbytes, void *tweak);
-typedef void (*speck64_crypt_one_t)(const struct speck64_tfm_ctx *,
-                                    u8 *, const u8 *);
-typedef void (*speck64_xts_crypt_many_t)(const u32 *, int, void *,
-                                         const void *, unsigned int, void *);
-static __always_inline int
-__speck64_xts_crypt(struct skcipher_request *req, speck64_crypt_one_t crypt_one,
-                    speck64_xts_crypt_many_t crypt_many)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        const struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct skcipher_walk walk;
-        __le64 tweak;
-        int err;
-        err = skcipher_walk_virt(&walk, req, true);
-        crypto_speck64_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
-        while (walk.nbytes > 0) {
-                unsigned int nbytes = walk.nbytes;
-                u8 *dst = walk.dst.virt.addr;
-                const u8 *src = walk.src.virt.addr;
-                if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
-                        unsigned int count;
-                        count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
-                        kernel_neon_begin();
-                        (*crypt_many)(ctx->main_key.round_keys,
-                                      ctx->main_key.nrounds,
-                                      dst, src, count, &tweak);
-                        kernel_neon_end();
-                        dst += count;
-                        src += count;
-                        nbytes -= count;
-                }
-                /* Handle any remainder with generic code */
-                while (nbytes >= sizeof(tweak)) {
-                        *(__le64 *)dst = *(__le64 *)src ^ tweak;
-                        (*crypt_one)(&ctx->main_key, dst, dst);
-                        *(__le64 *)dst ^= tweak;
-                        tweak = cpu_to_le64((le64_to_cpu(tweak) << 1) ^
-                                            ((tweak & cpu_to_le64(1ULL << 63)) ?
-                                             0x1B : 0));
-                        dst += sizeof(tweak);
-                        src += sizeof(tweak);
-                        nbytes -= sizeof(tweak);
-                }
-                err = skcipher_walk_done(&walk, nbytes);
-        }
-        return err;
-}
-static int speck64_xts_encrypt(struct skcipher_request *req)
-{
-        return __speck64_xts_crypt(req, crypto_speck64_encrypt,
-                                   speck64_xts_encrypt_neon);
-}
-static int speck64_xts_decrypt(struct skcipher_request *req)
-{
-        return __speck64_xts_crypt(req, crypto_speck64_decrypt,
-                                   speck64_xts_decrypt_neon);
-}
-static int speck64_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
-                              unsigned int keylen)
-{
-        struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-        int err;
-        err = xts_verify_key(tfm, key, keylen);
-        if (err)
-                return err;
-        keylen /= 2;
-        err = crypto_speck64_setkey(&ctx->main_key, key, keylen);
-        if (err)
-                return err;
-        return crypto_speck64_setkey(&ctx->tweak_key, key + keylen, keylen);
-}
-static struct skcipher_alg speck_algs[] = {
-        {
-                .base.cra_name          = "xts(speck128)",
-                .base.cra_driver_name   = "xts-speck128-neon",
-                .base.cra_priority      = 300,
-                .base.cra_blocksize     = SPECK128_BLOCK_SIZE,
-                .base.cra_ctxsize       = sizeof(struct speck128_xts_tfm_ctx),
-                .base.cra_alignmask     = 7,
-                .base.cra_module        = THIS_MODULE,
-                .min_keysize            = 2 * SPECK128_128_KEY_SIZE,
-                .max_keysize            = 2 * SPECK128_256_KEY_SIZE,
-                .ivsize                 = SPECK128_BLOCK_SIZE,
-                .walksize               = SPECK_NEON_CHUNK_SIZE,
-                .setkey                 = speck128_xts_setkey,
-                .encrypt                = speck128_xts_encrypt,
-                .decrypt                = speck128_xts_decrypt,
-        }, {
-                .base.cra_name          = "xts(speck64)",
-                .base.cra_driver_name   = "xts-speck64-neon",
-                .base.cra_priority      = 300,
-                .base.cra_blocksize     = SPECK64_BLOCK_SIZE,
-                .base.cra_ctxsize       = sizeof(struct speck64_xts_tfm_ctx),
-                .base.cra_alignmask     = 7,
-                .base.cra_module        = THIS_MODULE,
-                .min_keysize            = 2 * SPECK64_96_KEY_SIZE,
-                .max_keysize            = 2 * SPECK64_128_KEY_SIZE,
-                .ivsize                 = SPECK64_BLOCK_SIZE,
-                .walksize               = SPECK_NEON_CHUNK_SIZE,
-                .setkey                 = speck64_xts_setkey,
-                .encrypt                = speck64_xts_encrypt,
-                .decrypt                = speck64_xts_decrypt,
-        }
-};
-static int __init speck_neon_module_init(void)
-{
-        if (!(elf_hwcap & HWCAP_ASIMD))
-                return -ENODEV;
-        return crypto_register_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
-}
-static void __exit speck_neon_module_exit(void)
-{
-        crypto_unregister_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
-}
-module_init(speck_neon_module_init);
-module_exit(speck_neon_module_exit);
-MODULE_DESCRIPTION("Speck block cipher (NEON-accelerated)");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("xts(speck128)");
-MODULE_ALIAS_CRYPTO("xts-speck128-neon");
-MODULE_ALIAS_CRYPTO("xts(speck64)");
-MODULE_ALIAS_CRYPTO("xts-speck64-neon");
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index 1d5483f6e457..85904b73e261 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -621,7 +621,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -657,7 +656,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index 52a0af127951..9b3818bbb68b 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -578,7 +578,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -614,7 +613,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index b3103e51268a..769677809945 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -599,7 +599,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -635,7 +634,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index fb7d651a4cab..7dd264ddf2ea 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -606,7 +605,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 6b37f5537c39..515f7439c755 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -580,7 +580,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -616,7 +615,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index c717bf879449..8e1038ceb407 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -602,7 +602,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -638,7 +637,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index 226c994ce794..62c8aaa15cc7 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -684,7 +684,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -720,7 +719,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index b383327fd77a..733973f91297 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -606,7 +605,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 9783d3deb9e9..fee30cc9ac16 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -606,7 +605,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index a35d10ee10cb..eebf9c9088e7 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -593,7 +593,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -629,7 +628,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index 573bf922d448..dabc54318c09 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -571,7 +571,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -607,7 +606,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index efb27a7fcc55..0d9a5c2a311a 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -572,7 +572,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_USER=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_AEGIS128=m
@@ -608,7 +607,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 941d8cc6c9f5..259d1698ac50 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -668,7 +668,6 @@ CONFIG_CRYPTO_USER=m
 # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set
 CONFIG_CRYPTO_PCRYPT=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_LRW=m
diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig
index eb6f75f24208..37fd60c20e22 100644
--- a/arch/s390/configs/performance_defconfig
+++ b/arch/s390/configs/performance_defconfig
@@ -610,7 +610,6 @@ CONFIG_CRYPTO_USER=m
 # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set
 CONFIG_CRYPTO_PCRYPT=m
 CONFIG_CRYPTO_CRYPTD=m
-CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
 CONFIG_CRYPTO_LRW=m
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c
index c54cb26eb7f5..812d9498d97b 100644
--- a/arch/s390/crypto/aes_s390.c
+++ b/arch/s390/crypto/aes_s390.c
@@ -44,7 +44,7 @@ struct s390_aes_ctx {
        int key_len;
        unsigned long fc;
        union {
-                struct crypto_skcipher *blk;
+                struct crypto_sync_skcipher *blk;
                struct crypto_cipher *cip;
        } fallback;
 };
@@ -54,7 +54,7 @@ struct s390_xts_ctx {
        u8 pcc_key[32];
        int key_len;
        unsigned long fc;
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
 };
 struct gcm_sg_walk {
@@ -184,14 +184,15 @@ static int setkey_fallback_blk(struct crypto_tfm *tfm, const u8 *key,
        struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
        unsigned int ret;
-        crypto_skcipher_clear_flags(sctx->fallback.blk, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(sctx->fallback.blk,
-        crypto_skcipher_set_flags(sctx->fallback.blk, tfm->crt_flags &
+                                         CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_set_flags(sctx->fallback.blk, tfm->crt_flags &
                                                      CRYPTO_TFM_REQ_MASK);
-        ret = crypto_skcipher_setkey(sctx->fallback.blk, key, len);
+        ret = crypto_sync_skcipher_setkey(sctx->fallback.blk, key, len);
        tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
-        tfm->crt_flags |= crypto_skcipher_get_flags(sctx->fallback.blk) &
+        tfm->crt_flags |= crypto_sync_skcipher_get_flags(sctx->fallback.blk) &
                          CRYPTO_TFM_RES_MASK;
        return ret;
@@ -204,9 +205,9 @@ static int fallback_blk_dec(struct blkcipher_desc *desc,
        unsigned int ret;
        struct crypto_blkcipher *tfm = desc->tfm;
        struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm);
-        SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
-        skcipher_request_set_tfm(req, sctx->fallback.blk);
+        skcipher_request_set_sync_tfm(req, sctx->fallback.blk);
        skcipher_request_set_callback(req, desc->flags, NULL, NULL);
        skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -223,9 +224,9 @@ static int fallback_blk_enc(struct blkcipher_desc *desc,
        unsigned int ret;
        struct crypto_blkcipher *tfm = desc->tfm;
        struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm);
-        SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
-        skcipher_request_set_tfm(req, sctx->fallback.blk);
+        skcipher_request_set_sync_tfm(req, sctx->fallback.blk);
        skcipher_request_set_callback(req, desc->flags, NULL, NULL);
        skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -306,8 +307,7 @@ static int fallback_init_blk(struct crypto_tfm *tfm)
        const char *name = tfm->__crt_alg->cra_name;
        struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-        sctx->fallback.blk = crypto_alloc_skcipher(name, 0,
+        sctx->fallback.blk = crypto_alloc_sync_skcipher(name, 0,
-                                                   CRYPTO_ALG_ASYNC |
                                                   CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(sctx->fallback.blk)) {
@@ -323,7 +323,7 @@ static void fallback_exit_blk(struct crypto_tfm *tfm)
 {
        struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(sctx->fallback.blk);
+        crypto_free_sync_skcipher(sctx->fallback.blk);
 }
 static struct crypto_alg ecb_aes_alg = {
@@ -453,14 +453,15 @@ static int xts_fallback_setkey(struct crypto_tfm *tfm, const u8 *key,
        struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
        unsigned int ret;
-        crypto_skcipher_clear_flags(xts_ctx->fallback, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(xts_ctx->fallback,
-        crypto_skcipher_set_flags(xts_ctx->fallback, tfm->crt_flags &
+                                         CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_set_flags(xts_ctx->fallback, tfm->crt_flags &
                                                     CRYPTO_TFM_REQ_MASK);
-        ret = crypto_skcipher_setkey(xts_ctx->fallback, key, len);
+        ret = crypto_sync_skcipher_setkey(xts_ctx->fallback, key, len);
        tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
-        tfm->crt_flags |= crypto_skcipher_get_flags(xts_ctx->fallback) &
+        tfm->crt_flags |= crypto_sync_skcipher_get_flags(xts_ctx->fallback) &
                          CRYPTO_TFM_RES_MASK;
        return ret;
@@ -472,10 +473,10 @@ static int xts_fallback_decrypt(struct blkcipher_desc *desc,
 {
        struct crypto_blkcipher *tfm = desc->tfm;
        struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm);
-        SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
        unsigned int ret;
-        skcipher_request_set_tfm(req, xts_ctx->fallback);
+        skcipher_request_set_sync_tfm(req, xts_ctx->fallback);
        skcipher_request_set_callback(req, desc->flags, NULL, NULL);
        skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -491,10 +492,10 @@ static int xts_fallback_encrypt(struct blkcipher_desc *desc,
 {
        struct crypto_blkcipher *tfm = desc->tfm;
        struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm);
-        SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
        unsigned int ret;
-        skcipher_request_set_tfm(req, xts_ctx->fallback);
+        skcipher_request_set_sync_tfm(req, xts_ctx->fallback);
        skcipher_request_set_callback(req, desc->flags, NULL, NULL);
        skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
@@ -611,8 +612,7 @@ static int xts_fallback_init(struct crypto_tfm *tfm)
        const char *name = tfm->__crt_alg->cra_name;
        struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
-        xts_ctx->fallback = crypto_alloc_skcipher(name, 0,
+        xts_ctx->fallback = crypto_alloc_sync_skcipher(name, 0,
-                                                  CRYPTO_ALG_ASYNC |
                                                  CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(xts_ctx->fallback)) {
@@ -627,7 +627,7 @@ static void xts_fallback_exit(struct crypto_tfm *tfm)
 {
        struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(xts_ctx->fallback);
+        crypto_free_sync_skcipher(xts_ctx->fallback);
 }
 static struct crypto_alg xts_aes_alg = {
diff --git a/arch/s390/defconfig b/arch/s390/defconfig
index 20add000dd6d..7cb6a52f727d 100644
--- a/arch/s390/defconfig
+++ b/arch/s390/defconfig
@@ -221,7 +221,6 @@ CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4=m
-CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_DEFLATE=m
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index a450ad573dcb..a4b0007a54e1 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -60,9 +60,6 @@ endif
 ifeq ($(avx2_supported),yes)
        obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o
        obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o
-        obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/
-        obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/
-        obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/
        obj-$(CONFIG_CRYPTO_MORUS1280_AVX2) += morus1280-avx2.o
 endif
@@ -106,7 +103,7 @@ ifeq ($(avx2_supported),yes)
        morus1280-avx2-y := morus1280-avx2-asm.o morus1280-avx2-glue.o
 endif
-aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
+aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
 aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o
 ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
 sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index acbe7e8336d8..661f7daf43da 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -102,9 +102,6 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);
-int crypto_fpu_init(void);
-void crypto_fpu_exit(void);
 #define AVX_GEN2_OPTSIZE 640
 #define AVX_GEN4_OPTSIZE 4096
@@ -817,7 +814,7 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
        /* Linearize assoc, if not already linear */
        if (req->src->length >= assoclen && req->src->length &&
                (!PageHighMem(sg_page(req->src)) ||
-                        req->src->offset + req->src->length < PAGE_SIZE)) {
+                        req->src->offset + req->src->length <= PAGE_SIZE)) {
                scatterwalk_start(&assoc_sg_walk, req->src);
                assoc = scatterwalk_map(&assoc_sg_walk);
        } else {
@@ -1253,22 +1250,6 @@ static struct skcipher_alg aesni_skciphers[] = {
 static
 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
-static struct {
-        const char *algname;
-        const char *drvname;
-        const char *basename;
-        struct simd_skcipher_alg *simd;
-} aesni_simd_skciphers2[] = {
-#if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
-    IS_BUILTIN(CONFIG_CRYPTO_PCBC)
-        {
-                .algname        = "pcbc(aes)",
-                .drvname        = "pcbc-aes-aesni",
-                .basename       = "fpu(pcbc(__aes-aesni))",
-        },
-#endif
-};
 #ifdef CONFIG_X86_64
 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
                                  unsigned int key_len)
@@ -1422,10 +1403,6 @@ static void aesni_free_simds(void)
        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
                    aesni_simd_skciphers[i]; i++)
                simd_skcipher_free(aesni_simd_skciphers[i]);
-        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
-                if (aesni_simd_skciphers2[i].simd)
-                        simd_skcipher_free(aesni_simd_skciphers2[i].simd);
 }
 static int __init aesni_init(void)
@@ -1469,13 +1446,9 @@ static int __init aesni_init(void)
 #endif
 #endif
-        err = crypto_fpu_init();
-        if (err)
-                return err;
        err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
        if (err)
-                goto fpu_exit;
+                return err;
        err = crypto_register_skciphers(aesni_skciphers,
                                        ARRAY_SIZE(aesni_skciphers));
@@ -1499,18 +1472,6 @@ static int __init aesni_init(void)
                aesni_simd_skciphers[i] = simd;
        }
-        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
-                algname = aesni_simd_skciphers2[i].algname;
-                drvname = aesni_simd_skciphers2[i].drvname;
-                basename = aesni_simd_skciphers2[i].basename;
-                simd = simd_skcipher_create_compat(algname, drvname, basename);
-                err = PTR_ERR(simd);
-                if (IS_ERR(simd))
-                        continue;
-                aesni_simd_skciphers2[i].simd = simd;
-        }
        return 0;
 unregister_simds:
@@ -1521,8 +1482,6 @@ unregister_skciphers:
                                    ARRAY_SIZE(aesni_skciphers));
 unregister_algs:
        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
-fpu_exit:
-        crypto_fpu_exit();
        return err;
 }
@@ -1533,8 +1492,6 @@ static void __exit aesni_exit(void)
        crypto_unregister_skciphers(aesni_skciphers,
                                    ARRAY_SIZE(aesni_skciphers));
        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
-        crypto_fpu_exit();
 }
 late_initcall(aesni_init);
diff --git a/arch/x86/crypto/fpu.c b/arch/x86/crypto/fpu.c
deleted file mode 100644
index 406680476c52..000000000000
--- a/arch/x86/crypto/fpu.c
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * FPU: Wrapper for blkcipher touching fpu
- *
- * Copyright (c) Intel Corp.
- *   Author: Huang Ying <ying.huang@intel.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- */
-#include <crypto/internal/skcipher.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <asm/fpu/api.h>
-struct crypto_fpu_ctx {
-        struct crypto_skcipher *child;
-};
-static int crypto_fpu_setkey(struct crypto_skcipher *parent, const u8 *key,
-                             unsigned int keylen)
-{
-        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(parent);
-        struct crypto_skcipher *child = ctx->child;
-        int err;
-        crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-        crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
-                                         CRYPTO_TFM_REQ_MASK);
-        err = crypto_skcipher_setkey(child, key, keylen);
-        crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
-                                          CRYPTO_TFM_RES_MASK);
-        return err;
-}
-static int crypto_fpu_encrypt(struct skcipher_request *req)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct crypto_skcipher *child = ctx->child;
-        SKCIPHER_REQUEST_ON_STACK(subreq, child);
-        int err;
-        skcipher_request_set_tfm(subreq, child);
-        skcipher_request_set_callback(subreq, 0, NULL, NULL);
-        skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
-                                   req->iv);
-        kernel_fpu_begin();
-        err = crypto_skcipher_encrypt(subreq);
-        kernel_fpu_end();
-        skcipher_request_zero(subreq);
-        return err;
-}
-static int crypto_fpu_decrypt(struct skcipher_request *req)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct crypto_skcipher *child = ctx->child;
-        SKCIPHER_REQUEST_ON_STACK(subreq, child);
-        int err;
-        skcipher_request_set_tfm(subreq, child);
-        skcipher_request_set_callback(subreq, 0, NULL, NULL);
-        skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
-                                   req->iv);
-        kernel_fpu_begin();
-        err = crypto_skcipher_decrypt(subreq);
-        kernel_fpu_end();
-        skcipher_request_zero(subreq);
-        return err;
-}
-static int crypto_fpu_init_tfm(struct crypto_skcipher *tfm)
-{
-        struct skcipher_instance *inst = skcipher_alg_instance(tfm);
-        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct crypto_skcipher_spawn *spawn;
-        struct crypto_skcipher *cipher;
-        spawn = skcipher_instance_ctx(inst);
-        cipher = crypto_spawn_skcipher(spawn);
-        if (IS_ERR(cipher))
-                return PTR_ERR(cipher);
-        ctx->child = cipher;
-        return 0;
-}
-static void crypto_fpu_exit_tfm(struct crypto_skcipher *tfm)
-{
-        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
-        crypto_free_skcipher(ctx->child);
-}
-static void crypto_fpu_free(struct skcipher_instance *inst)
-{
-        crypto_drop_skcipher(skcipher_instance_ctx(inst));
-        kfree(inst);
-}
-static int crypto_fpu_create(struct crypto_template *tmpl, struct rtattr **tb)
-{
-        struct crypto_skcipher_spawn *spawn;
-        struct skcipher_instance *inst;
-        struct crypto_attr_type *algt;
-        struct skcipher_alg *alg;
-        const char *cipher_name;
-        int err;
-        algt = crypto_get_attr_type(tb);
-        if (IS_ERR(algt))
-                return PTR_ERR(algt);
-        if ((algt->type ^ (CRYPTO_ALG_INTERNAL | CRYPTO_ALG_TYPE_SKCIPHER)) &
-            algt->mask)
-                return -EINVAL;
-        if (!(algt->mask & CRYPTO_ALG_INTERNAL))
-                return -EINVAL;
-        cipher_name = crypto_attr_alg_name(tb[1]);
-        if (IS_ERR(cipher_name))
-                return PTR_ERR(cipher_name);
-        inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
-        if (!inst)
-                return -ENOMEM;
-        spawn = skcipher_instance_ctx(inst);
-        crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
-        err = crypto_grab_skcipher(spawn, cipher_name, CRYPTO_ALG_INTERNAL,
-                                   CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
-        if (err)
-                goto out_free_inst;
-        alg = crypto_skcipher_spawn_alg(spawn);
-        err = crypto_inst_setname(skcipher_crypto_instance(inst), "fpu",
-                                  &alg->base);
-        if (err)
-                goto out_drop_skcipher;
-        inst->alg.base.cra_flags = CRYPTO_ALG_INTERNAL;
-        inst->alg.base.cra_priority = alg->base.cra_priority;
-        inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
-        inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
-        inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
-        inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
-        inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
-        inst->alg.base.cra_ctxsize = sizeof(struct crypto_fpu_ctx);
-        inst->alg.init = crypto_fpu_init_tfm;
-        inst->alg.exit = crypto_fpu_exit_tfm;
-        inst->alg.setkey = crypto_fpu_setkey;
-        inst->alg.encrypt = crypto_fpu_encrypt;
-        inst->alg.decrypt = crypto_fpu_decrypt;
-        inst->free = crypto_fpu_free;
-        err = skcipher_register_instance(tmpl, inst);
-        if (err)
-                goto out_drop_skcipher;
-out:
-        return err;
-out_drop_skcipher:
-        crypto_drop_skcipher(spawn);
-out_free_inst:
-        kfree(inst);
-        goto out;
-}
-static struct crypto_template crypto_fpu_tmpl = {
-        .name = "fpu",
-        .create = crypto_fpu_create,
-        .module = THIS_MODULE,
-};
-int __init crypto_fpu_init(void)
-{
-        return crypto_register_template(&crypto_fpu_tmpl);
-}
-void crypto_fpu_exit(void)
-{
-        crypto_unregister_template(&crypto_fpu_tmpl);
-}
-MODULE_ALIAS_CRYPTO("fpu");
diff --git a/arch/x86/crypto/sha1-mb/Makefile b/arch/x86/crypto/sha1-mb/Makefile
deleted file mode 100644
index 815ded3ba90e..000000000000
--- a/arch/x86/crypto/sha1-mb/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Arch-specific CryptoAPI modules.
-#
-OBJECT_FILES_NON_STANDARD := y
-avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
-                                $(comma)4)$(comma)%ymm2,yes,no)
-ifeq ($(avx2_supported),yes)
-        obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb.o
-        sha1-mb-y := sha1_mb.o sha1_mb_mgr_flush_avx2.o \
-             sha1_mb_mgr_init_avx2.o sha1_mb_mgr_submit_avx2.o sha1_x8_avx2.o
-endif
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb.c b/arch/x86/crypto/sha1-mb/sha1_mb.c
deleted file mode 100644
index b93805664c1d..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb.c
+++ /dev/null
@@ -1,1011 +0,0 @@
-/*
- * Multi buffer SHA1 algorithm Glue Code
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
-#include <crypto/internal/hash.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/cryptohash.h>
-#include <linux/types.h>
-#include <linux/list.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/sha.h>
-#include <crypto/mcryptd.h>
-#include <crypto/crypto_wq.h>
-#include <asm/byteorder.h>
-#include <linux/hardirq.h>
-#include <asm/fpu/api.h>
-#include "sha1_mb_ctx.h"
-#define FLUSH_INTERVAL 1000 /* in usec */
-static struct mcryptd_alg_state sha1_mb_alg_state;
-struct sha1_mb_ctx {
-        struct mcryptd_ahash *mcryptd_tfm;
-};
-static inline struct mcryptd_hash_request_ctx
-                *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx)
-{
-        struct ahash_request *areq;
-        areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
-        return container_of(areq, struct mcryptd_hash_request_ctx, areq);
-}
-static inline struct ahash_request
-                *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
-{
-        return container_of((void *) ctx, struct ahash_request, __ctx);
-}
-static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
-                                struct ahash_request *areq)
-{
-        rctx->flag = HASH_UPDATE;
-}
-static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state);
-static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)
-                        (struct sha1_mb_mgr *state, struct job_sha1 *job);
-static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)
-                                                (struct sha1_mb_mgr *state);
-static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)
-                                                (struct sha1_mb_mgr *state);
-static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2],
-                         uint64_t total_len)
-{
-        uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1);
-        memset(&padblock[i], 0, SHA1_BLOCK_SIZE);
-        padblock[i] = 0x80;
-        i += ((SHA1_BLOCK_SIZE - 1) &
-              (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1)))
-             + 1 + SHA1_PADLENGTHFIELD_SIZE;
-#if SHA1_PADLENGTHFIELD_SIZE == 16
-        *((uint64_t *) &padblock[i - 16]) = 0;
-#endif
-        *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
-        /* Number of extra blocks to hash */
-        return i >> SHA1_LOG2_BLOCK_SIZE;
-}
-static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr,
-                                                struct sha1_hash_ctx *ctx)
-{
-        while (ctx) {
-                if (ctx->status & HASH_CTX_STS_COMPLETE) {
-                        /* Clear PROCESSING bit */
-                        ctx->status = HASH_CTX_STS_COMPLETE;
-                        return ctx;
-                }
-                /*
-                 * If the extra blocks are empty, begin hashing what remains
-                 * in the user's buffer.
-                 */
-                if (ctx->partial_block_buffer_length == 0 &&
-                    ctx->incoming_buffer_length) {
-                        const void *buffer = ctx->incoming_buffer;
-                        uint32_t len = ctx->incoming_buffer_length;
-                        uint32_t copy_len;
-                        /*
-                         * Only entire blocks can be hashed.
-                         * Copy remainder to extra blocks buffer.
-                         */
-                        copy_len = len & (SHA1_BLOCK_SIZE-1);
-                        if (copy_len) {
-                                len -= copy_len;
-                                memcpy(ctx->partial_block_buffer,
-                                       ((const char *) buffer + len),
-                                       copy_len);
-                                ctx->partial_block_buffer_length = copy_len;
-                        }
-                        ctx->incoming_buffer_length = 0;
-                        /* len should be a multiple of the block size now */
-                        assert((len % SHA1_BLOCK_SIZE) == 0);
-                        /* Set len to the number of blocks to be hashed */
-                        len >>= SHA1_LOG2_BLOCK_SIZE;
-                        if (len) {
-                                ctx->job.buffer = (uint8_t *) buffer;
-                                ctx->job.len = len;
-                                ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr,
-                                                                                &ctx->job);
-                                continue;
-                        }
-                }
-                /*
-                 * If the extra blocks are not empty, then we are
-                 * either on the last block(s) or we need more
-                 * user input before continuing.
-                 */
-                if (ctx->status & HASH_CTX_STS_LAST) {
-                        uint8_t *buf = ctx->partial_block_buffer;
-                        uint32_t n_extra_blocks =
-                                        sha1_pad(buf, ctx->total_length);
-                        ctx->status = (HASH_CTX_STS_PROCESSING |
-                                       HASH_CTX_STS_COMPLETE);
-                        ctx->job.buffer = buf;
-                        ctx->job.len = (uint32_t) n_extra_blocks;
-                        ctx = (struct sha1_hash_ctx *)
-                                sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
-                        continue;
-                }
-                ctx->status = HASH_CTX_STS_IDLE;
-                return ctx;
-        }
-        return NULL;
-}
-static struct sha1_hash_ctx
-                        *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr)
-{
-        /*
-         * If get_comp_job returns NULL, there are no jobs complete.
-         * If get_comp_job returns a job, verify that it is safe to return to
-         * the user.
-         * If it is not ready, resubmit the job to finish processing.
-         * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
-         * Otherwise, all jobs currently being managed by the hash_ctx_mgr
-         * still need processing.
-         */
-        struct sha1_hash_ctx *ctx;
-        ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr);
-        return sha1_ctx_mgr_resubmit(mgr, ctx);
-}
-static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr)
-{
-        sha1_job_mgr_init(&mgr->mgr);
-}
-static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr,
-                                          struct sha1_hash_ctx *ctx,
-                                          const void *buffer,
-                                          uint32_t len,
-                                          int flags)
-{
-        if (flags & ~(HASH_UPDATE | HASH_LAST)) {
-                /* User should not pass anything other than UPDATE or LAST */
-                ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
-                return ctx;
-        }
-        if (ctx->status & HASH_CTX_STS_PROCESSING) {
-                /* Cannot submit to a currently processing job. */
-                ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
-                return ctx;
-        }
-        if (ctx->status & HASH_CTX_STS_COMPLETE) {
-                /* Cannot update a finished job. */
-                ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
-                return ctx;
-        }
-        /*
-         * If we made it here, there were no errors during this call to
-         * submit
-         */
-        ctx->error = HASH_CTX_ERROR_NONE;
-        /* Store buffer ptr info from user */
-        ctx->incoming_buffer = buffer;
-        ctx->incoming_buffer_length = len;
-        /*
-         * Store the user's request flags and mark this ctx as currently
-         * being processed.
-         */
-        ctx->status = (flags & HASH_LAST) ?
-                        (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
-                        HASH_CTX_STS_PROCESSING;
-        /* Advance byte counter */
-        ctx->total_length += len;
-        /*
-         * If there is anything currently buffered in the extra blocks,
-         * append to it until it contains a whole block.
-         * Or if the user's buffer contains less than a whole block,
-         * append as much as possible to the extra block.
-         */
-        if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) {
-                /*
-                 * Compute how many bytes to copy from user buffer into
-                 * extra block
-                 */
-                uint32_t copy_len = SHA1_BLOCK_SIZE -
-                                        ctx->partial_block_buffer_length;
-                if (len < copy_len)
-                        copy_len = len;
-                if (copy_len) {
-                        /* Copy and update relevant pointers and counters */
-                        memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
-                                buffer, copy_len);
-                        ctx->partial_block_buffer_length += copy_len;
-                        ctx->incoming_buffer = (const void *)
-                                        ((const char *)buffer + copy_len);
-                        ctx->incoming_buffer_length = len - copy_len;
-                }
-                /*
-                 * The extra block should never contain more than 1 block
-                 * here
-                 */
-                assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE);
-                /*
-                 * If the extra block buffer contains exactly 1 block, it can
-                 * be hashed.
-                 */
-                if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) {
-                        ctx->partial_block_buffer_length = 0;
-                        ctx->job.buffer = ctx->partial_block_buffer;
-                        ctx->job.len = 1;
-                        ctx = (struct sha1_hash_ctx *)
-                                sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
-                }
-        }
-        return sha1_ctx_mgr_resubmit(mgr, ctx);
-}
-static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr)
-{
-        struct sha1_hash_ctx *ctx;
-        while (1) {
-                ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr);
-                /* If flush returned 0, there are no more jobs in flight. */
-                if (!ctx)
-                        return NULL;
-                /*
-                 * If flush returned a job, resubmit the job to finish
-                 * processing.
-                 */
-                ctx = sha1_ctx_mgr_resubmit(mgr, ctx);
-                /*
-                 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be
-                 * returned. Otherwise, all jobs currently being managed by the
-                 * sha1_ctx_mgr still need processing. Loop.
-                 */
-                if (ctx)
-                        return ctx;
-        }
-}
-static int sha1_mb_init(struct ahash_request *areq)
-{
-        struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
-        hash_ctx_init(sctx);
-        sctx->job.result_digest[0] = SHA1_H0;
-        sctx->job.result_digest[1] = SHA1_H1;
-        sctx->job.result_digest[2] = SHA1_H2;
-        sctx->job.result_digest[3] = SHA1_H3;
-        sctx->job.result_digest[4] = SHA1_H4;
-        sctx->total_length = 0;
-        sctx->partial_block_buffer_length = 0;
-        sctx->status = HASH_CTX_STS_IDLE;
-        return 0;
-}
-static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
-{
-        int     i;
-        struct  sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
-        __be32  *dst = (__be32 *) rctx->out;
-        for (i = 0; i < 5; ++i)
-                dst[i] = cpu_to_be32(sctx->job.result_digest[i]);
-        return 0;
-}
-static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
-                        struct mcryptd_alg_cstate *cstate, bool flush)
-{
-        int     flag = HASH_UPDATE;
-        int     nbytes, err = 0;
-        struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
-        struct sha1_hash_ctx *sha_ctx;
-        /* more work ? */
-        while (!(rctx->flag & HASH_DONE)) {
-                nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
-                if (nbytes < 0) {
-                        err = nbytes;
-                        goto out;
-                }
-                /* check if the walk is done */
-                if (crypto_ahash_walk_last(&rctx->walk)) {
-                        rctx->flag |= HASH_DONE;
-                        if (rctx->flag & HASH_FINAL)
-                                flag |= HASH_LAST;
-                }
-                sha_ctx = (struct sha1_hash_ctx *)
-                                                ahash_request_ctx(&rctx->areq);
-                kernel_fpu_begin();
-                sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx,
-                                                rctx->walk.data, nbytes, flag);
-                if (!sha_ctx) {
-                        if (flush)
-                                sha_ctx = sha1_ctx_mgr_flush(cstate->mgr);
-                }
-                kernel_fpu_end();
-                if (sha_ctx)
-                        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                else {
-                        rctx = NULL;
-                        goto out;
-                }
-        }
-        /* copy the results */
-        if (rctx->flag & HASH_FINAL)
-                sha1_mb_set_results(rctx);
-out:
-        *ret_rctx = rctx;
-        return err;
-}
-static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
-                            struct mcryptd_alg_cstate *cstate,
-                            int err)
-{
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha1_hash_ctx *sha_ctx;
-        struct mcryptd_hash_request_ctx *req_ctx;
-        int ret;
-        /* remove from work list */
-        spin_lock(&cstate->work_lock);
-        list_del(&rctx->waiter);
-        spin_unlock(&cstate->work_lock);
-        if (irqs_disabled())
-                rctx->complete(&req->base, err);
-        else {
-                local_bh_disable();
-                rctx->complete(&req->base, err);
-                local_bh_enable();
-        }
-        /* check to see if there are other jobs that are done */
-        sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
-        while (sha_ctx) {
-                req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                ret = sha_finish_walk(&req_ctx, cstate, false);
-                if (req_ctx) {
-                        spin_lock(&cstate->work_lock);
-                        list_del(&req_ctx->waiter);
-                        spin_unlock(&cstate->work_lock);
-                        req = cast_mcryptd_ctx_to_req(req_ctx);
-                        if (irqs_disabled())
-                                req_ctx->complete(&req->base, ret);
-                        else {
-                                local_bh_disable();
-                                req_ctx->complete(&req->base, ret);
-                                local_bh_enable();
-                        }
-                }
-                sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
-        }
-        return 0;
-}
-static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
-                             struct mcryptd_alg_cstate *cstate)
-{
-        unsigned long next_flush;
-        unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
-        /* initialize tag */
-        rctx->tag.arrival = jiffies;    /* tag the arrival time */
-        rctx->tag.seq_num = cstate->next_seq_num++;
-        next_flush = rctx->tag.arrival + delay;
-        rctx->tag.expire = next_flush;
-        spin_lock(&cstate->work_lock);
-        list_add_tail(&rctx->waiter, &cstate->work_list);
-        spin_unlock(&cstate->work_lock);
-        mcryptd_arm_flusher(cstate, delay);
-}
-static int sha1_mb_update(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                container_of(areq, struct mcryptd_hash_request_ctx, areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha1_hash_ctx *sha_ctx;
-        int ret = 0, nbytes;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        nbytes = crypto_ahash_walk_first(req, &rctx->walk);
-        if (nbytes < 0) {
-                ret = nbytes;
-                goto done;
-        }
-        if (crypto_ahash_walk_last(&rctx->walk))
-                rctx->flag |= HASH_DONE;
-        /* submit */
-        sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
-        sha1_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
-                                                        nbytes, HASH_UPDATE);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha1_mb_finup(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                container_of(areq, struct mcryptd_hash_request_ctx, areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha1_hash_ctx *sha_ctx;
-        int ret = 0, flag = HASH_UPDATE, nbytes;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        nbytes = crypto_ahash_walk_first(req, &rctx->walk);
-        if (nbytes < 0) {
-                ret = nbytes;
-                goto done;
-        }
-        if (crypto_ahash_walk_last(&rctx->walk)) {
-                rctx->flag |= HASH_DONE;
-                flag = HASH_LAST;
-        }
-        /* submit */
-        rctx->flag |= HASH_FINAL;
-        sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
-        sha1_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
-                                                                nbytes, flag);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha1_mb_final(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                container_of(areq, struct mcryptd_hash_request_ctx, areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
-        struct sha1_hash_ctx *sha_ctx;
-        int ret = 0;
-        u8 data;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        rctx->flag |= HASH_DONE | HASH_FINAL;
-        sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
-        /* flag HASH_FINAL and 0 data size */
-        sha1_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
-                                                                HASH_LAST);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha1_mb_export(struct ahash_request *areq, void *out)
-{
-        struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
-        memcpy(out, sctx, sizeof(*sctx));
-        return 0;
-}
-static int sha1_mb_import(struct ahash_request *areq, const void *in)
-{
-        struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
-        memcpy(sctx, in, sizeof(*sctx));
-        return 0;
-}
-static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm)
-{
-        struct mcryptd_ahash *mcryptd_tfm;
-        struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct mcryptd_hash_ctx *mctx;
-        mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb",
-                                                CRYPTO_ALG_INTERNAL,
-                                                CRYPTO_ALG_INTERNAL);
-        if (IS_ERR(mcryptd_tfm))
-                return PTR_ERR(mcryptd_tfm);
-        mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
-        mctx->alg_state = &sha1_mb_alg_state;
-        ctx->mcryptd_tfm = mcryptd_tfm;
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                sizeof(struct ahash_request) +
-                                crypto_ahash_reqsize(&mcryptd_tfm->base));
-        return 0;
-}
-static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        mcryptd_free_ahash(ctx->mcryptd_tfm);
-}
-static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm)
-{
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                sizeof(struct ahash_request) +
-                                sizeof(struct sha1_hash_ctx));
-        return 0;
-}
-static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        mcryptd_free_ahash(ctx->mcryptd_tfm);
-}
-static struct ahash_alg sha1_mb_areq_alg = {
-        .init           =       sha1_mb_init,
-        .update         =       sha1_mb_update,
-        .final          =       sha1_mb_final,
-        .finup          =       sha1_mb_finup,
-        .export         =       sha1_mb_export,
-        .import         =       sha1_mb_import,
-        .halg           =       {
-                .digestsize     =       SHA1_DIGEST_SIZE,
-                .statesize      =       sizeof(struct sha1_hash_ctx),
-                .base           =       {
-                        .cra_name        = "__sha1-mb",
-                        .cra_driver_name = "__intel_sha1-mb",
-                        .cra_priority    = 100,
-                        /*
-                         * use ASYNC flag as some buffers in multi-buffer
-                         * algo may not have completed before hashing thread
-                         * sleep
-                         */
-                        .cra_flags      = CRYPTO_ALG_ASYNC |
-                                          CRYPTO_ALG_INTERNAL,
-                        .cra_blocksize  = SHA1_BLOCK_SIZE,
-                        .cra_module     = THIS_MODULE,
-                        .cra_list       = LIST_HEAD_INIT
-                                        (sha1_mb_areq_alg.halg.base.cra_list),
-                        .cra_init       = sha1_mb_areq_init_tfm,
-                        .cra_exit       = sha1_mb_areq_exit_tfm,
-                        .cra_ctxsize    = sizeof(struct sha1_hash_ctx),
-                }
-        }
-};
-static int sha1_mb_async_init(struct ahash_request *req)
-{
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_init(mcryptd_req);
-}
-static int sha1_mb_async_update(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_update(mcryptd_req);
-}
-static int sha1_mb_async_finup(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_finup(mcryptd_req);
-}
-static int sha1_mb_async_final(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_final(mcryptd_req);
-}
-static int sha1_mb_async_digest(struct ahash_request *req)
-{
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_digest(mcryptd_req);
-}
-static int sha1_mb_async_export(struct ahash_request *req, void *out)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_export(mcryptd_req, out);
-}
-static int sha1_mb_async_import(struct ahash_request *req, const void *in)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
-        struct mcryptd_hash_request_ctx *rctx;
-        struct ahash_request *areq;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        rctx = ahash_request_ctx(mcryptd_req);
-        areq = &rctx->areq;
-        ahash_request_set_tfm(areq, child);
-        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
-                                        rctx->complete, req);
-        return crypto_ahash_import(mcryptd_req, in);
-}
-static struct ahash_alg sha1_mb_async_alg = {
-        .init           = sha1_mb_async_init,
-        .update         = sha1_mb_async_update,
-        .final          = sha1_mb_async_final,
-        .finup          = sha1_mb_async_finup,
-        .digest         = sha1_mb_async_digest,
-        .export         = sha1_mb_async_export,
-        .import         = sha1_mb_async_import,
-        .halg = {
-                .digestsize     = SHA1_DIGEST_SIZE,
-                .statesize      = sizeof(struct sha1_hash_ctx),
-                .base = {
-                        .cra_name               = "sha1",
-                        .cra_driver_name        = "sha1_mb",
-                        /*
-                         * Low priority, since with few concurrent hash requests
-                         * this is extremely slow due to the flush delay.  Users
-                         * whose workloads would benefit from this can request
-                         * it explicitly by driver name, or can increase its
-                         * priority at runtime using NETLINK_CRYPTO.
-                         */
-                        .cra_priority           = 50,
-                        .cra_flags              = CRYPTO_ALG_ASYNC,
-                        .cra_blocksize          = SHA1_BLOCK_SIZE,
-                        .cra_module             = THIS_MODULE,
-                        .cra_list               = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list),
-                        .cra_init               = sha1_mb_async_init_tfm,
-                        .cra_exit               = sha1_mb_async_exit_tfm,
-                        .cra_ctxsize            = sizeof(struct sha1_mb_ctx),
-                        .cra_alignmask          = 0,
-                },
-        },
-};
-static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate)
-{
-        struct mcryptd_hash_request_ctx *rctx;
-        unsigned long cur_time;
-        unsigned long next_flush = 0;
-        struct sha1_hash_ctx *sha_ctx;
-        cur_time = jiffies;
-        while (!list_empty(&cstate->work_list)) {
-                rctx = list_entry(cstate->work_list.next,
-                                struct mcryptd_hash_request_ctx, waiter);
-                if (time_before(cur_time, rctx->tag.expire))
-                        break;
-                kernel_fpu_begin();
-                sha_ctx = (struct sha1_hash_ctx *)
-                                        sha1_ctx_mgr_flush(cstate->mgr);
-                kernel_fpu_end();
-                if (!sha_ctx) {
-                        pr_err("sha1_mb error: nothing got flushed for non-empty list\n");
-                        break;
-                }
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                sha_finish_walk(&rctx, cstate, true);
-                sha_complete_job(rctx, cstate, 0);
-        }
-        if (!list_empty(&cstate->work_list)) {
-                rctx = list_entry(cstate->work_list.next,
-                                struct mcryptd_hash_request_ctx, waiter);
-                /* get the hash context and then flush time */
-                next_flush = rctx->tag.expire;
-                mcryptd_arm_flusher(cstate, get_delay(next_flush));
-        }
-        return next_flush;
-}
-static int __init sha1_mb_mod_init(void)
-{
-        int cpu;
-        int err;
-        struct mcryptd_alg_cstate *cpu_state;
-        /* check for dependent cpu features */
-        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
-            !boot_cpu_has(X86_FEATURE_BMI2))
-                return -ENODEV;
-        /* initialize multibuffer structures */
-        sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate);
-        sha1_job_mgr_init = sha1_mb_mgr_init_avx2;
-        sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2;
-        sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2;
-        sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2;
-        if (!sha1_mb_alg_state.alg_cstate)
-                return -ENOMEM;
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
-                cpu_state->next_flush = 0;
-                cpu_state->next_seq_num = 0;
-                cpu_state->flusher_engaged = false;
-                INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
-                cpu_state->cpu = cpu;
-                cpu_state->alg_state = &sha1_mb_alg_state;
-                cpu_state->mgr = kzalloc(sizeof(struct sha1_ctx_mgr),
-                                        GFP_KERNEL);
-                if (!cpu_state->mgr)
-                        goto err2;
-                sha1_ctx_mgr_init(cpu_state->mgr);
-                INIT_LIST_HEAD(&cpu_state->work_list);
-                spin_lock_init(&cpu_state->work_lock);
-        }
-        sha1_mb_alg_state.flusher = &sha1_mb_flusher;
-        err = crypto_register_ahash(&sha1_mb_areq_alg);
-        if (err)
-                goto err2;
-        err = crypto_register_ahash(&sha1_mb_async_alg);
-        if (err)
-                goto err1;
-        return 0;
-err1:
-        crypto_unregister_ahash(&sha1_mb_areq_alg);
-err2:
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
-                kfree(cpu_state->mgr);
-        }
-        free_percpu(sha1_mb_alg_state.alg_cstate);
-        return -ENODEV;
-}
-static void __exit sha1_mb_mod_fini(void)
-{
-        int cpu;
-        struct mcryptd_alg_cstate *cpu_state;
-        crypto_unregister_ahash(&sha1_mb_async_alg);
-        crypto_unregister_ahash(&sha1_mb_areq_alg);
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
-                kfree(cpu_state->mgr);
-        }
-        free_percpu(sha1_mb_alg_state.alg_cstate);
-}
-module_init(sha1_mb_mod_init);
-module_exit(sha1_mb_mod_fini);
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated");
-MODULE_ALIAS_CRYPTO("sha1");
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h
deleted file mode 100644
index 9454bd16f9f8..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Header file for multi buffer SHA context
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef _SHA_MB_CTX_INTERNAL_H
-#define _SHA_MB_CTX_INTERNAL_H
-#include "sha1_mb_mgr.h"
-#define HASH_UPDATE          0x00
-#define HASH_LAST            0x01
-#define HASH_DONE            0x02
-#define HASH_FINAL           0x04
-#define HASH_CTX_STS_IDLE       0x00
-#define HASH_CTX_STS_PROCESSING 0x01
-#define HASH_CTX_STS_LAST       0x02
-#define HASH_CTX_STS_COMPLETE   0x04
-enum hash_ctx_error {
-        HASH_CTX_ERROR_NONE               =  0,
-        HASH_CTX_ERROR_INVALID_FLAGS      = -1,
-        HASH_CTX_ERROR_ALREADY_PROCESSING = -2,
-        HASH_CTX_ERROR_ALREADY_COMPLETED  = -3,
-#ifdef HASH_CTX_DEBUG
-        HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4,
-#endif
-};
-#define hash_ctx_user_data(ctx)  ((ctx)->user_data)
-#define hash_ctx_digest(ctx)     ((ctx)->job.result_digest)
-#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING)
-#define hash_ctx_complete(ctx)   ((ctx)->status == HASH_CTX_STS_COMPLETE)
-#define hash_ctx_status(ctx)     ((ctx)->status)
-#define hash_ctx_error(ctx)      ((ctx)->error)
-#define hash_ctx_init(ctx) \
-        do { \
-                (ctx)->error = HASH_CTX_ERROR_NONE; \
-                (ctx)->status = HASH_CTX_STS_COMPLETE; \
-        } while (0)
-/* Hash Constants and Typedefs */
-#define SHA1_DIGEST_LENGTH          5
-#define SHA1_LOG2_BLOCK_SIZE        6
-#define SHA1_PADLENGTHFIELD_SIZE    8
-#ifdef SHA_MB_DEBUG
-#define assert(expr) \
-do { \
-        if (unlikely(!(expr))) { \
-                printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
-                #expr, __FILE__, __func__, __LINE__); \
-        } \
-} while (0)
-#else
-#define assert(expr) do {} while (0)
-#endif
-struct sha1_ctx_mgr {
-        struct sha1_mb_mgr mgr;
-};
-/* typedef struct sha1_ctx_mgr sha1_ctx_mgr; */
-struct sha1_hash_ctx {
-        /* Must be at struct offset 0 */
-        struct job_sha1       job;
-        /* status flag */
-        int status;
-        /* error flag */
-        int error;
-        uint64_t        total_length;
-        const void      *incoming_buffer;
-        uint32_t        incoming_buffer_length;
-        uint8_t         partial_block_buffer[SHA1_BLOCK_SIZE * 2];
-        uint32_t        partial_block_buffer_length;
-        void            *user_data;
-};
-#endif
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h b/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h
deleted file mode 100644
index 08ad1a9acfd7..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Header file for multi buffer SHA1 algorithm manager
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      James Guilford <james.guilford@intel.com>
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef __SHA_MB_MGR_H
-#define __SHA_MB_MGR_H
-#include <linux/types.h>
-#define NUM_SHA1_DIGEST_WORDS 5
-enum job_sts {  STS_UNKNOWN = 0,
-                STS_BEING_PROCESSED = 1,
-                STS_COMPLETED = 2,
-                STS_INTERNAL_ERROR = 3,
-                STS_ERROR = 4
-};
-struct job_sha1 {
-        u8      *buffer;
-        u32     len;
-        u32     result_digest[NUM_SHA1_DIGEST_WORDS] __aligned(32);
-        enum    job_sts status;
-        void    *user_data;
-};
-/* SHA1 out-of-order scheduler */
-/* typedef uint32_t sha1_digest_array[5][8]; */
-struct sha1_args_x8 {
-        uint32_t        digest[5][8];
-        uint8_t         *data_ptr[8];
-};
-struct sha1_lane_data {
-        struct job_sha1 *job_in_lane;
-};
-struct sha1_mb_mgr {
-        struct sha1_args_x8 args;
-        uint32_t lens[8];
-        /* each byte is index (0...7) of unused lanes */
-        uint64_t unused_lanes;
-        /* byte 4 is set to FF as a flag */
-        struct sha1_lane_data ldata[8];
-};
-#define SHA1_MB_MGR_NUM_LANES_AVX2 8
-void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state);
-struct job_sha1 *sha1_mb_mgr_submit_avx2(struct sha1_mb_mgr *state,
-                                         struct job_sha1 *job);
-struct job_sha1 *sha1_mb_mgr_flush_avx2(struct sha1_mb_mgr *state);
-struct job_sha1 *sha1_mb_mgr_get_comp_job_avx2(struct sha1_mb_mgr *state);
-#endif
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S
deleted file mode 100644
index 86688c6e7a25..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Header file for multi buffer SHA1 algorithm data structure
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      James Guilford <james.guilford@intel.com>
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-# Macros for defining data structures
-# Usage example
-#START_FIELDS   # JOB_AES
-###     name            size    align
-#FIELD  _plaintext,     8,      8       # pointer to plaintext
-#FIELD  _ciphertext,    8,      8       # pointer to ciphertext
-#FIELD  _IV,            16,     8       # IV
-#FIELD  _keys,          8,      8       # pointer to keys
-#FIELD  _len,           4,      4       # length in bytes
-#FIELD  _status,        4,      4       # status enumeration
-#FIELD  _user_data,     8,      8       # pointer to user data
-#UNION  _union,         size1,  align1, \
-#                       size2,  align2, \
-#                       size3,  align3, \
-#                       ...
-#END_FIELDS
-#%assign _JOB_AES_size  _FIELD_OFFSET
-#%assign _JOB_AES_align _STRUCT_ALIGN
-#########################################################################
-# Alternate "struc-like" syntax:
-#       STRUCT job_aes2
-#       RES_Q   .plaintext,     1
-#       RES_Q   .ciphertext,    1
-#       RES_DQ  .IV,            1
-#       RES_B   .nested,        _JOB_AES_SIZE, _JOB_AES_ALIGN
-#       RES_U   .union,         size1, align1, \
-#                               size2, align2, \
-#                               ...
-#       ENDSTRUCT
-#       # Following only needed if nesting
-#       %assign job_aes2_size   _FIELD_OFFSET
-#       %assign job_aes2_align  _STRUCT_ALIGN
-#
-# RES_* macros take a name, a count and an optional alignment.
-# The count in in terms of the base size of the macro, and the
-# default alignment is the base size.
-# The macros are:
-# Macro    Base size
-# RES_B     1
-# RES_W     2
-# RES_D     4
-# RES_Q     8
-# RES_DQ   16
-# RES_Y    32
-# RES_Z    64
-#
-# RES_U defines a union. It's arguments are a name and two or more
-# pairs of "size, alignment"
-#
-# The two assigns are only needed if this structure is being nested
-# within another. Even if the assigns are not done, one can still use
-# STRUCT_NAME_size as the size of the structure.
-#
-# Note that for nesting, you still need to assign to STRUCT_NAME_size.
-#
-# The differences between this and using "struc" directly are that each
-# type is implicitly aligned to its natural length (although this can be
-# over-ridden with an explicit third parameter), and that the structure
-# is padded at the end to its overall alignment.
-#
-#########################################################################
-#ifndef _SHA1_MB_MGR_DATASTRUCT_ASM_
-#define _SHA1_MB_MGR_DATASTRUCT_ASM_
-## START_FIELDS
-.macro START_FIELDS
- _FIELD_OFFSET = 0
- _STRUCT_ALIGN = 0
-.endm
-## FIELD name size align
-.macro FIELD name size align
- _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
- \name  = _FIELD_OFFSET
- _FIELD_OFFSET = _FIELD_OFFSET + (\size)
-.if (\align > _STRUCT_ALIGN)
- _STRUCT_ALIGN = \align
-.endif
-.endm
-## END_FIELDS
-.macro END_FIELDS
- _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
-.endm
-########################################################################
-.macro STRUCT p1
-START_FIELDS
-.struc \p1
-.endm
-.macro ENDSTRUCT
- tmp = _FIELD_OFFSET
- END_FIELDS
- tmp = (_FIELD_OFFSET - %%tmp)
-.if (tmp > 0)
-        .lcomm  tmp
-.endif
-.endstruc
-.endm
-## RES_int name size align
-.macro RES_int p1 p2 p3
- name = \p1
- size = \p2
- align = .\p3
- _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
-.align align
-.lcomm name size
- _FIELD_OFFSET = _FIELD_OFFSET + (size)
-.if (align > _STRUCT_ALIGN)
- _STRUCT_ALIGN = align
-.endif
-.endm
-# macro RES_B name, size [, align]
-.macro RES_B _name, _size, _align=1
-RES_int _name _size _align
-.endm
-# macro RES_W name, size [, align]
-.macro RES_W _name, _size, _align=2
-RES_int _name 2*(_size) _align
-.endm
-# macro RES_D name, size [, align]
-.macro RES_D _name, _size, _align=4
-RES_int _name 4*(_size) _align
-.endm
-# macro RES_Q name, size [, align]
-.macro RES_Q _name, _size, _align=8
-RES_int _name 8*(_size) _align
-.endm
-# macro RES_DQ name, size [, align]
-.macro RES_DQ _name, _size, _align=16
-RES_int _name 16*(_size) _align
-.endm
-# macro RES_Y name, size [, align]
-.macro RES_Y _name, _size, _align=32
-RES_int _name 32*(_size) _align
-.endm
-# macro RES_Z name, size [, align]
-.macro RES_Z _name, _size, _align=64
-RES_int _name 64*(_size) _align
-.endm
-#endif
-########################################################################
-#### Define constants
-########################################################################
-########################################################################
-#### Define SHA1 Out Of Order Data Structures
-########################################################################
-START_FIELDS    # LANE_DATA
-###     name            size    align
-FIELD   _job_in_lane,   8,      8       # pointer to job object
-END_FIELDS
-_LANE_DATA_size = _FIELD_OFFSET
-_LANE_DATA_align = _STRUCT_ALIGN
-########################################################################
-START_FIELDS    # SHA1_ARGS_X8
-###     name            size    align
-FIELD   _digest,        4*5*8,  16      # transposed digest
-FIELD   _data_ptr,      8*8,    8       # array of pointers to data
-END_FIELDS
-_SHA1_ARGS_X4_size =     _FIELD_OFFSET
-_SHA1_ARGS_X4_align =    _STRUCT_ALIGN
-_SHA1_ARGS_X8_size =     _FIELD_OFFSET
-_SHA1_ARGS_X8_align =    _STRUCT_ALIGN
-########################################################################
-START_FIELDS    # MB_MGR
-###     name            size    align
-FIELD   _args,          _SHA1_ARGS_X4_size, _SHA1_ARGS_X4_align
-FIELD   _lens,          4*8,    8
-FIELD   _unused_lanes,  8,      8
-FIELD   _ldata,         _LANE_DATA_size*8, _LANE_DATA_align
-END_FIELDS
-_MB_MGR_size =   _FIELD_OFFSET
-_MB_MGR_align =  _STRUCT_ALIGN
-_args_digest    =     _args + _digest
-_args_data_ptr  =     _args + _data_ptr
-########################################################################
-#### Define constants
-########################################################################
-#define STS_UNKNOWN             0
-#define STS_BEING_PROCESSED     1
-#define STS_COMPLETED           2
-########################################################################
-#### Define JOB_SHA1 structure
-########################################################################
-START_FIELDS    # JOB_SHA1
-###     name                            size    align
-FIELD   _buffer,                        8,      8       # pointer to buffer
-FIELD   _len,                           4,      4       # length in bytes
-FIELD   _result_digest,                 5*4,    32      # Digest (output)
-FIELD   _status,                        4,      4
-FIELD   _user_data,                     8,      8
-END_FIELDS
-_JOB_SHA1_size =  _FIELD_OFFSET
-_JOB_SHA1_align = _STRUCT_ALIGN
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S
deleted file mode 100644
index 7cfba738f104..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Flush routine for SHA1 multibuffer
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      James Guilford <james.guilford@intel.com>
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-#include "sha1_mb_mgr_datastruct.S"
-.extern sha1_x8_avx2
-# LINUX register definitions
-#define arg1    %rdi
-#define arg2    %rsi
-# Common definitions
-#define state   arg1
-#define job     arg2
-#define len2    arg2
-# idx must be a register not clobbered by sha1_x8_avx2
-#define idx             %r8
-#define DWORD_idx       %r8d
-#define unused_lanes    %rbx
-#define lane_data       %rbx
-#define tmp2            %rbx
-#define tmp2_w          %ebx
-#define job_rax         %rax
-#define tmp1            %rax
-#define size_offset     %rax
-#define tmp             %rax
-#define start_offset    %rax
-#define tmp3            %arg1
-#define extra_blocks    %arg2
-#define p               %arg2
-.macro LABEL prefix n
-\prefix\n\():
-.endm
-.macro JNE_SKIP i
-jne     skip_\i
-.endm
-.altmacro
-.macro SET_OFFSET _offset
-offset = \_offset
-.endm
-.noaltmacro
-# JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state)
-# arg 1 : rcx : state
-ENTRY(sha1_mb_mgr_flush_avx2)
-        FRAME_BEGIN
-        push    %rbx
-        # If bit (32+3) is set, then all lanes are empty
-        mov     _unused_lanes(state), unused_lanes
-        bt      $32+3, unused_lanes
-        jc      return_null
-        # find a lane with a non-null job
-        xor     idx, idx
-        offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  one(%rip), idx
-        offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  two(%rip), idx
-        offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  three(%rip), idx
-        offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  four(%rip), idx
-        offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  five(%rip), idx
-        offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  six(%rip), idx
-        offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  seven(%rip), idx
-        # copy idx to empty lanes
-copy_lane_data:
-        offset =  (_args + _data_ptr)
-        mov     offset(state,idx,8), tmp
-        I = 0
-.rep 8
-        offset =  (_ldata + I * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-.altmacro
-        JNE_SKIP %I
-        offset =  (_args + _data_ptr + 8*I)
-        mov     tmp, offset(state)
-        offset =  (_lens + 4*I)
-        movl    $0xFFFFFFFF, offset(state)
-LABEL skip_ %I
-        I = (I+1)
-.noaltmacro
-.endr
-        # Find min length
-        vmovdqu _lens+0*16(state), %xmm0
-        vmovdqu _lens+1*16(state), %xmm1
-        vpminud %xmm1, %xmm0, %xmm2     # xmm2 has {D,C,B,A}
-        vpalignr $8, %xmm2, %xmm3, %xmm3   # xmm3 has {x,x,D,C}
-        vpminud %xmm3, %xmm2, %xmm2        # xmm2 has {x,x,E,F}
-        vpalignr $4, %xmm2, %xmm3, %xmm3    # xmm3 has {x,x,x,E}
-        vpminud %xmm3, %xmm2, %xmm2        # xmm2 has min value in low dword
-        vmovd   %xmm2, DWORD_idx
-        mov     idx, len2
-        and     $0xF, idx
-        shr     $4, len2
-        jz      len_is_0
-        vpand   clear_low_nibble(%rip), %xmm2, %xmm2
-        vpshufd $0, %xmm2, %xmm2
-        vpsubd  %xmm2, %xmm0, %xmm0
-        vpsubd  %xmm2, %xmm1, %xmm1
-        vmovdqu %xmm0, _lens+0*16(state)
-        vmovdqu %xmm1, _lens+1*16(state)
-        # "state" and "args" are the same address, arg1
-        # len is arg2
-        call    sha1_x8_avx2
-        # state and idx are intact
-len_is_0:
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        movq    $0, _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        mov     _unused_lanes(state), unused_lanes
-        shl     $4, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens(state, idx, 4)
-        vmovd    _args_digest(state , idx, 4) , %xmm0
-        vpinsrd  $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd  $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd  $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
-        movl    _args_digest+4*32(state, idx, 4), tmp2_w
-        vmovdqu  %xmm0, _result_digest(job_rax)
-        offset =  (_result_digest + 1*16)
-        mov     tmp2_w, offset(job_rax)
-return:
-        pop     %rbx
-        FRAME_END
-        ret
-return_null:
-        xor     job_rax, job_rax
-        jmp     return
-ENDPROC(sha1_mb_mgr_flush_avx2)
-#################################################################
-.align 16
-ENTRY(sha1_mb_mgr_get_comp_job_avx2)
-        push    %rbx
-        ## if bit 32+3 is set, then all lanes are empty
-        mov     _unused_lanes(state), unused_lanes
-        bt      $(32+3), unused_lanes
-        jc      .return_null
-        # Find min length
-        vmovdqu _lens(state), %xmm0
-        vmovdqu _lens+1*16(state), %xmm1
-        vpminud %xmm1, %xmm0, %xmm2        # xmm2 has {D,C,B,A}
-        vpalignr $8, %xmm2, %xmm3, %xmm3   # xmm3 has {x,x,D,C}
-        vpminud %xmm3, %xmm2, %xmm2        # xmm2 has {x,x,E,F}
-        vpalignr $4, %xmm2, %xmm3, %xmm3    # xmm3 has {x,x,x,E}
-        vpminud %xmm3, %xmm2, %xmm2        # xmm2 has min value in low dword
-        vmovd   %xmm2, DWORD_idx
-        test    $~0xF, idx
-        jnz     .return_null
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        movq    $0,  _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        mov     _unused_lanes(state), unused_lanes
-        shl     $4, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens(state,  idx, 4)
-        vmovd   _args_digest(state, idx, 4), %xmm0
-        vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
-        movl    _args_digest+4*32(state, idx, 4), tmp2_w
-        vmovdqu %xmm0, _result_digest(job_rax)
-        movl    tmp2_w, _result_digest+1*16(job_rax)
-        pop     %rbx
-        ret
-.return_null:
-        xor     job_rax, job_rax
-        pop     %rbx
-        ret
-ENDPROC(sha1_mb_mgr_get_comp_job_avx2)
-.section        .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
-.align 16
-clear_low_nibble:
-.octa   0x000000000000000000000000FFFFFFF0
-.section        .rodata.cst8, "aM", @progbits, 8
-.align 8
-one:
-.quad  1
-two:
-.quad  2
-three:
-.quad  3
-four:
-.quad  4
-five:
-.quad  5
-six:
-.quad  6
-seven:
-.quad  7
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c
deleted file mode 100644
index d2add0d35f43..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Initialization code for multi buffer SHA1 algorithm for AVX2
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include "sha1_mb_mgr.h"
-void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state)
-{
-        unsigned int j;
-        state->unused_lanes = 0xF76543210ULL;
-        for (j = 0; j < 8; j++) {
-                state->lens[j] = 0xFFFFFFFF;
-                state->ldata[j].job_in_lane = NULL;
-        }
-}
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S
deleted file mode 100644
index 7a93b1c0d69a..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Buffer submit code for multi buffer SHA1 algorithm
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      James Guilford <james.guilford@intel.com>
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-#include "sha1_mb_mgr_datastruct.S"
-.extern sha1_x8_avx
-# LINUX register definitions
-arg1    = %rdi
-arg2    = %rsi
-size_offset     = %rcx
-tmp2            = %rcx
-extra_blocks    = %rdx
-# Common definitions
-#define state   arg1
-#define job     %rsi
-#define len2    arg2
-#define p2      arg2
-# idx must be a register not clobberred by sha1_x8_avx2
-idx             = %r8
-DWORD_idx       = %r8d
-last_len        = %r8
-p               = %r11
-start_offset    = %r11
-unused_lanes    = %rbx
-BYTE_unused_lanes = %bl
-job_rax         = %rax
-len             = %rax
-DWORD_len       = %eax
-lane            = %r12
-tmp3            = %r12
-tmp             = %r9
-DWORD_tmp       = %r9d
-lane_data       = %r10
-# JOB* submit_mb_mgr_submit_avx2(MB_MGR *state, job_sha1 *job)
-# arg 1 : rcx : state
-# arg 2 : rdx : job
-ENTRY(sha1_mb_mgr_submit_avx2)
-        FRAME_BEGIN
-        push    %rbx
-        push    %r12
-        mov     _unused_lanes(state), unused_lanes
-        mov     unused_lanes, lane
-        and     $0xF, lane
-        shr     $4, unused_lanes
-        imul    $_LANE_DATA_size, lane, lane_data
-        movl    $STS_BEING_PROCESSED, _status(job)
-        lea     _ldata(state, lane_data), lane_data
-        mov     unused_lanes, _unused_lanes(state)
-        movl    _len(job),  DWORD_len
-        mov     job, _job_in_lane(lane_data)
-        shl     $4, len
-        or      lane, len
-        movl    DWORD_len,  _lens(state , lane, 4)
-        # Load digest words from result_digest
-        vmovdqu _result_digest(job), %xmm0
-        mov     _result_digest+1*16(job), DWORD_tmp
-        vmovd    %xmm0, _args_digest(state, lane, 4)
-        vpextrd  $1, %xmm0, _args_digest+1*32(state , lane, 4)
-        vpextrd  $2, %xmm0, _args_digest+2*32(state , lane, 4)
-        vpextrd  $3, %xmm0, _args_digest+3*32(state , lane, 4)
-        movl    DWORD_tmp, _args_digest+4*32(state , lane, 4)
-        mov     _buffer(job), p
-        mov     p, _args_data_ptr(state, lane, 8)
-        cmp     $0xF, unused_lanes
-        jne     return_null
-start_loop:
-        # Find min length
-        vmovdqa _lens(state), %xmm0
-        vmovdqa _lens+1*16(state), %xmm1
-        vpminud %xmm1, %xmm0, %xmm2        # xmm2 has {D,C,B,A}
-        vpalignr $8, %xmm2, %xmm3, %xmm3   # xmm3 has {x,x,D,C}
-        vpminud %xmm3, %xmm2, %xmm2        # xmm2 has {x,x,E,F}
-        vpalignr $4, %xmm2, %xmm3, %xmm3   # xmm3 has {x,x,x,E}
-        vpminud %xmm3, %xmm2, %xmm2        # xmm2 has min value in low dword
-        vmovd   %xmm2, DWORD_idx
-        mov    idx, len2
-        and    $0xF, idx
-        shr    $4, len2
-        jz     len_is_0
-        vpand   clear_low_nibble(%rip), %xmm2, %xmm2
-        vpshufd $0, %xmm2, %xmm2
-        vpsubd  %xmm2, %xmm0, %xmm0
-        vpsubd  %xmm2, %xmm1, %xmm1
-        vmovdqa %xmm0, _lens + 0*16(state)
-        vmovdqa %xmm1, _lens + 1*16(state)
-        # "state" and "args" are the same address, arg1
-        # len is arg2
-        call    sha1_x8_avx2
-        # state and idx are intact
-len_is_0:
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        mov     _unused_lanes(state), unused_lanes
-        movq    $0, _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        shl     $4, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens(state, idx, 4)
-        vmovd    _args_digest(state, idx, 4), %xmm0
-        vpinsrd  $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0
-        vpinsrd  $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0
-        vpinsrd  $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0
-        movl     _args_digest+4*32(state, idx, 4), DWORD_tmp
-        vmovdqu  %xmm0, _result_digest(job_rax)
-        movl    DWORD_tmp, _result_digest+1*16(job_rax)
-return:
-        pop     %r12
-        pop     %rbx
-        FRAME_END
-        ret
-return_null:
-        xor     job_rax, job_rax
-        jmp     return
-ENDPROC(sha1_mb_mgr_submit_avx2)
-.section        .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
-.align 16
-clear_low_nibble:
-        .octa   0x000000000000000000000000FFFFFFF0
diff --git a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S
deleted file mode 100644
index 20f77aa633de..000000000000
--- a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S
+++ /dev/null
@@ -1,492 +0,0 @@
-/*
- * Multi-buffer SHA1 algorithm hash compute routine
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      James Guilford <james.guilford@intel.com>
- *      Tim Chen <tim.c.chen@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2014 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include "sha1_mb_mgr_datastruct.S"
-## code to compute oct SHA1 using SSE-256
-## outer calling routine takes care of save and restore of XMM registers
-## Function clobbers: rax, rcx, rdx,   rbx, rsi, rdi, r9-r15# ymm0-15
-##
-## Linux clobbers:    rax rbx rcx rdx rsi            r9 r10 r11 r12 r13 r14 r15
-## Linux preserves:                       rdi rbp r8
-##
-## clobbers ymm0-15
-# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1
-# "transpose" data in {r0...r7} using temps {t0...t1}
-# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
-# r0 = {a7 a6 a5 a4   a3 a2 a1 a0}
-# r1 = {b7 b6 b5 b4   b3 b2 b1 b0}
-# r2 = {c7 c6 c5 c4   c3 c2 c1 c0}
-# r3 = {d7 d6 d5 d4   d3 d2 d1 d0}
-# r4 = {e7 e6 e5 e4   e3 e2 e1 e0}
-# r5 = {f7 f6 f5 f4   f3 f2 f1 f0}
-# r6 = {g7 g6 g5 g4   g3 g2 g1 g0}
-# r7 = {h7 h6 h5 h4   h3 h2 h1 h0}
-#
-# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
-# r0 = {h0 g0 f0 e0   d0 c0 b0 a0}
-# r1 = {h1 g1 f1 e1   d1 c1 b1 a1}
-# r2 = {h2 g2 f2 e2   d2 c2 b2 a2}
-# r3 = {h3 g3 f3 e3   d3 c3 b3 a3}
-# r4 = {h4 g4 f4 e4   d4 c4 b4 a4}
-# r5 = {h5 g5 f5 e5   d5 c5 b5 a5}
-# r6 = {h6 g6 f6 e6   d6 c6 b6 a6}
-# r7 = {h7 g7 f7 e7   d7 c7 b7 a7}
-#
-.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1
-        # process top half (r0..r3) {a...d}
-        vshufps  $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4   b1 b0 a1 a0}
-        vshufps  $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6   b3 b2 a3 a2}
-        vshufps  $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4   d1 d0 c1 c0}
-        vshufps  $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6   d3 d2 c3 c2}
-        vshufps  $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5   d1 c1 b1 a1}
-        vshufps  $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6   d2 c2 b2 a2}
-        vshufps  $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7   d3 c3 b3 a3}
-        vshufps  $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4   d0 c0 b0 a0}
-        # use r2 in place of t0
-        # process bottom half (r4..r7) {e...h}
-        vshufps  $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4   f1 f0 e1 e0}
-        vshufps  $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6   f3 f2 e3 e2}
-        vshufps  $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4   h1 h0 g1 g0}
-        vshufps  $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6   h3 h2 g3 g2}
-        vshufps  $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5   h1 g1 f1 e1}
-        vshufps  $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6   h2 g2 f2 e2}
-        vshufps  $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7   h3 g3 f3 e3}
-        vshufps  $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4   h0 g0 f0 e0}
-        vperm2f128      $0x13, \r1, \r5, \r6  # h6...a6
-        vperm2f128      $0x02, \r1, \r5, \r2  # h2...a2
-        vperm2f128      $0x13, \r3, \r7, \r5  # h5...a5
-        vperm2f128      $0x02, \r3, \r7, \r1  # h1...a1
-        vperm2f128      $0x13, \r0, \r4, \r7  # h7...a7
-        vperm2f128      $0x02, \r0, \r4, \r3  # h3...a3
-        vperm2f128      $0x13, \t0, \t1, \r4  # h4...a4
-        vperm2f128      $0x02, \t0, \t1, \r0  # h0...a0
-.endm
-##
-## Magic functions defined in FIPS 180-1
-##
-# macro MAGIC_F0 F,B,C,D,T   ## F = (D ^ (B & (C ^ D)))
-.macro MAGIC_F0 regF regB regC regD regT
-    vpxor \regD, \regC, \regF
-    vpand \regB, \regF, \regF
-    vpxor \regD, \regF, \regF
-.endm
-# macro MAGIC_F1 F,B,C,D,T   ## F = (B ^ C ^ D)
-.macro MAGIC_F1 regF regB regC regD regT
-    vpxor  \regC, \regD, \regF
-    vpxor  \regB, \regF, \regF
-.endm
-# macro MAGIC_F2 F,B,C,D,T   ## F = ((B & C) | (B & D) | (C & D))
-.macro MAGIC_F2 regF regB regC regD regT
-    vpor  \regC, \regB, \regF
-    vpand \regC, \regB, \regT
-    vpand \regD, \regF, \regF
-    vpor  \regT, \regF, \regF
-.endm
-# macro MAGIC_F3 F,B,C,D,T   ## F = (B ^ C ^ D)
-.macro MAGIC_F3 regF regB regC regD regT
-    MAGIC_F1 \regF,\regB,\regC,\regD,\regT
-.endm
-# PROLD reg, imm, tmp
-.macro PROLD reg imm tmp
-        vpsrld  $(32-\imm), \reg, \tmp
-        vpslld  $\imm, \reg, \reg
-        vpor    \tmp, \reg, \reg
-.endm
-.macro PROLD_nd reg imm tmp src
-        vpsrld  $(32-\imm), \src, \tmp
-        vpslld  $\imm, \src, \reg
-        vpor    \tmp, \reg, \reg
-.endm
-.macro SHA1_STEP_00_15 regA regB regC regD regE regT regF memW immCNT MAGIC
-        vpaddd  \immCNT, \regE, \regE
-        vpaddd  \memW*32(%rsp), \regE, \regE
-        PROLD_nd \regT, 5, \regF, \regA
-        vpaddd  \regT, \regE, \regE
-        \MAGIC  \regF, \regB, \regC, \regD, \regT
-        PROLD   \regB, 30, \regT
-        vpaddd  \regF, \regE, \regE
-.endm
-.macro SHA1_STEP_16_79 regA regB regC regD regE regT regF memW immCNT MAGIC
-        vpaddd  \immCNT, \regE, \regE
-        offset = ((\memW - 14) & 15) * 32
-        vmovdqu offset(%rsp), W14
-        vpxor   W14, W16, W16
-        offset = ((\memW -  8) & 15) * 32
-        vpxor   offset(%rsp), W16, W16
-        offset = ((\memW -  3) & 15) * 32
-        vpxor   offset(%rsp), W16, W16
-        vpsrld  $(32-1), W16, \regF
-        vpslld  $1, W16, W16
-        vpor    W16, \regF, \regF
-        ROTATE_W
-        offset = ((\memW - 0) & 15) * 32
-        vmovdqu \regF, offset(%rsp)
-        vpaddd  \regF, \regE, \regE
-        PROLD_nd \regT, 5, \regF, \regA
-        vpaddd  \regT, \regE, \regE
-        \MAGIC \regF,\regB,\regC,\regD,\regT      ## FUN  = MAGIC_Fi(B,C,D)
-        PROLD   \regB,30, \regT
-        vpaddd  \regF, \regE, \regE
-.endm
-########################################################################
-########################################################################
-########################################################################
-## FRAMESZ plus pushes must be an odd multiple of 8
-YMM_SAVE = (15-15)*32
-FRAMESZ = 32*16 + YMM_SAVE
-_YMM  =   FRAMESZ - YMM_SAVE
-#define VMOVPS   vmovups
-IDX  = %rax
-inp0 = %r9
-inp1 = %r10
-inp2 = %r11
-inp3 = %r12
-inp4 = %r13
-inp5 = %r14
-inp6 = %r15
-inp7 = %rcx
-arg1 = %rdi
-arg2 = %rsi
-RSP_SAVE = %rdx
-# ymm0 A
-# ymm1 B
-# ymm2 C
-# ymm3 D
-# ymm4 E
-# ymm5         F       AA
-# ymm6         T0      BB
-# ymm7         T1      CC
-# ymm8         T2      DD
-# ymm9         T3      EE
-# ymm10                T4      TMP
-# ymm11                T5      FUN
-# ymm12                T6      K
-# ymm13                T7      W14
-# ymm14                T8      W15
-# ymm15                T9      W16
-A  =     %ymm0
-B  =     %ymm1
-C  =     %ymm2
-D  =     %ymm3
-E  =     %ymm4
-F  =     %ymm5
-T0 =     %ymm6
-T1 =     %ymm7
-T2 =     %ymm8
-T3 =     %ymm9
-T4 =     %ymm10
-T5 =     %ymm11
-T6 =     %ymm12
-T7 =     %ymm13
-T8  =     %ymm14
-T9  =     %ymm15
-AA  =     %ymm5
-BB  =     %ymm6
-CC  =     %ymm7
-DD  =     %ymm8
-EE  =     %ymm9
-TMP =     %ymm10
-FUN =     %ymm11
-K   =     %ymm12
-W14 =     %ymm13
-W15 =     %ymm14
-W16 =     %ymm15
-.macro ROTATE_ARGS
- TMP_ = E
- E = D
- D = C
- C = B
- B = A
- A = TMP_
-.endm
-.macro ROTATE_W
-TMP_  = W16
-W16  = W15
-W15  = W14
-W14  = TMP_
-.endm
-# 8 streams x 5 32bit words per digest x 4 bytes per word
-#define DIGEST_SIZE (8*5*4)
-.align 32
-# void sha1_x8_avx2(void **input_data, UINT128 *digest, UINT32 size)
-# arg 1 : pointer to array[4] of pointer to input data
-# arg 2 : size (in blocks) ;; assumed to be >= 1
-#
-ENTRY(sha1_x8_avx2)
-        # save callee-saved clobbered registers to comply with C function ABI
-        push    %r12
-        push    %r13
-        push    %r14
-        push    %r15
-        #save rsp
-        mov     %rsp, RSP_SAVE
-        sub     $FRAMESZ, %rsp
-        #align rsp to 32 Bytes
-        and     $~0x1F, %rsp
-        ## Initialize digests
-        vmovdqu  0*32(arg1), A
-        vmovdqu  1*32(arg1), B
-        vmovdqu  2*32(arg1), C
-        vmovdqu  3*32(arg1), D
-        vmovdqu  4*32(arg1), E
-        ## transpose input onto stack
-        mov     _data_ptr+0*8(arg1),inp0
-        mov     _data_ptr+1*8(arg1),inp1
-        mov     _data_ptr+2*8(arg1),inp2
-        mov     _data_ptr+3*8(arg1),inp3
-        mov     _data_ptr+4*8(arg1),inp4
-        mov     _data_ptr+5*8(arg1),inp5
-        mov     _data_ptr+6*8(arg1),inp6
-        mov     _data_ptr+7*8(arg1),inp7
-        xor     IDX, IDX
-lloop:
-        vmovdqu  PSHUFFLE_BYTE_FLIP_MASK(%rip), F
-        I=0
-.rep 2
-        VMOVPS   (inp0, IDX), T0
-        VMOVPS   (inp1, IDX), T1
-        VMOVPS   (inp2, IDX), T2
-        VMOVPS   (inp3, IDX), T3
-        VMOVPS   (inp4, IDX), T4
-        VMOVPS   (inp5, IDX), T5
-        VMOVPS   (inp6, IDX), T6
-        VMOVPS   (inp7, IDX), T7
-        TRANSPOSE8       T0, T1, T2, T3, T4, T5, T6, T7, T8, T9
-        vpshufb  F, T0, T0
-        vmovdqu  T0, (I*8)*32(%rsp)
-        vpshufb  F, T1, T1
-        vmovdqu  T1, (I*8+1)*32(%rsp)
-        vpshufb  F, T2, T2
-        vmovdqu  T2, (I*8+2)*32(%rsp)
-        vpshufb  F, T3, T3
-        vmovdqu  T3, (I*8+3)*32(%rsp)
-        vpshufb  F, T4, T4
-        vmovdqu  T4, (I*8+4)*32(%rsp)
-        vpshufb  F, T5, T5
-        vmovdqu  T5, (I*8+5)*32(%rsp)
-        vpshufb  F, T6, T6
-        vmovdqu  T6, (I*8+6)*32(%rsp)
-        vpshufb  F, T7, T7
-        vmovdqu  T7, (I*8+7)*32(%rsp)
-        add     $32, IDX
-        I = (I+1)
-.endr
-        # save old digests
-        vmovdqu  A,AA
-        vmovdqu  B,BB
-        vmovdqu  C,CC
-        vmovdqu  D,DD
-        vmovdqu  E,EE
-##
-## perform 0-79 steps
-##
-        vmovdqu  K00_19(%rip), K
-## do rounds 0...15
-        I = 0
-.rep 16
-        SHA1_STEP_00_15 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0
-        ROTATE_ARGS
-        I = (I+1)
-.endr
-## do rounds 16...19
-        vmovdqu  ((16 - 16) & 15) * 32 (%rsp), W16
-        vmovdqu  ((16 - 15) & 15) * 32 (%rsp), W15
-.rep 4
-        SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0
-        ROTATE_ARGS
-        I = (I+1)
-.endr
-## do rounds 20...39
-        vmovdqu  K20_39(%rip), K
-.rep 20
-        SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F1
-        ROTATE_ARGS
-        I = (I+1)
-.endr
-## do rounds 40...59
-        vmovdqu  K40_59(%rip), K
-.rep 20
-        SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F2
-        ROTATE_ARGS
-        I = (I+1)
-.endr
-## do rounds 60...79
-        vmovdqu  K60_79(%rip), K
-.rep 20
-        SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F3
-        ROTATE_ARGS
-        I = (I+1)
-.endr
-        vpaddd   AA,A,A
-        vpaddd   BB,B,B
-        vpaddd   CC,C,C
-        vpaddd   DD,D,D
-        vpaddd   EE,E,E
-        sub     $1, arg2
-        jne     lloop
-        # write out digests
-        vmovdqu  A, 0*32(arg1)
-        vmovdqu  B, 1*32(arg1)
-        vmovdqu  C, 2*32(arg1)
-        vmovdqu  D, 3*32(arg1)
-        vmovdqu  E, 4*32(arg1)
-        # update input pointers
-        add     IDX, inp0
-        add     IDX, inp1
-        add     IDX, inp2
-        add     IDX, inp3
-        add     IDX, inp4
-        add     IDX, inp5
-        add     IDX, inp6
-        add     IDX, inp7
-        mov     inp0, _data_ptr (arg1)
-        mov     inp1, _data_ptr + 1*8(arg1)
-        mov     inp2, _data_ptr + 2*8(arg1)
-        mov     inp3, _data_ptr + 3*8(arg1)
-        mov     inp4, _data_ptr + 4*8(arg1)
-        mov     inp5, _data_ptr + 5*8(arg1)
-        mov     inp6, _data_ptr + 6*8(arg1)
-        mov     inp7, _data_ptr + 7*8(arg1)
-        ################
-        ## Postamble
-        mov     RSP_SAVE, %rsp
-        # restore callee-saved clobbered registers
-        pop     %r15
-        pop     %r14
-        pop     %r13
-        pop     %r12
-        ret
-ENDPROC(sha1_x8_avx2)
-.section        .rodata.cst32.K00_19, "aM", @progbits, 32
-.align 32
-K00_19:
-.octa 0x5A8279995A8279995A8279995A827999
-.octa 0x5A8279995A8279995A8279995A827999
-.section        .rodata.cst32.K20_39, "aM", @progbits, 32
-.align 32
-K20_39:
-.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1
-.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1
-.section        .rodata.cst32.K40_59, "aM", @progbits, 32
-.align 32
-K40_59:
-.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC
-.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC
-.section        .rodata.cst32.K60_79, "aM", @progbits, 32
-.align 32
-K60_79:
-.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6
-.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6
-.section        .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
-.align 32
-PSHUFFLE_BYTE_FLIP_MASK:
-.octa 0x0c0d0e0f08090a0b0405060700010203
-.octa 0x0c0d0e0f08090a0b0405060700010203
diff --git a/arch/x86/crypto/sha256-mb/Makefile b/arch/x86/crypto/sha256-mb/Makefile
deleted file mode 100644
index 53ad6e7db747..000000000000
--- a/arch/x86/crypto/sha256-mb/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Arch-specific CryptoAPI modules.
-#
-OBJECT_FILES_NON_STANDARD := y
-avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
-                                $(comma)4)$(comma)%ymm2,yes,no)
-ifeq ($(avx2_supported),yes)
-        obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb.o
-        sha256-mb-y := sha256_mb.o sha256_mb_mgr_flush_avx2.o \
-             sha256_mb_mgr_init_avx2.o sha256_mb_mgr_submit_avx2.o sha256_x8_avx2.o
-endif
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb.c b/arch/x86/crypto/sha256-mb/sha256_mb.c
deleted file mode 100644
index 97c5fc43e115..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb.c
+++ /dev/null
@@ -1,1013 +0,0 @@
-/*
- * Multi buffer SHA256 algorithm Glue Code
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
-#include <crypto/internal/hash.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/cryptohash.h>
-#include <linux/types.h>
-#include <linux/list.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/sha.h>
-#include <crypto/mcryptd.h>
-#include <crypto/crypto_wq.h>
-#include <asm/byteorder.h>
-#include <linux/hardirq.h>
-#include <asm/fpu/api.h>
-#include "sha256_mb_ctx.h"
-#define FLUSH_INTERVAL 1000 /* in usec */
-static struct mcryptd_alg_state sha256_mb_alg_state;
-struct sha256_mb_ctx {
-        struct mcryptd_ahash *mcryptd_tfm;
-};
-static inline struct mcryptd_hash_request_ctx
-                *cast_hash_to_mcryptd_ctx(struct sha256_hash_ctx *hash_ctx)
-{
-        struct ahash_request *areq;
-        areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
-        return container_of(areq, struct mcryptd_hash_request_ctx, areq);
-}
-static inline struct ahash_request
-                *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
-{
-        return container_of((void *) ctx, struct ahash_request, __ctx);
-}
-static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
-                                struct ahash_request *areq)
-{
-        rctx->flag = HASH_UPDATE;
-}
-static asmlinkage void (*sha256_job_mgr_init)(struct sha256_mb_mgr *state);
-static asmlinkage struct job_sha256* (*sha256_job_mgr_submit)
-                        (struct sha256_mb_mgr *state, struct job_sha256 *job);
-static asmlinkage struct job_sha256* (*sha256_job_mgr_flush)
-                        (struct sha256_mb_mgr *state);
-static asmlinkage struct job_sha256* (*sha256_job_mgr_get_comp_job)
-                        (struct sha256_mb_mgr *state);
-inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2],
-                         uint64_t total_len)
-{
-        uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1);
-        memset(&padblock[i], 0, SHA256_BLOCK_SIZE);
-        padblock[i] = 0x80;
-        i += ((SHA256_BLOCK_SIZE - 1) &
-              (0 - (total_len + SHA256_PADLENGTHFIELD_SIZE + 1)))
-             + 1 + SHA256_PADLENGTHFIELD_SIZE;
-#if SHA256_PADLENGTHFIELD_SIZE == 16
-        *((uint64_t *) &padblock[i - 16]) = 0;
-#endif
-        *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
-        /* Number of extra blocks to hash */
-        return i >> SHA256_LOG2_BLOCK_SIZE;
-}
-static struct sha256_hash_ctx
-                *sha256_ctx_mgr_resubmit(struct sha256_ctx_mgr *mgr,
-                                        struct sha256_hash_ctx *ctx)
-{
-        while (ctx) {
-                if (ctx->status & HASH_CTX_STS_COMPLETE) {
-                        /* Clear PROCESSING bit */
-                        ctx->status = HASH_CTX_STS_COMPLETE;
-                        return ctx;
-                }
-                /*
-                 * If the extra blocks are empty, begin hashing what remains
-                 * in the user's buffer.
-                 */
-                if (ctx->partial_block_buffer_length == 0 &&
-                    ctx->incoming_buffer_length) {
-                        const void *buffer = ctx->incoming_buffer;
-                        uint32_t len = ctx->incoming_buffer_length;
-                        uint32_t copy_len;
-                        /*
-                         * Only entire blocks can be hashed.
-                         * Copy remainder to extra blocks buffer.
-                         */
-                        copy_len = len & (SHA256_BLOCK_SIZE-1);
-                        if (copy_len) {
-                                len -= copy_len;
-                                memcpy(ctx->partial_block_buffer,
-                                       ((const char *) buffer + len),
-                                       copy_len);
-                                ctx->partial_block_buffer_length = copy_len;
-                        }
-                        ctx->incoming_buffer_length = 0;
-                        /* len should be a multiple of the block size now */
-                        assert((len % SHA256_BLOCK_SIZE) == 0);
-                        /* Set len to the number of blocks to be hashed */
-                        len >>= SHA256_LOG2_BLOCK_SIZE;
-                        if (len) {
-                                ctx->job.buffer = (uint8_t *) buffer;
-                                ctx->job.len = len;
-                                ctx = (struct sha256_hash_ctx *)
-                                sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
-                                continue;
-                        }
-                }
-                /*
-                 * If the extra blocks are not empty, then we are
-                 * either on the last block(s) or we need more
-                 * user input before continuing.
-                 */
-                if (ctx->status & HASH_CTX_STS_LAST) {
-                        uint8_t *buf = ctx->partial_block_buffer;
-                        uint32_t n_extra_blocks =
-                                sha256_pad(buf, ctx->total_length);
-                        ctx->status = (HASH_CTX_STS_PROCESSING |
-                                       HASH_CTX_STS_COMPLETE);
-                        ctx->job.buffer = buf;
-                        ctx->job.len = (uint32_t) n_extra_blocks;
-                        ctx = (struct sha256_hash_ctx *)
-                                sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
-                        continue;
-                }
-                ctx->status = HASH_CTX_STS_IDLE;
-                return ctx;
-        }
-        return NULL;
-}
-static struct sha256_hash_ctx
-                *sha256_ctx_mgr_get_comp_ctx(struct sha256_ctx_mgr *mgr)
-{
-        /*
-         * If get_comp_job returns NULL, there are no jobs complete.
-         * If get_comp_job returns a job, verify that it is safe to return to
-         * the user. If it is not ready, resubmit the job to finish processing.
-         * If sha256_ctx_mgr_resubmit returned a job, it is ready to be
-         * returned. Otherwise, all jobs currently being managed by the
-         * hash_ctx_mgr still need processing.
-         */
-        struct sha256_hash_ctx *ctx;
-        ctx = (struct sha256_hash_ctx *) sha256_job_mgr_get_comp_job(&mgr->mgr);
-        return sha256_ctx_mgr_resubmit(mgr, ctx);
-}
-static void sha256_ctx_mgr_init(struct sha256_ctx_mgr *mgr)
-{
-        sha256_job_mgr_init(&mgr->mgr);
-}
-static struct sha256_hash_ctx *sha256_ctx_mgr_submit(struct sha256_ctx_mgr *mgr,
-                                          struct sha256_hash_ctx *ctx,
-                                          const void *buffer,
-                                          uint32_t len,
-                                          int flags)
-{
-        if (flags & ~(HASH_UPDATE | HASH_LAST)) {
-                /* User should not pass anything other than UPDATE or LAST */
-                ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
-                return ctx;
-        }
-        if (ctx->status & HASH_CTX_STS_PROCESSING) {
-                /* Cannot submit to a currently processing job. */
-                ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
-                return ctx;
-        }
-        if (ctx->status & HASH_CTX_STS_COMPLETE) {
-                /* Cannot update a finished job. */
-                ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
-                return ctx;
-        }
-        /* If we made it here, there was no error during this call to submit */
-        ctx->error = HASH_CTX_ERROR_NONE;
-        /* Store buffer ptr info from user */
-        ctx->incoming_buffer = buffer;
-        ctx->incoming_buffer_length = len;
-        /*
-         * Store the user's request flags and mark this ctx as currently
-         * being processed.
-         */
-        ctx->status = (flags & HASH_LAST) ?
-                        (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
-                        HASH_CTX_STS_PROCESSING;
-        /* Advance byte counter */
-        ctx->total_length += len;
-        /*
-         * If there is anything currently buffered in the extra blocks,
-         * append to it until it contains a whole block.
-         * Or if the user's buffer contains less than a whole block,
-         * append as much as possible to the extra block.
-         */
-        if (ctx->partial_block_buffer_length || len < SHA256_BLOCK_SIZE) {
-                /*
-                 * Compute how many bytes to copy from user buffer into
-                 * extra block
-                 */
-                uint32_t copy_len = SHA256_BLOCK_SIZE -
-                                        ctx->partial_block_buffer_length;
-                if (len < copy_len)
-                        copy_len = len;
-                if (copy_len) {
-                        /* Copy and update relevant pointers and counters */
-                        memcpy(
-                &ctx->partial_block_buffer[ctx->partial_block_buffer_length],
-                                buffer, copy_len);
-                        ctx->partial_block_buffer_length += copy_len;
-                        ctx->incoming_buffer = (const void *)
-                                        ((const char *)buffer + copy_len);
-                        ctx->incoming_buffer_length = len - copy_len;
-                }
-                /* The extra block should never contain more than 1 block */
-                assert(ctx->partial_block_buffer_length <= SHA256_BLOCK_SIZE);
-                /*
-                 * If the extra block buffer contains exactly 1 block,
-                 * it can be hashed.
-                 */
-                if (ctx->partial_block_buffer_length >= SHA256_BLOCK_SIZE) {
-                        ctx->partial_block_buffer_length = 0;
-                        ctx->job.buffer = ctx->partial_block_buffer;
-                        ctx->job.len = 1;
-                        ctx = (struct sha256_hash_ctx *)
-                                sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
-                }
-        }
-        return sha256_ctx_mgr_resubmit(mgr, ctx);
-}
-static struct sha256_hash_ctx *sha256_ctx_mgr_flush(struct sha256_ctx_mgr *mgr)
-{
-        struct sha256_hash_ctx *ctx;
-        while (1) {
-                ctx = (struct sha256_hash_ctx *)
-                                        sha256_job_mgr_flush(&mgr->mgr);
-                /* If flush returned 0, there are no more jobs in flight. */
-                if (!ctx)
-                        return NULL;
-                /*
-                 * If flush returned a job, resubmit the job to finish
-                 * processing.
-                 */
-                ctx = sha256_ctx_mgr_resubmit(mgr, ctx);
-                /*
-                 * If sha256_ctx_mgr_resubmit returned a job, it is ready to
-                 * be returned. Otherwise, all jobs currently being managed by
-                 * the sha256_ctx_mgr still need processing. Loop.
-                 */
-                if (ctx)
-                        return ctx;
-        }
-}
-static int sha256_mb_init(struct ahash_request *areq)
-{
-        struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);
-        hash_ctx_init(sctx);
-        sctx->job.result_digest[0] = SHA256_H0;
-        sctx->job.result_digest[1] = SHA256_H1;
-        sctx->job.result_digest[2] = SHA256_H2;
-        sctx->job.result_digest[3] = SHA256_H3;
-        sctx->job.result_digest[4] = SHA256_H4;
-        sctx->job.result_digest[5] = SHA256_H5;
-        sctx->job.result_digest[6] = SHA256_H6;
-        sctx->job.result_digest[7] = SHA256_H7;
-        sctx->total_length = 0;
-        sctx->partial_block_buffer_length = 0;
-        sctx->status = HASH_CTX_STS_IDLE;
-        return 0;
-}
-static int sha256_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
-{
-        int     i;
-        struct  sha256_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
-        __be32  *dst = (__be32 *) rctx->out;
-        for (i = 0; i < 8; ++i)
-                dst[i] = cpu_to_be32(sctx->job.result_digest[i]);
-        return 0;
-}
-static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
-                        struct mcryptd_alg_cstate *cstate, bool flush)
-{
-        int     flag = HASH_UPDATE;
-        int     nbytes, err = 0;
-        struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
-        struct sha256_hash_ctx *sha_ctx;
-        /* more work ? */
-        while (!(rctx->flag & HASH_DONE)) {
-                nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
-                if (nbytes < 0) {
-                        err = nbytes;
-                        goto out;
-                }
-                /* check if the walk is done */
-                if (crypto_ahash_walk_last(&rctx->walk)) {
-                        rctx->flag |= HASH_DONE;
-                        if (rctx->flag & HASH_FINAL)
-                                flag |= HASH_LAST;
-                }
-                sha_ctx = (struct sha256_hash_ctx *)
-                                                ahash_request_ctx(&rctx->areq);
-                kernel_fpu_begin();
-                sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx,
-                                                rctx->walk.data, nbytes, flag);
-                if (!sha_ctx) {
-                        if (flush)
-                                sha_ctx = sha256_ctx_mgr_flush(cstate->mgr);
-                }
-                kernel_fpu_end();
-                if (sha_ctx)
-                        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                else {
-                        rctx = NULL;
-                        goto out;
-                }
-        }
-        /* copy the results */
-        if (rctx->flag & HASH_FINAL)
-                sha256_mb_set_results(rctx);
-out:
-        *ret_rctx = rctx;
-        return err;
-}
-static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
-                            struct mcryptd_alg_cstate *cstate,
-                            int err)
-{
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha256_hash_ctx *sha_ctx;
-        struct mcryptd_hash_request_ctx *req_ctx;
-        int ret;
-        /* remove from work list */
-        spin_lock(&cstate->work_lock);
-        list_del(&rctx->waiter);
-        spin_unlock(&cstate->work_lock);
-        if (irqs_disabled())
-                rctx->complete(&req->base, err);
-        else {
-                local_bh_disable();
-                rctx->complete(&req->base, err);
-                local_bh_enable();
-        }
-        /* check to see if there are other jobs that are done */
-        sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr);
-        while (sha_ctx) {
-                req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                ret = sha_finish_walk(&req_ctx, cstate, false);
-                if (req_ctx) {
-                        spin_lock(&cstate->work_lock);
-                        list_del(&req_ctx->waiter);
-                        spin_unlock(&cstate->work_lock);
-                        req = cast_mcryptd_ctx_to_req(req_ctx);
-                        if (irqs_disabled())
-                                req_ctx->complete(&req->base, ret);
-                        else {
-                                local_bh_disable();
-                                req_ctx->complete(&req->base, ret);
-                                local_bh_enable();
-                        }
-                }
-                sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr);
-        }
-        return 0;
-}
-static void sha256_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
-                             struct mcryptd_alg_cstate *cstate)
-{
-        unsigned long next_flush;
-        unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
-        /* initialize tag */
-        rctx->tag.arrival = jiffies;    /* tag the arrival time */
-        rctx->tag.seq_num = cstate->next_seq_num++;
-        next_flush = rctx->tag.arrival + delay;
-        rctx->tag.expire = next_flush;
-        spin_lock(&cstate->work_lock);
-        list_add_tail(&rctx->waiter, &cstate->work_list);
-        spin_unlock(&cstate->work_lock);
-        mcryptd_arm_flusher(cstate, delay);
-}
-static int sha256_mb_update(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                container_of(areq, struct mcryptd_hash_request_ctx, areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha256_mb_alg_state.alg_cstate);
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha256_hash_ctx *sha_ctx;
-        int ret = 0, nbytes;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        nbytes = crypto_ahash_walk_first(req, &rctx->walk);
-        if (nbytes < 0) {
-                ret = nbytes;
-                goto done;
-        }
-        if (crypto_ahash_walk_last(&rctx->walk))
-                rctx->flag |= HASH_DONE;
-        /* submit */
-        sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
-        sha256_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
-                                                        nbytes, HASH_UPDATE);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha256_mb_finup(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                container_of(areq, struct mcryptd_hash_request_ctx, areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha256_mb_alg_state.alg_cstate);
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha256_hash_ctx *sha_ctx;
-        int ret = 0, flag = HASH_UPDATE, nbytes;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        nbytes = crypto_ahash_walk_first(req, &rctx->walk);
-        if (nbytes < 0) {
-                ret = nbytes;
-                goto done;
-        }
-        if (crypto_ahash_walk_last(&rctx->walk)) {
-                rctx->flag |= HASH_DONE;
-                flag = HASH_LAST;
-        }
-        /* submit */
-        rctx->flag |= HASH_FINAL;
-        sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
-        sha256_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
-                                                                nbytes, flag);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha256_mb_final(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                        container_of(areq, struct mcryptd_hash_request_ctx,
-                        areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha256_mb_alg_state.alg_cstate);
-        struct sha256_hash_ctx *sha_ctx;
-        int ret = 0;
-        u8 data;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        rctx->flag |= HASH_DONE | HASH_FINAL;
-        sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
-        /* flag HASH_FINAL and 0 data size */
-        sha256_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
-                                                                HASH_LAST);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha256_mb_export(struct ahash_request *areq, void *out)
-{
-        struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);
-        memcpy(out, sctx, sizeof(*sctx));
-        return 0;
-}
-static int sha256_mb_import(struct ahash_request *areq, const void *in)
-{
-        struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);
-        memcpy(sctx, in, sizeof(*sctx));
-        return 0;
-}
-static int sha256_mb_async_init_tfm(struct crypto_tfm *tfm)
-{
-        struct mcryptd_ahash *mcryptd_tfm;
-        struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct mcryptd_hash_ctx *mctx;
-        mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha256-mb",
-                                                CRYPTO_ALG_INTERNAL,
-                                                CRYPTO_ALG_INTERNAL);
-        if (IS_ERR(mcryptd_tfm))
-                return PTR_ERR(mcryptd_tfm);
-        mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
-        mctx->alg_state = &sha256_mb_alg_state;
-        ctx->mcryptd_tfm = mcryptd_tfm;
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                sizeof(struct ahash_request) +
-                                crypto_ahash_reqsize(&mcryptd_tfm->base));
-        return 0;
-}
-static void sha256_mb_async_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        mcryptd_free_ahash(ctx->mcryptd_tfm);
-}
-static int sha256_mb_areq_init_tfm(struct crypto_tfm *tfm)
-{
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                sizeof(struct ahash_request) +
-                                sizeof(struct sha256_hash_ctx));
-        return 0;
-}
-static void sha256_mb_areq_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        mcryptd_free_ahash(ctx->mcryptd_tfm);
-}
-static struct ahash_alg sha256_mb_areq_alg = {
-        .init           =       sha256_mb_init,
-        .update         =       sha256_mb_update,
-        .final          =       sha256_mb_final,
-        .finup          =       sha256_mb_finup,
-        .export         =       sha256_mb_export,
-        .import         =       sha256_mb_import,
-        .halg           =       {
-        .digestsize     =       SHA256_DIGEST_SIZE,
-        .statesize      =       sizeof(struct sha256_hash_ctx),
-                .base           =       {
-                        .cra_name        = "__sha256-mb",
-                        .cra_driver_name = "__intel_sha256-mb",
-                        .cra_priority    = 100,
-                        /*
-                         * use ASYNC flag as some buffers in multi-buffer
-                         * algo may not have completed before hashing thread
-                         * sleep
-                         */
-                        .cra_flags      = CRYPTO_ALG_ASYNC |
-                                          CRYPTO_ALG_INTERNAL,
-                        .cra_blocksize  = SHA256_BLOCK_SIZE,
-                        .cra_module     = THIS_MODULE,
-                        .cra_list       = LIST_HEAD_INIT
-                                        (sha256_mb_areq_alg.halg.base.cra_list),
-                        .cra_init       = sha256_mb_areq_init_tfm,
-                        .cra_exit       = sha256_mb_areq_exit_tfm,
-                        .cra_ctxsize    = sizeof(struct sha256_hash_ctx),
-                }
-        }
-};
-static int sha256_mb_async_init(struct ahash_request *req)
-{
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_init(mcryptd_req);
-}
-static int sha256_mb_async_update(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_update(mcryptd_req);
-}
-static int sha256_mb_async_finup(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_finup(mcryptd_req);
-}
-static int sha256_mb_async_final(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_final(mcryptd_req);
-}
-static int sha256_mb_async_digest(struct ahash_request *req)
-{
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_digest(mcryptd_req);
-}
-static int sha256_mb_async_export(struct ahash_request *req, void *out)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_export(mcryptd_req, out);
-}
-static int sha256_mb_async_import(struct ahash_request *req, const void *in)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
-        struct mcryptd_hash_request_ctx *rctx;
-        struct ahash_request *areq;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        rctx = ahash_request_ctx(mcryptd_req);
-        areq = &rctx->areq;
-        ahash_request_set_tfm(areq, child);
-        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
-                                        rctx->complete, req);
-        return crypto_ahash_import(mcryptd_req, in);
-}
-static struct ahash_alg sha256_mb_async_alg = {
-        .init           = sha256_mb_async_init,
-        .update         = sha256_mb_async_update,
-        .final          = sha256_mb_async_final,
-        .finup          = sha256_mb_async_finup,
-        .export         = sha256_mb_async_export,
-        .import         = sha256_mb_async_import,
-        .digest         = sha256_mb_async_digest,
-        .halg = {
-                .digestsize     = SHA256_DIGEST_SIZE,
-                .statesize      = sizeof(struct sha256_hash_ctx),
-                .base = {
-                        .cra_name               = "sha256",
-                        .cra_driver_name        = "sha256_mb",
-                        /*
-                         * Low priority, since with few concurrent hash requests
-                         * this is extremely slow due to the flush delay.  Users
-                         * whose workloads would benefit from this can request
-                         * it explicitly by driver name, or can increase its
-                         * priority at runtime using NETLINK_CRYPTO.
-                         */
-                        .cra_priority           = 50,
-                        .cra_flags              = CRYPTO_ALG_ASYNC,
-                        .cra_blocksize          = SHA256_BLOCK_SIZE,
-                        .cra_module             = THIS_MODULE,
-                        .cra_list               = LIST_HEAD_INIT
-                                (sha256_mb_async_alg.halg.base.cra_list),
-                        .cra_init               = sha256_mb_async_init_tfm,
-                        .cra_exit               = sha256_mb_async_exit_tfm,
-                        .cra_ctxsize            = sizeof(struct sha256_mb_ctx),
-                        .cra_alignmask          = 0,
-                },
-        },
-};
-static unsigned long sha256_mb_flusher(struct mcryptd_alg_cstate *cstate)
-{
-        struct mcryptd_hash_request_ctx *rctx;
-        unsigned long cur_time;
-        unsigned long next_flush = 0;
-        struct sha256_hash_ctx *sha_ctx;
-        cur_time = jiffies;
-        while (!list_empty(&cstate->work_list)) {
-                rctx = list_entry(cstate->work_list.next,
-                                struct mcryptd_hash_request_ctx, waiter);
-                if (time_before(cur_time, rctx->tag.expire))
-                        break;
-                kernel_fpu_begin();
-                sha_ctx = (struct sha256_hash_ctx *)
-                                        sha256_ctx_mgr_flush(cstate->mgr);
-                kernel_fpu_end();
-                if (!sha_ctx) {
-                        pr_err("sha256_mb error: nothing got"
-                                        " flushed for non-empty list\n");
-                        break;
-                }
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                sha_finish_walk(&rctx, cstate, true);
-                sha_complete_job(rctx, cstate, 0);
-        }
-        if (!list_empty(&cstate->work_list)) {
-                rctx = list_entry(cstate->work_list.next,
-                                struct mcryptd_hash_request_ctx, waiter);
-                /* get the hash context and then flush time */
-                next_flush = rctx->tag.expire;
-                mcryptd_arm_flusher(cstate, get_delay(next_flush));
-        }
-        return next_flush;
-}
-static int __init sha256_mb_mod_init(void)
-{
-        int cpu;
-        int err;
-        struct mcryptd_alg_cstate *cpu_state;
-        /* check for dependent cpu features */
-        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
-            !boot_cpu_has(X86_FEATURE_BMI2))
-                return -ENODEV;
-        /* initialize multibuffer structures */
-        sha256_mb_alg_state.alg_cstate = alloc_percpu
-                                                (struct mcryptd_alg_cstate);
-        sha256_job_mgr_init = sha256_mb_mgr_init_avx2;
-        sha256_job_mgr_submit = sha256_mb_mgr_submit_avx2;
-        sha256_job_mgr_flush = sha256_mb_mgr_flush_avx2;
-        sha256_job_mgr_get_comp_job = sha256_mb_mgr_get_comp_job_avx2;
-        if (!sha256_mb_alg_state.alg_cstate)
-                return -ENOMEM;
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
-                cpu_state->next_flush = 0;
-                cpu_state->next_seq_num = 0;
-                cpu_state->flusher_engaged = false;
-                INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
-                cpu_state->cpu = cpu;
-                cpu_state->alg_state = &sha256_mb_alg_state;
-                cpu_state->mgr = kzalloc(sizeof(struct sha256_ctx_mgr),
-                                        GFP_KERNEL);
-                if (!cpu_state->mgr)
-                        goto err2;
-                sha256_ctx_mgr_init(cpu_state->mgr);
-                INIT_LIST_HEAD(&cpu_state->work_list);
-                spin_lock_init(&cpu_state->work_lock);
-        }
-        sha256_mb_alg_state.flusher = &sha256_mb_flusher;
-        err = crypto_register_ahash(&sha256_mb_areq_alg);
-        if (err)
-                goto err2;
-        err = crypto_register_ahash(&sha256_mb_async_alg);
-        if (err)
-                goto err1;
-        return 0;
-err1:
-        crypto_unregister_ahash(&sha256_mb_areq_alg);
-err2:
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
-                kfree(cpu_state->mgr);
-        }
-        free_percpu(sha256_mb_alg_state.alg_cstate);
-        return -ENODEV;
-}
-static void __exit sha256_mb_mod_fini(void)
-{
-        int cpu;
-        struct mcryptd_alg_cstate *cpu_state;
-        crypto_unregister_ahash(&sha256_mb_async_alg);
-        crypto_unregister_ahash(&sha256_mb_areq_alg);
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
-                kfree(cpu_state->mgr);
-        }
-        free_percpu(sha256_mb_alg_state.alg_cstate);
-}
-module_init(sha256_mb_mod_init);
-module_exit(sha256_mb_mod_fini);
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, multi buffer accelerated");
-MODULE_ALIAS_CRYPTO("sha256");
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h
deleted file mode 100644
index 7c432543dc7f..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Header file for multi buffer SHA256 context
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef _SHA_MB_CTX_INTERNAL_H
-#define _SHA_MB_CTX_INTERNAL_H
-#include "sha256_mb_mgr.h"
-#define HASH_UPDATE          0x00
-#define HASH_LAST            0x01
-#define HASH_DONE            0x02
-#define HASH_FINAL           0x04
-#define HASH_CTX_STS_IDLE       0x00
-#define HASH_CTX_STS_PROCESSING 0x01
-#define HASH_CTX_STS_LAST       0x02
-#define HASH_CTX_STS_COMPLETE   0x04
-enum hash_ctx_error {
-        HASH_CTX_ERROR_NONE               =  0,
-        HASH_CTX_ERROR_INVALID_FLAGS      = -1,
-        HASH_CTX_ERROR_ALREADY_PROCESSING = -2,
-        HASH_CTX_ERROR_ALREADY_COMPLETED  = -3,
-#ifdef HASH_CTX_DEBUG
-        HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4,
-#endif
-};
-#define hash_ctx_user_data(ctx)  ((ctx)->user_data)
-#define hash_ctx_digest(ctx)     ((ctx)->job.result_digest)
-#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING)
-#define hash_ctx_complete(ctx)   ((ctx)->status == HASH_CTX_STS_COMPLETE)
-#define hash_ctx_status(ctx)     ((ctx)->status)
-#define hash_ctx_error(ctx)      ((ctx)->error)
-#define hash_ctx_init(ctx) \
-        do { \
-                (ctx)->error = HASH_CTX_ERROR_NONE; \
-                (ctx)->status = HASH_CTX_STS_COMPLETE; \
-        } while (0)
-/* Hash Constants and Typedefs */
-#define SHA256_DIGEST_LENGTH        8
-#define SHA256_LOG2_BLOCK_SIZE        6
-#define SHA256_PADLENGTHFIELD_SIZE    8
-#ifdef SHA_MB_DEBUG
-#define assert(expr) \
-do { \
-        if (unlikely(!(expr))) { \
-                printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
-                #expr, __FILE__, __func__, __LINE__); \
-        } \
-} while (0)
-#else
-#define assert(expr) do {} while (0)
-#endif
-struct sha256_ctx_mgr {
-        struct sha256_mb_mgr mgr;
-};
-/* typedef struct sha256_ctx_mgr sha256_ctx_mgr; */
-struct sha256_hash_ctx {
-        /* Must be at struct offset 0 */
-        struct job_sha256       job;
-        /* status flag */
-        int status;
-        /* error flag */
-        int error;
-        uint64_t        total_length;
-        const void      *incoming_buffer;
-        uint32_t        incoming_buffer_length;
-        uint8_t         partial_block_buffer[SHA256_BLOCK_SIZE * 2];
-        uint32_t        partial_block_buffer_length;
-        void            *user_data;
-};
-#endif
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h b/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h
deleted file mode 100644
index b01ae408c56d..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Header file for multi buffer SHA256 algorithm manager
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef __SHA_MB_MGR_H
-#define __SHA_MB_MGR_H
-#include <linux/types.h>
-#define NUM_SHA256_DIGEST_WORDS 8
-enum job_sts {  STS_UNKNOWN = 0,
-                STS_BEING_PROCESSED = 1,
-                STS_COMPLETED = 2,
-                STS_INTERNAL_ERROR = 3,
-                STS_ERROR = 4
-};
-struct job_sha256 {
-        u8      *buffer;
-        u32     len;
-        u32     result_digest[NUM_SHA256_DIGEST_WORDS] __aligned(32);
-        enum    job_sts status;
-        void    *user_data;
-};
-/* SHA256 out-of-order scheduler */
-/* typedef uint32_t sha8_digest_array[8][8]; */
-struct sha256_args_x8 {
-        uint32_t        digest[8][8];
-        uint8_t         *data_ptr[8];
-};
-struct sha256_lane_data {
-        struct job_sha256 *job_in_lane;
-};
-struct sha256_mb_mgr {
-        struct sha256_args_x8 args;
-        uint32_t lens[8];
-        /* each byte is index (0...7) of unused lanes */
-        uint64_t unused_lanes;
-        /* byte 4 is set to FF as a flag */
-        struct sha256_lane_data ldata[8];
-};
-#define SHA256_MB_MGR_NUM_LANES_AVX2 8
-void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state);
-struct job_sha256 *sha256_mb_mgr_submit_avx2(struct sha256_mb_mgr *state,
-                                         struct job_sha256 *job);
-struct job_sha256 *sha256_mb_mgr_flush_avx2(struct sha256_mb_mgr *state);
-struct job_sha256 *sha256_mb_mgr_get_comp_job_avx2(struct sha256_mb_mgr *state);
-#endif
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S
deleted file mode 100644
index 5c377bac21d0..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Header file for multi buffer SHA256 algorithm data structure
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- *     Megha Dey <megha.dey@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-# Macros for defining data structures
-# Usage example
-#START_FIELDS   # JOB_AES
-###     name            size    align
-#FIELD  _plaintext,     8,      8       # pointer to plaintext
-#FIELD  _ciphertext,    8,      8       # pointer to ciphertext
-#FIELD  _IV,            16,     8       # IV
-#FIELD  _keys,          8,      8       # pointer to keys
-#FIELD  _len,           4,      4       # length in bytes
-#FIELD  _status,        4,      4       # status enumeration
-#FIELD  _user_data,     8,      8       # pointer to user data
-#UNION  _union,         size1,  align1, \
-#                       size2,  align2, \
-#                       size3,  align3, \
-#                       ...
-#END_FIELDS
-#%assign _JOB_AES_size  _FIELD_OFFSET
-#%assign _JOB_AES_align _STRUCT_ALIGN
-#########################################################################
-# Alternate "struc-like" syntax:
-#       STRUCT job_aes2
-#       RES_Q   .plaintext,     1
-#       RES_Q   .ciphertext,    1
-#       RES_DQ  .IV,            1
-#       RES_B   .nested,        _JOB_AES_SIZE, _JOB_AES_ALIGN
-#       RES_U   .union,         size1, align1, \
-#                               size2, align2, \
-#                               ...
-#       ENDSTRUCT
-#       # Following only needed if nesting
-#       %assign job_aes2_size   _FIELD_OFFSET
-#       %assign job_aes2_align  _STRUCT_ALIGN
-#
-# RES_* macros take a name, a count and an optional alignment.
-# The count in in terms of the base size of the macro, and the
-# default alignment is the base size.
-# The macros are:
-# Macro    Base size
-# RES_B     1
-# RES_W     2
-# RES_D     4
-# RES_Q     8
-# RES_DQ   16
-# RES_Y    32
-# RES_Z    64
-#
-# RES_U defines a union. It's arguments are a name and two or more
-# pairs of "size, alignment"
-#
-# The two assigns are only needed if this structure is being nested
-# within another. Even if the assigns are not done, one can still use
-# STRUCT_NAME_size as the size of the structure.
-#
-# Note that for nesting, you still need to assign to STRUCT_NAME_size.
-#
-# The differences between this and using "struc" directly are that each
-# type is implicitly aligned to its natural length (although this can be
-# over-ridden with an explicit third parameter), and that the structure
-# is padded at the end to its overall alignment.
-#
-#########################################################################
-#ifndef _DATASTRUCT_ASM_
-#define _DATASTRUCT_ASM_
-#define SZ8                     8*SHA256_DIGEST_WORD_SIZE
-#define ROUNDS                  64*SZ8
-#define PTR_SZ                  8
-#define SHA256_DIGEST_WORD_SIZE 4
-#define MAX_SHA256_LANES        8
-#define SHA256_DIGEST_WORDS 8
-#define SHA256_DIGEST_ROW_SIZE  (MAX_SHA256_LANES * SHA256_DIGEST_WORD_SIZE)
-#define SHA256_DIGEST_SIZE      (SHA256_DIGEST_ROW_SIZE * SHA256_DIGEST_WORDS)
-#define SHA256_BLK_SZ           64
-# START_FIELDS
-.macro START_FIELDS
- _FIELD_OFFSET = 0
- _STRUCT_ALIGN = 0
-.endm
-# FIELD name size align
-.macro FIELD name size align
- _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
- \name  = _FIELD_OFFSET
- _FIELD_OFFSET = _FIELD_OFFSET + (\size)
-.if (\align > _STRUCT_ALIGN)
- _STRUCT_ALIGN = \align
-.endif
-.endm
-# END_FIELDS
-.macro END_FIELDS
- _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
-.endm
-########################################################################
-.macro STRUCT p1
-START_FIELDS
-.struc \p1
-.endm
-.macro ENDSTRUCT
- tmp = _FIELD_OFFSET
- END_FIELDS
- tmp = (_FIELD_OFFSET - %%tmp)
-.if (tmp > 0)
-        .lcomm  tmp
-.endif
-.endstruc
-.endm
-## RES_int name size align
-.macro RES_int p1 p2 p3
- name = \p1
- size = \p2
- align = .\p3
- _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
-.align align
-.lcomm name size
- _FIELD_OFFSET = _FIELD_OFFSET + (size)
-.if (align > _STRUCT_ALIGN)
- _STRUCT_ALIGN = align
-.endif
-.endm
-# macro RES_B name, size [, align]
-.macro RES_B _name, _size, _align=1
-RES_int _name _size _align
-.endm
-# macro RES_W name, size [, align]
-.macro RES_W _name, _size, _align=2
-RES_int _name 2*(_size) _align
-.endm
-# macro RES_D name, size [, align]
-.macro RES_D _name, _size, _align=4
-RES_int _name 4*(_size) _align
-.endm
-# macro RES_Q name, size [, align]
-.macro RES_Q _name, _size, _align=8
-RES_int _name 8*(_size) _align
-.endm
-# macro RES_DQ name, size [, align]
-.macro RES_DQ _name, _size, _align=16
-RES_int _name 16*(_size) _align
-.endm
-# macro RES_Y name, size [, align]
-.macro RES_Y _name, _size, _align=32
-RES_int _name 32*(_size) _align
-.endm
-# macro RES_Z name, size [, align]
-.macro RES_Z _name, _size, _align=64
-RES_int _name 64*(_size) _align
-.endm
-#endif
-########################################################################
-#### Define SHA256 Out Of Order Data Structures
-########################################################################
-START_FIELDS    # LANE_DATA
-###     name            size    align
-FIELD   _job_in_lane,   8,      8       # pointer to job object
-END_FIELDS
- _LANE_DATA_size = _FIELD_OFFSET
- _LANE_DATA_align = _STRUCT_ALIGN
-########################################################################
-START_FIELDS    # SHA256_ARGS_X4
-###     name            size    align
-FIELD   _digest,        4*8*8,  4       # transposed digest
-FIELD   _data_ptr,      8*8,    8       # array of pointers to data
-END_FIELDS
- _SHA256_ARGS_X4_size  =  _FIELD_OFFSET
- _SHA256_ARGS_X4_align = _STRUCT_ALIGN
- _SHA256_ARGS_X8_size  =        _FIELD_OFFSET
- _SHA256_ARGS_X8_align =        _STRUCT_ALIGN
-#######################################################################
-START_FIELDS    # MB_MGR
-###     name            size    align
-FIELD   _args,          _SHA256_ARGS_X4_size, _SHA256_ARGS_X4_align
-FIELD   _lens,          4*8,    8
-FIELD   _unused_lanes,  8,      8
-FIELD   _ldata,         _LANE_DATA_size*8, _LANE_DATA_align
-END_FIELDS
- _MB_MGR_size  =  _FIELD_OFFSET
- _MB_MGR_align =  _STRUCT_ALIGN
-_args_digest   =     _args + _digest
-_args_data_ptr =     _args + _data_ptr
-#######################################################################
-START_FIELDS    #STACK_FRAME
-###     name            size    align
-FIELD   _data,          16*SZ8,   1       # transposed digest
-FIELD   _digest,         8*SZ8,   1       # array of pointers to data
-FIELD   _ytmp,           4*SZ8,   1
-FIELD   _rsp,            8,       1
-END_FIELDS
- _STACK_FRAME_size  =  _FIELD_OFFSET
- _STACK_FRAME_align =  _STRUCT_ALIGN
-#######################################################################
-########################################################################
-#### Define constants
-########################################################################
-#define STS_UNKNOWN             0
-#define STS_BEING_PROCESSED     1
-#define STS_COMPLETED           2
-########################################################################
-#### Define JOB_SHA256 structure
-########################################################################
-START_FIELDS    # JOB_SHA256
-###     name                            size    align
-FIELD   _buffer,                        8,      8       # pointer to buffer
-FIELD   _len,                           8,      8       # length in bytes
-FIELD   _result_digest,                 8*4,    32      # Digest (output)
-FIELD   _status,                        4,      4
-FIELD   _user_data,                     8,      8
-END_FIELDS
- _JOB_SHA256_size = _FIELD_OFFSET
- _JOB_SHA256_align = _STRUCT_ALIGN
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S
deleted file mode 100644
index d2364c55bbde..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Flush routine for SHA256 multibuffer
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-#include "sha256_mb_mgr_datastruct.S"
-.extern sha256_x8_avx2
-#LINUX register definitions
-#define arg1    %rdi
-#define arg2    %rsi
-# Common register definitions
-#define state   arg1
-#define job     arg2
-#define len2    arg2
-# idx must be a register not clobberred by sha1_mult
-#define idx             %r8
-#define DWORD_idx       %r8d
-#define unused_lanes    %rbx
-#define lane_data       %rbx
-#define tmp2            %rbx
-#define tmp2_w          %ebx
-#define job_rax         %rax
-#define tmp1            %rax
-#define size_offset     %rax
-#define tmp             %rax
-#define start_offset    %rax
-#define tmp3            %arg1
-#define extra_blocks    %arg2
-#define p               %arg2
-.macro LABEL prefix n
-\prefix\n\():
-.endm
-.macro JNE_SKIP i
-jne     skip_\i
-.endm
-.altmacro
-.macro SET_OFFSET _offset
-offset = \_offset
-.endm
-.noaltmacro
-# JOB_SHA256* sha256_mb_mgr_flush_avx2(MB_MGR *state)
-# arg 1 : rcx : state
-ENTRY(sha256_mb_mgr_flush_avx2)
-        FRAME_BEGIN
-        push    %rbx
-        # If bit (32+3) is set, then all lanes are empty
-        mov     _unused_lanes(state), unused_lanes
-        bt      $32+3, unused_lanes
-        jc      return_null
-        # find a lane with a non-null job
-        xor     idx, idx
-        offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  one(%rip), idx
-        offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  two(%rip), idx
-        offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  three(%rip), idx
-        offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  four(%rip), idx
-        offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  five(%rip), idx
-        offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  six(%rip), idx
-        offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  seven(%rip), idx
-        # copy idx to empty lanes
-copy_lane_data:
-        offset =  (_args + _data_ptr)
-        mov     offset(state,idx,8), tmp
-        I = 0
-.rep 8
-        offset = (_ldata + I * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-.altmacro
-        JNE_SKIP %I
-        offset =  (_args + _data_ptr + 8*I)
-        mov     tmp, offset(state)
-        offset =  (_lens + 4*I)
-        movl    $0xFFFFFFFF, offset(state)
-LABEL skip_ %I
-        I = (I+1)
-.noaltmacro
-.endr
-        # Find min length
-        vmovdqu _lens+0*16(state), %xmm0
-        vmovdqu _lens+1*16(state), %xmm1
-        vpminud %xmm1, %xmm0, %xmm2             # xmm2 has {D,C,B,A}
-        vpalignr $8, %xmm2, %xmm3, %xmm3        # xmm3 has {x,x,D,C}
-        vpminud %xmm3, %xmm2, %xmm2             # xmm2 has {x,x,E,F}
-        vpalignr $4, %xmm2, %xmm3, %xmm3        # xmm3 has {x,x,x,E}
-        vpminud %xmm3, %xmm2, %xmm2             # xmm2 has min val in low dword
-        vmovd   %xmm2, DWORD_idx
-        mov     idx, len2
-        and     $0xF, idx
-        shr     $4, len2
-        jz      len_is_0
-        vpand   clear_low_nibble(%rip), %xmm2, %xmm2
-        vpshufd $0, %xmm2, %xmm2
-        vpsubd  %xmm2, %xmm0, %xmm0
-        vpsubd  %xmm2, %xmm1, %xmm1
-        vmovdqu %xmm0, _lens+0*16(state)
-        vmovdqu %xmm1, _lens+1*16(state)
-        # "state" and "args" are the same address, arg1
-        # len is arg2
-        call    sha256_x8_avx2
-        # state and idx are intact
-len_is_0:
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        movq    $0, _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        mov     _unused_lanes(state), unused_lanes
-        shl     $4, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens(state,idx,4)
-        vmovd   _args_digest(state , idx, 4) , %xmm0
-        vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
-        vmovd   _args_digest+4*32(state, idx, 4), %xmm1
-        vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1
-        vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1
-        vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1
-        vmovdqu %xmm0, _result_digest(job_rax)
-        offset =  (_result_digest + 1*16)
-        vmovdqu %xmm1, offset(job_rax)
-return:
-        pop     %rbx
-        FRAME_END
-        ret
-return_null:
-        xor     job_rax, job_rax
-        jmp     return
-ENDPROC(sha256_mb_mgr_flush_avx2)
-##############################################################################
-.align 16
-ENTRY(sha256_mb_mgr_get_comp_job_avx2)
-        push    %rbx
-        ## if bit 32+3 is set, then all lanes are empty
-        mov     _unused_lanes(state), unused_lanes
-        bt      $(32+3), unused_lanes
-        jc      .return_null
-        # Find min length
-        vmovdqu _lens(state), %xmm0
-        vmovdqu _lens+1*16(state), %xmm1
-        vpminud %xmm1, %xmm0, %xmm2             # xmm2 has {D,C,B,A}
-        vpalignr $8, %xmm2, %xmm3, %xmm3        # xmm3 has {x,x,D,C}
-        vpminud %xmm3, %xmm2, %xmm2             # xmm2 has {x,x,E,F}
-        vpalignr $4, %xmm2, %xmm3, %xmm3        # xmm3 has {x,x,x,E}
-        vpminud %xmm3, %xmm2, %xmm2             # xmm2 has min val in low dword
-        vmovd   %xmm2, DWORD_idx
-        test    $~0xF, idx
-        jnz     .return_null
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        movq    $0,  _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        mov     _unused_lanes(state), unused_lanes
-        shl     $4, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens(state,  idx, 4)
-        vmovd   _args_digest(state, idx, 4), %xmm0
-        vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
-        vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
-        vmovd   _args_digest+4*32(state, idx, 4), %xmm1
-        vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1
-        vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1
-        vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1
-        vmovdqu %xmm0, _result_digest(job_rax)
-        offset =  (_result_digest + 1*16)
-        vmovdqu %xmm1, offset(job_rax)
-        pop     %rbx
-        ret
-.return_null:
-        xor     job_rax, job_rax
-        pop     %rbx
-        ret
-ENDPROC(sha256_mb_mgr_get_comp_job_avx2)
-.section        .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
-.align 16
-clear_low_nibble:
-.octa   0x000000000000000000000000FFFFFFF0
-.section        .rodata.cst8, "aM", @progbits, 8
-.align 8
-one:
-.quad   1
-two:
-.quad   2
-three:
-.quad   3
-four:
-.quad   4
-five:
-.quad   5
-six:
-.quad   6
-seven:
-.quad  7
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c
deleted file mode 100644
index b0c498371e67..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Initialization code for multi buffer SHA256 algorithm for AVX2
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include "sha256_mb_mgr.h"
-void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state)
-{
-        unsigned int j;
-        state->unused_lanes = 0xF76543210ULL;
-        for (j = 0; j < 8; j++) {
-                state->lens[j] = 0xFFFFFFFF;
-                state->ldata[j].job_in_lane = NULL;
-        }
-}
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S
deleted file mode 100644
index b36ae7454084..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Buffer submit code for multi buffer SHA256 algorithm
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-#include "sha256_mb_mgr_datastruct.S"
-.extern sha256_x8_avx2
-# LINUX register definitions
-arg1            = %rdi
-arg2            = %rsi
-size_offset     = %rcx
-tmp2            = %rcx
-extra_blocks    = %rdx
-# Common definitions
-#define state   arg1
-#define job     %rsi
-#define len2    arg2
-#define p2      arg2
-# idx must be a register not clobberred by sha1_x8_avx2
-idx             = %r8
-DWORD_idx       = %r8d
-last_len        = %r8
-p               = %r11
-start_offset    = %r11
-unused_lanes    = %rbx
-BYTE_unused_lanes = %bl
-job_rax         = %rax
-len             = %rax
-DWORD_len       = %eax
-lane            = %r12
-tmp3            = %r12
-tmp             = %r9
-DWORD_tmp       = %r9d
-lane_data       = %r10
-# JOB* sha256_mb_mgr_submit_avx2(MB_MGR *state, JOB_SHA256 *job)
-# arg 1 : rcx : state
-# arg 2 : rdx : job
-ENTRY(sha256_mb_mgr_submit_avx2)
-        FRAME_BEGIN
-        push    %rbx
-        push    %r12
-        mov     _unused_lanes(state), unused_lanes
-        mov     unused_lanes, lane
-        and     $0xF, lane
-        shr     $4, unused_lanes
-        imul    $_LANE_DATA_size, lane, lane_data
-        movl    $STS_BEING_PROCESSED, _status(job)
-        lea     _ldata(state, lane_data), lane_data
-        mov     unused_lanes, _unused_lanes(state)
-        movl    _len(job),  DWORD_len
-        mov     job, _job_in_lane(lane_data)
-        shl     $4, len
-        or      lane, len
-        movl    DWORD_len,  _lens(state , lane, 4)
-        # Load digest words from result_digest
-        vmovdqu _result_digest(job), %xmm0
-        vmovdqu _result_digest+1*16(job), %xmm1
-        vmovd   %xmm0, _args_digest(state, lane, 4)
-        vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4)
-        vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4)
-        vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4)
-        vmovd   %xmm1, _args_digest+4*32(state , lane, 4)
-        vpextrd $1, %xmm1, _args_digest+5*32(state , lane, 4)
-        vpextrd $2, %xmm1, _args_digest+6*32(state , lane, 4)
-        vpextrd $3, %xmm1, _args_digest+7*32(state , lane, 4)
-        mov     _buffer(job), p
-        mov     p, _args_data_ptr(state, lane, 8)
-        cmp     $0xF, unused_lanes
-        jne     return_null
-start_loop:
-        # Find min length
-        vmovdqa _lens(state), %xmm0
-        vmovdqa _lens+1*16(state), %xmm1
-        vpminud %xmm1, %xmm0, %xmm2             # xmm2 has {D,C,B,A}
-        vpalignr $8, %xmm2, %xmm3, %xmm3        # xmm3 has {x,x,D,C}
-        vpminud %xmm3, %xmm2, %xmm2             # xmm2 has {x,x,E,F}
-        vpalignr $4, %xmm2, %xmm3, %xmm3        # xmm3 has {x,x,x,E}
-        vpminud %xmm3, %xmm2, %xmm2             # xmm2 has min val in low dword
-        vmovd   %xmm2, DWORD_idx
-        mov     idx, len2
-        and     $0xF, idx
-        shr     $4, len2
-        jz      len_is_0
-        vpand   clear_low_nibble(%rip), %xmm2, %xmm2
-        vpshufd $0, %xmm2, %xmm2
-        vpsubd  %xmm2, %xmm0, %xmm0
-        vpsubd  %xmm2, %xmm1, %xmm1
-        vmovdqa %xmm0, _lens + 0*16(state)
-        vmovdqa %xmm1, _lens + 1*16(state)
-        # "state" and "args" are the same address, arg1
-        # len is arg2
-        call    sha256_x8_avx2
-        # state and idx are intact
-len_is_0:
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        mov     _unused_lanes(state), unused_lanes
-        movq    $0, _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        shl     $4, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens(state,idx,4)
-        vmovd   _args_digest(state, idx, 4), %xmm0
-        vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0
-        vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0
-        vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0
-        vmovd   _args_digest+4*32(state, idx, 4), %xmm1
-        vpinsrd $1, _args_digest+5*32(state , idx, 4), %xmm1, %xmm1
-        vpinsrd $2, _args_digest+6*32(state , idx, 4), %xmm1, %xmm1
-        vpinsrd $3, _args_digest+7*32(state , idx, 4), %xmm1, %xmm1
-        vmovdqu %xmm0, _result_digest(job_rax)
-        vmovdqu %xmm1, _result_digest+1*16(job_rax)
-return:
-        pop     %r12
-        pop     %rbx
-        FRAME_END
-        ret
-return_null:
-        xor     job_rax, job_rax
-        jmp     return
-ENDPROC(sha256_mb_mgr_submit_avx2)
-.section        .rodata.cst16.clear_low_nibble, "aM", @progbits, 16
-.align 16
-clear_low_nibble:
-        .octa   0x000000000000000000000000FFFFFFF0
diff --git a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S
deleted file mode 100644
index 1687c80c5995..000000000000
--- a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
- * Multi-buffer SHA256 algorithm hash compute routine
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include "sha256_mb_mgr_datastruct.S"
-## code to compute oct SHA256 using SSE-256
-## outer calling routine takes care of save and restore of XMM registers
-## Logic designed/laid out by JDG
-## Function clobbers: rax, rcx, rdx,   rbx, rsi, rdi, r9-r15; %ymm0-15
-## Linux clobbers:    rax rbx rcx rdx rsi            r9 r10 r11 r12 r13 r14 r15
-## Linux preserves:                       rdi rbp r8
-##
-## clobbers %ymm0-15
-arg1 = %rdi
-arg2 = %rsi
-reg3 = %rcx
-reg4 = %rdx
-# Common definitions
-STATE = arg1
-INP_SIZE = arg2
-IDX = %rax
-ROUND = %rbx
-TBL = reg3
-inp0 = %r9
-inp1 = %r10
-inp2 = %r11
-inp3 = %r12
-inp4 = %r13
-inp5 = %r14
-inp6 = %r15
-inp7 = reg4
-a = %ymm0
-b = %ymm1
-c = %ymm2
-d = %ymm3
-e = %ymm4
-f = %ymm5
-g = %ymm6
-h = %ymm7
-T1 = %ymm8
-a0 = %ymm12
-a1 = %ymm13
-a2 = %ymm14
-TMP = %ymm15
-TMP0 = %ymm6
-TMP1 = %ymm7
-TT0 = %ymm8
-TT1 = %ymm9
-TT2 = %ymm10
-TT3 = %ymm11
-TT4 = %ymm12
-TT5 = %ymm13
-TT6 = %ymm14
-TT7 = %ymm15
-# Define stack usage
-# Assume stack aligned to 32 bytes before call
-# Therefore FRAMESZ mod 32 must be 32-8 = 24
-#define FRAMESZ 0x388
-#define VMOVPS  vmovups
-# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1
-# "transpose" data in {r0...r7} using temps {t0...t1}
-# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
-# r0 = {a7 a6 a5 a4   a3 a2 a1 a0}
-# r1 = {b7 b6 b5 b4   b3 b2 b1 b0}
-# r2 = {c7 c6 c5 c4   c3 c2 c1 c0}
-# r3 = {d7 d6 d5 d4   d3 d2 d1 d0}
-# r4 = {e7 e6 e5 e4   e3 e2 e1 e0}
-# r5 = {f7 f6 f5 f4   f3 f2 f1 f0}
-# r6 = {g7 g6 g5 g4   g3 g2 g1 g0}
-# r7 = {h7 h6 h5 h4   h3 h2 h1 h0}
-#
-# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
-# r0 = {h0 g0 f0 e0   d0 c0 b0 a0}
-# r1 = {h1 g1 f1 e1   d1 c1 b1 a1}
-# r2 = {h2 g2 f2 e2   d2 c2 b2 a2}
-# r3 = {h3 g3 f3 e3   d3 c3 b3 a3}
-# r4 = {h4 g4 f4 e4   d4 c4 b4 a4}
-# r5 = {h5 g5 f5 e5   d5 c5 b5 a5}
-# r6 = {h6 g6 f6 e6   d6 c6 b6 a6}
-# r7 = {h7 g7 f7 e7   d7 c7 b7 a7}
-#
-.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1
-        # process top half (r0..r3) {a...d}
-        vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4   b1 b0 a1 a0}
-        vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6   b3 b2 a3 a2}
-        vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4   d1 d0 c1 c0}
-        vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6   d3 d2 c3 c2}
-        vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5   d1 c1 b1 a1}
-        vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6   d2 c2 b2 a2}
-        vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7   d3 c3 b3 a3}
-        vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4   d0 c0 b0 a0}
-        # use r2 in place of t0
-        # process bottom half (r4..r7) {e...h}
-        vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4   f1 f0 e1 e0}
-        vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6   f3 f2 e3 e2}
-        vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4   h1 h0 g1 g0}
-        vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6   h3 h2 g3 g2}
-        vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5   h1 g1 f1 e1}
-        vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6   h2 g2 f2 e2}
-        vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7   h3 g3 f3 e3}
-        vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4   h0 g0 f0 e0}
-        vperm2f128      $0x13, \r1, \r5, \r6  # h6...a6
-        vperm2f128      $0x02, \r1, \r5, \r2  # h2...a2
-        vperm2f128      $0x13, \r3, \r7, \r5  # h5...a5
-        vperm2f128      $0x02, \r3, \r7, \r1  # h1...a1
-        vperm2f128      $0x13, \r0, \r4, \r7  # h7...a7
-        vperm2f128      $0x02, \r0, \r4, \r3  # h3...a3
-        vperm2f128      $0x13, \t0, \t1, \r4  # h4...a4
-        vperm2f128      $0x02, \t0, \t1, \r0  # h0...a0
-.endm
-.macro ROTATE_ARGS
-TMP_ = h
-h = g
-g = f
-f = e
-e = d
-d = c
-c = b
-b = a
-a = TMP_
-.endm
-.macro _PRORD reg imm tmp
-        vpslld  $(32-\imm),\reg,\tmp
-        vpsrld  $\imm,\reg, \reg
-        vpor    \tmp,\reg, \reg
-.endm
-# PRORD_nd reg, imm, tmp, src
-.macro _PRORD_nd reg imm tmp src
-        vpslld  $(32-\imm), \src, \tmp
-        vpsrld  $\imm, \src, \reg
-        vpor    \tmp, \reg, \reg
-.endm
-# PRORD dst/src, amt
-.macro PRORD reg imm
-        _PRORD  \reg,\imm,TMP
-.endm
-# PRORD_nd dst, src, amt
-.macro PRORD_nd reg tmp imm
-        _PRORD_nd       \reg, \imm, TMP, \tmp
-.endm
-# arguments passed implicitly in preprocessor symbols i, a...h
-.macro ROUND_00_15 _T1 i
-        PRORD_nd        a0,e,5  # sig1: a0 = (e >> 5)
-        vpxor   g, f, a2        # ch: a2 = f^g
-        vpand   e,a2, a2        # ch: a2 = (f^g)&e
-        vpxor   g, a2, a2       # a2 = ch
-        PRORD_nd        a1,e,25 # sig1: a1 = (e >> 25)
-        vmovdqu \_T1,(SZ8*(\i & 0xf))(%rsp)
-        vpaddd  (TBL,ROUND,1), \_T1, \_T1       # T1 = W + K
-        vpxor   e,a0, a0        # sig1: a0 = e ^ (e >> 5)
-        PRORD   a0, 6           # sig1: a0 = (e >> 6) ^ (e >> 11)
-        vpaddd  a2, h, h        # h = h + ch
-        PRORD_nd        a2,a,11 # sig0: a2 = (a >> 11)
-        vpaddd  \_T1,h, h       # h = h + ch + W + K
-        vpxor   a1, a0, a0      # a0 = sigma1
-        PRORD_nd        a1,a,22 # sig0: a1 = (a >> 22)
-        vpxor   c, a, \_T1      # maj: T1 = a^c
-        add     $SZ8, ROUND     # ROUND++
-        vpand   b, \_T1, \_T1   # maj: T1 = (a^c)&b
-        vpaddd  a0, h, h
-        vpaddd  h, d, d
-        vpxor   a, a2, a2       # sig0: a2 = a ^ (a >> 11)
-        PRORD   a2,2            # sig0: a2 = (a >> 2) ^ (a >> 13)
-        vpxor   a1, a2, a2      # a2 = sig0
-        vpand   c, a, a1        # maj: a1 = a&c
-        vpor    \_T1, a1, a1    # a1 = maj
-        vpaddd  a1, h, h        # h = h + ch + W + K + maj
-        vpaddd  a2, h, h        # h = h + ch + W + K + maj + sigma0
-        ROTATE_ARGS
-.endm
-# arguments passed implicitly in preprocessor symbols i, a...h
-.macro ROUND_16_XX _T1 i
-        vmovdqu (SZ8*((\i-15)&0xf))(%rsp), \_T1
-        vmovdqu (SZ8*((\i-2)&0xf))(%rsp), a1
-        vmovdqu \_T1, a0
-        PRORD   \_T1,11
-        vmovdqu a1, a2
-        PRORD   a1,2
-        vpxor   a0, \_T1, \_T1
-        PRORD   \_T1, 7
-        vpxor   a2, a1, a1
-        PRORD   a1, 17
-        vpsrld  $3, a0, a0
-        vpxor   a0, \_T1, \_T1
-        vpsrld  $10, a2, a2
-        vpxor   a2, a1, a1
-        vpaddd  (SZ8*((\i-16)&0xf))(%rsp), \_T1, \_T1
-        vpaddd  (SZ8*((\i-7)&0xf))(%rsp), a1, a1
-        vpaddd  a1, \_T1, \_T1
-        ROUND_00_15 \_T1,\i
-.endm
-# SHA256_ARGS:
-#   UINT128 digest[8];  // transposed digests
-#   UINT8  *data_ptr[4];
-# void sha256_x8_avx2(SHA256_ARGS *args, UINT64 bytes);
-# arg 1 : STATE : pointer to array of pointers to input data
-# arg 2 : INP_SIZE  : size of input in blocks
-        # general registers preserved in outer calling routine
-        # outer calling routine saves all the XMM registers
-        # save rsp, allocate 32-byte aligned for local variables
-ENTRY(sha256_x8_avx2)
-        # save callee-saved clobbered registers to comply with C function ABI
-        push    %r12
-        push    %r13
-        push    %r14
-        push    %r15
-        mov     %rsp, IDX
-        sub     $FRAMESZ, %rsp
-        and     $~0x1F, %rsp
-        mov     IDX, _rsp(%rsp)
-        # Load the pre-transposed incoming digest.
-        vmovdqu 0*SHA256_DIGEST_ROW_SIZE(STATE),a
-        vmovdqu 1*SHA256_DIGEST_ROW_SIZE(STATE),b
-        vmovdqu 2*SHA256_DIGEST_ROW_SIZE(STATE),c
-        vmovdqu 3*SHA256_DIGEST_ROW_SIZE(STATE),d
-        vmovdqu 4*SHA256_DIGEST_ROW_SIZE(STATE),e
-        vmovdqu 5*SHA256_DIGEST_ROW_SIZE(STATE),f
-        vmovdqu 6*SHA256_DIGEST_ROW_SIZE(STATE),g
-        vmovdqu 7*SHA256_DIGEST_ROW_SIZE(STATE),h
-        lea     K256_8(%rip),TBL
-        # load the address of each of the 4 message lanes
-        # getting ready to transpose input onto stack
-        mov     _args_data_ptr+0*PTR_SZ(STATE),inp0
-        mov     _args_data_ptr+1*PTR_SZ(STATE),inp1
-        mov     _args_data_ptr+2*PTR_SZ(STATE),inp2
-        mov     _args_data_ptr+3*PTR_SZ(STATE),inp3
-        mov     _args_data_ptr+4*PTR_SZ(STATE),inp4
-        mov     _args_data_ptr+5*PTR_SZ(STATE),inp5
-        mov     _args_data_ptr+6*PTR_SZ(STATE),inp6
-        mov     _args_data_ptr+7*PTR_SZ(STATE),inp7
-        xor     IDX, IDX
-lloop:
-        xor     ROUND, ROUND
-        # save old digest
-        vmovdqu a, _digest(%rsp)
-        vmovdqu b, _digest+1*SZ8(%rsp)
-        vmovdqu c, _digest+2*SZ8(%rsp)
-        vmovdqu d, _digest+3*SZ8(%rsp)
-        vmovdqu e, _digest+4*SZ8(%rsp)
-        vmovdqu f, _digest+5*SZ8(%rsp)
-        vmovdqu g, _digest+6*SZ8(%rsp)
-        vmovdqu h, _digest+7*SZ8(%rsp)
-        i = 0
-.rep 2
-        VMOVPS  i*32(inp0, IDX), TT0
-        VMOVPS  i*32(inp1, IDX), TT1
-        VMOVPS  i*32(inp2, IDX), TT2
-        VMOVPS  i*32(inp3, IDX), TT3
-        VMOVPS  i*32(inp4, IDX), TT4
-        VMOVPS  i*32(inp5, IDX), TT5
-        VMOVPS  i*32(inp6, IDX), TT6
-        VMOVPS  i*32(inp7, IDX), TT7
-        vmovdqu g, _ytmp(%rsp)
-        vmovdqu h, _ytmp+1*SZ8(%rsp)
-        TRANSPOSE8      TT0, TT1, TT2, TT3, TT4, TT5, TT6, TT7,   TMP0, TMP1
-        vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP1
-        vmovdqu _ytmp(%rsp), g
-        vpshufb TMP1, TT0, TT0
-        vpshufb TMP1, TT1, TT1
-        vpshufb TMP1, TT2, TT2
-        vpshufb TMP1, TT3, TT3
-        vpshufb TMP1, TT4, TT4
-        vpshufb TMP1, TT5, TT5
-        vpshufb TMP1, TT6, TT6
-        vpshufb TMP1, TT7, TT7
-        vmovdqu _ytmp+1*SZ8(%rsp), h
-        vmovdqu TT4, _ytmp(%rsp)
-        vmovdqu TT5, _ytmp+1*SZ8(%rsp)
-        vmovdqu TT6, _ytmp+2*SZ8(%rsp)
-        vmovdqu TT7, _ytmp+3*SZ8(%rsp)
-        ROUND_00_15     TT0,(i*8+0)
-        vmovdqu _ytmp(%rsp), TT0
-        ROUND_00_15     TT1,(i*8+1)
-        vmovdqu _ytmp+1*SZ8(%rsp), TT1
-        ROUND_00_15     TT2,(i*8+2)
-        vmovdqu _ytmp+2*SZ8(%rsp), TT2
-        ROUND_00_15     TT3,(i*8+3)
-        vmovdqu _ytmp+3*SZ8(%rsp), TT3
-        ROUND_00_15     TT0,(i*8+4)
-        ROUND_00_15     TT1,(i*8+5)
-        ROUND_00_15     TT2,(i*8+6)
-        ROUND_00_15     TT3,(i*8+7)
-        i = (i+1)
-.endr
-        add     $64, IDX
-        i = (i*8)
-        jmp     Lrounds_16_xx
-.align 16
-Lrounds_16_xx:
-.rep 16
-        ROUND_16_XX     T1, i
-        i = (i+1)
-.endr
-        cmp     $ROUNDS,ROUND
-        jb      Lrounds_16_xx
-        # add old digest
-        vpaddd  _digest+0*SZ8(%rsp), a, a
-        vpaddd  _digest+1*SZ8(%rsp), b, b
-        vpaddd  _digest+2*SZ8(%rsp), c, c
-        vpaddd  _digest+3*SZ8(%rsp), d, d
-        vpaddd  _digest+4*SZ8(%rsp), e, e
-        vpaddd  _digest+5*SZ8(%rsp), f, f
-        vpaddd  _digest+6*SZ8(%rsp), g, g
-        vpaddd  _digest+7*SZ8(%rsp), h, h
-        sub     $1, INP_SIZE  # unit is blocks
-        jne     lloop
-        # write back to memory (state object) the transposed digest
-        vmovdqu a, 0*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu b, 1*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu c, 2*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu d, 3*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu e, 4*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu f, 5*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu g, 6*SHA256_DIGEST_ROW_SIZE(STATE)
-        vmovdqu h, 7*SHA256_DIGEST_ROW_SIZE(STATE)
-        # update input pointers
-        add     IDX, inp0
-        mov     inp0, _args_data_ptr+0*8(STATE)
-        add     IDX, inp1
-        mov     inp1, _args_data_ptr+1*8(STATE)
-        add     IDX, inp2
-        mov     inp2, _args_data_ptr+2*8(STATE)
-        add     IDX, inp3
-        mov     inp3, _args_data_ptr+3*8(STATE)
-        add     IDX, inp4
-        mov     inp4, _args_data_ptr+4*8(STATE)
-        add     IDX, inp5
-        mov     inp5, _args_data_ptr+5*8(STATE)
-        add     IDX, inp6
-        mov     inp6, _args_data_ptr+6*8(STATE)
-        add     IDX, inp7
-        mov     inp7, _args_data_ptr+7*8(STATE)
-        # Postamble
-        mov     _rsp(%rsp), %rsp
-        # restore callee-saved clobbered registers
-        pop     %r15
-        pop     %r14
-        pop     %r13
-        pop     %r12
-        ret
-ENDPROC(sha256_x8_avx2)
-.section        .rodata.K256_8, "a", @progbits
-.align 64
-K256_8:
-        .octa   0x428a2f98428a2f98428a2f98428a2f98
-        .octa   0x428a2f98428a2f98428a2f98428a2f98
-        .octa   0x71374491713744917137449171374491
-        .octa   0x71374491713744917137449171374491
-        .octa   0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf
-        .octa   0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf
-        .octa   0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5
-        .octa   0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5
-        .octa   0x3956c25b3956c25b3956c25b3956c25b
-        .octa   0x3956c25b3956c25b3956c25b3956c25b
-        .octa   0x59f111f159f111f159f111f159f111f1
-        .octa   0x59f111f159f111f159f111f159f111f1
-        .octa   0x923f82a4923f82a4923f82a4923f82a4
-        .octa   0x923f82a4923f82a4923f82a4923f82a4
-        .octa   0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5
-        .octa   0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5
-        .octa   0xd807aa98d807aa98d807aa98d807aa98
-        .octa   0xd807aa98d807aa98d807aa98d807aa98
-        .octa   0x12835b0112835b0112835b0112835b01
-        .octa   0x12835b0112835b0112835b0112835b01
-        .octa   0x243185be243185be243185be243185be
-        .octa   0x243185be243185be243185be243185be
-        .octa   0x550c7dc3550c7dc3550c7dc3550c7dc3
-        .octa   0x550c7dc3550c7dc3550c7dc3550c7dc3
-        .octa   0x72be5d7472be5d7472be5d7472be5d74
-        .octa   0x72be5d7472be5d7472be5d7472be5d74
-        .octa   0x80deb1fe80deb1fe80deb1fe80deb1fe
-        .octa   0x80deb1fe80deb1fe80deb1fe80deb1fe
-        .octa   0x9bdc06a79bdc06a79bdc06a79bdc06a7
-        .octa   0x9bdc06a79bdc06a79bdc06a79bdc06a7
-        .octa   0xc19bf174c19bf174c19bf174c19bf174
-        .octa   0xc19bf174c19bf174c19bf174c19bf174
-        .octa   0xe49b69c1e49b69c1e49b69c1e49b69c1
-        .octa   0xe49b69c1e49b69c1e49b69c1e49b69c1
-        .octa   0xefbe4786efbe4786efbe4786efbe4786
-        .octa   0xefbe4786efbe4786efbe4786efbe4786
-        .octa   0x0fc19dc60fc19dc60fc19dc60fc19dc6
-        .octa   0x0fc19dc60fc19dc60fc19dc60fc19dc6
-        .octa   0x240ca1cc240ca1cc240ca1cc240ca1cc
-        .octa   0x240ca1cc240ca1cc240ca1cc240ca1cc
-        .octa   0x2de92c6f2de92c6f2de92c6f2de92c6f
-        .octa   0x2de92c6f2de92c6f2de92c6f2de92c6f
-        .octa   0x4a7484aa4a7484aa4a7484aa4a7484aa
-        .octa   0x4a7484aa4a7484aa4a7484aa4a7484aa
-        .octa   0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc
-        .octa   0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc
-        .octa   0x76f988da76f988da76f988da76f988da
-        .octa   0x76f988da76f988da76f988da76f988da
-        .octa   0x983e5152983e5152983e5152983e5152
-        .octa   0x983e5152983e5152983e5152983e5152
-        .octa   0xa831c66da831c66da831c66da831c66d
-        .octa   0xa831c66da831c66da831c66da831c66d
-        .octa   0xb00327c8b00327c8b00327c8b00327c8
-        .octa   0xb00327c8b00327c8b00327c8b00327c8
-        .octa   0xbf597fc7bf597fc7bf597fc7bf597fc7
-        .octa   0xbf597fc7bf597fc7bf597fc7bf597fc7
-        .octa   0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3
-        .octa   0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3
-        .octa   0xd5a79147d5a79147d5a79147d5a79147
-        .octa   0xd5a79147d5a79147d5a79147d5a79147
-        .octa   0x06ca635106ca635106ca635106ca6351
-        .octa   0x06ca635106ca635106ca635106ca6351
-        .octa   0x14292967142929671429296714292967
-        .octa   0x14292967142929671429296714292967
-        .octa   0x27b70a8527b70a8527b70a8527b70a85
-        .octa   0x27b70a8527b70a8527b70a8527b70a85
-        .octa   0x2e1b21382e1b21382e1b21382e1b2138
-        .octa   0x2e1b21382e1b21382e1b21382e1b2138
-        .octa   0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc
-        .octa   0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc
-        .octa   0x53380d1353380d1353380d1353380d13
-        .octa   0x53380d1353380d1353380d1353380d13
-        .octa   0x650a7354650a7354650a7354650a7354
-        .octa   0x650a7354650a7354650a7354650a7354
-        .octa   0x766a0abb766a0abb766a0abb766a0abb
-        .octa   0x766a0abb766a0abb766a0abb766a0abb
-        .octa   0x81c2c92e81c2c92e81c2c92e81c2c92e
-        .octa   0x81c2c92e81c2c92e81c2c92e81c2c92e
-        .octa   0x92722c8592722c8592722c8592722c85
-        .octa   0x92722c8592722c8592722c8592722c85
-        .octa   0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1
-        .octa   0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1
-        .octa   0xa81a664ba81a664ba81a664ba81a664b
-        .octa   0xa81a664ba81a664ba81a664ba81a664b
-        .octa   0xc24b8b70c24b8b70c24b8b70c24b8b70
-        .octa   0xc24b8b70c24b8b70c24b8b70c24b8b70
-        .octa   0xc76c51a3c76c51a3c76c51a3c76c51a3
-        .octa   0xc76c51a3c76c51a3c76c51a3c76c51a3
-        .octa   0xd192e819d192e819d192e819d192e819
-        .octa   0xd192e819d192e819d192e819d192e819
-        .octa   0xd6990624d6990624d6990624d6990624
-        .octa   0xd6990624d6990624d6990624d6990624
-        .octa   0xf40e3585f40e3585f40e3585f40e3585
-        .octa   0xf40e3585f40e3585f40e3585f40e3585
-        .octa   0x106aa070106aa070106aa070106aa070
-        .octa   0x106aa070106aa070106aa070106aa070
-        .octa   0x19a4c11619a4c11619a4c11619a4c116
-        .octa   0x19a4c11619a4c11619a4c11619a4c116
-        .octa   0x1e376c081e376c081e376c081e376c08
-        .octa   0x1e376c081e376c081e376c081e376c08
-        .octa   0x2748774c2748774c2748774c2748774c
-        .octa   0x2748774c2748774c2748774c2748774c
-        .octa   0x34b0bcb534b0bcb534b0bcb534b0bcb5
-        .octa   0x34b0bcb534b0bcb534b0bcb534b0bcb5
-        .octa   0x391c0cb3391c0cb3391c0cb3391c0cb3
-        .octa   0x391c0cb3391c0cb3391c0cb3391c0cb3
-        .octa   0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a
-        .octa   0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a
-        .octa   0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f
-        .octa   0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f
-        .octa   0x682e6ff3682e6ff3682e6ff3682e6ff3
-        .octa   0x682e6ff3682e6ff3682e6ff3682e6ff3
-        .octa   0x748f82ee748f82ee748f82ee748f82ee
-        .octa   0x748f82ee748f82ee748f82ee748f82ee
-        .octa   0x78a5636f78a5636f78a5636f78a5636f
-        .octa   0x78a5636f78a5636f78a5636f78a5636f
-        .octa   0x84c8781484c8781484c8781484c87814
-        .octa   0x84c8781484c8781484c8781484c87814
-        .octa   0x8cc702088cc702088cc702088cc70208
-        .octa   0x8cc702088cc702088cc702088cc70208
-        .octa   0x90befffa90befffa90befffa90befffa
-        .octa   0x90befffa90befffa90befffa90befffa
-        .octa   0xa4506ceba4506ceba4506ceba4506ceb
-        .octa   0xa4506ceba4506ceba4506ceba4506ceb
-        .octa   0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7
-        .octa   0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7
-        .octa   0xc67178f2c67178f2c67178f2c67178f2
-        .octa   0xc67178f2c67178f2c67178f2c67178f2
-.section        .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
-.align 32
-PSHUFFLE_BYTE_FLIP_MASK:
-.octa 0x0c0d0e0f08090a0b0405060700010203
-.octa 0x0c0d0e0f08090a0b0405060700010203
-.section        .rodata.cst256.K256, "aM", @progbits, 256
-.align 64
-.global K256
-K256:
-        .int    0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
-        .int    0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
-        .int    0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
-        .int    0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
-        .int    0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
-        .int    0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
-        .int    0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
-        .int    0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
-        .int    0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
-        .int    0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
-        .int    0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
-        .int    0xd192e819,0xd6990624,0xf40e3585,0x106aa070
-        .int    0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
-        .int    0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
-        .int    0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
-        .int    0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
diff --git a/arch/x86/crypto/sha512-mb/Makefile b/arch/x86/crypto/sha512-mb/Makefile
deleted file mode 100644
index 90f1ef69152e..000000000000
--- a/arch/x86/crypto/sha512-mb/Makefile
+++ /dev/null
@@ -1,12 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Arch-specific CryptoAPI modules.
-#
-avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
-                                $(comma)4)$(comma)%ymm2,yes,no)
-ifeq ($(avx2_supported),yes)
-        obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb.o
-        sha512-mb-y := sha512_mb.o sha512_mb_mgr_flush_avx2.o \
-             sha512_mb_mgr_init_avx2.o sha512_mb_mgr_submit_avx2.o sha512_x4_avx2.o
-endif
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c
deleted file mode 100644
index 26b85678012d..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb.c
+++ /dev/null
@@ -1,1047 +0,0 @@
-/*
- * Multi buffer SHA512 algorithm Glue Code
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
-#include <crypto/internal/hash.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/cryptohash.h>
-#include <linux/types.h>
-#include <linux/list.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/sha.h>
-#include <crypto/mcryptd.h>
-#include <crypto/crypto_wq.h>
-#include <asm/byteorder.h>
-#include <linux/hardirq.h>
-#include <asm/fpu/api.h>
-#include "sha512_mb_ctx.h"
-#define FLUSH_INTERVAL 1000 /* in usec */
-static struct mcryptd_alg_state sha512_mb_alg_state;
-struct sha512_mb_ctx {
-        struct mcryptd_ahash *mcryptd_tfm;
-};
-static inline struct mcryptd_hash_request_ctx
-                *cast_hash_to_mcryptd_ctx(struct sha512_hash_ctx *hash_ctx)
-{
-        struct ahash_request *areq;
-        areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
-        return container_of(areq, struct mcryptd_hash_request_ctx, areq);
-}
-static inline struct ahash_request
-                *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
-{
-        return container_of((void *) ctx, struct ahash_request, __ctx);
-}
-static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
-                                struct ahash_request *areq)
-{
-        rctx->flag = HASH_UPDATE;
-}
-static asmlinkage void (*sha512_job_mgr_init)(struct sha512_mb_mgr *state);
-static asmlinkage struct job_sha512* (*sha512_job_mgr_submit)
-                                                (struct sha512_mb_mgr *state,
-                                                struct job_sha512 *job);
-static asmlinkage struct job_sha512* (*sha512_job_mgr_flush)
-                                                (struct sha512_mb_mgr *state);
-static asmlinkage struct job_sha512* (*sha512_job_mgr_get_comp_job)
-                                                (struct sha512_mb_mgr *state);
-inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2],
-                         uint64_t total_len)
-{
-        uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1);
-        memset(&padblock[i], 0, SHA512_BLOCK_SIZE);
-        padblock[i] = 0x80;
-        i += ((SHA512_BLOCK_SIZE - 1) &
-              (0 - (total_len + SHA512_PADLENGTHFIELD_SIZE + 1)))
-             + 1 + SHA512_PADLENGTHFIELD_SIZE;
-#if SHA512_PADLENGTHFIELD_SIZE == 16
-        *((uint64_t *) &padblock[i - 16]) = 0;
-#endif
-        *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
-        /* Number of extra blocks to hash */
-        return i >> SHA512_LOG2_BLOCK_SIZE;
-}
-static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit
-                (struct sha512_ctx_mgr *mgr, struct sha512_hash_ctx *ctx)
-{
-        while (ctx) {
-                if (ctx->status & HASH_CTX_STS_COMPLETE) {
-                        /* Clear PROCESSING bit */
-                        ctx->status = HASH_CTX_STS_COMPLETE;
-                        return ctx;
-                }
-                /*
-                 * If the extra blocks are empty, begin hashing what remains
-                 * in the user's buffer.
-                 */
-                if (ctx->partial_block_buffer_length == 0 &&
-                    ctx->incoming_buffer_length) {
-                        const void *buffer = ctx->incoming_buffer;
-                        uint32_t len = ctx->incoming_buffer_length;
-                        uint32_t copy_len;
-                        /*
-                         * Only entire blocks can be hashed.
-                         * Copy remainder to extra blocks buffer.
-                         */
-                        copy_len = len & (SHA512_BLOCK_SIZE-1);
-                        if (copy_len) {
-                                len -= copy_len;
-                                memcpy(ctx->partial_block_buffer,
-                                       ((const char *) buffer + len),
-                                       copy_len);
-                                ctx->partial_block_buffer_length = copy_len;
-                        }
-                        ctx->incoming_buffer_length = 0;
-                        /* len should be a multiple of the block size now */
-                        assert((len % SHA512_BLOCK_SIZE) == 0);
-                        /* Set len to the number of blocks to be hashed */
-                        len >>= SHA512_LOG2_BLOCK_SIZE;
-                        if (len) {
-                                ctx->job.buffer = (uint8_t *) buffer;
-                                ctx->job.len = len;
-                                ctx = (struct sha512_hash_ctx *)
-                                        sha512_job_mgr_submit(&mgr->mgr,
-                                        &ctx->job);
-                                continue;
-                        }
-                }
-                /*
-                 * If the extra blocks are not empty, then we are
-                 * either on the last block(s) or we need more
-                 * user input before continuing.
-                 */
-                if (ctx->status & HASH_CTX_STS_LAST) {
-                        uint8_t *buf = ctx->partial_block_buffer;
-                        uint32_t n_extra_blocks =
-                                        sha512_pad(buf, ctx->total_length);
-                        ctx->status = (HASH_CTX_STS_PROCESSING |
-                                       HASH_CTX_STS_COMPLETE);
-                        ctx->job.buffer = buf;
-                        ctx->job.len = (uint32_t) n_extra_blocks;
-                        ctx = (struct sha512_hash_ctx *)
-                                sha512_job_mgr_submit(&mgr->mgr, &ctx->job);
-                        continue;
-                }
-                if (ctx)
-                        ctx->status = HASH_CTX_STS_IDLE;
-                return ctx;
-        }
-        return NULL;
-}
-static struct sha512_hash_ctx
-                *sha512_ctx_mgr_get_comp_ctx(struct mcryptd_alg_cstate *cstate)
-{
-        /*
-         * If get_comp_job returns NULL, there are no jobs complete.
-         * If get_comp_job returns a job, verify that it is safe to return to
-         * the user.
-         * If it is not ready, resubmit the job to finish processing.
-         * If sha512_ctx_mgr_resubmit returned a job, it is ready to be
-         * returned.
-         * Otherwise, all jobs currently being managed by the hash_ctx_mgr
-         * still need processing.
-         */
-        struct sha512_ctx_mgr *mgr;
-        struct sha512_hash_ctx *ctx;
-        unsigned long flags;
-        mgr = cstate->mgr;
-        spin_lock_irqsave(&cstate->work_lock, flags);
-        ctx = (struct sha512_hash_ctx *)
-                                sha512_job_mgr_get_comp_job(&mgr->mgr);
-        ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
-        spin_unlock_irqrestore(&cstate->work_lock, flags);
-        return ctx;
-}
-static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
-{
-        sha512_job_mgr_init(&mgr->mgr);
-}
-static struct sha512_hash_ctx
-                        *sha512_ctx_mgr_submit(struct mcryptd_alg_cstate *cstate,
-                                          struct sha512_hash_ctx *ctx,
-                                          const void *buffer,
-                                          uint32_t len,
-                                          int flags)
-{
-        struct sha512_ctx_mgr *mgr;
-        unsigned long irqflags;
-        mgr = cstate->mgr;
-        spin_lock_irqsave(&cstate->work_lock, irqflags);
-        if (flags & ~(HASH_UPDATE | HASH_LAST)) {
-                /* User should not pass anything other than UPDATE or LAST */
-                ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
-                goto unlock;
-        }
-        if (ctx->status & HASH_CTX_STS_PROCESSING) {
-                /* Cannot submit to a currently processing job. */
-                ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
-                goto unlock;
-        }
-        if (ctx->status & HASH_CTX_STS_COMPLETE) {
-                /* Cannot update a finished job. */
-                ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
-                goto unlock;
-        }
-        /*
-         * If we made it here, there were no errors during this call to
-         * submit
-         */
-        ctx->error = HASH_CTX_ERROR_NONE;
-        /* Store buffer ptr info from user */
-        ctx->incoming_buffer = buffer;
-        ctx->incoming_buffer_length = len;
-        /*
-         * Store the user's request flags and mark this ctx as currently being
-         * processed.
-         */
-        ctx->status = (flags & HASH_LAST) ?
-                        (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
-                        HASH_CTX_STS_PROCESSING;
-        /* Advance byte counter */
-        ctx->total_length += len;
-        /*
-         * If there is anything currently buffered in the extra blocks,
-         * append to it until it contains a whole block.
-         * Or if the user's buffer contains less than a whole block,
-         * append as much as possible to the extra block.
-         */
-        if (ctx->partial_block_buffer_length || len < SHA512_BLOCK_SIZE) {
-                /* Compute how many bytes to copy from user buffer into extra
-                 * block
-                 */
-                uint32_t copy_len = SHA512_BLOCK_SIZE -
-                                        ctx->partial_block_buffer_length;
-                if (len < copy_len)
-                        copy_len = len;
-                if (copy_len) {
-                        /* Copy and update relevant pointers and counters */
-                        memcpy
-                (&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
-                                buffer, copy_len);
-                        ctx->partial_block_buffer_length += copy_len;
-                        ctx->incoming_buffer = (const void *)
-                                        ((const char *)buffer + copy_len);
-                        ctx->incoming_buffer_length = len - copy_len;
-                }
-                /* The extra block should never contain more than 1 block
-                 * here
-                 */
-                assert(ctx->partial_block_buffer_length <= SHA512_BLOCK_SIZE);
-                /* If the extra block buffer contains exactly 1 block, it can
-                 * be hashed.
-                 */
-                if (ctx->partial_block_buffer_length >= SHA512_BLOCK_SIZE) {
-                        ctx->partial_block_buffer_length = 0;
-                        ctx->job.buffer = ctx->partial_block_buffer;
-                        ctx->job.len = 1;
-                        ctx = (struct sha512_hash_ctx *)
-                                sha512_job_mgr_submit(&mgr->mgr, &ctx->job);
-                }
-        }
-        ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
-unlock:
-        spin_unlock_irqrestore(&cstate->work_lock, irqflags);
-        return ctx;
-}
-static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct mcryptd_alg_cstate *cstate)
-{
-        struct sha512_ctx_mgr *mgr;
-        struct sha512_hash_ctx *ctx;
-        unsigned long flags;
-        mgr = cstate->mgr;
-        spin_lock_irqsave(&cstate->work_lock, flags);
-        while (1) {
-                ctx = (struct sha512_hash_ctx *)
-                                        sha512_job_mgr_flush(&mgr->mgr);
-                /* If flush returned 0, there are no more jobs in flight. */
-                if (!ctx)
-                        break;
-                /*
-                 * If flush returned a job, resubmit the job to finish
-                 * processing.
-                 */
-                ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
-                /*
-                 * If sha512_ctx_mgr_resubmit returned a job, it is ready to
-                 * be returned. Otherwise, all jobs currently being managed by
-                 * the sha512_ctx_mgr still need processing. Loop.
-                 */
-                if (ctx)
-                        break;
-        }
-        spin_unlock_irqrestore(&cstate->work_lock, flags);
-        return ctx;
-}
-static int sha512_mb_init(struct ahash_request *areq)
-{
-        struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);
-        hash_ctx_init(sctx);
-        sctx->job.result_digest[0] = SHA512_H0;
-        sctx->job.result_digest[1] = SHA512_H1;
-        sctx->job.result_digest[2] = SHA512_H2;
-        sctx->job.result_digest[3] = SHA512_H3;
-        sctx->job.result_digest[4] = SHA512_H4;
-        sctx->job.result_digest[5] = SHA512_H5;
-        sctx->job.result_digest[6] = SHA512_H6;
-        sctx->job.result_digest[7] = SHA512_H7;
-        sctx->total_length = 0;
-        sctx->partial_block_buffer_length = 0;
-        sctx->status = HASH_CTX_STS_IDLE;
-        return 0;
-}
-static int sha512_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
-{
-        int     i;
-        struct  sha512_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
-        __be64  *dst = (__be64 *) rctx->out;
-        for (i = 0; i < 8; ++i)
-                dst[i] = cpu_to_be64(sctx->job.result_digest[i]);
-        return 0;
-}
-static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
-                        struct mcryptd_alg_cstate *cstate, bool flush)
-{
-        int     flag = HASH_UPDATE;
-        int     nbytes, err = 0;
-        struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
-        struct sha512_hash_ctx *sha_ctx;
-        /* more work ? */
-        while (!(rctx->flag & HASH_DONE)) {
-                nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
-                if (nbytes < 0) {
-                        err = nbytes;
-                        goto out;
-                }
-                /* check if the walk is done */
-                if (crypto_ahash_walk_last(&rctx->walk)) {
-                        rctx->flag |= HASH_DONE;
-                        if (rctx->flag & HASH_FINAL)
-                                flag |= HASH_LAST;
-                }
-                sha_ctx = (struct sha512_hash_ctx *)
-                                                ahash_request_ctx(&rctx->areq);
-                kernel_fpu_begin();
-                sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx,
-                                                rctx->walk.data, nbytes, flag);
-                if (!sha_ctx) {
-                        if (flush)
-                                sha_ctx = sha512_ctx_mgr_flush(cstate);
-                }
-                kernel_fpu_end();
-                if (sha_ctx)
-                        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                else {
-                        rctx = NULL;
-                        goto out;
-                }
-        }
-        /* copy the results */
-        if (rctx->flag & HASH_FINAL)
-                sha512_mb_set_results(rctx);
-out:
-        *ret_rctx = rctx;
-        return err;
-}
-static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
-                            struct mcryptd_alg_cstate *cstate,
-                            int err)
-{
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha512_hash_ctx *sha_ctx;
-        struct mcryptd_hash_request_ctx *req_ctx;
-        int ret;
-        unsigned long flags;
-        /* remove from work list */
-        spin_lock_irqsave(&cstate->work_lock, flags);
-        list_del(&rctx->waiter);
-        spin_unlock_irqrestore(&cstate->work_lock, flags);
-        if (irqs_disabled())
-                rctx->complete(&req->base, err);
-        else {
-                local_bh_disable();
-                rctx->complete(&req->base, err);
-                local_bh_enable();
-        }
-        /* check to see if there are other jobs that are done */
-        sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
-        while (sha_ctx) {
-                req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                ret = sha_finish_walk(&req_ctx, cstate, false);
-                if (req_ctx) {
-                        spin_lock_irqsave(&cstate->work_lock, flags);
-                        list_del(&req_ctx->waiter);
-                        spin_unlock_irqrestore(&cstate->work_lock, flags);
-                        req = cast_mcryptd_ctx_to_req(req_ctx);
-                        if (irqs_disabled())
-                                req_ctx->complete(&req->base, ret);
-                        else {
-                                local_bh_disable();
-                                req_ctx->complete(&req->base, ret);
-                                local_bh_enable();
-                        }
-                }
-                sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
-        }
-        return 0;
-}
-static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
-                             struct mcryptd_alg_cstate *cstate)
-{
-        unsigned long next_flush;
-        unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
-        unsigned long flags;
-        /* initialize tag */
-        rctx->tag.arrival = jiffies;    /* tag the arrival time */
-        rctx->tag.seq_num = cstate->next_seq_num++;
-        next_flush = rctx->tag.arrival + delay;
-        rctx->tag.expire = next_flush;
-        spin_lock_irqsave(&cstate->work_lock, flags);
-        list_add_tail(&rctx->waiter, &cstate->work_list);
-        spin_unlock_irqrestore(&cstate->work_lock, flags);
-        mcryptd_arm_flusher(cstate, delay);
-}
-static int sha512_mb_update(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                        container_of(areq, struct mcryptd_hash_request_ctx,
-                                                                        areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha512_mb_alg_state.alg_cstate);
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha512_hash_ctx *sha_ctx;
-        int ret = 0, nbytes;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        nbytes = crypto_ahash_walk_first(req, &rctx->walk);
-        if (nbytes < 0) {
-                ret = nbytes;
-                goto done;
-        }
-        if (crypto_ahash_walk_last(&rctx->walk))
-                rctx->flag |= HASH_DONE;
-        /* submit */
-        sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
-        sha512_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
-                                                        nbytes, HASH_UPDATE);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha512_mb_finup(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                        container_of(areq, struct mcryptd_hash_request_ctx,
-                                                                        areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha512_mb_alg_state.alg_cstate);
-        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
-        struct sha512_hash_ctx *sha_ctx;
-        int ret = 0, flag = HASH_UPDATE, nbytes;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        nbytes = crypto_ahash_walk_first(req, &rctx->walk);
-        if (nbytes < 0) {
-                ret = nbytes;
-                goto done;
-        }
-        if (crypto_ahash_walk_last(&rctx->walk)) {
-                rctx->flag |= HASH_DONE;
-                flag = HASH_LAST;
-        }
-        /* submit */
-        rctx->flag |= HASH_FINAL;
-        sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
-        sha512_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
-                                                                nbytes, flag);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha512_mb_final(struct ahash_request *areq)
-{
-        struct mcryptd_hash_request_ctx *rctx =
-                        container_of(areq, struct mcryptd_hash_request_ctx,
-                                                                        areq);
-        struct mcryptd_alg_cstate *cstate =
-                                this_cpu_ptr(sha512_mb_alg_state.alg_cstate);
-        struct sha512_hash_ctx *sha_ctx;
-        int ret = 0;
-        u8 data;
-        /* sanity check */
-        if (rctx->tag.cpu != smp_processor_id()) {
-                pr_err("mcryptd error: cpu clash\n");
-                goto done;
-        }
-        /* need to init context */
-        req_ctx_init(rctx, areq);
-        rctx->flag |= HASH_DONE | HASH_FINAL;
-        sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
-        /* flag HASH_FINAL and 0 data size */
-        sha512_mb_add_list(rctx, cstate);
-        kernel_fpu_begin();
-        sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, &data, 0, HASH_LAST);
-        kernel_fpu_end();
-        /* check if anything is returned */
-        if (!sha_ctx)
-                return -EINPROGRESS;
-        if (sha_ctx->error) {
-                ret = sha_ctx->error;
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                goto done;
-        }
-        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-        ret = sha_finish_walk(&rctx, cstate, false);
-        if (!rctx)
-                return -EINPROGRESS;
-done:
-        sha_complete_job(rctx, cstate, ret);
-        return ret;
-}
-static int sha512_mb_export(struct ahash_request *areq, void *out)
-{
-        struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);
-        memcpy(out, sctx, sizeof(*sctx));
-        return 0;
-}
-static int sha512_mb_import(struct ahash_request *areq, const void *in)
-{
-        struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);
-        memcpy(sctx, in, sizeof(*sctx));
-        return 0;
-}
-static int sha512_mb_async_init_tfm(struct crypto_tfm *tfm)
-{
-        struct mcryptd_ahash *mcryptd_tfm;
-        struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct mcryptd_hash_ctx *mctx;
-        mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha512-mb",
-                                                CRYPTO_ALG_INTERNAL,
-                                                CRYPTO_ALG_INTERNAL);
-        if (IS_ERR(mcryptd_tfm))
-                return PTR_ERR(mcryptd_tfm);
-        mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
-        mctx->alg_state = &sha512_mb_alg_state;
-        ctx->mcryptd_tfm = mcryptd_tfm;
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                sizeof(struct ahash_request) +
-                                crypto_ahash_reqsize(&mcryptd_tfm->base));
-        return 0;
-}
-static void sha512_mb_async_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        mcryptd_free_ahash(ctx->mcryptd_tfm);
-}
-static int sha512_mb_areq_init_tfm(struct crypto_tfm *tfm)
-{
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                sizeof(struct ahash_request) +
-                                sizeof(struct sha512_hash_ctx));
-        return 0;
-}
-static void sha512_mb_areq_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-        mcryptd_free_ahash(ctx->mcryptd_tfm);
-}
-static struct ahash_alg sha512_mb_areq_alg = {
-        .init           =       sha512_mb_init,
-        .update         =       sha512_mb_update,
-        .final          =       sha512_mb_final,
-        .finup          =       sha512_mb_finup,
-        .export         =       sha512_mb_export,
-        .import         =       sha512_mb_import,
-        .halg           =       {
-        .digestsize     =       SHA512_DIGEST_SIZE,
-        .statesize      =       sizeof(struct sha512_hash_ctx),
-        .base           =       {
-                        .cra_name        = "__sha512-mb",
-                        .cra_driver_name = "__intel_sha512-mb",
-                        .cra_priority    = 100,
-                        /*
-                         * use ASYNC flag as some buffers in multi-buffer
-                         * algo may not have completed before hashing thread
-                         * sleep
-                         */
-                        .cra_flags      = CRYPTO_ALG_ASYNC |
-                                          CRYPTO_ALG_INTERNAL,
-                        .cra_blocksize  = SHA512_BLOCK_SIZE,
-                        .cra_module     = THIS_MODULE,
-                        .cra_list       = LIST_HEAD_INIT
-                                        (sha512_mb_areq_alg.halg.base.cra_list),
-                        .cra_init       = sha512_mb_areq_init_tfm,
-                        .cra_exit       = sha512_mb_areq_exit_tfm,
-                        .cra_ctxsize    = sizeof(struct sha512_hash_ctx),
-                }
-        }
-};
-static int sha512_mb_async_init(struct ahash_request *req)
-{
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_init(mcryptd_req);
-}
-static int sha512_mb_async_update(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_update(mcryptd_req);
-}
-static int sha512_mb_async_finup(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_finup(mcryptd_req);
-}
-static int sha512_mb_async_final(struct ahash_request *req)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_final(mcryptd_req);
-}
-static int sha512_mb_async_digest(struct ahash_request *req)
-{
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_digest(mcryptd_req);
-}
-static int sha512_mb_async_export(struct ahash_request *req, void *out)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        return crypto_ahash_export(mcryptd_req, out);
-}
-static int sha512_mb_async_import(struct ahash_request *req, const void *in)
-{
-        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
-        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-        struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
-        struct mcryptd_hash_request_ctx *rctx;
-        struct ahash_request *areq;
-        memcpy(mcryptd_req, req, sizeof(*req));
-        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
-        rctx = ahash_request_ctx(mcryptd_req);
-        areq = &rctx->areq;
-        ahash_request_set_tfm(areq, child);
-        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
-                                        rctx->complete, req);
-        return crypto_ahash_import(mcryptd_req, in);
-}
-static struct ahash_alg sha512_mb_async_alg = {
-        .init           = sha512_mb_async_init,
-        .update         = sha512_mb_async_update,
-        .final          = sha512_mb_async_final,
-        .finup          = sha512_mb_async_finup,
-        .digest         = sha512_mb_async_digest,
-        .export         = sha512_mb_async_export,
-        .import         = sha512_mb_async_import,
-        .halg = {
-                .digestsize     = SHA512_DIGEST_SIZE,
-                .statesize      = sizeof(struct sha512_hash_ctx),
-                .base = {
-                        .cra_name               = "sha512",
-                        .cra_driver_name        = "sha512_mb",
-                        /*
-                         * Low priority, since with few concurrent hash requests
-                         * this is extremely slow due to the flush delay.  Users
-                         * whose workloads would benefit from this can request
-                         * it explicitly by driver name, or can increase its
-                         * priority at runtime using NETLINK_CRYPTO.
-                         */
-                        .cra_priority           = 50,
-                        .cra_flags              = CRYPTO_ALG_ASYNC,
-                        .cra_blocksize          = SHA512_BLOCK_SIZE,
-                        .cra_module             = THIS_MODULE,
-                        .cra_list               = LIST_HEAD_INIT
-                                (sha512_mb_async_alg.halg.base.cra_list),
-                        .cra_init               = sha512_mb_async_init_tfm,
-                        .cra_exit               = sha512_mb_async_exit_tfm,
-                        .cra_ctxsize            = sizeof(struct sha512_mb_ctx),
-                        .cra_alignmask          = 0,
-                },
-        },
-};
-static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate)
-{
-        struct mcryptd_hash_request_ctx *rctx;
-        unsigned long cur_time;
-        unsigned long next_flush = 0;
-        struct sha512_hash_ctx *sha_ctx;
-        cur_time = jiffies;
-        while (!list_empty(&cstate->work_list)) {
-                rctx = list_entry(cstate->work_list.next,
-                                struct mcryptd_hash_request_ctx, waiter);
-                if time_before(cur_time, rctx->tag.expire)
-                        break;
-                kernel_fpu_begin();
-                sha_ctx = (struct sha512_hash_ctx *)
-                                        sha512_ctx_mgr_flush(cstate);
-                kernel_fpu_end();
-                if (!sha_ctx) {
-                        pr_err("sha512_mb error: nothing got flushed for"
-                                                        " non-empty list\n");
-                        break;
-                }
-                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
-                sha_finish_walk(&rctx, cstate, true);
-                sha_complete_job(rctx, cstate, 0);
-        }
-        if (!list_empty(&cstate->work_list)) {
-                rctx = list_entry(cstate->work_list.next,
-                                struct mcryptd_hash_request_ctx, waiter);
-                /* get the hash context and then flush time */
-                next_flush = rctx->tag.expire;
-                mcryptd_arm_flusher(cstate, get_delay(next_flush));
-        }
-        return next_flush;
-}
-static int __init sha512_mb_mod_init(void)
-{
-        int cpu;
-        int err;
-        struct mcryptd_alg_cstate *cpu_state;
-        /* check for dependent cpu features */
-        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
-            !boot_cpu_has(X86_FEATURE_BMI2))
-                return -ENODEV;
-        /* initialize multibuffer structures */
-        sha512_mb_alg_state.alg_cstate =
-                                alloc_percpu(struct mcryptd_alg_cstate);
-        sha512_job_mgr_init = sha512_mb_mgr_init_avx2;
-        sha512_job_mgr_submit = sha512_mb_mgr_submit_avx2;
-        sha512_job_mgr_flush = sha512_mb_mgr_flush_avx2;
-        sha512_job_mgr_get_comp_job = sha512_mb_mgr_get_comp_job_avx2;
-        if (!sha512_mb_alg_state.alg_cstate)
-                return -ENOMEM;
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
-                cpu_state->next_flush = 0;
-                cpu_state->next_seq_num = 0;
-                cpu_state->flusher_engaged = false;
-                INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
-                cpu_state->cpu = cpu;
-                cpu_state->alg_state = &sha512_mb_alg_state;
-                cpu_state->mgr = kzalloc(sizeof(struct sha512_ctx_mgr),
-                                                                GFP_KERNEL);
-                if (!cpu_state->mgr)
-                        goto err2;
-                sha512_ctx_mgr_init(cpu_state->mgr);
-                INIT_LIST_HEAD(&cpu_state->work_list);
-                spin_lock_init(&cpu_state->work_lock);
-        }
-        sha512_mb_alg_state.flusher = &sha512_mb_flusher;
-        err = crypto_register_ahash(&sha512_mb_areq_alg);
-        if (err)
-                goto err2;
-        err = crypto_register_ahash(&sha512_mb_async_alg);
-        if (err)
-                goto err1;
-        return 0;
-err1:
-        crypto_unregister_ahash(&sha512_mb_areq_alg);
-err2:
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
-                kfree(cpu_state->mgr);
-        }
-        free_percpu(sha512_mb_alg_state.alg_cstate);
-        return -ENODEV;
-}
-static void __exit sha512_mb_mod_fini(void)
-{
-        int cpu;
-        struct mcryptd_alg_cstate *cpu_state;
-        crypto_unregister_ahash(&sha512_mb_async_alg);
-        crypto_unregister_ahash(&sha512_mb_areq_alg);
-        for_each_possible_cpu(cpu) {
-                cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
-                kfree(cpu_state->mgr);
-        }
-        free_percpu(sha512_mb_alg_state.alg_cstate);
-}
-module_init(sha512_mb_mod_init);
-module_exit(sha512_mb_mod_fini);
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, multi buffer accelerated");
-MODULE_ALIAS("sha512");
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h
deleted file mode 100644
index e5c465bd821e..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Header file for multi buffer SHA512 context
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef _SHA_MB_CTX_INTERNAL_H
-#define _SHA_MB_CTX_INTERNAL_H
-#include "sha512_mb_mgr.h"
-#define HASH_UPDATE          0x00
-#define HASH_LAST            0x01
-#define HASH_DONE            0x02
-#define HASH_FINAL           0x04
-#define HASH_CTX_STS_IDLE       0x00
-#define HASH_CTX_STS_PROCESSING 0x01
-#define HASH_CTX_STS_LAST       0x02
-#define HASH_CTX_STS_COMPLETE   0x04
-enum hash_ctx_error {
-        HASH_CTX_ERROR_NONE               =  0,
-        HASH_CTX_ERROR_INVALID_FLAGS      = -1,
-        HASH_CTX_ERROR_ALREADY_PROCESSING = -2,
-        HASH_CTX_ERROR_ALREADY_COMPLETED  = -3,
-};
-#define hash_ctx_user_data(ctx)  ((ctx)->user_data)
-#define hash_ctx_digest(ctx)     ((ctx)->job.result_digest)
-#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING)
-#define hash_ctx_complete(ctx)   ((ctx)->status == HASH_CTX_STS_COMPLETE)
-#define hash_ctx_status(ctx)     ((ctx)->status)
-#define hash_ctx_error(ctx)      ((ctx)->error)
-#define hash_ctx_init(ctx) \
-        do { \
-                (ctx)->error = HASH_CTX_ERROR_NONE; \
-                (ctx)->status = HASH_CTX_STS_COMPLETE; \
-        } while (0)
-/* Hash Constants and Typedefs */
-#define SHA512_DIGEST_LENGTH          8
-#define SHA512_LOG2_BLOCK_SIZE        7
-#define SHA512_PADLENGTHFIELD_SIZE    16
-#ifdef SHA_MB_DEBUG
-#define assert(expr) \
-do { \
-        if (unlikely(!(expr))) { \
-                printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
-                #expr, __FILE__, __func__, __LINE__); \
-        } \
-} while (0)
-#else
-#define assert(expr) do {} while (0)
-#endif
-struct sha512_ctx_mgr {
-        struct sha512_mb_mgr mgr;
-};
-/* typedef struct sha512_ctx_mgr sha512_ctx_mgr; */
-struct sha512_hash_ctx {
-        /* Must be at struct offset 0 */
-        struct job_sha512       job;
-        /* status flag */
-        int status;
-        /* error flag */
-        int error;
-        uint64_t        total_length;
-        const void      *incoming_buffer;
-        uint32_t        incoming_buffer_length;
-        uint8_t         partial_block_buffer[SHA512_BLOCK_SIZE * 2];
-        uint32_t        partial_block_buffer_length;
-        void            *user_data;
-};
-#endif
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h b/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h
deleted file mode 100644
index 178f17eef382..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Header file for multi buffer SHA512 algorithm manager
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef __SHA_MB_MGR_H
-#define __SHA_MB_MGR_H
-#include <linux/types.h>
-#define NUM_SHA512_DIGEST_WORDS 8
-enum job_sts {STS_UNKNOWN = 0,
-        STS_BEING_PROCESSED = 1,
-        STS_COMPLETED =       2,
-        STS_INTERNAL_ERROR = 3,
-        STS_ERROR = 4
-};
-struct job_sha512 {
-        u8  *buffer;
-        u64  len;
-        u64  result_digest[NUM_SHA512_DIGEST_WORDS] __aligned(32);
-        enum job_sts status;
-        void   *user_data;
-};
-struct sha512_args_x4 {
-        uint64_t        digest[8][4];
-        uint8_t         *data_ptr[4];
-};
-struct sha512_lane_data {
-        struct job_sha512 *job_in_lane;
-};
-struct sha512_mb_mgr {
-        struct sha512_args_x4 args;
-        uint64_t lens[4];
-        /* each byte is index (0...7) of unused lanes */
-        uint64_t unused_lanes;
-        /* byte 4 is set to FF as a flag */
-        struct sha512_lane_data ldata[4];
-};
-#define SHA512_MB_MGR_NUM_LANES_AVX2 4
-void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state);
-struct job_sha512 *sha512_mb_mgr_submit_avx2(struct sha512_mb_mgr *state,
-                                                struct job_sha512 *job);
-struct job_sha512 *sha512_mb_mgr_flush_avx2(struct sha512_mb_mgr *state);
-struct job_sha512 *sha512_mb_mgr_get_comp_job_avx2(struct sha512_mb_mgr *state);
-#endif
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S
deleted file mode 100644
index cf2636d4c9ba..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Header file for multi buffer SHA256 algorithm data structure
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of version 2 of the GNU General Public License as
- *  published by the Free Software Foundation.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  Contact Information:
- *      Megha Dey <megha.dey@linux.intel.com>
- *
- *  BSD LICENSE
- *
- *  Copyright(c) 2016 Intel Corporation.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions
- *  are met:
- *
- *    * Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *    * Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in
- *      the documentation and/or other materials provided with the
- *      distribution.
- *    * Neither the name of Intel Corporation nor the names of its
- *      contributors may be used to endorse or promote products derived
- *      from this software without specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-# Macros for defining data structures
-# Usage example
-#START_FIELDS   # JOB_AES
-###     name            size    align
-#FIELD  _plaintext,     8,      8       # pointer to plaintext
-#FIELD  _ciphertext,    8,      8       # pointer to ciphertext
-#FIELD  _IV,            16,     8       # IV
-#FIELD  _keys,          8,      8       # pointer to keys
-#FIELD  _len,           4,      4       # length in bytes
-#FIELD  _status,        4,      4       # status enumeration
-#FIELD  _user_data,     8,      8       # pointer to user data
-#UNION  _union,         size1,  align1, \
-#                       size2,  align2, \
-#                       size3,  align3, \
-#                       ...
-#END_FIELDS
-#%assign _JOB_AES_size  _FIELD_OFFSET
-#%assign _JOB_AES_align _STRUCT_ALIGN
-#########################################################################
-# Alternate "struc-like" syntax:
-#       STRUCT job_aes2
-#       RES_Q   .plaintext,     1
-#       RES_Q   .ciphertext,    1
-#       RES_DQ  .IV,            1
-#       RES_B   .nested,        _JOB_AES_SIZE, _JOB_AES_ALIGN
-#       RES_U   .union,         size1, align1, \
-#                               size2, align2, \
-#                               ...
-#       ENDSTRUCT
-#       # Following only needed if nesting
-#       %assign job_aes2_size   _FIELD_OFFSET
-#       %assign job_aes2_align  _STRUCT_ALIGN
-#
-# RES_* macros take a name, a count and an optional alignment.
-# The count in in terms of the base size of the macro, and the
-# default alignment is the base size.
-# The macros are:
-# Macro    Base size
-# RES_B     1
-# RES_W     2
-# RES_D     4
-# RES_Q     8
-# RES_DQ   16
-# RES_Y    32
-# RES_Z    64
-#
-# RES_U defines a union. It's arguments are a name and two or more
-# pairs of "size, alignment"
-#
-# The two assigns are only needed if this structure is being nested
-# within another. Even if the assigns are not done, one can still use
-# STRUCT_NAME_size as the size of the structure.
-#
-# Note that for nesting, you still need to assign to STRUCT_NAME_size.
-#
-# The differences between this and using "struc" directly are that each
-# type is implicitly aligned to its natural length (although this can be
-# over-ridden with an explicit third parameter), and that the structure
-# is padded at the end to its overall alignment.
-#
-#########################################################################
-#ifndef _DATASTRUCT_ASM_
-#define _DATASTRUCT_ASM_
-#define PTR_SZ                  8
-#define SHA512_DIGEST_WORD_SIZE 8
-#define SHA512_MB_MGR_NUM_LANES_AVX2 4
-#define NUM_SHA512_DIGEST_WORDS 8
-#define SZ4                     4*SHA512_DIGEST_WORD_SIZE
-#define ROUNDS                  80*SZ4
-#define SHA512_DIGEST_ROW_SIZE  (SHA512_MB_MGR_NUM_LANES_AVX2 * 8)
-# START_FIELDS
-.macro START_FIELDS
- _FIELD_OFFSET = 0
- _STRUCT_ALIGN = 0
-.endm
-# FIELD name size align
-.macro FIELD name size align
- _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
- \name  = _FIELD_OFFSET
- _FIELD_OFFSET = _FIELD_OFFSET + (\size)
-.if (\align > _STRUCT_ALIGN)
- _STRUCT_ALIGN = \align
-.endif
-.endm
-# END_FIELDS
-.macro END_FIELDS
- _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
-.endm
-.macro STRUCT p1
-START_FIELDS
-.struc \p1
-.endm
-.macro ENDSTRUCT
- tmp = _FIELD_OFFSET
- END_FIELDS
- tmp = (_FIELD_OFFSET - ##tmp)
-.if (tmp > 0)
-        .lcomm  tmp
-.endm
-## RES_int name size align
-.macro RES_int p1 p2 p3
- name = \p1
- size = \p2
- align = .\p3
- _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
-.align align
-.lcomm name size
- _FIELD_OFFSET = _FIELD_OFFSET + (size)
-.if (align > _STRUCT_ALIGN)
- _STRUCT_ALIGN = align
-.endif
-.endm
-# macro RES_B name, size [, align]
-.macro RES_B _name, _size, _align=1
-RES_int _name _size _align
-.endm
-# macro RES_W name, size [, align]
-.macro RES_W _name, _size, _align=2
-RES_int _name 2*(_size) _align
-.endm
-# macro RES_D name, size [, align]
-.macro RES_D _name, _size, _align=4
-RES_int _name 4*(_size) _align
-.endm
-# macro RES_Q name, size [, align]
-.macro RES_Q _name, _size, _align=8
-RES_int _name 8*(_size) _align
-.endm
-# macro RES_DQ name, size [, align]
-.macro RES_DQ _name, _size, _align=16
-RES_int _name 16*(_size) _align
-.endm
-# macro RES_Y name, size [, align]
-.macro RES_Y _name, _size, _align=32
-RES_int _name 32*(_size) _align
-.endm
-# macro RES_Z name, size [, align]
-.macro RES_Z _name, _size, _align=64
-RES_int _name 64*(_size) _align
-.endm
-#endif
-###################################################################
-### Define SHA512 Out Of Order Data Structures
-###################################################################
-START_FIELDS    # LANE_DATA
-###     name            size    align
-FIELD   _job_in_lane,   8,      8       # pointer to job object
-END_FIELDS
- _LANE_DATA_size = _FIELD_OFFSET
- _LANE_DATA_align = _STRUCT_ALIGN
-####################################################################
-START_FIELDS    # SHA512_ARGS_X4
-###     name            size    align
-FIELD   _digest,        8*8*4,  4      # transposed digest
-FIELD   _data_ptr,      8*4,    8       # array of pointers to data
-END_FIELDS
- _SHA512_ARGS_X4_size  =  _FIELD_OFFSET
- _SHA512_ARGS_X4_align =  _STRUCT_ALIGN
-#####################################################################
-START_FIELDS    # MB_MGR
-###     name            size    align
-FIELD   _args,          _SHA512_ARGS_X4_size, _SHA512_ARGS_X4_align
-FIELD   _lens,          8*4,    8
-FIELD   _unused_lanes,  8,      8
-FIELD   _ldata,         _LANE_DATA_size*4, _LANE_DATA_align
-END_FIELDS
- _MB_MGR_size  =  _FIELD_OFFSET
- _MB_MGR_align =  _STRUCT_ALIGN
-_args_digest = _args + _digest
-_args_data_ptr = _args + _data_ptr
-#######################################################################
-#######################################################################
-#### Define constants
-#######################################################################
-#define STS_UNKNOWN             0
-#define STS_BEING_PROCESSED     1
-#define STS_COMPLETED           2
-#######################################################################
-#### Define JOB_SHA512 structure
-#######################################################################
-START_FIELDS    # JOB_SHA512
-###     name                            size    align
-FIELD   _buffer,                        8,      8       # pointer to buffer
-FIELD   _len,                           8,      8       # length in bytes
-FIELD   _result_digest,                 8*8,    32      # Digest (output)
-FIELD   _status,                        4,      4
-FIELD   _user_data,                     8,      8
-END_FIELDS
- _JOB_SHA512_size = _FIELD_OFFSET
- _JOB_SHA512_align = _STRUCT_ALIGN
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S
deleted file mode 100644
index 7c629caebc05..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Flush routine for SHA512 multibuffer
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- *     Megha Dey <megha.dey@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-#include "sha512_mb_mgr_datastruct.S"
-.extern sha512_x4_avx2
-# LINUX register definitions
-#define arg1    %rdi
-#define arg2    %rsi
-# idx needs to be other than arg1, arg2, rbx, r12
-#define idx     %rdx
-# Common definitions
-#define state   arg1
-#define job     arg2
-#define len2    arg2
-#define unused_lanes    %rbx
-#define lane_data       %rbx
-#define tmp2            %rbx
-#define job_rax         %rax
-#define tmp1            %rax
-#define size_offset     %rax
-#define tmp             %rax
-#define start_offset    %rax
-#define tmp3            arg1
-#define extra_blocks    arg2
-#define p               arg2
-#define tmp4            %r8
-#define lens0           %r8
-#define lens1           %r9
-#define lens2           %r10
-#define lens3           %r11
-.macro LABEL prefix n
-\prefix\n\():
-.endm
-.macro JNE_SKIP i
-jne     skip_\i
-.endm
-.altmacro
-.macro SET_OFFSET _offset
-offset = \_offset
-.endm
-.noaltmacro
-# JOB* sha512_mb_mgr_flush_avx2(MB_MGR *state)
-# arg 1 : rcx : state
-ENTRY(sha512_mb_mgr_flush_avx2)
-        FRAME_BEGIN
-        push    %rbx
-        # If bit (32+3) is set, then all lanes are empty
-        mov     _unused_lanes(state), unused_lanes
-        bt      $32+7, unused_lanes
-        jc      return_null
-        # find a lane with a non-null job
-        xor     idx, idx
-        offset = (_ldata + 1*_LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  one(%rip), idx
-        offset = (_ldata + 2*_LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  two(%rip), idx
-        offset = (_ldata + 3*_LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-        cmovne  three(%rip), idx
-        # copy idx to empty lanes
-copy_lane_data:
-        offset =  (_args + _data_ptr)
-        mov     offset(state,idx,8), tmp
-        I = 0
-.rep 4
-        offset =  (_ldata + I * _LANE_DATA_size + _job_in_lane)
-        cmpq    $0, offset(state)
-.altmacro
-        JNE_SKIP %I
-        offset =  (_args + _data_ptr + 8*I)
-        mov     tmp, offset(state)
-        offset =  (_lens + 8*I +4)
-        movl    $0xFFFFFFFF, offset(state)
-LABEL skip_ %I
-        I = (I+1)
-.noaltmacro
-.endr
-        # Find min length
-        mov     _lens + 0*8(state),lens0
-        mov     lens0,idx
-        mov     _lens + 1*8(state),lens1
-        cmp     idx,lens1
-        cmovb   lens1,idx
-        mov     _lens + 2*8(state),lens2
-        cmp     idx,lens2
-        cmovb   lens2,idx
-        mov     _lens + 3*8(state),lens3
-        cmp     idx,lens3
-        cmovb   lens3,idx
-        mov     idx,len2
-        and     $0xF,idx
-        and     $~0xFF,len2
-        jz      len_is_0
-        sub     len2, lens0
-        sub     len2, lens1
-        sub     len2, lens2
-        sub     len2, lens3
-        shr     $32,len2
-        mov     lens0, _lens + 0*8(state)
-        mov     lens1, _lens + 1*8(state)
-        mov     lens2, _lens + 2*8(state)
-        mov     lens3, _lens + 3*8(state)
-        # "state" and "args" are the same address, arg1
-        # len is arg2
-        call    sha512_x4_avx2
-        # state and idx are intact
-len_is_0:
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        movq    $0,  _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        mov     _unused_lanes(state), unused_lanes
-        shl     $8, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens+4(state,  idx, 8)
-        vmovq _args_digest+0*32(state, idx, 8), %xmm0
-        vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0
-        vmovq _args_digest+2*32(state, idx, 8), %xmm1
-        vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1
-        vmovq _args_digest+4*32(state, idx, 8), %xmm2
-        vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2
-        vmovq _args_digest+6*32(state, idx, 8), %xmm3
-        vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3
-        vmovdqu %xmm0, _result_digest(job_rax)
-        vmovdqu %xmm1, _result_digest+1*16(job_rax)
-        vmovdqu %xmm2, _result_digest+2*16(job_rax)
-        vmovdqu %xmm3, _result_digest+3*16(job_rax)
-return:
-        pop     %rbx
-        FRAME_END
-        ret
-return_null:
-        xor     job_rax, job_rax
-        jmp     return
-ENDPROC(sha512_mb_mgr_flush_avx2)
-.align 16
-ENTRY(sha512_mb_mgr_get_comp_job_avx2)
-        push    %rbx
-        mov     _unused_lanes(state), unused_lanes
-        bt      $(32+7), unused_lanes
-        jc      .return_null
-        # Find min length
-        mov     _lens(state),lens0
-        mov     lens0,idx
-        mov     _lens+1*8(state),lens1
-        cmp     idx,lens1
-        cmovb   lens1,idx
-        mov     _lens+2*8(state),lens2
-        cmp     idx,lens2
-        cmovb   lens2,idx
-        mov     _lens+3*8(state),lens3
-        cmp     idx,lens3
-        cmovb   lens3,idx
-        test    $~0xF,idx
-        jnz     .return_null
-        and     $0xF,idx
-        #process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        movq    $0,  _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        mov     _unused_lanes(state), unused_lanes
-        shl     $8, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF, _lens+4(state,  idx, 8)
-        vmovq   _args_digest(state, idx, 8), %xmm0
-        vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0
-        vmovq    _args_digest+2*32(state, idx, 8), %xmm1
-        vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1
-        vmovq    _args_digest+4*32(state, idx, 8), %xmm2
-        vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2
-        vmovq    _args_digest+6*32(state, idx, 8), %xmm3
-        vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3
-        vmovdqu %xmm0, _result_digest+0*16(job_rax)
-        vmovdqu %xmm1, _result_digest+1*16(job_rax)
-        vmovdqu %xmm2, _result_digest+2*16(job_rax)
-        vmovdqu %xmm3, _result_digest+3*16(job_rax)
-        pop     %rbx
-        ret
-.return_null:
-        xor     job_rax, job_rax
-        pop     %rbx
-        ret
-ENDPROC(sha512_mb_mgr_get_comp_job_avx2)
-.section        .rodata.cst8.one, "aM", @progbits, 8
-.align 8
-one:
-.quad  1
-.section        .rodata.cst8.two, "aM", @progbits, 8
-.align 8
-two:
-.quad  2
-.section        .rodata.cst8.three, "aM", @progbits, 8
-.align 8
-three:
-.quad  3
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c
deleted file mode 100644
index d08805032f01..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Initialization code for multi buffer SHA256 algorithm for AVX2
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- *     Megha Dey <megha.dey@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include "sha512_mb_mgr.h"
-void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state)
-{
-        unsigned int j;
-        /* initially all lanes are unused */
-        state->lens[0] = 0xFFFFFFFF00000000;
-        state->lens[1] = 0xFFFFFFFF00000001;
-        state->lens[2] = 0xFFFFFFFF00000002;
-        state->lens[3] = 0xFFFFFFFF00000003;
-        state->unused_lanes = 0xFF03020100;
-        for (j = 0; j < 4; j++)
-                state->ldata[j].job_in_lane = NULL;
-}
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S
deleted file mode 100644
index 4ba709ba78e5..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Buffer submit code for multi buffer SHA512 algorithm
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- *     Megha Dey <megha.dey@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-#include "sha512_mb_mgr_datastruct.S"
-.extern sha512_x4_avx2
-#define arg1    %rdi
-#define arg2    %rsi
-#define idx             %rdx
-#define last_len        %rdx
-#define size_offset     %rcx
-#define tmp2            %rcx
-# Common definitions
-#define state   arg1
-#define job     arg2
-#define len2    arg2
-#define p2      arg2
-#define p               %r11
-#define start_offset    %r11
-#define unused_lanes    %rbx
-#define job_rax         %rax
-#define len             %rax
-#define lane            %r12
-#define tmp3            %r12
-#define lens3           %r12
-#define extra_blocks    %r8
-#define lens0           %r8
-#define tmp             %r9
-#define lens1           %r9
-#define lane_data       %r10
-#define lens2           %r10
-#define DWORD_len %eax
-# JOB* sha512_mb_mgr_submit_avx2(MB_MGR *state, JOB *job)
-# arg 1 : rcx : state
-# arg 2 : rdx : job
-ENTRY(sha512_mb_mgr_submit_avx2)
-        FRAME_BEGIN
-        push    %rbx
-        push    %r12
-        mov     _unused_lanes(state), unused_lanes
-        movzb     %bl,lane
-        shr     $8, unused_lanes
-        imul    $_LANE_DATA_size, lane,lane_data
-        movl    $STS_BEING_PROCESSED, _status(job)
-        lea     _ldata(state, lane_data), lane_data
-        mov     unused_lanes, _unused_lanes(state)
-        movl    _len(job),  DWORD_len
-        mov     job, _job_in_lane(lane_data)
-        movl    DWORD_len,_lens+4(state , lane, 8)
-        # Load digest words from result_digest
-        vmovdqu _result_digest+0*16(job), %xmm0
-        vmovdqu _result_digest+1*16(job), %xmm1
-        vmovdqu _result_digest+2*16(job), %xmm2
-        vmovdqu _result_digest+3*16(job), %xmm3
-        vmovq    %xmm0, _args_digest(state, lane, 8)
-        vpextrq  $1, %xmm0, _args_digest+1*32(state , lane, 8)
-        vmovq    %xmm1, _args_digest+2*32(state , lane, 8)
-        vpextrq  $1, %xmm1, _args_digest+3*32(state , lane, 8)
-        vmovq    %xmm2, _args_digest+4*32(state , lane, 8)
-        vpextrq  $1, %xmm2, _args_digest+5*32(state , lane, 8)
-        vmovq    %xmm3, _args_digest+6*32(state , lane, 8)
-        vpextrq  $1, %xmm3, _args_digest+7*32(state , lane, 8)
-        mov     _buffer(job), p
-        mov     p, _args_data_ptr(state, lane, 8)
-        cmp     $0xFF, unused_lanes
-        jne     return_null
-start_loop:
-        # Find min length
-        mov     _lens+0*8(state),lens0
-        mov     lens0,idx
-        mov     _lens+1*8(state),lens1
-        cmp     idx,lens1
-        cmovb   lens1, idx
-        mov     _lens+2*8(state),lens2
-        cmp     idx,lens2
-        cmovb   lens2,idx
-        mov     _lens+3*8(state),lens3
-        cmp     idx,lens3
-        cmovb   lens3,idx
-        mov     idx,len2
-        and     $0xF,idx
-        and     $~0xFF,len2
-        jz      len_is_0
-        sub     len2,lens0
-        sub     len2,lens1
-        sub     len2,lens2
-        sub     len2,lens3
-        shr     $32,len2
-        mov     lens0, _lens + 0*8(state)
-        mov     lens1, _lens + 1*8(state)
-        mov     lens2, _lens + 2*8(state)
-        mov     lens3, _lens + 3*8(state)
-        # "state" and "args" are the same address, arg1
-        # len is arg2
-        call    sha512_x4_avx2
-        # state and idx are intact
-len_is_0:
-        # process completed job "idx"
-        imul    $_LANE_DATA_size, idx, lane_data
-        lea     _ldata(state, lane_data), lane_data
-        mov     _job_in_lane(lane_data), job_rax
-        mov     _unused_lanes(state), unused_lanes
-        movq    $0, _job_in_lane(lane_data)
-        movl    $STS_COMPLETED, _status(job_rax)
-        shl     $8, unused_lanes
-        or      idx, unused_lanes
-        mov     unused_lanes, _unused_lanes(state)
-        movl    $0xFFFFFFFF,_lens+4(state,idx,8)
-        vmovq    _args_digest+0*32(state , idx, 8), %xmm0
-        vpinsrq  $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0
-        vmovq    _args_digest+2*32(state , idx, 8), %xmm1
-        vpinsrq  $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1
-        vmovq    _args_digest+4*32(state , idx, 8), %xmm2
-        vpinsrq  $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2
-        vmovq    _args_digest+6*32(state , idx, 8), %xmm3
-        vpinsrq  $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3
-        vmovdqu  %xmm0, _result_digest + 0*16(job_rax)
-        vmovdqu  %xmm1, _result_digest + 1*16(job_rax)
-        vmovdqu  %xmm2, _result_digest + 2*16(job_rax)
-        vmovdqu  %xmm3, _result_digest + 3*16(job_rax)
-return:
-        pop     %r12
-        pop     %rbx
-        FRAME_END
-        ret
-return_null:
-        xor     job_rax, job_rax
-        jmp     return
-ENDPROC(sha512_mb_mgr_submit_avx2)
-/* UNUSED?
-.section        .rodata.cst16, "aM", @progbits, 16
-.align 16
-H0:     .int  0x6a09e667
-H1:     .int  0xbb67ae85
-H2:     .int  0x3c6ef372
-H3:     .int  0xa54ff53a
-H4:     .int  0x510e527f
-H5:     .int  0x9b05688c
-H6:     .int  0x1f83d9ab
-H7:     .int  0x5be0cd19
-*/
diff --git a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S
deleted file mode 100644
index e22e907643a6..000000000000
--- a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S
+++ /dev/null
@@ -1,531 +0,0 @@
-/*
- * Multi-buffer SHA512 algorithm hash compute routine
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- *     Megha Dey <megha.dey@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2016 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-# code to compute quad SHA512 using AVX2
-# use YMMs to tackle the larger digest size
-# outer calling routine takes care of save and restore of XMM registers
-# Logic designed/laid out by JDG
-# Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; ymm0-15
-# Stack must be aligned to 32 bytes before call
-# Linux clobbers: rax rbx rcx rsi r8 r9 r10 r11 r12
-# Linux preserves: rcx rdx rdi rbp r13 r14 r15
-# clobbers ymm0-15
-#include <linux/linkage.h>
-#include "sha512_mb_mgr_datastruct.S"
-arg1 = %rdi
-arg2 = %rsi
-# Common definitions
-STATE = arg1
-INP_SIZE = arg2
-IDX = %rax
-ROUND = %rbx
-TBL = %r8
-inp0 = %r9
-inp1 = %r10
-inp2 = %r11
-inp3 = %r12
-a = %ymm0
-b = %ymm1
-c = %ymm2
-d = %ymm3
-e = %ymm4
-f = %ymm5
-g = %ymm6
-h = %ymm7
-a0 = %ymm8
-a1 = %ymm9
-a2 = %ymm10
-TT0 = %ymm14
-TT1 = %ymm13
-TT2 = %ymm12
-TT3 = %ymm11
-TT4 = %ymm10
-TT5 = %ymm9
-T1 = %ymm14
-TMP = %ymm15
-# Define stack usage
-STACK_SPACE1 = SZ4*16 + NUM_SHA512_DIGEST_WORDS*SZ4 + 24
-#define VMOVPD  vmovupd
-_digest = SZ4*16
-# transpose r0, r1, r2, r3, t0, t1
-# "transpose" data in {r0..r3} using temps {t0..t3}
-# Input looks like: {r0 r1 r2 r3}
-# r0 = {a7 a6 a5 a4 a3 a2 a1 a0}
-# r1 = {b7 b6 b5 b4 b3 b2 b1 b0}
-# r2 = {c7 c6 c5 c4 c3 c2 c1 c0}
-# r3 = {d7 d6 d5 d4 d3 d2 d1 d0}
-#
-# output looks like: {t0 r1 r0 r3}
-# t0 = {d1 d0 c1 c0 b1 b0 a1 a0}
-# r1 = {d3 d2 c3 c2 b3 b2 a3 a2}
-# r0 = {d5 d4 c5 c4 b5 b4 a5 a4}
-# r3 = {d7 d6 c7 c6 b7 b6 a7 a6}
-.macro TRANSPOSE r0 r1 r2 r3 t0 t1
-        vshufps  $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4   b1 b0 a1 a0}
-        vshufps  $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6   b3 b2 a3 a2}
-        vshufps  $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4   d1 d0 c1 c0}
-        vshufps  $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6   d3 d2 c3 c2}
-        vperm2f128      $0x20, \r2, \r0, \r1  # h6...a6
-        vperm2f128      $0x31, \r2, \r0, \r3  # h2...a2
-        vperm2f128      $0x31, \t1, \t0, \r0  # h5...a5
-        vperm2f128      $0x20, \t1, \t0, \t0  # h1...a1
-.endm
-.macro ROTATE_ARGS
-TMP_ = h
-h = g
-g = f
-f = e
-e = d
-d = c
-c = b
-b = a
-a = TMP_
-.endm
-# PRORQ reg, imm, tmp
-# packed-rotate-right-double
-# does a rotate by doing two shifts and an or
-.macro _PRORQ reg imm tmp
-        vpsllq  $(64-\imm),\reg,\tmp
-        vpsrlq  $\imm,\reg, \reg
-        vpor    \tmp,\reg, \reg
-.endm
-# non-destructive
-# PRORQ_nd reg, imm, tmp, src
-.macro _PRORQ_nd reg imm tmp src
-        vpsllq  $(64-\imm), \src, \tmp
-        vpsrlq  $\imm, \src, \reg
-        vpor    \tmp, \reg, \reg
-.endm
-# PRORQ dst/src, amt
-.macro PRORQ reg imm
-        _PRORQ  \reg, \imm, TMP
-.endm
-# PRORQ_nd dst, src, amt
-.macro PRORQ_nd reg tmp imm
-        _PRORQ_nd       \reg, \imm, TMP, \tmp
-.endm
-#; arguments passed implicitly in preprocessor symbols i, a...h
-.macro ROUND_00_15 _T1 i
-        PRORQ_nd a0, e, (18-14) # sig1: a0 = (e >> 4)
-        vpxor   g, f, a2        # ch: a2 = f^g
-        vpand   e,a2, a2                # ch: a2 = (f^g)&e
-        vpxor   g, a2, a2               # a2 = ch
-        PRORQ_nd        a1,e,41         # sig1: a1 = (e >> 25)
-        offset = SZ4*(\i & 0xf)
-        vmovdqu \_T1,offset(%rsp)
-        vpaddq  (TBL,ROUND,1), \_T1, \_T1       # T1 = W + K
-        vpxor   e,a0, a0        # sig1: a0 = e ^ (e >> 5)
-        PRORQ   a0, 14           # sig1: a0 = (e >> 6) ^ (e >> 11)
-        vpaddq  a2, h, h        # h = h + ch
-        PRORQ_nd        a2,a,6  # sig0: a2 = (a >> 11)
-        vpaddq  \_T1,h, h       # h = h + ch + W + K
-        vpxor   a1, a0, a0      # a0 = sigma1
-        vmovdqu a,\_T1
-        PRORQ_nd        a1,a,39 # sig0: a1 = (a >> 22)
-        vpxor   c, \_T1, \_T1      # maj: T1 = a^c
-        add     $SZ4, ROUND     # ROUND++
-        vpand   b, \_T1, \_T1   # maj: T1 = (a^c)&b
-        vpaddq  a0, h, h
-        vpaddq  h, d, d
-        vpxor   a, a2, a2       # sig0: a2 = a ^ (a >> 11)
-        PRORQ   a2,28            # sig0: a2 = (a >> 2) ^ (a >> 13)
-        vpxor   a1, a2, a2      # a2 = sig0
-        vpand   c, a, a1        # maj: a1 = a&c
-        vpor    \_T1, a1, a1    # a1 = maj
-        vpaddq  a1, h, h        # h = h + ch + W + K + maj
-        vpaddq  a2, h, h        # h = h + ch + W + K + maj + sigma0
-        ROTATE_ARGS
-.endm
-#; arguments passed implicitly in preprocessor symbols i, a...h
-.macro ROUND_16_XX _T1 i
-        vmovdqu SZ4*((\i-15)&0xf)(%rsp), \_T1
-        vmovdqu SZ4*((\i-2)&0xf)(%rsp), a1
-        vmovdqu \_T1, a0
-        PRORQ   \_T1,7
-        vmovdqu a1, a2
-        PRORQ   a1,42
-        vpxor   a0, \_T1, \_T1
-        PRORQ   \_T1, 1
-        vpxor   a2, a1, a1
-        PRORQ   a1, 19
-        vpsrlq  $7, a0, a0
-        vpxor   a0, \_T1, \_T1
-        vpsrlq  $6, a2, a2
-        vpxor   a2, a1, a1
-        vpaddq  SZ4*((\i-16)&0xf)(%rsp), \_T1, \_T1
-        vpaddq  SZ4*((\i-7)&0xf)(%rsp), a1, a1
-        vpaddq  a1, \_T1, \_T1
-        ROUND_00_15 \_T1,\i
-.endm
-# void sha512_x4_avx2(void *STATE, const int INP_SIZE)
-# arg 1 : STATE    : pointer to input data
-# arg 2 : INP_SIZE : size of data in blocks (assumed >= 1)
-ENTRY(sha512_x4_avx2)
-        # general registers preserved in outer calling routine
-        # outer calling routine saves all the XMM registers
-        # save callee-saved clobbered registers to comply with C function ABI
-        push    %r12
-        push    %r13
-        push    %r14
-        push    %r15
-        sub     $STACK_SPACE1, %rsp
-        # Load the pre-transposed incoming digest.
-        vmovdqu 0*SHA512_DIGEST_ROW_SIZE(STATE),a
-        vmovdqu 1*SHA512_DIGEST_ROW_SIZE(STATE),b
-        vmovdqu 2*SHA512_DIGEST_ROW_SIZE(STATE),c
-        vmovdqu 3*SHA512_DIGEST_ROW_SIZE(STATE),d
-        vmovdqu 4*SHA512_DIGEST_ROW_SIZE(STATE),e
-        vmovdqu 5*SHA512_DIGEST_ROW_SIZE(STATE),f
-        vmovdqu 6*SHA512_DIGEST_ROW_SIZE(STATE),g
-        vmovdqu 7*SHA512_DIGEST_ROW_SIZE(STATE),h
-        lea     K512_4(%rip),TBL
-        # load the address of each of the 4 message lanes
-        # getting ready to transpose input onto stack
-        mov     _data_ptr+0*PTR_SZ(STATE),inp0
-        mov     _data_ptr+1*PTR_SZ(STATE),inp1
-        mov     _data_ptr+2*PTR_SZ(STATE),inp2
-        mov     _data_ptr+3*PTR_SZ(STATE),inp3
-        xor     IDX, IDX
-lloop:
-        xor     ROUND, ROUND
-        # save old digest
-        vmovdqu a, _digest(%rsp)
-        vmovdqu b, _digest+1*SZ4(%rsp)
-        vmovdqu c, _digest+2*SZ4(%rsp)
-        vmovdqu d, _digest+3*SZ4(%rsp)
-        vmovdqu e, _digest+4*SZ4(%rsp)
-        vmovdqu f, _digest+5*SZ4(%rsp)
-        vmovdqu g, _digest+6*SZ4(%rsp)
-        vmovdqu h, _digest+7*SZ4(%rsp)
-        i = 0
-.rep 4
-        vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP
-        VMOVPD  i*32(inp0, IDX), TT2
-        VMOVPD  i*32(inp1, IDX), TT1
-        VMOVPD  i*32(inp2, IDX), TT4
-        VMOVPD  i*32(inp3, IDX), TT3
-        TRANSPOSE       TT2, TT1, TT4, TT3, TT0, TT5
-        vpshufb TMP, TT0, TT0
-        vpshufb TMP, TT1, TT1
-        vpshufb TMP, TT2, TT2
-        vpshufb TMP, TT3, TT3
-        ROUND_00_15     TT0,(i*4+0)
-        ROUND_00_15     TT1,(i*4+1)
-        ROUND_00_15     TT2,(i*4+2)
-        ROUND_00_15     TT3,(i*4+3)
-        i = (i+1)
-.endr
-        add     $128, IDX
-        i = (i*4)
-        jmp     Lrounds_16_xx
-.align 16
-Lrounds_16_xx:
-.rep 16
-        ROUND_16_XX     T1, i
-        i = (i+1)
-.endr
-        cmp     $0xa00,ROUND
-        jb      Lrounds_16_xx
-        # add old digest
-        vpaddq  _digest(%rsp), a, a
-        vpaddq  _digest+1*SZ4(%rsp), b, b
-        vpaddq  _digest+2*SZ4(%rsp), c, c
-        vpaddq  _digest+3*SZ4(%rsp), d, d
-        vpaddq  _digest+4*SZ4(%rsp), e, e
-        vpaddq  _digest+5*SZ4(%rsp), f, f
-        vpaddq  _digest+6*SZ4(%rsp), g, g
-        vpaddq  _digest+7*SZ4(%rsp), h, h
-        sub     $1, INP_SIZE  # unit is blocks
-        jne     lloop
-        # write back to memory (state object) the transposed digest
-        vmovdqu a, 0*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu b, 1*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu c, 2*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu d, 3*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu e, 4*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu f, 5*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu g, 6*SHA512_DIGEST_ROW_SIZE(STATE)
-        vmovdqu h, 7*SHA512_DIGEST_ROW_SIZE(STATE)
-        # update input data pointers
-        add     IDX, inp0
-        mov     inp0, _data_ptr+0*PTR_SZ(STATE)
-        add     IDX, inp1
-        mov     inp1, _data_ptr+1*PTR_SZ(STATE)
-        add     IDX, inp2
-        mov     inp2, _data_ptr+2*PTR_SZ(STATE)
-        add     IDX, inp3
-        mov     inp3, _data_ptr+3*PTR_SZ(STATE)
-        #;;;;;;;;;;;;;;;
-        #; Postamble
-        add $STACK_SPACE1, %rsp
-        # restore callee-saved clobbered registers
-        pop     %r15
-        pop     %r14
-        pop     %r13
-        pop     %r12
-        # outer calling routine restores XMM and other GP registers
-        ret
-ENDPROC(sha512_x4_avx2)
-.section        .rodata.K512_4, "a", @progbits
-.align 64
-K512_4:
-        .octa 0x428a2f98d728ae22428a2f98d728ae22,\
-                0x428a2f98d728ae22428a2f98d728ae22
-        .octa 0x7137449123ef65cd7137449123ef65cd,\
-                0x7137449123ef65cd7137449123ef65cd
-        .octa 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f,\
-                0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f
-        .octa 0xe9b5dba58189dbbce9b5dba58189dbbc,\
-                0xe9b5dba58189dbbce9b5dba58189dbbc
-        .octa 0x3956c25bf348b5383956c25bf348b538,\
-                0x3956c25bf348b5383956c25bf348b538
-        .octa 0x59f111f1b605d01959f111f1b605d019,\
-                0x59f111f1b605d01959f111f1b605d019
-        .octa 0x923f82a4af194f9b923f82a4af194f9b,\
-                0x923f82a4af194f9b923f82a4af194f9b
-        .octa 0xab1c5ed5da6d8118ab1c5ed5da6d8118,\
-                0xab1c5ed5da6d8118ab1c5ed5da6d8118
-        .octa 0xd807aa98a3030242d807aa98a3030242,\
-                0xd807aa98a3030242d807aa98a3030242
-        .octa 0x12835b0145706fbe12835b0145706fbe,\
-                0x12835b0145706fbe12835b0145706fbe
-        .octa 0x243185be4ee4b28c243185be4ee4b28c,\
-                0x243185be4ee4b28c243185be4ee4b28c
-        .octa 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2,\
-                0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2
-        .octa 0x72be5d74f27b896f72be5d74f27b896f,\
-                0x72be5d74f27b896f72be5d74f27b896f
-        .octa 0x80deb1fe3b1696b180deb1fe3b1696b1,\
-                0x80deb1fe3b1696b180deb1fe3b1696b1
-        .octa 0x9bdc06a725c712359bdc06a725c71235,\
-                0x9bdc06a725c712359bdc06a725c71235
-        .octa 0xc19bf174cf692694c19bf174cf692694,\
-                0xc19bf174cf692694c19bf174cf692694
-        .octa 0xe49b69c19ef14ad2e49b69c19ef14ad2,\
-                0xe49b69c19ef14ad2e49b69c19ef14ad2
-        .octa 0xefbe4786384f25e3efbe4786384f25e3,\
-                0xefbe4786384f25e3efbe4786384f25e3
-        .octa 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5,\
-                0x0fc19dc68b8cd5b50fc19dc68b8cd5b5
-        .octa 0x240ca1cc77ac9c65240ca1cc77ac9c65,\
-                0x240ca1cc77ac9c65240ca1cc77ac9c65
-        .octa 0x2de92c6f592b02752de92c6f592b0275,\
-                0x2de92c6f592b02752de92c6f592b0275
-        .octa 0x4a7484aa6ea6e4834a7484aa6ea6e483,\
-                0x4a7484aa6ea6e4834a7484aa6ea6e483
-        .octa 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4,\
-                0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4
-        .octa 0x76f988da831153b576f988da831153b5,\
-                0x76f988da831153b576f988da831153b5
-        .octa 0x983e5152ee66dfab983e5152ee66dfab,\
-                0x983e5152ee66dfab983e5152ee66dfab
-        .octa 0xa831c66d2db43210a831c66d2db43210,\
-                0xa831c66d2db43210a831c66d2db43210
-        .octa 0xb00327c898fb213fb00327c898fb213f,\
-                0xb00327c898fb213fb00327c898fb213f
-        .octa 0xbf597fc7beef0ee4bf597fc7beef0ee4,\
-                0xbf597fc7beef0ee4bf597fc7beef0ee4
-        .octa 0xc6e00bf33da88fc2c6e00bf33da88fc2,\
-                0xc6e00bf33da88fc2c6e00bf33da88fc2
-        .octa 0xd5a79147930aa725d5a79147930aa725,\
-                0xd5a79147930aa725d5a79147930aa725
-        .octa 0x06ca6351e003826f06ca6351e003826f,\
-                0x06ca6351e003826f06ca6351e003826f
-        .octa 0x142929670a0e6e70142929670a0e6e70,\
-                0x142929670a0e6e70142929670a0e6e70
-        .octa 0x27b70a8546d22ffc27b70a8546d22ffc,\
-                0x27b70a8546d22ffc27b70a8546d22ffc
-        .octa 0x2e1b21385c26c9262e1b21385c26c926,\
-                0x2e1b21385c26c9262e1b21385c26c926
-        .octa 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed,\
-                0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed
-        .octa 0x53380d139d95b3df53380d139d95b3df,\
-                0x53380d139d95b3df53380d139d95b3df
-        .octa 0x650a73548baf63de650a73548baf63de,\
-                0x650a73548baf63de650a73548baf63de
-        .octa 0x766a0abb3c77b2a8766a0abb3c77b2a8,\
-                0x766a0abb3c77b2a8766a0abb3c77b2a8
-        .octa 0x81c2c92e47edaee681c2c92e47edaee6,\
-                0x81c2c92e47edaee681c2c92e47edaee6
-        .octa 0x92722c851482353b92722c851482353b,\
-                0x92722c851482353b92722c851482353b
-        .octa 0xa2bfe8a14cf10364a2bfe8a14cf10364,\
-                0xa2bfe8a14cf10364a2bfe8a14cf10364
-        .octa 0xa81a664bbc423001a81a664bbc423001,\
-                0xa81a664bbc423001a81a664bbc423001
-        .octa 0xc24b8b70d0f89791c24b8b70d0f89791,\
-                0xc24b8b70d0f89791c24b8b70d0f89791
-        .octa 0xc76c51a30654be30c76c51a30654be30,\
-                0xc76c51a30654be30c76c51a30654be30
-        .octa 0xd192e819d6ef5218d192e819d6ef5218,\
-                0xd192e819d6ef5218d192e819d6ef5218
-        .octa 0xd69906245565a910d69906245565a910,\
-                0xd69906245565a910d69906245565a910
-        .octa 0xf40e35855771202af40e35855771202a,\
-                0xf40e35855771202af40e35855771202a
-        .octa 0x106aa07032bbd1b8106aa07032bbd1b8,\
-                0x106aa07032bbd1b8106aa07032bbd1b8
-        .octa 0x19a4c116b8d2d0c819a4c116b8d2d0c8,\
-                0x19a4c116b8d2d0c819a4c116b8d2d0c8
-        .octa 0x1e376c085141ab531e376c085141ab53,\
-                0x1e376c085141ab531e376c085141ab53
-        .octa 0x2748774cdf8eeb992748774cdf8eeb99,\
-                0x2748774cdf8eeb992748774cdf8eeb99
-        .octa 0x34b0bcb5e19b48a834b0bcb5e19b48a8,\
-                0x34b0bcb5e19b48a834b0bcb5e19b48a8
-        .octa 0x391c0cb3c5c95a63391c0cb3c5c95a63,\
-                0x391c0cb3c5c95a63391c0cb3c5c95a63
-        .octa 0x4ed8aa4ae3418acb4ed8aa4ae3418acb,\
-                0x4ed8aa4ae3418acb4ed8aa4ae3418acb
-        .octa 0x5b9cca4f7763e3735b9cca4f7763e373,\
-                0x5b9cca4f7763e3735b9cca4f7763e373
-        .octa 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3,\
-                0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3
-        .octa 0x748f82ee5defb2fc748f82ee5defb2fc,\
-                0x748f82ee5defb2fc748f82ee5defb2fc
-        .octa 0x78a5636f43172f6078a5636f43172f60,\
-                0x78a5636f43172f6078a5636f43172f60
-        .octa 0x84c87814a1f0ab7284c87814a1f0ab72,\
-                0x84c87814a1f0ab7284c87814a1f0ab72
-        .octa 0x8cc702081a6439ec8cc702081a6439ec,\
-                0x8cc702081a6439ec8cc702081a6439ec
-        .octa 0x90befffa23631e2890befffa23631e28,\
-                0x90befffa23631e2890befffa23631e28
-        .octa 0xa4506cebde82bde9a4506cebde82bde9,\
-                0xa4506cebde82bde9a4506cebde82bde9
-        .octa 0xbef9a3f7b2c67915bef9a3f7b2c67915,\
-                0xbef9a3f7b2c67915bef9a3f7b2c67915
-        .octa 0xc67178f2e372532bc67178f2e372532b,\
-                0xc67178f2e372532bc67178f2e372532b
-        .octa 0xca273eceea26619cca273eceea26619c,\
-                0xca273eceea26619cca273eceea26619c
-        .octa 0xd186b8c721c0c207d186b8c721c0c207,\
-                0xd186b8c721c0c207d186b8c721c0c207
-        .octa 0xeada7dd6cde0eb1eeada7dd6cde0eb1e,\
-                0xeada7dd6cde0eb1eeada7dd6cde0eb1e
-        .octa 0xf57d4f7fee6ed178f57d4f7fee6ed178,\
-                0xf57d4f7fee6ed178f57d4f7fee6ed178
-        .octa 0x06f067aa72176fba06f067aa72176fba,\
-                0x06f067aa72176fba06f067aa72176fba
-        .octa 0x0a637dc5a2c898a60a637dc5a2c898a6,\
-                0x0a637dc5a2c898a60a637dc5a2c898a6
-        .octa 0x113f9804bef90dae113f9804bef90dae,\
-                0x113f9804bef90dae113f9804bef90dae
-        .octa 0x1b710b35131c471b1b710b35131c471b,\
-                0x1b710b35131c471b1b710b35131c471b
-        .octa 0x28db77f523047d8428db77f523047d84,\
-                0x28db77f523047d8428db77f523047d84
-        .octa 0x32caab7b40c7249332caab7b40c72493,\
-                0x32caab7b40c7249332caab7b40c72493
-        .octa 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc,\
-                0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc
-        .octa 0x431d67c49c100d4c431d67c49c100d4c,\
-                0x431d67c49c100d4c431d67c49c100d4c
-        .octa 0x4cc5d4becb3e42b64cc5d4becb3e42b6,\
-                0x4cc5d4becb3e42b64cc5d4becb3e42b6
-        .octa 0x597f299cfc657e2a597f299cfc657e2a,\
-                0x597f299cfc657e2a597f299cfc657e2a
-        .octa 0x5fcb6fab3ad6faec5fcb6fab3ad6faec,\
-                0x5fcb6fab3ad6faec5fcb6fab3ad6faec
-        .octa 0x6c44198c4a4758176c44198c4a475817,\
-                0x6c44198c4a4758176c44198c4a475817
-.section        .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
-.align 32
-PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607
-                         .octa 0x18191a1b1c1d1e1f1011121314151617
diff --git a/crypto/Kconfig b/crypto/Kconfig
index f3e40ac56d93..f7a235db56aa 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -213,20 +213,6 @@ config CRYPTO_CRYPTD
          converts an arbitrary synchronous software crypto algorithm
          into an asynchronous algorithm that executes in a kernel thread.
-config CRYPTO_MCRYPTD
-        tristate "Software async multi-buffer crypto daemon"
-        select CRYPTO_BLKCIPHER
-        select CRYPTO_HASH
-        select CRYPTO_MANAGER
-        select CRYPTO_WORKQUEUE
-        help
-          This is a generic software asynchronous crypto daemon that
-          provides the kernel thread to assist multi-buffer crypto
-          algorithms for submitting jobs and flushing jobs in multi-buffer
-          crypto algorithms.  Multi-buffer crypto algorithms are executed
-          in the context of this kernel thread and drivers can post
-          their crypto request asynchronously to be processed by this daemon.
 config CRYPTO_AUTHENC
        tristate "Authenc support"
        select CRYPTO_AEAD
@@ -470,6 +456,18 @@ config CRYPTO_LRW
          The first 128, 192 or 256 bits in the key are used for AES and the
          rest is used to tie each cipher block to its logical position.
+config CRYPTO_OFB
+        tristate "OFB support"
+        select CRYPTO_BLKCIPHER
+        select CRYPTO_MANAGER
+        help
+          OFB: the Output Feedback mode makes a block cipher into a synchronous
+          stream cipher. It generates keystream blocks, which are then XORed
+          with the plaintext blocks to get the ciphertext. Flipping a bit in the
+          ciphertext produces a flipped bit in the plaintext at the same
+          location. This property allows many error correcting codes to function
+          normally even when applied before encryption.
 config CRYPTO_PCBC
        tristate "PCBC support"
        select CRYPTO_BLKCIPHER
@@ -848,54 +846,6 @@ config CRYPTO_SHA1_PPC_SPE
          SHA-1 secure hash standard (DFIPS 180-4) implemented
          using powerpc SPE SIMD instruction set.
-config CRYPTO_SHA1_MB
-        tristate "SHA1 digest algorithm (x86_64 Multi-Buffer, Experimental)"
-        depends on X86 && 64BIT
-        select CRYPTO_SHA1
-        select CRYPTO_HASH
-        select CRYPTO_MCRYPTD
-        help
-          SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
-          using multi-buffer technique.  This algorithm computes on
-          multiple data lanes concurrently with SIMD instructions for
-          better throughput.  It should not be enabled by default but
-          used when there is significant amount of work to keep the keep
-          the data lanes filled to get performance benefit.  If the data
-          lanes remain unfilled, a flush operation will be initiated to
-          process the crypto jobs, adding a slight latency.
-config CRYPTO_SHA256_MB
-        tristate "SHA256 digest algorithm (x86_64 Multi-Buffer, Experimental)"
-        depends on X86 && 64BIT
-        select CRYPTO_SHA256
-        select CRYPTO_HASH
-        select CRYPTO_MCRYPTD
-        help
-          SHA-256 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
-          using multi-buffer technique.  This algorithm computes on
-          multiple data lanes concurrently with SIMD instructions for
-          better throughput.  It should not be enabled by default but
-          used when there is significant amount of work to keep the keep
-          the data lanes filled to get performance benefit.  If the data
-          lanes remain unfilled, a flush operation will be initiated to
-          process the crypto jobs, adding a slight latency.
-config CRYPTO_SHA512_MB
-        tristate "SHA512 digest algorithm (x86_64 Multi-Buffer, Experimental)"
-        depends on X86 && 64BIT
-        select CRYPTO_SHA512
-        select CRYPTO_HASH
-        select CRYPTO_MCRYPTD
-        help
-          SHA-512 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
-          using multi-buffer technique.  This algorithm computes on
-          multiple data lanes concurrently with SIMD instructions for
-          better throughput.  It should not be enabled by default but
-          used when there is significant amount of work to keep the keep
-          the data lanes filled to get performance benefit.  If the data
-          lanes remain unfilled, a flush operation will be initiated to
-          process the crypto jobs, adding a slight latency.
 config CRYPTO_SHA256
        tristate "SHA224 and SHA256 digest algorithm"
        select CRYPTO_HASH
@@ -1133,7 +1083,7 @@ config CRYPTO_AES_NI_INTEL
          In addition to AES cipher algorithm support, the acceleration
          for some popular block cipher mode is supported too, including
-          ECB, CBC, LRW, PCBC, XTS. The 64 bit version has additional
+          ECB, CBC, LRW, XTS. The 64 bit version has additional
          acceleration for CTR.
 config CRYPTO_AES_SPARC64
@@ -1590,20 +1540,6 @@ config CRYPTO_SM4
          If unsure, say N.
-config CRYPTO_SPECK
-        tristate "Speck cipher algorithm"
-        select CRYPTO_ALGAPI
-        help
-          Speck is a lightweight block cipher that is tuned for optimal
-          performance in software (rather than hardware).
-          Speck may not be as secure as AES, and should only be used on systems
-          where AES is not fast enough.
-          See also: <https://eprint.iacr.org/2013/404.pdf>
-          If unsure, say N.
 config CRYPTO_TEA
        tristate "TEA, XTEA and XETA cipher algorithms"
        select CRYPTO_ALGAPI
@@ -1875,6 +1811,17 @@ config CRYPTO_USER_API_AEAD
          This option enables the user-spaces interface for AEAD
          cipher algorithms.
+config CRYPTO_STATS
+        bool "Crypto usage statistics for User-space"
+        help
+          This option enables the gathering of crypto stats.
+          This will collect:
+          - encrypt/decrypt size and numbers of symmeric operations
+          - compress/decompress size and numbers of compress operations
+          - size and numbers of hash operations
+          - encrypt/decrypt/sign/verify numbers for asymmetric operations
+          - generate/seed numbers for rng operations
 config CRYPTO_HASH_INFO
        bool
diff --git a/crypto/Makefile b/crypto/Makefile
index 6d1d40eeb964..5c207c76abf7 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -54,6 +54,7 @@ cryptomgr-y := algboss.o testmgr.o
 obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o
 obj-$(CONFIG_CRYPTO_USER) += crypto_user.o
+crypto_user-y := crypto_user_base.o crypto_user_stat.o
 obj-$(CONFIG_CRYPTO_CMAC) += cmac.o
 obj-$(CONFIG_CRYPTO_HMAC) += hmac.o
 obj-$(CONFIG_CRYPTO_VMAC) += vmac.o
@@ -93,7 +94,6 @@ obj-$(CONFIG_CRYPTO_MORUS640) += morus640.o
 obj-$(CONFIG_CRYPTO_MORUS1280) += morus1280.o
 obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
 obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
-obj-$(CONFIG_CRYPTO_MCRYPTD) += mcryptd.o
 obj-$(CONFIG_CRYPTO_DES) += des_generic.o
 obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o
 obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish_generic.o
@@ -115,7 +115,6 @@ obj-$(CONFIG_CRYPTO_TEA) += tea.o
 obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o
 obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o
 obj-$(CONFIG_CRYPTO_SEED) += seed.o
-obj-$(CONFIG_CRYPTO_SPECK) += speck.o
 obj-$(CONFIG_CRYPTO_SALSA20) += salsa20_generic.o
 obj-$(CONFIG_CRYPTO_CHACHA20) += chacha20_generic.o
 obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o
@@ -143,6 +142,7 @@ obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
 obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
 obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o
 obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
+obj-$(CONFIG_CRYPTO_OFB) += ofb.o
 ecdh_generic-y := ecc.o
 ecdh_generic-y += ecdh.o
diff --git a/crypto/aegis.h b/crypto/aegis.h
index f1c6900ddb80..405e025fc906 100644
--- a/crypto/aegis.h
+++ b/crypto/aegis.h
@@ -21,7 +21,7 @@
 union aegis_block {
        __le64 words64[AEGIS_BLOCK_SIZE / sizeof(__le64)];
-        u32 words32[AEGIS_BLOCK_SIZE / sizeof(u32)];
+        __le32 words32[AEGIS_BLOCK_SIZE / sizeof(__le32)];
        u8 bytes[AEGIS_BLOCK_SIZE];
 };
@@ -57,24 +57,22 @@ static void crypto_aegis_aesenc(union aegis_block *dst,
                                const union aegis_block *src,
                                const union aegis_block *key)
 {
-        u32 *d = dst->words32;
        const u8  *s  = src->bytes;
-        const u32 *k  = key->words32;
        const u32 *t0 = crypto_ft_tab[0];
        const u32 *t1 = crypto_ft_tab[1];
        const u32 *t2 = crypto_ft_tab[2];
        const u32 *t3 = crypto_ft_tab[3];
        u32 d0, d1, d2, d3;
-        d0 = t0[s[ 0]] ^ t1[s[ 5]] ^ t2[s[10]] ^ t3[s[15]] ^ k[0];
+        d0 = t0[s[ 0]] ^ t1[s[ 5]] ^ t2[s[10]] ^ t3[s[15]];
-        d1 = t0[s[ 4]] ^ t1[s[ 9]] ^ t2[s[14]] ^ t3[s[ 3]] ^ k[1];
+        d1 = t0[s[ 4]] ^ t1[s[ 9]] ^ t2[s[14]] ^ t3[s[ 3]];
-        d2 = t0[s[ 8]] ^ t1[s[13]] ^ t2[s[ 2]] ^ t3[s[ 7]] ^ k[2];
+        d2 = t0[s[ 8]] ^ t1[s[13]] ^ t2[s[ 2]] ^ t3[s[ 7]];
-        d3 = t0[s[12]] ^ t1[s[ 1]] ^ t2[s[ 6]] ^ t3[s[11]] ^ k[3];
+        d3 = t0[s[12]] ^ t1[s[ 1]] ^ t2[s[ 6]] ^ t3[s[11]];
-        d[0] = d0;
+        dst->words32[0] = cpu_to_le32(d0) ^ key->words32[0];
-        d[1] = d1;
+        dst->words32[1] = cpu_to_le32(d1) ^ key->words32[1];
-        d[2] = d2;
+        dst->words32[2] = cpu_to_le32(d2) ^ key->words32[2];
-        d[3] = d3;
+        dst->words32[3] = cpu_to_le32(d3) ^ key->words32[3];
 }
 #endif /* _CRYPTO_AEGIS_H */
diff --git a/crypto/ahash.c b/crypto/ahash.c
index a64c143165b1..e21667b4e10a 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -364,24 +364,35 @@ static int crypto_ahash_op(struct ahash_request *req,
 int crypto_ahash_final(struct ahash_request *req)
 {
-        return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final);
+        int ret;
+        ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final);
+        crypto_stat_ahash_final(req, ret);
+        return ret;
 }
 EXPORT_SYMBOL_GPL(crypto_ahash_final);
 int crypto_ahash_finup(struct ahash_request *req)
 {
-        return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup);
+        int ret;
+        ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup);
+        crypto_stat_ahash_final(req, ret);
+        return ret;
 }
 EXPORT_SYMBOL_GPL(crypto_ahash_finup);
 int crypto_ahash_digest(struct ahash_request *req)
 {
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+        int ret;
        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
-                return -ENOKEY;
+                ret = -ENOKEY;
+        else
-        return crypto_ahash_op(req, tfm->digest);
+                ret = crypto_ahash_op(req, tfm->digest);
+        crypto_stat_ahash_final(req, ret);
+        return ret;
 }
 EXPORT_SYMBOL_GPL(crypto_ahash_digest);
@@ -550,8 +561,8 @@ static int ahash_prepare_alg(struct ahash_alg *alg)
 {
        struct crypto_alg *base = &alg->halg.base;
-        if (alg->halg.digestsize > PAGE_SIZE / 8 ||
+        if (alg->halg.digestsize > HASH_MAX_DIGESTSIZE ||
-            alg->halg.statesize > PAGE_SIZE / 8 ||
+            alg->halg.statesize > HASH_MAX_STATESIZE ||
            alg->halg.statesize == 0)
                return -EINVAL;
diff --git a/crypto/algapi.c b/crypto/algapi.c
index c0755cf4f53f..2545c5f89c4c 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -57,9 +57,14 @@ static int crypto_check_alg(struct crypto_alg *alg)
        if (alg->cra_alignmask & (alg->cra_alignmask + 1))
                return -EINVAL;
-        if (alg->cra_blocksize > PAGE_SIZE / 8)
+        /* General maximums for all algs. */
+        if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
                return -EINVAL;
+        if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
+                return -EINVAL;
+        /* Lower maximums for specific alg types. */
        if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
                               CRYPTO_ALG_TYPE_CIPHER) {
                if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
@@ -253,6 +258,14 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
        list_add(&alg->cra_list, &crypto_alg_list);
        list_add(&larval->alg.cra_list, &crypto_alg_list);
+        atomic_set(&alg->encrypt_cnt, 0);
+        atomic_set(&alg->decrypt_cnt, 0);
+        atomic64_set(&alg->encrypt_tlen, 0);
+        atomic64_set(&alg->decrypt_tlen, 0);
+        atomic_set(&alg->verify_cnt, 0);
+        atomic_set(&alg->cipher_err_cnt, 0);
+        atomic_set(&alg->sign_cnt, 0);
 out:
        return larval;
@@ -367,6 +380,8 @@ static void crypto_wait_for_test(struct crypto_larval *larval)
        err = wait_for_completion_killable(&larval->completion);
        WARN_ON(err);
+        if (!err)
+                crypto_probing_notify(CRYPTO_MSG_ALG_LOADED, larval);
 out:
        crypto_larval_kill(&larval->alg);
diff --git a/crypto/algboss.c b/crypto/algboss.c
index 5e6df2a087fa..527b44d0af21 100644
--- a/crypto/algboss.c
+++ b/crypto/algboss.c
@@ -274,6 +274,8 @@ static int cryptomgr_notify(struct notifier_block *this, unsigned long msg,
                return cryptomgr_schedule_probe(data);
        case CRYPTO_MSG_ALG_REGISTER:
                return cryptomgr_schedule_test(data);
+        case CRYPTO_MSG_ALG_LOADED:
+                break;
        }
        return NOTIFY_DONE;
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index c40a8c7ee8ae..eb100a04ce9f 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -42,7 +42,7 @@
 struct aead_tfm {
        struct crypto_aead *aead;
-        struct crypto_skcipher *null_tfm;
+        struct crypto_sync_skcipher *null_tfm;
 };
 static inline bool aead_sufficient_data(struct sock *sk)
@@ -75,13 +75,13 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
        return af_alg_sendmsg(sock, msg, size, ivsize);
 }
-static int crypto_aead_copy_sgl(struct crypto_skcipher *null_tfm,
+static int crypto_aead_copy_sgl(struct crypto_sync_skcipher *null_tfm,
                                struct scatterlist *src,
                                struct scatterlist *dst, unsigned int len)
 {
-        SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
-        skcipher_request_set_tfm(skreq, null_tfm);
+        skcipher_request_set_sync_tfm(skreq, null_tfm);
        skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_BACKLOG,
                                      NULL, NULL);
        skcipher_request_set_crypt(skreq, src, dst, len, NULL);
@@ -99,7 +99,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
        struct af_alg_ctx *ctx = ask->private;
        struct aead_tfm *aeadc = pask->private;
        struct crypto_aead *tfm = aeadc->aead;
-        struct crypto_skcipher *null_tfm = aeadc->null_tfm;
+        struct crypto_sync_skcipher *null_tfm = aeadc->null_tfm;
        unsigned int i, as = crypto_aead_authsize(tfm);
        struct af_alg_async_req *areq;
        struct af_alg_tsgl *tsgl, *tmp;
@@ -478,7 +478,7 @@ static void *aead_bind(const char *name, u32 type, u32 mask)
 {
        struct aead_tfm *tfm;
        struct crypto_aead *aead;
-        struct crypto_skcipher *null_tfm;
+        struct crypto_sync_skcipher *null_tfm;
        tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
        if (!tfm)
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index bfcf595fd8f9..d0cde541beb6 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -239,7 +239,7 @@ static int hash_accept(struct socket *sock, struct socket *newsock, int flags,
        struct alg_sock *ask = alg_sk(sk);
        struct hash_ctx *ctx = ask->private;
        struct ahash_request *req = &ctx->req;
-        char state[crypto_ahash_statesize(crypto_ahash_reqtfm(req)) ? : 1];
+        char state[HASH_MAX_STATESIZE];
        struct sock *sk2;
        struct alg_sock *ask2;
        struct hash_ctx *ctx2;
diff --git a/crypto/authenc.c b/crypto/authenc.c
index 4fa8d40d947b..37f54d1b2f66 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -33,7 +33,7 @@ struct authenc_instance_ctx {
 struct crypto_authenc_ctx {
        struct crypto_ahash *auth;
        struct crypto_skcipher *enc;
-        struct crypto_skcipher *null;
+        struct crypto_sync_skcipher *null;
 };
 struct authenc_request_ctx {
@@ -185,9 +185,9 @@ static int crypto_authenc_copy_assoc(struct aead_request *req)
 {
        struct crypto_aead *authenc = crypto_aead_reqtfm(req);
        struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
-        SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
-        skcipher_request_set_tfm(skreq, ctx->null);
+        skcipher_request_set_sync_tfm(skreq, ctx->null);
        skcipher_request_set_callback(skreq, aead_request_flags(req),
                                      NULL, NULL);
        skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
@@ -318,7 +318,7 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm)
        struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm);
        struct crypto_ahash *auth;
        struct crypto_skcipher *enc;
-        struct crypto_skcipher *null;
+        struct crypto_sync_skcipher *null;
        int err;
        auth = crypto_spawn_ahash(&ictx->auth);
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 50b804747e20..80a25cc04aec 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -36,7 +36,7 @@ struct crypto_authenc_esn_ctx {
        unsigned int reqoff;
        struct crypto_ahash *auth;
        struct crypto_skcipher *enc;
-        struct crypto_skcipher *null;
+        struct crypto_sync_skcipher *null;
 };
 struct authenc_esn_request_ctx {
@@ -183,9 +183,9 @@ static int crypto_authenc_esn_copy(struct aead_request *req, unsigned int len)
 {
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
-        SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
-        skcipher_request_set_tfm(skreq, ctx->null);
+        skcipher_request_set_sync_tfm(skreq, ctx->null);
        skcipher_request_set_callback(skreq, aead_request_flags(req),
                                      NULL, NULL);
        skcipher_request_set_crypt(skreq, req->src, req->dst, len, NULL);
@@ -341,7 +341,7 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm)
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);
        struct crypto_ahash *auth;
        struct crypto_skcipher *enc;
-        struct crypto_skcipher *null;
+        struct crypto_sync_skcipher *null;
        int err;
        auth = crypto_spawn_ahash(&ictx->auth);
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 0a083342ec8c..b242fd0d3262 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -50,7 +50,10 @@ struct crypto_ccm_req_priv_ctx {
        u32 flags;
        struct scatterlist src[3];
        struct scatterlist dst[3];
-        struct skcipher_request skreq;
+        union {
+                struct ahash_request ahreq;
+                struct skcipher_request skreq;
+        };
 };
 struct cbcmac_tfm_ctx {
@@ -181,7 +184,7 @@ static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
        struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
        struct crypto_aead *aead = crypto_aead_reqtfm(req);
        struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
-        AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
+        struct ahash_request *ahreq = &pctx->ahreq;
        unsigned int assoclen = req->assoclen;
        struct scatterlist sg[3];
        u8 *odata = pctx->odata;
@@ -427,7 +430,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
        crypto_aead_set_reqsize(
                tfm,
                align + sizeof(struct crypto_ccm_req_priv_ctx) +
-                crypto_skcipher_reqsize(ctr));
+                max(crypto_ahash_reqsize(mac), crypto_skcipher_reqsize(ctr)));
        return 0;
diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c
index e451c3cb6a56..3ae96587caf9 100644
--- a/crypto/chacha20_generic.c
+++ b/crypto/chacha20_generic.c
@@ -18,20 +18,21 @@
 static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src,
                             unsigned int bytes)
 {
-        u32 stream[CHACHA20_BLOCK_WORDS];
+        /* aligned to potentially speed up crypto_xor() */
+        u8 stream[CHACHA20_BLOCK_SIZE] __aligned(sizeof(long));
        if (dst != src)
                memcpy(dst, src, bytes);
        while (bytes >= CHACHA20_BLOCK_SIZE) {
                chacha20_block(state, stream);
-                crypto_xor(dst, (const u8 *)stream, CHACHA20_BLOCK_SIZE);
+                crypto_xor(dst, stream, CHACHA20_BLOCK_SIZE);
                bytes -= CHACHA20_BLOCK_SIZE;
                dst += CHACHA20_BLOCK_SIZE;
        }
        if (bytes) {
                chacha20_block(state, stream);
-                crypto_xor(dst, (const u8 *)stream, bytes);
+                crypto_xor(dst, stream, bytes);
        }
 }
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index addca7bae33f..7118fb5efbaa 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -76,7 +76,7 @@ struct cryptd_blkcipher_request_ctx {
 struct cryptd_skcipher_ctx {
        atomic_t refcnt;
-        struct crypto_skcipher *child;
+        struct crypto_sync_skcipher *child;
 };
 struct cryptd_skcipher_request_ctx {
@@ -449,14 +449,16 @@ static int cryptd_skcipher_setkey(struct crypto_skcipher *parent,
                                  const u8 *key, unsigned int keylen)
 {
        struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent);
-        struct crypto_skcipher *child = ctx->child;
+        struct crypto_sync_skcipher *child = ctx->child;
        int err;
-        crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-        crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+        crypto_sync_skcipher_set_flags(child,
+                                       crypto_skcipher_get_flags(parent) &
                                         CRYPTO_TFM_REQ_MASK);
-        err = crypto_skcipher_setkey(child, key, keylen);
+        err = crypto_sync_skcipher_setkey(child, key, keylen);
-        crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
+        crypto_skcipher_set_flags(parent,
+                                  crypto_sync_skcipher_get_flags(child) &
                                          CRYPTO_TFM_RES_MASK);
        return err;
 }
@@ -483,13 +485,13 @@ static void cryptd_skcipher_encrypt(struct crypto_async_request *base,
        struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct crypto_skcipher *child = ctx->child;
+        struct crypto_sync_skcipher *child = ctx->child;
-        SKCIPHER_REQUEST_ON_STACK(subreq, child);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child);
        if (unlikely(err == -EINPROGRESS))
                goto out;
-        skcipher_request_set_tfm(subreq, child);
+        skcipher_request_set_sync_tfm(subreq, child);
        skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
                                      NULL, NULL);
        skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
@@ -511,13 +513,13 @@ static void cryptd_skcipher_decrypt(struct crypto_async_request *base,
        struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
-        struct crypto_skcipher *child = ctx->child;
+        struct crypto_sync_skcipher *child = ctx->child;
-        SKCIPHER_REQUEST_ON_STACK(subreq, child);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child);
        if (unlikely(err == -EINPROGRESS))
                goto out;
-        skcipher_request_set_tfm(subreq, child);
+        skcipher_request_set_sync_tfm(subreq, child);
        skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
                                      NULL, NULL);
        skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
@@ -568,7 +570,7 @@ static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm)
        if (IS_ERR(cipher))
                return PTR_ERR(cipher);
-        ctx->child = cipher;
+        ctx->child = (struct crypto_sync_skcipher *)cipher;
        crypto_skcipher_set_reqsize(
                tfm, sizeof(struct cryptd_skcipher_request_ctx));
        return 0;
@@ -578,7 +580,7 @@ static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm)
 {
        struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
-        crypto_free_skcipher(ctx->child);
+        crypto_free_sync_skcipher(ctx->child);
 }
 static void cryptd_skcipher_free(struct skcipher_instance *inst)
@@ -1243,7 +1245,7 @@ struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm)
 {
        struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
-        return ctx->child;
+        return &ctx->child->base;
 }
 EXPORT_SYMBOL_GPL(cryptd_skcipher_child);
diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c
index 0959b268966c..0bae59922a80 100644
--- a/crypto/crypto_null.c
+++ b/crypto/crypto_null.c
@@ -26,7 +26,7 @@
 #include <linux/string.h>
 static DEFINE_MUTEX(crypto_default_null_skcipher_lock);
-static struct crypto_skcipher *crypto_default_null_skcipher;
+static struct crypto_sync_skcipher *crypto_default_null_skcipher;
 static int crypto_default_null_skcipher_refcnt;
 static int null_compress(struct crypto_tfm *tfm, const u8 *src,
@@ -152,16 +152,15 @@ MODULE_ALIAS_CRYPTO("compress_null");
 MODULE_ALIAS_CRYPTO("digest_null");
 MODULE_ALIAS_CRYPTO("cipher_null");
-struct crypto_skcipher *crypto_get_default_null_skcipher(void)
+struct crypto_sync_skcipher *crypto_get_default_null_skcipher(void)
 {
-        struct crypto_skcipher *tfm;
+        struct crypto_sync_skcipher *tfm;
        mutex_lock(&crypto_default_null_skcipher_lock);
        tfm = crypto_default_null_skcipher;
        if (!tfm) {
-                tfm = crypto_alloc_skcipher("ecb(cipher_null)",
+                tfm = crypto_alloc_sync_skcipher("ecb(cipher_null)", 0, 0);
-                                            0, CRYPTO_ALG_ASYNC);
                if (IS_ERR(tfm))
                        goto unlock;
@@ -181,7 +180,7 @@ void crypto_put_default_null_skcipher(void)
 {
        mutex_lock(&crypto_default_null_skcipher_lock);
        if (!--crypto_default_null_skcipher_refcnt) {
-                crypto_free_skcipher(crypto_default_null_skcipher);
+                crypto_free_sync_skcipher(crypto_default_null_skcipher);
                crypto_default_null_skcipher = NULL;
        }
        mutex_unlock(&crypto_default_null_skcipher_lock);
diff --git a/crypto/crypto_user.c b/crypto/crypto_user_base.c
index 0e89b5457cab..e41f6cc33fff 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user_base.c
@@ -29,6 +29,7 @@
 #include <crypto/internal/rng.h>
 #include <crypto/akcipher.h>
 #include <crypto/kpp.h>
+#include <crypto/internal/cryptouser.h>
 #include "internal.h"
@@ -37,7 +38,7 @@
 static DEFINE_MUTEX(crypto_cfg_mutex);
 /* The crypto netlink socket */
-static struct sock *crypto_nlsk;
+struct sock *crypto_nlsk;
 struct crypto_dump_info {
        struct sk_buff *in_skb;
@@ -46,7 +47,7 @@ struct crypto_dump_info {
        u16 nlmsg_flags;
 };
-static struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact)
+struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact)
 {
        struct crypto_alg *q, *alg = NULL;
@@ -461,6 +462,7 @@ static const int crypto_msg_min[CRYPTO_NR_MSGTYPES] = {
        [CRYPTO_MSG_UPDATEALG   - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_GETALG      - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_DELRNG      - CRYPTO_MSG_BASE] = 0,
+        [CRYPTO_MSG_GETSTAT     - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
 };
 static const struct nla_policy crypto_policy[CRYPTOCFGA_MAX+1] = {
@@ -481,6 +483,9 @@ static const struct crypto_link {
                                                       .dump = crypto_dump_report,
                                                       .done = crypto_dump_report_done},
        [CRYPTO_MSG_DELRNG      - CRYPTO_MSG_BASE] = { .doit = crypto_del_rng },
+        [CRYPTO_MSG_GETSTAT     - CRYPTO_MSG_BASE] = { .doit = crypto_reportstat,
+                                                       .dump = crypto_dump_reportstat,
+                                                       .done = crypto_dump_reportstat_done},
 };
 static int crypto_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
diff --git a/crypto/crypto_user_stat.c b/crypto/crypto_user_stat.c
new file mode 100644
index 000000000000..021ad06bbb62
--- /dev/null
+++ b/crypto/crypto_user_stat.c
@@ -0,0 +1,463 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Crypto user configuration API.
+ *
+ * Copyright (C) 2017-2018 Corentin Labbe <clabbe@baylibre.com>
+ *
+ */
+#include <linux/crypto.h>
+#include <linux/cryptouser.h>
+#include <linux/sched.h>
+#include <net/netlink.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/internal/rng.h>
+#include <crypto/akcipher.h>
+#include <crypto/kpp.h>
+#include <crypto/internal/cryptouser.h>
+#include "internal.h"
+#define null_terminated(x)      (strnlen(x, sizeof(x)) < sizeof(x))
+static DEFINE_MUTEX(crypto_cfg_mutex);
+extern struct sock *crypto_nlsk;
+struct crypto_dump_info {
+        struct sk_buff *in_skb;
+        struct sk_buff *out_skb;
+        u32 nlmsg_seq;
+        u16 nlmsg_flags;
+};
+static int crypto_report_aead(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat raead;
+        u64 v64;
+        u32 v32;
+        strncpy(raead.type, "aead", sizeof(raead.type));
+        v32 = atomic_read(&alg->encrypt_cnt);
+        raead.stat_encrypt_cnt = v32;
+        v64 = atomic64_read(&alg->encrypt_tlen);
+        raead.stat_encrypt_tlen = v64;
+        v32 = atomic_read(&alg->decrypt_cnt);
+        raead.stat_decrypt_cnt = v32;
+        v64 = atomic64_read(&alg->decrypt_tlen);
+        raead.stat_decrypt_tlen = v64;
+        v32 = atomic_read(&alg->aead_err_cnt);
+        raead.stat_aead_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_AEAD,
+                    sizeof(struct crypto_stat), &raead))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rcipher;
+        u64 v64;
+        u32 v32;
+        strlcpy(rcipher.type, "cipher", sizeof(rcipher.type));
+        v32 = atomic_read(&alg->encrypt_cnt);
+        rcipher.stat_encrypt_cnt = v32;
+        v64 = atomic64_read(&alg->encrypt_tlen);
+        rcipher.stat_encrypt_tlen = v64;
+        v32 = atomic_read(&alg->decrypt_cnt);
+        rcipher.stat_decrypt_cnt = v32;
+        v64 = atomic64_read(&alg->decrypt_tlen);
+        rcipher.stat_decrypt_tlen = v64;
+        v32 = atomic_read(&alg->cipher_err_cnt);
+        rcipher.stat_cipher_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_CIPHER,
+                    sizeof(struct crypto_stat), &rcipher))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_comp(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rcomp;
+        u64 v64;
+        u32 v32;
+        strlcpy(rcomp.type, "compression", sizeof(rcomp.type));
+        v32 = atomic_read(&alg->compress_cnt);
+        rcomp.stat_compress_cnt = v32;
+        v64 = atomic64_read(&alg->compress_tlen);
+        rcomp.stat_compress_tlen = v64;
+        v32 = atomic_read(&alg->decompress_cnt);
+        rcomp.stat_decompress_cnt = v32;
+        v64 = atomic64_read(&alg->decompress_tlen);
+        rcomp.stat_decompress_tlen = v64;
+        v32 = atomic_read(&alg->cipher_err_cnt);
+        rcomp.stat_compress_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_COMPRESS,
+                    sizeof(struct crypto_stat), &rcomp))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_acomp(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat racomp;
+        u64 v64;
+        u32 v32;
+        strlcpy(racomp.type, "acomp", sizeof(racomp.type));
+        v32 = atomic_read(&alg->compress_cnt);
+        racomp.stat_compress_cnt = v32;
+        v64 = atomic64_read(&alg->compress_tlen);
+        racomp.stat_compress_tlen = v64;
+        v32 = atomic_read(&alg->decompress_cnt);
+        racomp.stat_decompress_cnt = v32;
+        v64 = atomic64_read(&alg->decompress_tlen);
+        racomp.stat_decompress_tlen = v64;
+        v32 = atomic_read(&alg->cipher_err_cnt);
+        racomp.stat_compress_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_ACOMP,
+                    sizeof(struct crypto_stat), &racomp))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_akcipher(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rakcipher;
+        u64 v64;
+        u32 v32;
+        strncpy(rakcipher.type, "akcipher", sizeof(rakcipher.type));
+        v32 = atomic_read(&alg->encrypt_cnt);
+        rakcipher.stat_encrypt_cnt = v32;
+        v64 = atomic64_read(&alg->encrypt_tlen);
+        rakcipher.stat_encrypt_tlen = v64;
+        v32 = atomic_read(&alg->decrypt_cnt);
+        rakcipher.stat_decrypt_cnt = v32;
+        v64 = atomic64_read(&alg->decrypt_tlen);
+        rakcipher.stat_decrypt_tlen = v64;
+        v32 = atomic_read(&alg->sign_cnt);
+        rakcipher.stat_sign_cnt = v32;
+        v32 = atomic_read(&alg->verify_cnt);
+        rakcipher.stat_verify_cnt = v32;
+        v32 = atomic_read(&alg->akcipher_err_cnt);
+        rakcipher.stat_akcipher_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_AKCIPHER,
+                    sizeof(struct crypto_stat), &rakcipher))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_kpp(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rkpp;
+        u32 v;
+        strlcpy(rkpp.type, "kpp", sizeof(rkpp.type));
+        v = atomic_read(&alg->setsecret_cnt);
+        rkpp.stat_setsecret_cnt = v;
+        v = atomic_read(&alg->generate_public_key_cnt);
+        rkpp.stat_generate_public_key_cnt = v;
+        v = atomic_read(&alg->compute_shared_secret_cnt);
+        rkpp.stat_compute_shared_secret_cnt = v;
+        v = atomic_read(&alg->kpp_err_cnt);
+        rkpp.stat_kpp_err_cnt = v;
+        if (nla_put(skb, CRYPTOCFGA_STAT_KPP,
+                    sizeof(struct crypto_stat), &rkpp))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_ahash(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rhash;
+        u64 v64;
+        u32 v32;
+        strncpy(rhash.type, "ahash", sizeof(rhash.type));
+        v32 = atomic_read(&alg->hash_cnt);
+        rhash.stat_hash_cnt = v32;
+        v64 = atomic64_read(&alg->hash_tlen);
+        rhash.stat_hash_tlen = v64;
+        v32 = atomic_read(&alg->hash_err_cnt);
+        rhash.stat_hash_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_HASH,
+                    sizeof(struct crypto_stat), &rhash))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_shash(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rhash;
+        u64 v64;
+        u32 v32;
+        strncpy(rhash.type, "shash", sizeof(rhash.type));
+        v32 = atomic_read(&alg->hash_cnt);
+        rhash.stat_hash_cnt = v32;
+        v64 = atomic64_read(&alg->hash_tlen);
+        rhash.stat_hash_tlen = v64;
+        v32 = atomic_read(&alg->hash_err_cnt);
+        rhash.stat_hash_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_HASH,
+                    sizeof(struct crypto_stat), &rhash))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_report_rng(struct sk_buff *skb, struct crypto_alg *alg)
+{
+        struct crypto_stat rrng;
+        u64 v64;
+        u32 v32;
+        strncpy(rrng.type, "rng", sizeof(rrng.type));
+        v32 = atomic_read(&alg->generate_cnt);
+        rrng.stat_generate_cnt = v32;
+        v64 = atomic64_read(&alg->generate_tlen);
+        rrng.stat_generate_tlen = v64;
+        v32 = atomic_read(&alg->seed_cnt);
+        rrng.stat_seed_cnt = v32;
+        v32 = atomic_read(&alg->hash_err_cnt);
+        rrng.stat_rng_err_cnt = v32;
+        if (nla_put(skb, CRYPTOCFGA_STAT_RNG,
+                    sizeof(struct crypto_stat), &rrng))
+                goto nla_put_failure;
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_reportstat_one(struct crypto_alg *alg,
+                                 struct crypto_user_alg *ualg,
+                                 struct sk_buff *skb)
+{
+        strlcpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name));
+        strlcpy(ualg->cru_driver_name, alg->cra_driver_name,
+                sizeof(ualg->cru_driver_name));
+        strlcpy(ualg->cru_module_name, module_name(alg->cra_module),
+                sizeof(ualg->cru_module_name));
+        ualg->cru_type = 0;
+        ualg->cru_mask = 0;
+        ualg->cru_flags = alg->cra_flags;
+        ualg->cru_refcnt = refcount_read(&alg->cra_refcnt);
+        if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority))
+                goto nla_put_failure;
+        if (alg->cra_flags & CRYPTO_ALG_LARVAL) {
+                struct crypto_stat rl;
+                strlcpy(rl.type, "larval", sizeof(rl.type));
+                if (nla_put(skb, CRYPTOCFGA_STAT_LARVAL,
+                            sizeof(struct crypto_stat), &rl))
+                        goto nla_put_failure;
+                goto out;
+        }
+        switch (alg->cra_flags & (CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_LARVAL)) {
+        case CRYPTO_ALG_TYPE_AEAD:
+                if (crypto_report_aead(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_SKCIPHER:
+                if (crypto_report_cipher(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_BLKCIPHER:
+                if (crypto_report_cipher(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_CIPHER:
+                if (crypto_report_cipher(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_COMPRESS:
+                if (crypto_report_comp(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_ACOMPRESS:
+                if (crypto_report_acomp(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_SCOMPRESS:
+                if (crypto_report_acomp(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_AKCIPHER:
+                if (crypto_report_akcipher(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_KPP:
+                if (crypto_report_kpp(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_AHASH:
+                if (crypto_report_ahash(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_HASH:
+                if (crypto_report_shash(skb, alg))
+                        goto nla_put_failure;
+                break;
+        case CRYPTO_ALG_TYPE_RNG:
+                if (crypto_report_rng(skb, alg))
+                        goto nla_put_failure;
+                break;
+        default:
+                pr_err("ERROR: Unhandled alg %d in %s\n",
+                       alg->cra_flags & (CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_LARVAL),
+                       __func__);
+        }
+out:
+        return 0;
+nla_put_failure:
+        return -EMSGSIZE;
+}
+static int crypto_reportstat_alg(struct crypto_alg *alg,
+                                 struct crypto_dump_info *info)
+{
+        struct sk_buff *in_skb = info->in_skb;
+        struct sk_buff *skb = info->out_skb;
+        struct nlmsghdr *nlh;
+        struct crypto_user_alg *ualg;
+        int err = 0;
+        nlh = nlmsg_put(skb, NETLINK_CB(in_skb).portid, info->nlmsg_seq,
+                        CRYPTO_MSG_GETSTAT, sizeof(*ualg), info->nlmsg_flags);
+        if (!nlh) {
+                err = -EMSGSIZE;
+                goto out;
+        }
+        ualg = nlmsg_data(nlh);
+        err = crypto_reportstat_one(alg, ualg, skb);
+        if (err) {
+                nlmsg_cancel(skb, nlh);
+                goto out;
+        }
+        nlmsg_end(skb, nlh);
+out:
+        return err;
+}
+int crypto_reportstat(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
+                      struct nlattr **attrs)
+{
+        struct crypto_user_alg *p = nlmsg_data(in_nlh);
+        struct crypto_alg *alg;
+        struct sk_buff *skb;
+        struct crypto_dump_info info;
+        int err;
+        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
+                return -EINVAL;
+        alg = crypto_alg_match(p, 0);
+        if (!alg)
+                return -ENOENT;
+        err = -ENOMEM;
+        skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+        if (!skb)
+                goto drop_alg;
+        info.in_skb = in_skb;
+        info.out_skb = skb;
+        info.nlmsg_seq = in_nlh->nlmsg_seq;
+        info.nlmsg_flags = 0;
+        err = crypto_reportstat_alg(alg, &info);
+drop_alg:
+        crypto_mod_put(alg);
+        if (err)
+                return err;
+        return nlmsg_unicast(crypto_nlsk, skb, NETLINK_CB(in_skb).portid);
+}
+int crypto_dump_reportstat(struct sk_buff *skb, struct netlink_callback *cb)
+{
+        struct crypto_alg *alg;
+        struct crypto_dump_info info;
+        int err;
+        if (cb->args[0])
+                goto out;
+        cb->args[0] = 1;
+        info.in_skb = cb->skb;
+        info.out_skb = skb;
+        info.nlmsg_seq = cb->nlh->nlmsg_seq;
+        info.nlmsg_flags = NLM_F_MULTI;
+        list_for_each_entry(alg, &crypto_alg_list, cra_list) {
+                err = crypto_reportstat_alg(alg, &info);
+                if (err)
+                        goto out_err;
+        }
+out:
+        return skb->len;
+out_err:
+        return err;
+}
+int crypto_dump_reportstat_done(struct netlink_callback *cb)
+{
+        return 0;
+}
+MODULE_LICENSE("GPL");
diff --git a/crypto/echainiv.c b/crypto/echainiv.c
index 45819e6015bf..77e607fdbfb7 100644
--- a/crypto/echainiv.c
+++ b/crypto/echainiv.c
@@ -47,9 +47,9 @@ static int echainiv_encrypt(struct aead_request *req)
        info = req->iv;
        if (req->src != req->dst) {
-                SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
-                skcipher_request_set_tfm(nreq, ctx->sknull);
+                skcipher_request_set_sync_tfm(nreq, ctx->sknull);
                skcipher_request_set_callback(nreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(nreq, req->src, req->dst,
diff --git a/crypto/gcm.c b/crypto/gcm.c
index 0ad879e1f9b2..e438492db2ca 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -50,7 +50,7 @@ struct crypto_rfc4543_instance_ctx {
 struct crypto_rfc4543_ctx {
        struct crypto_aead *child;
-        struct crypto_skcipher *null;
+        struct crypto_sync_skcipher *null;
        u8 nonce[4];
 };
@@ -1067,9 +1067,9 @@ static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
        unsigned int authsize = crypto_aead_authsize(aead);
        unsigned int nbytes = req->assoclen + req->cryptlen -
                              (enc ? 0 : authsize);
-        SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
-        skcipher_request_set_tfm(nreq, ctx->null);
+        skcipher_request_set_sync_tfm(nreq, ctx->null);
        skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
        skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
@@ -1093,7 +1093,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
        struct crypto_aead_spawn *spawn = &ictx->aead;
        struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
        struct crypto_aead *aead;
-        struct crypto_skcipher *null;
+        struct crypto_sync_skcipher *null;
        unsigned long align;
        int err = 0;
diff --git a/crypto/internal.h b/crypto/internal.h
index 9a3f39939fba..ef769b5e8ad3 100644
--- a/crypto/internal.h
+++ b/crypto/internal.h
@@ -26,12 +26,6 @@
 #include <linux/rwsem.h>
 #include <linux/slab.h>
-/* Crypto notification events. */
-enum {
-        CRYPTO_MSG_ALG_REQUEST,
-        CRYPTO_MSG_ALG_REGISTER,
-};
 struct crypto_instance;
 struct crypto_template;
@@ -90,8 +84,6 @@ struct crypto_alg *crypto_find_alg(const char *alg_name,
 void *crypto_alloc_tfm(const char *alg_name,
                       const struct crypto_type *frontend, u32 type, u32 mask);
-int crypto_register_notifier(struct notifier_block *nb);
-int crypto_unregister_notifier(struct notifier_block *nb);
 int crypto_probing_notify(unsigned long val, void *v);
 unsigned int crypto_alg_extsize(struct crypto_alg *alg);
diff --git a/crypto/lrw.c b/crypto/lrw.c
index 393a782679c7..0430ccd08728 100644
--- a/crypto/lrw.c
+++ b/crypto/lrw.c
@@ -29,8 +29,6 @@
 #include <crypto/b128ops.h>
 #include <crypto/gf128mul.h>
-#define LRW_BUFFER_SIZE 128u
 #define LRW_BLOCK_SIZE 16
 struct priv {
@@ -56,19 +54,7 @@ struct priv {
 };
 struct rctx {
-        be128 buf[LRW_BUFFER_SIZE / sizeof(be128)];
        be128 t;
-        be128 *ext;
-        struct scatterlist srcbuf[2];
-        struct scatterlist dstbuf[2];
-        struct scatterlist *src;
-        struct scatterlist *dst;
-        unsigned int left;
        struct skcipher_request subreq;
 };
@@ -120,112 +106,68 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key,
        return 0;
 }
-static inline void inc(be128 *iv)
+/*
-{
+ * Returns the number of trailing '1' bits in the words of the counter, which is
-        be64_add_cpu(&iv->b, 1);
+ * represented by 4 32-bit words, arranged from least to most significant.
-        if (!iv->b)
+ * At the same time, increments the counter by one.
-                be64_add_cpu(&iv->a, 1);
+ *
-}
+ * For example:
+ *
-/* this returns the number of consequative 1 bits starting
+ * u32 counter[4] = { 0xFFFFFFFF, 0x1, 0x0, 0x0 };
- * from the right, get_index128(00 00 00 00 00 00 ... 00 00 10 FB) = 2 */
+ * int i = next_index(&counter);
-static inline int get_index128(be128 *block)
+ * // i == 33, counter == { 0x0, 0x2, 0x0, 0x0 }
+ */
+static int next_index(u32 *counter)
 {
-        int x;
+        int i, res = 0;
-        __be32 *p = (__be32 *) block;
-        for (p += 3, x = 0; x < 128; p--, x += 32) {
+        for (i = 0; i < 4; i++) {
-                u32 val = be32_to_cpup(p);
+                if (counter[i] + 1 != 0)
+                        return res + ffz(counter[i]++);
-                if (!~val)
+                counter[i] = 0;
-                        continue;
+                res += 32;
-                return x + ffz(val);
        }
-        return x;
+        /*
+         * If we get here, then x == 128 and we are incrementing the counter
+         * from all ones to all zeros. This means we must return index 127, i.e.
+         * the one corresponding to key2*{ 1,...,1 }.
+         */
+        return 127;
 }
-static int post_crypt(struct skcipher_request *req)
+/*
+ * We compute the tweak masks twice (both before and after the ECB encryption or
+ * decryption) to avoid having to allocate a temporary buffer and/or make
+ * mutliple calls to the 'ecb(..)' instance, which usually would be slower than
+ * just doing the next_index() calls again.
+ */
+static int xor_tweak(struct skcipher_request *req, bool second_pass)
 {
-        struct rctx *rctx = skcipher_request_ctx(req);
-        be128 *buf = rctx->ext ?: rctx->buf;
-        struct skcipher_request *subreq;
        const int bs = LRW_BLOCK_SIZE;
-        struct skcipher_walk w;
-        struct scatterlist *sg;
-        unsigned offset;
-        int err;
-        subreq = &rctx->subreq;
-        err = skcipher_walk_virt(&w, subreq, false);
-        while (w.nbytes) {
-                unsigned int avail = w.nbytes;
-                be128 *wdst;
-                wdst = w.dst.virt.addr;
-                do {
-                        be128_xor(wdst, buf++, wdst);
-                        wdst++;
-                } while ((avail -= bs) >= bs);
-                err = skcipher_walk_done(&w, avail);
-        }
-        rctx->left -= subreq->cryptlen;
-        if (err || !rctx->left)
-                goto out;
-        rctx->dst = rctx->dstbuf;
-        scatterwalk_done(&w.out, 0, 1);
-        sg = w.out.sg;
-        offset = w.out.offset;
-        if (rctx->dst != sg) {
-                rctx->dst[0] = *sg;
-                sg_unmark_end(rctx->dst);
-                scatterwalk_crypto_chain(rctx->dst, sg_next(sg), 2);
-        }
-        rctx->dst[0].length -= offset - sg->offset;
-        rctx->dst[0].offset = offset;
-out:
-        return err;
-}
-static int pre_crypt(struct skcipher_request *req)
-{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        struct rctx *rctx = skcipher_request_ctx(req);
        struct priv *ctx = crypto_skcipher_ctx(tfm);
-        be128 *buf = rctx->ext ?: rctx->buf;
+        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
+        be128 t = rctx->t;
-        const int bs = LRW_BLOCK_SIZE;
        struct skcipher_walk w;
-        struct scatterlist *sg;
+        __be32 *iv;
-        unsigned cryptlen;
+        u32 counter[4];
-        unsigned offset;
-        be128 *iv;
-        bool more;
        int err;
-        subreq = &rctx->subreq;
+        if (second_pass) {
-        skcipher_request_set_tfm(subreq, tfm);
+                req = &rctx->subreq;
+                /* set to our TFM to enforce correct alignment: */
-        cryptlen = subreq->cryptlen;
+                skcipher_request_set_tfm(req, tfm);
-        more = rctx->left > cryptlen;
+        }
-        if (!more)
-                cryptlen = rctx->left;
-        skcipher_request_set_crypt(subreq, rctx->src, rctx->dst,
+        err = skcipher_walk_virt(&w, req, false);
-                                   cryptlen, req->iv);
+        iv = (__be32 *)w.iv;
-        err = skcipher_walk_virt(&w, subreq, false);
+        counter[0] = be32_to_cpu(iv[3]);
-        iv = w.iv;
+        counter[1] = be32_to_cpu(iv[2]);
+        counter[2] = be32_to_cpu(iv[1]);
+        counter[3] = be32_to_cpu(iv[0]);
        while (w.nbytes) {
                unsigned int avail = w.nbytes;
@@ -236,188 +178,85 @@ static int pre_crypt(struct skcipher_request *req)
                wdst = w.dst.virt.addr;
                do {
-                        *buf++ = rctx->t;
+                        be128_xor(wdst++, &t, wsrc++);
-                        be128_xor(wdst++, &rctx->t, wsrc++);
                        /* T <- I*Key2, using the optimization
                         * discussed in the specification */
-                        be128_xor(&rctx->t, &rctx->t,
+                        be128_xor(&t, &t, &ctx->mulinc[next_index(counter)]);
-                                  &ctx->mulinc[get_index128(iv)]);
-                        inc(iv);
                } while ((avail -= bs) >= bs);
-                err = skcipher_walk_done(&w, avail);
+                if (second_pass && w.nbytes == w.total) {
-        }
+                        iv[0] = cpu_to_be32(counter[3]);
+                        iv[1] = cpu_to_be32(counter[2]);
-        skcipher_request_set_tfm(subreq, ctx->child);
+                        iv[2] = cpu_to_be32(counter[1]);
-        skcipher_request_set_crypt(subreq, rctx->dst, rctx->dst,
+                        iv[3] = cpu_to_be32(counter[0]);
-                                   cryptlen, NULL);
+                }
-        if (err || !more)
-                goto out;
-        rctx->src = rctx->srcbuf;
-        scatterwalk_done(&w.in, 0, 1);
-        sg = w.in.sg;
-        offset = w.in.offset;
-        if (rctx->src != sg) {
+                err = skcipher_walk_done(&w, avail);
-                rctx->src[0] = *sg;
-                sg_unmark_end(rctx->src);
-                scatterwalk_crypto_chain(rctx->src, sg_next(sg), 2);
        }
-        rctx->src[0].length -= offset - sg->offset;
-        rctx->src[0].offset = offset;
-out:
        return err;
 }
-static int init_crypt(struct skcipher_request *req, crypto_completion_t done)
+static int xor_tweak_pre(struct skcipher_request *req)
 {
-        struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+        return xor_tweak(req, false);
-        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
-        gfp_t gfp;
-        subreq = &rctx->subreq;
-        skcipher_request_set_callback(subreq, req->base.flags, done, req);
-        gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
-                                                           GFP_ATOMIC;
-        rctx->ext = NULL;
-        subreq->cryptlen = LRW_BUFFER_SIZE;
-        if (req->cryptlen > LRW_BUFFER_SIZE) {
-                unsigned int n = min(req->cryptlen, (unsigned int)PAGE_SIZE);
-                rctx->ext = kmalloc(n, gfp);
-                if (rctx->ext)
-                        subreq->cryptlen = n;
-        }
-        rctx->src = req->src;
-        rctx->dst = req->dst;
-        rctx->left = req->cryptlen;
-        /* calculate first value of T */
-        memcpy(&rctx->t, req->iv, sizeof(rctx->t));
-        /* T <- I*Key2 */
-        gf128mul_64k_bbe(&rctx->t, ctx->table);
-        return 0;
 }
-static void exit_crypt(struct skcipher_request *req)
+static int xor_tweak_post(struct skcipher_request *req)
 {
-        struct rctx *rctx = skcipher_request_ctx(req);
+        return xor_tweak(req, true);
-        rctx->left = 0;
-        if (rctx->ext)
-                kzfree(rctx->ext);
 }
-static int do_encrypt(struct skcipher_request *req, int err)
+static void crypt_done(struct crypto_async_request *areq, int err)
-{
-        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
-        subreq = &rctx->subreq;
-        while (!err && rctx->left) {
-                err = pre_crypt(req) ?:
-                      crypto_skcipher_encrypt(subreq) ?:
-                      post_crypt(req);
-                if (err == -EINPROGRESS || err == -EBUSY)
-                        return err;
-        }
-        exit_crypt(req);
-        return err;
-}
-static void encrypt_done(struct crypto_async_request *areq, int err)
 {
        struct skcipher_request *req = areq->data;
-        struct skcipher_request *subreq;
-        struct rctx *rctx;
-        rctx = skcipher_request_ctx(req);
+        if (!err)
+                err = xor_tweak_post(req);
-        if (err == -EINPROGRESS) {
-                if (rctx->left != req->cryptlen)
-                        return;
-                goto out;
-        }
-        subreq = &rctx->subreq;
-        subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
-        err = do_encrypt(req, err ?: post_crypt(req));
-        if (rctx->left)
-                return;
-out:
        skcipher_request_complete(req, err);
 }
-static int encrypt(struct skcipher_request *req)
+static void init_crypt(struct skcipher_request *req)
-{
-        return do_encrypt(req, init_crypt(req, encrypt_done));
-}
-static int do_decrypt(struct skcipher_request *req, int err)
 {
+        struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
+        struct skcipher_request *subreq = &rctx->subreq;
-        subreq = &rctx->subreq;
-        while (!err && rctx->left) {
+        skcipher_request_set_tfm(subreq, ctx->child);
-                err = pre_crypt(req) ?:
+        skcipher_request_set_callback(subreq, req->base.flags, crypt_done, req);
-                      crypto_skcipher_decrypt(subreq) ?:
+        /* pass req->iv as IV (will be used by xor_tweak, ECB will ignore it) */
-                      post_crypt(req);
+        skcipher_request_set_crypt(subreq, req->dst, req->dst,
+                                   req->cryptlen, req->iv);
-                if (err == -EINPROGRESS || err == -EBUSY)
+        /* calculate first value of T */
-                        return err;
+        memcpy(&rctx->t, req->iv, sizeof(rctx->t));
-        }
-        exit_crypt(req);
+        /* T <- I*Key2 */
-        return err;
+        gf128mul_64k_bbe(&rctx->t, ctx->table);
 }
-static void decrypt_done(struct crypto_async_request *areq, int err)
+static int encrypt(struct skcipher_request *req)
 {
-        struct skcipher_request *req = areq->data;
+        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
+        struct skcipher_request *subreq = &rctx->subreq;
-        struct rctx *rctx;
-        rctx = skcipher_request_ctx(req);
-        if (err == -EINPROGRESS) {
-                if (rctx->left != req->cryptlen)
-                        return;
-                goto out;
-        }
-        subreq = &rctx->subreq;
-        subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
-        err = do_decrypt(req, err ?: post_crypt(req));
-        if (rctx->left)
-                return;
-out:
+        init_crypt(req);
-        skcipher_request_complete(req, err);
+        return xor_tweak_pre(req) ?:
+                crypto_skcipher_encrypt(subreq) ?:
+                xor_tweak_post(req);
 }
 static int decrypt(struct skcipher_request *req)
 {
-        return do_decrypt(req, init_crypt(req, decrypt_done));
+        struct rctx *rctx = skcipher_request_ctx(req);
+        struct skcipher_request *subreq = &rctx->subreq;
+        init_crypt(req);
+        return xor_tweak_pre(req) ?:
+                crypto_skcipher_decrypt(subreq) ?:
+                xor_tweak_post(req);
 }
 static int init_tfm(struct crypto_skcipher *tfm)
@@ -543,7 +382,7 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb)
        inst->alg.base.cra_priority = alg->base.cra_priority;
        inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE;
        inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
-                                       (__alignof__(u64) - 1);
+                                       (__alignof__(__be32) - 1);
        inst->alg.ivsize = LRW_BLOCK_SIZE;
        inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) +
diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c
deleted file mode 100644
index f14152147ce8..000000000000
--- a/crypto/mcryptd.c
+++ /dev/null
@@ -1,675 +0,0 @@
-/*
- * Software multibuffer async crypto daemon.
- *
- * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
- *
- * Adapted from crypto daemon.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- */
-#include <crypto/algapi.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/aead.h>
-#include <crypto/mcryptd.h>
-#include <crypto/crypto_wq.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/scatterlist.h>
-#include <linux/sched.h>
-#include <linux/sched/stat.h>
-#include <linux/slab.h>
-#define MCRYPTD_MAX_CPU_QLEN 100
-#define MCRYPTD_BATCH 9
-static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
-                                   unsigned int tail);
-struct mcryptd_flush_list {
-        struct list_head list;
-        struct mutex lock;
-};
-static struct mcryptd_flush_list __percpu *mcryptd_flist;
-struct hashd_instance_ctx {
-        struct crypto_ahash_spawn spawn;
-        struct mcryptd_queue *queue;
-};
-static void mcryptd_queue_worker(struct work_struct *work);
-void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
-{
-        struct mcryptd_flush_list *flist;
-        if (!cstate->flusher_engaged) {
-                /* put the flusher on the flush list */
-                flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
-                mutex_lock(&flist->lock);
-                list_add_tail(&cstate->flush_list, &flist->list);
-                cstate->flusher_engaged = true;
-                cstate->next_flush = jiffies + delay;
-                queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
-                        &cstate->flush, delay);
-                mutex_unlock(&flist->lock);
-        }
-}
-EXPORT_SYMBOL(mcryptd_arm_flusher);
-static int mcryptd_init_queue(struct mcryptd_queue *queue,
-                             unsigned int max_cpu_qlen)
-{
-        int cpu;
-        struct mcryptd_cpu_queue *cpu_queue;
-        queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
-        pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
-        if (!queue->cpu_queue)
-                return -ENOMEM;
-        for_each_possible_cpu(cpu) {
-                cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
-                pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
-                crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
-                INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
-                spin_lock_init(&cpu_queue->q_lock);
-        }
-        return 0;
-}
-static void mcryptd_fini_queue(struct mcryptd_queue *queue)
-{
-        int cpu;
-        struct mcryptd_cpu_queue *cpu_queue;
-        for_each_possible_cpu(cpu) {
-                cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
-                BUG_ON(cpu_queue->queue.qlen);
-        }
-        free_percpu(queue->cpu_queue);
-}
-static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
-                                  struct crypto_async_request *request,
-                                  struct mcryptd_hash_request_ctx *rctx)
-{
-        int cpu, err;
-        struct mcryptd_cpu_queue *cpu_queue;
-        cpu_queue = raw_cpu_ptr(queue->cpu_queue);
-        spin_lock(&cpu_queue->q_lock);
-        cpu = smp_processor_id();
-        rctx->tag.cpu = smp_processor_id();
-        err = crypto_enqueue_request(&cpu_queue->queue, request);
-        pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
-                 cpu, cpu_queue, request);
-        spin_unlock(&cpu_queue->q_lock);
-        queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
-        return err;
-}
-/*
- * Try to opportunisticlly flush the partially completed jobs if
- * crypto daemon is the only task running.
- */
-static void mcryptd_opportunistic_flush(void)
-{
-        struct mcryptd_flush_list *flist;
-        struct mcryptd_alg_cstate *cstate;
-        flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
-        while (single_task_running()) {
-                mutex_lock(&flist->lock);
-                cstate = list_first_entry_or_null(&flist->list,
-                                struct mcryptd_alg_cstate, flush_list);
-                if (!cstate || !cstate->flusher_engaged) {
-                        mutex_unlock(&flist->lock);
-                        return;
-                }
-                list_del(&cstate->flush_list);
-                cstate->flusher_engaged = false;
-                mutex_unlock(&flist->lock);
-                cstate->alg_state->flusher(cstate);
-        }
-}
-/*
- * Called in workqueue context, do one real cryption work (via
- * req->complete) and reschedule itself if there are more work to
- * do.
- */
-static void mcryptd_queue_worker(struct work_struct *work)
-{
-        struct mcryptd_cpu_queue *cpu_queue;
-        struct crypto_async_request *req, *backlog;
-        int i;
-        /*
-         * Need to loop through more than once for multi-buffer to
-         * be effective.
-         */
-        cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
-        i = 0;
-        while (i < MCRYPTD_BATCH || single_task_running()) {
-                spin_lock_bh(&cpu_queue->q_lock);
-                backlog = crypto_get_backlog(&cpu_queue->queue);
-                req = crypto_dequeue_request(&cpu_queue->queue);
-                spin_unlock_bh(&cpu_queue->q_lock);
-                if (!req) {
-                        mcryptd_opportunistic_flush();
-                        return;
-                }
-                if (backlog)
-                        backlog->complete(backlog, -EINPROGRESS);
-                req->complete(req, 0);
-                if (!cpu_queue->queue.qlen)
-                        return;
-                ++i;
-        }
-        if (cpu_queue->queue.qlen)
-                queue_work_on(smp_processor_id(), kcrypto_wq, &cpu_queue->work);
-}
-void mcryptd_flusher(struct work_struct *__work)
-{
-        struct  mcryptd_alg_cstate      *alg_cpu_state;
-        struct  mcryptd_alg_state       *alg_state;
-        struct  mcryptd_flush_list      *flist;
-        int     cpu;
-        cpu = smp_processor_id();
-        alg_cpu_state = container_of(to_delayed_work(__work),
-                                     struct mcryptd_alg_cstate, flush);
-        alg_state = alg_cpu_state->alg_state;
-        if (alg_cpu_state->cpu != cpu)
-                pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
-                                cpu, alg_cpu_state->cpu);
-        if (alg_cpu_state->flusher_engaged) {
-                flist = per_cpu_ptr(mcryptd_flist, cpu);
-                mutex_lock(&flist->lock);
-                list_del(&alg_cpu_state->flush_list);
-                alg_cpu_state->flusher_engaged = false;
-                mutex_unlock(&flist->lock);
-                alg_state->flusher(alg_cpu_state);
-        }
-}
-EXPORT_SYMBOL_GPL(mcryptd_flusher);
-static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
-{
-        struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
-        struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
-        return ictx->queue;
-}
-static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
-                                   unsigned int tail)
-{
-        char *p;
-        struct crypto_instance *inst;
-        int err;
-        p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
-        if (!p)
-                return ERR_PTR(-ENOMEM);
-        inst = (void *)(p + head);
-        err = -ENAMETOOLONG;
-        if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
-                    "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
-                goto out_free_inst;
-        memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
-        inst->alg.cra_priority = alg->cra_priority + 50;
-        inst->alg.cra_blocksize = alg->cra_blocksize;
-        inst->alg.cra_alignmask = alg->cra_alignmask;
-out:
-        return p;
-out_free_inst:
-        kfree(p);
-        p = ERR_PTR(err);
-        goto out;
-}
-static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type,
-                                          u32 *mask)
-{
-        struct crypto_attr_type *algt;
-        algt = crypto_get_attr_type(tb);
-        if (IS_ERR(algt))
-                return false;
-        *type |= algt->type & CRYPTO_ALG_INTERNAL;
-        *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
-        if (*type & *mask & CRYPTO_ALG_INTERNAL)
-                return true;
-        else
-                return false;
-}
-static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
-{
-        struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
-        struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
-        struct crypto_ahash_spawn *spawn = &ictx->spawn;
-        struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct crypto_ahash *hash;
-        hash = crypto_spawn_ahash(spawn);
-        if (IS_ERR(hash))
-                return PTR_ERR(hash);
-        ctx->child = hash;
-        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-                                 sizeof(struct mcryptd_hash_request_ctx) +
-                                 crypto_ahash_reqsize(hash));
-        return 0;
-}
-static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
-{
-        struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
-        crypto_free_ahash(ctx->child);
-}
-static int mcryptd_hash_setkey(struct crypto_ahash *parent,
-                                   const u8 *key, unsigned int keylen)
-{
-        struct mcryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
-        struct crypto_ahash *child = ctx->child;
-        int err;
-        crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-        crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) &
-                                      CRYPTO_TFM_REQ_MASK);
-        err = crypto_ahash_setkey(child, key, keylen);
-        crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) &
-                                       CRYPTO_TFM_RES_MASK);
-        return err;
-}
-static int mcryptd_hash_enqueue(struct ahash_request *req,
-                                crypto_completion_t complete)
-{
-        int ret;
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-        struct mcryptd_queue *queue =
-                mcryptd_get_queue(crypto_ahash_tfm(tfm));
-        rctx->complete = req->base.complete;
-        req->base.complete = complete;
-        ret = mcryptd_enqueue_request(queue, &req->base, rctx);
-        return ret;
-}
-static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
-{
-        struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
-        struct crypto_ahash *child = ctx->child;
-        struct ahash_request *req = ahash_request_cast(req_async);
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        struct ahash_request *desc = &rctx->areq;
-        if (unlikely(err == -EINPROGRESS))
-                goto out;
-        ahash_request_set_tfm(desc, child);
-        ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
-                                                rctx->complete, req_async);
-        rctx->out = req->result;
-        err = crypto_ahash_init(desc);
-out:
-        local_bh_disable();
-        rctx->complete(&req->base, err);
-        local_bh_enable();
-}
-static int mcryptd_hash_init_enqueue(struct ahash_request *req)
-{
-        return mcryptd_hash_enqueue(req, mcryptd_hash_init);
-}
-static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
-{
-        struct ahash_request *req = ahash_request_cast(req_async);
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        if (unlikely(err == -EINPROGRESS))
-                goto out;
-        rctx->out = req->result;
-        err = crypto_ahash_update(&rctx->areq);
-        if (err) {
-                req->base.complete = rctx->complete;
-                goto out;
-        }
-        return;
-out:
-        local_bh_disable();
-        rctx->complete(&req->base, err);
-        local_bh_enable();
-}
-static int mcryptd_hash_update_enqueue(struct ahash_request *req)
-{
-        return mcryptd_hash_enqueue(req, mcryptd_hash_update);
-}
-static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
-{
-        struct ahash_request *req = ahash_request_cast(req_async);
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        if (unlikely(err == -EINPROGRESS))
-                goto out;
-        rctx->out = req->result;
-        err = crypto_ahash_final(&rctx->areq);
-        if (err) {
-                req->base.complete = rctx->complete;
-                goto out;
-        }
-        return;
-out:
-        local_bh_disable();
-        rctx->complete(&req->base, err);
-        local_bh_enable();
-}
-static int mcryptd_hash_final_enqueue(struct ahash_request *req)
-{
-        return mcryptd_hash_enqueue(req, mcryptd_hash_final);
-}
-static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
-{
-        struct ahash_request *req = ahash_request_cast(req_async);
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        if (unlikely(err == -EINPROGRESS))
-                goto out;
-        rctx->out = req->result;
-        err = crypto_ahash_finup(&rctx->areq);
-        if (err) {
-                req->base.complete = rctx->complete;
-                goto out;
-        }
-        return;
-out:
-        local_bh_disable();
-        rctx->complete(&req->base, err);
-        local_bh_enable();
-}
-static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
-{
-        return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
-}
-static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
-{
-        struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
-        struct crypto_ahash *child = ctx->child;
-        struct ahash_request *req = ahash_request_cast(req_async);
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        struct ahash_request *desc = &rctx->areq;
-        if (unlikely(err == -EINPROGRESS))
-                goto out;
-        ahash_request_set_tfm(desc, child);
-        ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
-                                                rctx->complete, req_async);
-        rctx->out = req->result;
-        err = crypto_ahash_init(desc) ?: crypto_ahash_finup(desc);
-out:
-        local_bh_disable();
-        rctx->complete(&req->base, err);
-        local_bh_enable();
-}
-static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
-{
-        return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
-}
-static int mcryptd_hash_export(struct ahash_request *req, void *out)
-{
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        return crypto_ahash_export(&rctx->areq, out);
-}
-static int mcryptd_hash_import(struct ahash_request *req, const void *in)
-{
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        return crypto_ahash_import(&rctx->areq, in);
-}
-static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
-                              struct mcryptd_queue *queue)
-{
-        struct hashd_instance_ctx *ctx;
-        struct ahash_instance *inst;
-        struct hash_alg_common *halg;
-        struct crypto_alg *alg;
-        u32 type = 0;
-        u32 mask = 0;
-        int err;
-        if (!mcryptd_check_internal(tb, &type, &mask))
-                return -EINVAL;
-        halg = ahash_attr_alg(tb[1], type, mask);
-        if (IS_ERR(halg))
-                return PTR_ERR(halg);
-        alg = &halg->base;
-        pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
-        inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
-                                        sizeof(*ctx));
-        err = PTR_ERR(inst);
-        if (IS_ERR(inst))
-                goto out_put_alg;
-        ctx = ahash_instance_ctx(inst);
-        ctx->queue = queue;
-        err = crypto_init_ahash_spawn(&ctx->spawn, halg,
-                                      ahash_crypto_instance(inst));
-        if (err)
-                goto out_free_inst;
-        inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC |
-                (alg->cra_flags & (CRYPTO_ALG_INTERNAL |
-                                   CRYPTO_ALG_OPTIONAL_KEY));
-        inst->alg.halg.digestsize = halg->digestsize;
-        inst->alg.halg.statesize = halg->statesize;
-        inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
-        inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
-        inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
-        inst->alg.init   = mcryptd_hash_init_enqueue;
-        inst->alg.update = mcryptd_hash_update_enqueue;
-        inst->alg.final  = mcryptd_hash_final_enqueue;
-        inst->alg.finup  = mcryptd_hash_finup_enqueue;
-        inst->alg.export = mcryptd_hash_export;
-        inst->alg.import = mcryptd_hash_import;
-        if (crypto_hash_alg_has_setkey(halg))
-                inst->alg.setkey = mcryptd_hash_setkey;
-        inst->alg.digest = mcryptd_hash_digest_enqueue;
-        err = ahash_register_instance(tmpl, inst);
-        if (err) {
-                crypto_drop_ahash(&ctx->spawn);
-out_free_inst:
-                kfree(inst);
-        }
-out_put_alg:
-        crypto_mod_put(alg);
-        return err;
-}
-static struct mcryptd_queue mqueue;
-static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
-{
-        struct crypto_attr_type *algt;
-        algt = crypto_get_attr_type(tb);
-        if (IS_ERR(algt))
-                return PTR_ERR(algt);
-        switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
-        case CRYPTO_ALG_TYPE_DIGEST:
-                return mcryptd_create_hash(tmpl, tb, &mqueue);
-        break;
-        }
-        return -EINVAL;
-}
-static void mcryptd_free(struct crypto_instance *inst)
-{
-        struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
-        struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
-        switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
-        case CRYPTO_ALG_TYPE_AHASH:
-                crypto_drop_ahash(&hctx->spawn);
-                kfree(ahash_instance(inst));
-                return;
-        default:
-                crypto_drop_spawn(&ctx->spawn);
-                kfree(inst);
-        }
-}
-static struct crypto_template mcryptd_tmpl = {
-        .name = "mcryptd",
-        .create = mcryptd_create,
-        .free = mcryptd_free,
-        .module = THIS_MODULE,
-};
-struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
-                                        u32 type, u32 mask)
-{
-        char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
-        struct crypto_ahash *tfm;
-        if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
-                     "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
-                return ERR_PTR(-EINVAL);
-        tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
-        if (IS_ERR(tfm))
-                return ERR_CAST(tfm);
-        if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
-                crypto_free_ahash(tfm);
-                return ERR_PTR(-EINVAL);
-        }
-        return __mcryptd_ahash_cast(tfm);
-}
-EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
-struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
-{
-        struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
-        return ctx->child;
-}
-EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
-struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req)
-{
-        struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
-        return &rctx->areq;
-}
-EXPORT_SYMBOL_GPL(mcryptd_ahash_desc);
-void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
-{
-        crypto_free_ahash(&tfm->base);
-}
-EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
-static int __init mcryptd_init(void)
-{
-        int err, cpu;
-        struct mcryptd_flush_list *flist;
-        mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
-        for_each_possible_cpu(cpu) {
-                flist = per_cpu_ptr(mcryptd_flist, cpu);
-                INIT_LIST_HEAD(&flist->list);
-                mutex_init(&flist->lock);
-        }
-        err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
-        if (err) {
-                free_percpu(mcryptd_flist);
-                return err;
-        }
-        err = crypto_register_template(&mcryptd_tmpl);
-        if (err) {
-                mcryptd_fini_queue(&mqueue);
-                free_percpu(mcryptd_flist);
-        }
-        return err;
-}
-static void __exit mcryptd_exit(void)
-{
-        mcryptd_fini_queue(&mqueue);
-        crypto_unregister_template(&mcryptd_tmpl);
-        free_percpu(mcryptd_flist);
-}
-subsys_initcall(mcryptd_init);
-module_exit(mcryptd_exit);
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
-MODULE_ALIAS_CRYPTO("mcryptd");
diff --git a/crypto/morus1280.c b/crypto/morus1280.c
index d057cf5ac4a8..3889c188f266 100644
--- a/crypto/morus1280.c
+++ b/crypto/morus1280.c
@@ -385,14 +385,11 @@ static void crypto_morus1280_final(struct morus1280_state *state,
                                   struct morus1280_block *tag_xor,
                                   u64 assoclen, u64 cryptlen)
 {
-        u64 assocbits = assoclen * 8;
-        u64 cryptbits = cryptlen * 8;
        struct morus1280_block tmp;
        unsigned int i;
-        tmp.words[0] = cpu_to_le64(assocbits);
+        tmp.words[0] = assoclen * 8;
-        tmp.words[1] = cpu_to_le64(cryptbits);
+        tmp.words[1] = cryptlen * 8;
        tmp.words[2] = 0;
        tmp.words[3] = 0;
diff --git a/crypto/morus640.c b/crypto/morus640.c
index 1ca76e54281b..da06ec2f6a80 100644
--- a/crypto/morus640.c
+++ b/crypto/morus640.c
@@ -384,21 +384,13 @@ static void crypto_morus640_final(struct morus640_state *state,
                                  struct morus640_block *tag_xor,
                                  u64 assoclen, u64 cryptlen)
 {
-        u64 assocbits = assoclen * 8;
-        u64 cryptbits = cryptlen * 8;
-        u32 assocbits_lo = (u32)assocbits;
-        u32 assocbits_hi = (u32)(assocbits >> 32);
-        u32 cryptbits_lo = (u32)cryptbits;
-        u32 cryptbits_hi = (u32)(cryptbits >> 32);
        struct morus640_block tmp;
        unsigned int i;
-        tmp.words[0] = cpu_to_le32(assocbits_lo);
+        tmp.words[0] = lower_32_bits(assoclen * 8);
-        tmp.words[1] = cpu_to_le32(assocbits_hi);
+        tmp.words[1] = upper_32_bits(assoclen * 8);
-        tmp.words[2] = cpu_to_le32(cryptbits_lo);
+        tmp.words[2] = lower_32_bits(cryptlen * 8);
-        tmp.words[3] = cpu_to_le32(cryptbits_hi);
+        tmp.words[3] = upper_32_bits(cryptlen * 8);
        for (i = 0; i < MORUS_BLOCK_WORDS; i++)
                state->s[4].words[i] ^= state->s[0].words[i];
diff --git a/crypto/ofb.c b/crypto/ofb.c
new file mode 100644
index 000000000000..886631708c5e
--- /dev/null
+++ b/crypto/ofb.c
@@ -0,0 +1,225 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * OFB: Output FeedBack mode
+ *
+ * Copyright (C) 2018 ARM Limited or its affiliates.
+ * All rights reserved.
+ *
+ * Based loosely on public domain code gleaned from libtomcrypt
+ * (https://github.com/libtom/libtomcrypt).
+ */
+#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+struct crypto_ofb_ctx {
+        struct crypto_cipher *child;
+        int cnt;
+};
+static int crypto_ofb_setkey(struct crypto_skcipher *parent, const u8 *key,
+                             unsigned int keylen)
+{
+        struct crypto_ofb_ctx *ctx = crypto_skcipher_ctx(parent);
+        struct crypto_cipher *child = ctx->child;
+        int err;
+        crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+        crypto_cipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+                                       CRYPTO_TFM_REQ_MASK);
+        err = crypto_cipher_setkey(child, key, keylen);
+        crypto_skcipher_set_flags(parent, crypto_cipher_get_flags(child) &
+                                  CRYPTO_TFM_RES_MASK);
+        return err;
+}
+static int crypto_ofb_encrypt_segment(struct crypto_ofb_ctx *ctx,
+                                      struct skcipher_walk *walk,
+                                      struct crypto_cipher *tfm)
+{
+        int bsize = crypto_cipher_blocksize(tfm);
+        int nbytes = walk->nbytes;
+        u8 *src = walk->src.virt.addr;
+        u8 *dst = walk->dst.virt.addr;
+        u8 *iv = walk->iv;
+        do {
+                if (ctx->cnt == bsize) {
+                        if (nbytes < bsize)
+                                break;
+                        crypto_cipher_encrypt_one(tfm, iv, iv);
+                        ctx->cnt = 0;
+                }
+                *dst = *src ^ iv[ctx->cnt];
+                src++;
+                dst++;
+                ctx->cnt++;
+        } while (--nbytes);
+        return nbytes;
+}
+static int crypto_ofb_encrypt(struct skcipher_request *req)
+{
+        struct skcipher_walk walk;
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        unsigned int bsize;
+        struct crypto_ofb_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct crypto_cipher *child = ctx->child;
+        int ret = 0;
+        bsize =  crypto_cipher_blocksize(child);
+        ctx->cnt = bsize;
+        ret = skcipher_walk_virt(&walk, req, false);
+        while (walk.nbytes) {
+                ret = crypto_ofb_encrypt_segment(ctx, &walk, child);
+                ret = skcipher_walk_done(&walk, ret);
+        }
+        return ret;
+}
+/* OFB encrypt and decrypt are identical */
+static int crypto_ofb_decrypt(struct skcipher_request *req)
+{
+        return crypto_ofb_encrypt(req);
+}
+static int crypto_ofb_init_tfm(struct crypto_skcipher *tfm)
+{
+        struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+        struct crypto_spawn *spawn = skcipher_instance_ctx(inst);
+        struct crypto_ofb_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct crypto_cipher *cipher;
+        cipher = crypto_spawn_cipher(spawn);
+        if (IS_ERR(cipher))
+                return PTR_ERR(cipher);
+        ctx->child = cipher;
+        return 0;
+}
+static void crypto_ofb_exit_tfm(struct crypto_skcipher *tfm)
+{
+        struct crypto_ofb_ctx *ctx = crypto_skcipher_ctx(tfm);
+        crypto_free_cipher(ctx->child);
+}
+static void crypto_ofb_free(struct skcipher_instance *inst)
+{
+        crypto_drop_skcipher(skcipher_instance_ctx(inst));
+        kfree(inst);
+}
+static int crypto_ofb_create(struct crypto_template *tmpl, struct rtattr **tb)
+{
+        struct skcipher_instance *inst;
+        struct crypto_attr_type *algt;
+        struct crypto_spawn *spawn;
+        struct crypto_alg *alg;
+        u32 mask;
+        int err;
+        err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER);
+        if (err)
+                return err;
+        inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
+        if (!inst)
+                return -ENOMEM;
+        algt = crypto_get_attr_type(tb);
+        err = PTR_ERR(algt);
+        if (IS_ERR(algt))
+                goto err_free_inst;
+        mask = CRYPTO_ALG_TYPE_MASK |
+                crypto_requires_off(algt->type, algt->mask,
+                                    CRYPTO_ALG_NEED_FALLBACK);
+        alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, mask);
+        err = PTR_ERR(alg);
+        if (IS_ERR(alg))
+                goto err_free_inst;
+        spawn = skcipher_instance_ctx(inst);
+        err = crypto_init_spawn(spawn, alg, skcipher_crypto_instance(inst),
+                                CRYPTO_ALG_TYPE_MASK);
+        crypto_mod_put(alg);
+        if (err)
+                goto err_free_inst;
+        err = crypto_inst_setname(skcipher_crypto_instance(inst), "ofb", alg);
+        if (err)
+                goto err_drop_spawn;
+        inst->alg.base.cra_priority = alg->cra_priority;
+        inst->alg.base.cra_blocksize = alg->cra_blocksize;
+        inst->alg.base.cra_alignmask = alg->cra_alignmask;
+        /* We access the data as u32s when xoring. */
+        inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
+        inst->alg.ivsize = alg->cra_blocksize;
+        inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
+        inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
+        inst->alg.base.cra_ctxsize = sizeof(struct crypto_ofb_ctx);
+        inst->alg.init = crypto_ofb_init_tfm;
+        inst->alg.exit = crypto_ofb_exit_tfm;
+        inst->alg.setkey = crypto_ofb_setkey;
+        inst->alg.encrypt = crypto_ofb_encrypt;
+        inst->alg.decrypt = crypto_ofb_decrypt;
+        inst->free = crypto_ofb_free;
+        err = skcipher_register_instance(tmpl, inst);
+        if (err)
+                goto err_drop_spawn;
+out:
+        return err;
+err_drop_spawn:
+        crypto_drop_spawn(spawn);
+err_free_inst:
+        kfree(inst);
+        goto out;
+}
+static struct crypto_template crypto_ofb_tmpl = {
+        .name = "ofb",
+        .create = crypto_ofb_create,
+        .module = THIS_MODULE,
+};
+static int __init crypto_ofb_module_init(void)
+{
+        return crypto_register_template(&crypto_ofb_tmpl);
+}
+static void __exit crypto_ofb_module_exit(void)
+{
+        crypto_unregister_template(&crypto_ofb_tmpl);
+}
+module_init(crypto_ofb_module_init);
+module_exit(crypto_ofb_module_exit);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("OFB block cipher algorithm");
+MODULE_ALIAS_CRYPTO("ofb");
diff --git a/crypto/rng.c b/crypto/rng.c
index b4a618668161..547f16ecbfb0 100644
--- a/crypto/rng.c
+++ b/crypto/rng.c
@@ -50,6 +50,7 @@ int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen)
        }
        err = crypto_rng_alg(tfm)->seed(tfm, seed, slen);
+        crypto_stat_rng_seed(tfm, err);
 out:
        kzfree(buf);
        return err;
diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c
index 9893dbfc1af4..812476e46821 100644
--- a/crypto/rsa-pkcs1pad.c
+++ b/crypto/rsa-pkcs1pad.c
@@ -261,15 +261,6 @@ static int pkcs1pad_encrypt(struct akcipher_request *req)
        pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf,
                        ctx->key_size - 1 - req->src_len, req->src);
-        req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL);
-        if (!req_ctx->out_buf) {
-                kfree(req_ctx->in_buf);
-                return -ENOMEM;
-        }
-        pkcs1pad_sg_set_buf(req_ctx->out_sg, req_ctx->out_buf,
-                        ctx->key_size, NULL);
        akcipher_request_set_tfm(&req_ctx->child_req, ctx->child);
        akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,
                        pkcs1pad_encrypt_sign_complete_cb, req);
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index 39dbf2f7e5f5..64a412be255e 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -73,9 +73,9 @@ static int seqiv_aead_encrypt(struct aead_request *req)
        info = req->iv;
        if (req->src != req->dst) {
-                SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
-                skcipher_request_set_tfm(nreq, ctx->sknull);
+                skcipher_request_set_sync_tfm(nreq, ctx->sknull);
                skcipher_request_set_callback(nreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(nreq, req->src, req->dst,
diff --git a/crypto/shash.c b/crypto/shash.c
index 5d732c6bb4b2..d21f04d70dce 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -73,13 +73,6 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key,
 }
 EXPORT_SYMBOL_GPL(crypto_shash_setkey);
-static inline unsigned int shash_align_buffer_size(unsigned len,
-                                                   unsigned long mask)
-{
-        typedef u8 __aligned_largest u8_aligned;
-        return len + (mask & ~(__alignof__(u8_aligned) - 1));
-}
 static int shash_update_unaligned(struct shash_desc *desc, const u8 *data,
                                  unsigned int len)
 {
@@ -88,11 +81,17 @@ static int shash_update_unaligned(struct shash_desc *desc, const u8 *data,
        unsigned long alignmask = crypto_shash_alignmask(tfm);
        unsigned int unaligned_len = alignmask + 1 -
                                     ((unsigned long)data & alignmask);
-        u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)]
+        /*
-                __aligned_largest;
+         * We cannot count on __aligned() working for large values:
+         * https://patchwork.kernel.org/patch/9507697/
+         */
+        u8 ubuf[MAX_ALGAPI_ALIGNMASK * 2];
        u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
        int err;
+        if (WARN_ON(buf + unaligned_len > ubuf + sizeof(ubuf)))
+                return -EINVAL;
        if (unaligned_len > len)
                unaligned_len = len;
@@ -124,11 +123,17 @@ static int shash_final_unaligned(struct shash_desc *desc, u8 *out)
        unsigned long alignmask = crypto_shash_alignmask(tfm);
        struct shash_alg *shash = crypto_shash_alg(tfm);
        unsigned int ds = crypto_shash_digestsize(tfm);
-        u8 ubuf[shash_align_buffer_size(ds, alignmask)]
+        /*
-                __aligned_largest;
+         * We cannot count on __aligned() working for large values:
+         * https://patchwork.kernel.org/patch/9507697/
+         */
+        u8 ubuf[MAX_ALGAPI_ALIGNMASK + HASH_MAX_DIGESTSIZE];
        u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
        int err;
+        if (WARN_ON(buf + ds > ubuf + sizeof(ubuf)))
+                return -EINVAL;
        err = shash->final(desc, buf);
        if (err)
                goto out;
@@ -458,9 +463,9 @@ static int shash_prepare_alg(struct shash_alg *alg)
 {
        struct crypto_alg *base = &alg->base;
-        if (alg->digestsize > PAGE_SIZE / 8 ||
+        if (alg->digestsize > HASH_MAX_DIGESTSIZE ||
-            alg->descsize > PAGE_SIZE / 8 ||
+            alg->descsize > HASH_MAX_DESCSIZE ||
-            alg->statesize > PAGE_SIZE / 8)
+            alg->statesize > HASH_MAX_STATESIZE)
                return -EINVAL;
        base->cra_type = &crypto_shash_type;
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index 0bd8c6caa498..4caab81d2d02 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -949,6 +949,30 @@ struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name,
 }
 EXPORT_SYMBOL_GPL(crypto_alloc_skcipher);
+struct crypto_sync_skcipher *crypto_alloc_sync_skcipher(
+                                const char *alg_name, u32 type, u32 mask)
+{
+        struct crypto_skcipher *tfm;
+        /* Only sync algorithms allowed. */
+        mask |= CRYPTO_ALG_ASYNC;
+        tfm = crypto_alloc_tfm(alg_name, &crypto_skcipher_type2, type, mask);
+        /*
+         * Make sure we do not allocate something that might get used with
+         * an on-stack request: check the request size.
+         */
+        if (!IS_ERR(tfm) && WARN_ON(crypto_skcipher_reqsize(tfm) >
+                                    MAX_SYNC_SKCIPHER_REQSIZE)) {
+                crypto_free_skcipher(tfm);
+                return ERR_PTR(-EINVAL);
+        }
+        return (struct crypto_sync_skcipher *)tfm;
+}
+EXPORT_SYMBOL_GPL(crypto_alloc_sync_skcipher);
 int crypto_has_skcipher2(const char *alg_name, u32 type, u32 mask)
 {
        return crypto_type_has_alg(alg_name, &crypto_skcipher_type2,
diff --git a/crypto/speck.c b/crypto/speck.c
deleted file mode 100644
index 58aa9f7f91f7..000000000000
--- a/crypto/speck.c
+++ /dev/null
@@ -1,307 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Speck: a lightweight block cipher
- *
- * Copyright (c) 2018 Google, Inc
- *
- * Speck has 10 variants, including 5 block sizes.  For now we only implement
- * the variants Speck128/128, Speck128/192, Speck128/256, Speck64/96, and
- * Speck64/128.   Speck${B}/${K} denotes the variant with a block size of B bits
- * and a key size of K bits.  The Speck128 variants are believed to be the most
- * secure variants, and they use the same block size and key sizes as AES.  The
- * Speck64 variants are less secure, but on 32-bit processors are usually
- * faster.  The remaining variants (Speck32, Speck48, and Speck96) are even less
- * secure and/or not as well suited for implementation on either 32-bit or
- * 64-bit processors, so are omitted.
- *
- * Reference: "The Simon and Speck Families of Lightweight Block Ciphers"
- * https://eprint.iacr.org/2013/404.pdf
- *
- * In a correspondence, the Speck designers have also clarified that the words
- * should be interpreted in little-endian format, and the words should be
- * ordered such that the first word of each block is 'y' rather than 'x', and
- * the first key word (rather than the last) becomes the first round key.
- */
-#include <asm/unaligned.h>
-#include <crypto/speck.h>
-#include <linux/bitops.h>
-#include <linux/crypto.h>
-#include <linux/init.h>
-#include <linux/module.h>
-/* Speck128 */
-static __always_inline void speck128_round(u64 *x, u64 *y, u64 k)
-{
-        *x = ror64(*x, 8);
-        *x += *y;
-        *x ^= k;
-        *y = rol64(*y, 3);
-        *y ^= *x;
-}
-static __always_inline void speck128_unround(u64 *x, u64 *y, u64 k)
-{
-        *y ^= *x;
-        *y = ror64(*y, 3);
-        *x ^= k;
-        *x -= *y;
-        *x = rol64(*x, 8);
-}
-void crypto_speck128_encrypt(const struct speck128_tfm_ctx *ctx,
-                             u8 *out, const u8 *in)
-{
-        u64 y = get_unaligned_le64(in);
-        u64 x = get_unaligned_le64(in + 8);
-        int i;
-        for (i = 0; i < ctx->nrounds; i++)
-                speck128_round(&x, &y, ctx->round_keys[i]);
-        put_unaligned_le64(y, out);
-        put_unaligned_le64(x, out + 8);
-}
-EXPORT_SYMBOL_GPL(crypto_speck128_encrypt);
-static void speck128_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-        crypto_speck128_encrypt(crypto_tfm_ctx(tfm), out, in);
-}
-void crypto_speck128_decrypt(const struct speck128_tfm_ctx *ctx,
-                             u8 *out, const u8 *in)
-{
-        u64 y = get_unaligned_le64(in);
-        u64 x = get_unaligned_le64(in + 8);
-        int i;
-        for (i = ctx->nrounds - 1; i >= 0; i--)
-                speck128_unround(&x, &y, ctx->round_keys[i]);
-        put_unaligned_le64(y, out);
-        put_unaligned_le64(x, out + 8);
-}
-EXPORT_SYMBOL_GPL(crypto_speck128_decrypt);
-static void speck128_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-        crypto_speck128_decrypt(crypto_tfm_ctx(tfm), out, in);
-}
-int crypto_speck128_setkey(struct speck128_tfm_ctx *ctx, const u8 *key,
-                           unsigned int keylen)
-{
-        u64 l[3];
-        u64 k;
-        int i;
-        switch (keylen) {
-        case SPECK128_128_KEY_SIZE:
-                k = get_unaligned_le64(key);
-                l[0] = get_unaligned_le64(key + 8);
-                ctx->nrounds = SPECK128_128_NROUNDS;
-                for (i = 0; i < ctx->nrounds; i++) {
-                        ctx->round_keys[i] = k;
-                        speck128_round(&l[0], &k, i);
-                }
-                break;
-        case SPECK128_192_KEY_SIZE:
-                k = get_unaligned_le64(key);
-                l[0] = get_unaligned_le64(key + 8);
-                l[1] = get_unaligned_le64(key + 16);
-                ctx->nrounds = SPECK128_192_NROUNDS;
-                for (i = 0; i < ctx->nrounds; i++) {
-                        ctx->round_keys[i] = k;
-                        speck128_round(&l[i % 2], &k, i);
-                }
-                break;
-        case SPECK128_256_KEY_SIZE:
-                k = get_unaligned_le64(key);
-                l[0] = get_unaligned_le64(key + 8);
-                l[1] = get_unaligned_le64(key + 16);
-                l[2] = get_unaligned_le64(key + 24);
-                ctx->nrounds = SPECK128_256_NROUNDS;
-                for (i = 0; i < ctx->nrounds; i++) {
-                        ctx->round_keys[i] = k;
-                        speck128_round(&l[i % 3], &k, i);
-                }
-                break;
-        default:
-                return -EINVAL;
-        }
-        return 0;
-}
-EXPORT_SYMBOL_GPL(crypto_speck128_setkey);
-static int speck128_setkey(struct crypto_tfm *tfm, const u8 *key,
-                           unsigned int keylen)
-{
-        return crypto_speck128_setkey(crypto_tfm_ctx(tfm), key, keylen);
-}
-/* Speck64 */
-static __always_inline void speck64_round(u32 *x, u32 *y, u32 k)
-{
-        *x = ror32(*x, 8);
-        *x += *y;
-        *x ^= k;
-        *y = rol32(*y, 3);
-        *y ^= *x;
-}
-static __always_inline void speck64_unround(u32 *x, u32 *y, u32 k)
-{
-        *y ^= *x;
-        *y = ror32(*y, 3);
-        *x ^= k;
-        *x -= *y;
-        *x = rol32(*x, 8);
-}
-void crypto_speck64_encrypt(const struct speck64_tfm_ctx *ctx,
-                            u8 *out, const u8 *in)
-{
-        u32 y = get_unaligned_le32(in);
-        u32 x = get_unaligned_le32(in + 4);
-        int i;
-        for (i = 0; i < ctx->nrounds; i++)
-                speck64_round(&x, &y, ctx->round_keys[i]);
-        put_unaligned_le32(y, out);
-        put_unaligned_le32(x, out + 4);
-}
-EXPORT_SYMBOL_GPL(crypto_speck64_encrypt);
-static void speck64_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-        crypto_speck64_encrypt(crypto_tfm_ctx(tfm), out, in);
-}
-void crypto_speck64_decrypt(const struct speck64_tfm_ctx *ctx,
-                            u8 *out, const u8 *in)
-{
-        u32 y = get_unaligned_le32(in);
-        u32 x = get_unaligned_le32(in + 4);
-        int i;
-        for (i = ctx->nrounds - 1; i >= 0; i--)
-                speck64_unround(&x, &y, ctx->round_keys[i]);
-        put_unaligned_le32(y, out);
-        put_unaligned_le32(x, out + 4);
-}
-EXPORT_SYMBOL_GPL(crypto_speck64_decrypt);
-static void speck64_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-        crypto_speck64_decrypt(crypto_tfm_ctx(tfm), out, in);
-}
-int crypto_speck64_setkey(struct speck64_tfm_ctx *ctx, const u8 *key,
-                          unsigned int keylen)
-{
-        u32 l[3];
-        u32 k;
-        int i;
-        switch (keylen) {
-        case SPECK64_96_KEY_SIZE:
-                k = get_unaligned_le32(key);
-                l[0] = get_unaligned_le32(key + 4);
-                l[1] = get_unaligned_le32(key + 8);
-                ctx->nrounds = SPECK64_96_NROUNDS;
-                for (i = 0; i < ctx->nrounds; i++) {
-                        ctx->round_keys[i] = k;
-                        speck64_round(&l[i % 2], &k, i);
-                }
-                break;
-        case SPECK64_128_KEY_SIZE:
-                k = get_unaligned_le32(key);
-                l[0] = get_unaligned_le32(key + 4);
-                l[1] = get_unaligned_le32(key + 8);
-                l[2] = get_unaligned_le32(key + 12);
-                ctx->nrounds = SPECK64_128_NROUNDS;
-                for (i = 0; i < ctx->nrounds; i++) {
-                        ctx->round_keys[i] = k;
-                        speck64_round(&l[i % 3], &k, i);
-                }
-                break;
-        default:
-                return -EINVAL;
-        }
-        return 0;
-}
-EXPORT_SYMBOL_GPL(crypto_speck64_setkey);
-static int speck64_setkey(struct crypto_tfm *tfm, const u8 *key,
-                          unsigned int keylen)
-{
-        return crypto_speck64_setkey(crypto_tfm_ctx(tfm), key, keylen);
-}
-/* Algorithm definitions */
-static struct crypto_alg speck_algs[] = {
-        {
-                .cra_name               = "speck128",
-                .cra_driver_name        = "speck128-generic",
-                .cra_priority           = 100,
-                .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
-                .cra_blocksize          = SPECK128_BLOCK_SIZE,
-                .cra_ctxsize            = sizeof(struct speck128_tfm_ctx),
-                .cra_module             = THIS_MODULE,
-                .cra_u                  = {
-                        .cipher = {
-                                .cia_min_keysize        = SPECK128_128_KEY_SIZE,
-                                .cia_max_keysize        = SPECK128_256_KEY_SIZE,
-                                .cia_setkey             = speck128_setkey,
-                                .cia_encrypt            = speck128_encrypt,
-                                .cia_decrypt            = speck128_decrypt
-                        }
-                }
-        }, {
-                .cra_name               = "speck64",
-                .cra_driver_name        = "speck64-generic",
-                .cra_priority           = 100,
-                .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
-                .cra_blocksize          = SPECK64_BLOCK_SIZE,
-                .cra_ctxsize            = sizeof(struct speck64_tfm_ctx),
-                .cra_module             = THIS_MODULE,
-                .cra_u                  = {
-                        .cipher = {
-                                .cia_min_keysize        = SPECK64_96_KEY_SIZE,
-                                .cia_max_keysize        = SPECK64_128_KEY_SIZE,
-                                .cia_setkey             = speck64_setkey,
-                                .cia_encrypt            = speck64_encrypt,
-                                .cia_decrypt            = speck64_decrypt
-                        }
-                }
-        }
-};
-static int __init speck_module_init(void)
-{
-        return crypto_register_algs(speck_algs, ARRAY_SIZE(speck_algs));
-}
-static void __exit speck_module_exit(void)
-{
-        crypto_unregister_algs(speck_algs, ARRAY_SIZE(speck_algs));
-}
-module_init(speck_module_init);
-module_exit(speck_module_exit);
-MODULE_DESCRIPTION("Speck block cipher (generic)");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("speck128");
-MODULE_ALIAS_CRYPTO("speck128-generic");
-MODULE_ALIAS_CRYPTO("speck64");
-MODULE_ALIAS_CRYPTO("speck64-generic");
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index bdde95e8d369..c20c9f5c18f2 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -76,8 +76,7 @@ static char *check[] = {
        "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",
        "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta",  "fcrypt",
        "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320",
-        "lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512",
+        "lzo", "cts", "sha3-224", "sha3-256", "sha3-384", "sha3-512", NULL
-        NULL
 };
 static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 };
@@ -1103,6 +1102,9 @@ static void test_ahash_speed_common(const char *algo, unsigned int secs,
                        break;
                }
+                if (speed[i].klen)
+                        crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
                pr_info("test%3u "
                        "(%5u byte blocks,%5u bytes per update,%4u updates): ",
                        i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
@@ -1733,6 +1735,7 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
                ret += tcrypt_test("xts(aes)");
                ret += tcrypt_test("ctr(aes)");
                ret += tcrypt_test("rfc3686(ctr(aes))");
+                ret += tcrypt_test("ofb(aes)");
                break;
        case 11:
@@ -1878,10 +1881,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
                ret += tcrypt_test("ecb(seed)");
                break;
-        case 44:
-                ret += tcrypt_test("zlib");
-                break;
        case 45:
                ret += tcrypt_test("rfc4309(ccm(aes))");
                break;
@@ -2033,6 +2032,8 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
                break;
        case 191:
                ret += tcrypt_test("ecb(sm4)");
+                ret += tcrypt_test("cbc(sm4)");
+                ret += tcrypt_test("ctr(sm4)");
                break;
        case 200:
                test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
@@ -2282,6 +2283,20 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
                                   num_mb);
                break;
+        case 218:
+                test_cipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
+                                speed_template_16);
+                test_cipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
+                                speed_template_16);
+                test_cipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
+                                speed_template_16);
+                test_cipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
+                                speed_template_16);
+                test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
+                                speed_template_16);
+                test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
+                                speed_template_16);
+                break;
        case 300:
                if (alg) {
                        test_hash_speed(alg, sec, generic_hash_speed_template);
diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h
index f0bfee1bb293..d09ea8b10b4f 100644
--- a/crypto/tcrypt.h
+++ b/crypto/tcrypt.h
@@ -51,6 +51,7 @@ static struct cipher_speed_template des3_speed_template[] = {
 * Cipher speed tests
 */
 static u8 speed_template_8[] = {8, 0};
+static u8 speed_template_16[] = {16, 0};
 static u8 speed_template_24[] = {24, 0};
 static u8 speed_template_8_16[] = {8, 16, 0};
 static u8 speed_template_8_32[] = {8, 32, 0};
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index a1d42245082a..b1f79c6bf409 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -1400,8 +1400,8 @@ static int test_comp(struct crypto_comp *tfm,
                int ilen;
                unsigned int dlen = COMP_BUF_SIZE;
-                memset(output, 0, sizeof(COMP_BUF_SIZE));
+                memset(output, 0, COMP_BUF_SIZE);
-                memset(decomp_output, 0, sizeof(COMP_BUF_SIZE));
+                memset(decomp_output, 0, COMP_BUF_SIZE);
                ilen = ctemplate[i].inlen;
                ret = crypto_comp_compress(tfm, ctemplate[i].input,
@@ -1445,7 +1445,7 @@ static int test_comp(struct crypto_comp *tfm,
                int ilen;
                unsigned int dlen = COMP_BUF_SIZE;
-                memset(decomp_output, 0, sizeof(COMP_BUF_SIZE));
+                memset(decomp_output, 0, COMP_BUF_SIZE);
                ilen = dtemplate[i].inlen;
                ret = crypto_comp_decompress(tfm, dtemplate[i].input,
@@ -2662,6 +2662,12 @@ static const struct alg_test_desc alg_test_descs[] = {
                        .cipher = __VECS(serpent_cbc_tv_template)
                },
        }, {
+                .alg = "cbc(sm4)",
+                .test = alg_test_skcipher,
+                .suite = {
+                        .cipher = __VECS(sm4_cbc_tv_template)
+                }
+        }, {
                .alg = "cbc(twofish)",
                .test = alg_test_skcipher,
                .suite = {
@@ -2785,6 +2791,12 @@ static const struct alg_test_desc alg_test_descs[] = {
                        .cipher = __VECS(serpent_ctr_tv_template)
                }
        }, {
+                .alg = "ctr(sm4)",
+                .test = alg_test_skcipher,
+                .suite = {
+                        .cipher = __VECS(sm4_ctr_tv_template)
+                }
+        }, {
                .alg = "ctr(twofish)",
                .test = alg_test_skcipher,
                .suite = {
@@ -3038,18 +3050,6 @@ static const struct alg_test_desc alg_test_descs[] = {
                        .cipher = __VECS(sm4_tv_template)
                }
        }, {
-                .alg = "ecb(speck128)",
-                .test = alg_test_skcipher,
-                .suite = {
-                        .cipher = __VECS(speck128_tv_template)
-                }
-        }, {
-                .alg = "ecb(speck64)",
-                .test = alg_test_skcipher,
-                .suite = {
-                        .cipher = __VECS(speck64_tv_template)
-                }
-        }, {
                .alg = "ecb(tea)",
                .test = alg_test_skcipher,
                .suite = {
@@ -3577,18 +3577,6 @@ static const struct alg_test_desc alg_test_descs[] = {
                        .cipher = __VECS(serpent_xts_tv_template)
                }
        }, {
-                .alg = "xts(speck128)",
-                .test = alg_test_skcipher,
-                .suite = {
-                        .cipher = __VECS(speck128_xts_tv_template)
-                }
-        }, {
-                .alg = "xts(speck64)",
-                .test = alg_test_skcipher,
-                .suite = {
-                        .cipher = __VECS(speck64_xts_tv_template)
-                }
-        }, {
                .alg = "xts(twofish)",
                .test = alg_test_skcipher,
                .suite = {
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 173111c70746..1fe7b97ba03f 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -24,8 +24,6 @@
 #ifndef _CRYPTO_TESTMGR_H
 #define _CRYPTO_TESTMGR_H
-#include <linux/netlink.h>
 #define MAX_DIGEST_SIZE         64
 #define MAX_TAP                 8
@@ -10133,12 +10131,13 @@ static const struct cipher_testvec serpent_xts_tv_template[] = {
 };
 /*
- * SM4 test vector taken from the draft RFC
+ * SM4 test vectors taken from the "The SM4 Blockcipher Algorithm And Its
- * https://tools.ietf.org/html/draft-crypto-sm4-00#ref-GBT.32907-2016
+ * Modes Of Operations" draft RFC
+ * https://datatracker.ietf.org/doc/draft-ribose-cfrg-sm4
 */
 static const struct cipher_testvec sm4_tv_template[] = {
-        { /* SM4 Appendix A: Example Calculations. Example 1. */
+        { /* GB/T 32907-2016 Example 1. */
                .key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
                          "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
                .klen   = 16,
@@ -10147,10 +10146,7 @@ static const struct cipher_testvec sm4_tv_template[] = {
                .ctext  = "\x68\x1E\xDF\x34\xD2\x06\x96\x5E"
                          "\x86\xB3\xE9\x4F\x53\x6E\x42\x46",
                .len    = 16,
-        }, { /*
+        }, { /* Last 10 iterations of GB/T 32907-2016 Example 2. */
-              *  SM4 Appendix A: Example Calculations.
-              *  Last 10 iterations of Example 2.
-              */
                .key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
                          "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
                .klen   = 16,
@@ -10195,744 +10191,116 @@ static const struct cipher_testvec sm4_tv_template[] = {
                          "\x59\x52\x98\xc7\xc6\xfd\x27\x1f"
                          "\x4\x2\xf8\x4\xc3\x3d\x3f\x66",
                .len    = 160
+        }, { /* A.2.1.1 SM4-ECB Example 1 */
+                .key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
+                          "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
+                .klen   = 16,
+                .ptext  = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb"
+                          "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd"
+                          "\xee\xee\xee\xee\xff\xff\xff\xff"
+                          "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb",
+                .ctext  = "\x5e\xc8\x14\x3d\xe5\x09\xcf\xf7"
+                          "\xb5\x17\x9f\x8f\x47\x4b\x86\x19"
+                          "\x2f\x1d\x30\x5a\x7f\xb1\x7d\xf9"
+                          "\x85\xf8\x1c\x84\x82\x19\x23\x04",
+                .len    = 32,
+        }, { /* A.2.1.2 SM4-ECB Example 2 */
+                .key    = "\xFE\xDC\xBA\x98\x76\x54\x32\x10"
+                          "\x01\x23\x45\x67\x89\xAB\xCD\xEF",
+                .klen   = 16,
+                .ptext  = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb"
+                          "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd"
+                          "\xee\xee\xee\xee\xff\xff\xff\xff"
+                          "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb",
+                .ctext  = "\xC5\x87\x68\x97\xE4\xA5\x9B\xBB"
+                          "\xA7\x2A\x10\xC8\x38\x72\x24\x5B"
+                          "\x12\xDD\x90\xBC\x2D\x20\x06\x92"
+                          "\xB5\x29\xA4\x15\x5A\xC9\xE6\x00",
+                .len    = 32,
        }
 };
-/*
+static const struct cipher_testvec sm4_cbc_tv_template[] = {
- * Speck test vectors taken from the original paper:
+        { /* A.2.2.1 SM4-CBC Example 1 */
- * "The Simon and Speck Families of Lightweight Block Ciphers"
+                .key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
- * https://eprint.iacr.org/2013/404.pdf
+                          "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
- *
- * Note that the paper does not make byte and word order clear.  But it was
- * confirmed with the authors that the intended orders are little endian byte
- * order and (y, x) word order.  Equivalently, the printed test vectors, when
- * looking at only the bytes (ignoring the whitespace that divides them into
- * words), are backwards: the left-most byte is actually the one with the
- * highest memory address, while the right-most byte is actually the one with
- * the lowest memory address.
- */
-static const struct cipher_testvec speck128_tv_template[] = {
-        { /* Speck128/128 */
-                .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
                .klen   = 16,
-                .ptext  = "\x20\x6d\x61\x64\x65\x20\x69\x74"
+                .ptext  = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb"
-                          "\x20\x65\x71\x75\x69\x76\x61\x6c",
+                          "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd"
-                .ctext  = "\x18\x0d\x57\x5c\xdf\xfe\x60\x78"
+                          "\xee\xee\xee\xee\xff\xff\xff\xff"
-                          "\x65\x32\x78\x79\x51\x98\x5d\xa6",
+                          "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb",
-                .len    = 16,
+                .iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-        }, { /* Speck128/192 */
+                          "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F",
-                .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
+                .ctext  = "\x78\xEB\xB1\x1C\xC4\x0B\x0A\x48"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+                          "\x31\x2A\xAE\xB2\x04\x02\x44\xCB"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17",
+                          "\x4C\xB7\x01\x69\x51\x90\x92\x26"
-                .klen   = 24,
+                          "\x97\x9B\x0D\x15\xDC\x6A\x8F\x6D",
-                .ptext  = "\x65\x6e\x74\x20\x74\x6f\x20\x43"
-                          "\x68\x69\x65\x66\x20\x48\x61\x72",
-                .ctext  = "\x86\x18\x3c\xe0\x5d\x18\xbc\xf9"
-                          "\x66\x55\x13\x13\x3a\xcf\xe4\x1b",
-                .len    = 16,
-        }, { /* Speck128/256 */
-                .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-                .klen   = 32,
-                .ptext  = "\x70\x6f\x6f\x6e\x65\x72\x2e\x20"
-                          "\x49\x6e\x20\x74\x68\x6f\x73\x65",
-                .ctext  = "\x43\x8f\x18\x9c\x8d\xb4\xee\x4e"
-                          "\x3e\xf5\xc0\x05\x04\x01\x09\x41",
-                .len    = 16,
-        },
-};
-/*
- * Speck128-XTS test vectors, taken from the AES-XTS test vectors with the
- * ciphertext recomputed with Speck128 as the cipher
- */
-static const struct cipher_testvec speck128_xts_tv_template[] = {
-        {
-                .key    = "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .klen   = 32,
-                .iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ctext  = "\xbe\xa0\xe7\x03\xd7\xfe\xab\x62"
-                          "\x3b\x99\x4a\x64\x74\x77\xac\xed"
-                          "\xd8\xf4\xa6\xcf\xae\xb9\x07\x42"
-                          "\x51\xd9\xb6\x1d\xe0\x5e\xbc\x54",
-                .len    = 32,
-        }, {
-                .key    = "\x11\x11\x11\x11\x11\x11\x11\x11"
-                          "\x11\x11\x11\x11\x11\x11\x11\x11"
-                          "\x22\x22\x22\x22\x22\x22\x22\x22"
-                          "\x22\x22\x22\x22\x22\x22\x22\x22",
-                .klen   = 32,
-                .iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x44\x44\x44\x44\x44\x44\x44\x44"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44",
-                .ctext  = "\xfb\x53\x81\x75\x6f\x9f\x34\xad"
-                          "\x7e\x01\xed\x7b\xcc\xda\x4e\x4a"
-                          "\xd4\x84\xa4\x53\xd5\x88\x73\x1b"
-                          "\xfd\xcb\xae\x0d\xf3\x04\xee\xe6",
                .len    = 32,
-        }, {
+        }, { /* A.2.2.2 SM4-CBC Example 2 */
-                .key    = "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+                .key    = "\xFE\xDC\xBA\x98\x76\x54\x32\x10"
-                          "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+                          "\x01\x23\x45\x67\x89\xAB\xCD\xEF",
-                          "\x22\x22\x22\x22\x22\x22\x22\x22"
+                .klen   = 16,
-                          "\x22\x22\x22\x22\x22\x22\x22\x22",
+                .ptext  = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb"
-                .klen   = 32,
+                          "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd"
-                .iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
+                          "\xee\xee\xee\xee\xff\xff\xff\xff"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
+                          "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb",
-                .ptext  = "\x44\x44\x44\x44\x44\x44\x44\x44"
+                .iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
+                          "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F",
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
+                .ctext  = "\x0d\x3a\x6d\xdc\x2d\x21\xc6\x98"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44",
+                          "\x85\x72\x15\x58\x7b\x7b\xb5\x9a"
-                .ctext  = "\x21\x52\x84\x15\xd1\xf7\x21\x55"
+                          "\x91\xf2\xc1\x47\x91\x1a\x41\x44"
-                          "\xd9\x75\x4a\xd3\xc5\xdb\x9f\x7d"
+                          "\x66\x5e\x1f\xa1\xd4\x0b\xae\x38",
-                          "\xda\x63\xb2\xf1\x82\xb0\x89\x59"
-                          "\x86\xd4\xaa\xaa\xdd\xff\x4f\x92",
                .len    = 32,
-        }, {
-                .key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
-                          "\x23\x53\x60\x28\x74\x71\x35\x26"
-                          "\x31\x41\x59\x26\x53\x58\x97\x93"
-                          "\x23\x84\x62\x64\x33\x83\x27\x95",
-                .klen   = 32,
-                .iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-                          "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-                .ctext  = "\x57\xb5\xf8\x71\x6e\x6d\xdd\x82"
-                          "\x53\xd0\xed\x2d\x30\xc1\x20\xef"
-                          "\x70\x67\x5e\xff\x09\x70\xbb\xc1"
-                          "\x3a\x7b\x48\x26\xd9\x0b\xf4\x48"
-                          "\xbe\xce\xb1\xc7\xb2\x67\xc4\xa7"
-                          "\x76\xf8\x36\x30\xb7\xb4\x9a\xd9"
-                          "\xf5\x9d\xd0\x7b\xc1\x06\x96\x44"
-                          "\x19\xc5\x58\x84\x63\xb9\x12\x68"
-                          "\x68\xc7\xaa\x18\x98\xf2\x1f\x5c"
-                          "\x39\xa6\xd8\x32\x2b\xc3\x51\xfd"
-                          "\x74\x79\x2e\xb4\x44\xd7\x69\xc4"
-                          "\xfc\x29\xe6\xed\x26\x1e\xa6\x9d"
-                          "\x1c\xbe\x00\x0e\x7f\x3a\xca\xfb"
-                          "\x6d\x13\x65\xa0\xf9\x31\x12\xe2"
-                          "\x26\xd1\xec\x2b\x0a\x8b\x59\x99"
-                          "\xa7\x49\xa0\x0e\x09\x33\x85\x50"
-                          "\xc3\x23\xca\x7a\xdd\x13\x45\x5f"
-                          "\xde\x4c\xa7\xcb\x00\x8a\x66\x6f"
-                          "\xa2\xb6\xb1\x2e\xe1\xa0\x18\xf6"
-                          "\xad\xf3\xbd\xeb\xc7\xef\x55\x4f"
-                          "\x79\x91\x8d\x36\x13\x7b\xd0\x4a"
-                          "\x6c\x39\xfb\x53\xb8\x6f\x02\x51"
-                          "\xa5\x20\xac\x24\x1c\x73\x59\x73"
-                          "\x58\x61\x3a\x87\x58\xb3\x20\x56"
-                          "\x39\x06\x2b\x4d\xd3\x20\x2b\x89"
-                          "\x3f\xa2\xf0\x96\xeb\x7f\xa4\xcd"
-                          "\x11\xae\xbd\xcb\x3a\xb4\xd9\x91"
-                          "\x09\x35\x71\x50\x65\xac\x92\xe3"
-                          "\x7b\x32\xc0\x7a\xdd\xd4\xc3\x92"
-                          "\x6f\xeb\x79\xde\x6f\xd3\x25\xc9"
-                          "\xcd\x63\xf5\x1e\x7a\x3b\x26\x9d"
-                          "\x77\x04\x80\xa9\xbf\x38\xb5\xbd"
-                          "\xb8\x05\x07\xbd\xfd\xab\x7b\xf8"
-                          "\x2a\x26\xcc\x49\x14\x6d\x55\x01"
-                          "\x06\x94\xd8\xb2\x2d\x53\x83\x1b"
-                          "\x8f\xd4\xdd\x57\x12\x7e\x18\xba"
-                          "\x8e\xe2\x4d\x80\xef\x7e\x6b\x9d"
-                          "\x24\xa9\x60\xa4\x97\x85\x86\x2a"
-                          "\x01\x00\x09\xf1\xcb\x4a\x24\x1c"
-                          "\xd8\xf6\xe6\x5b\xe7\x5d\xf2\xc4"
-                          "\x97\x1c\x10\xc6\x4d\x66\x4f\x98"
-                          "\x87\x30\xac\xd5\xea\x73\x49\x10"
-                          "\x80\xea\xe5\x5f\x4d\x5f\x03\x33"
-                          "\x66\x02\x35\x3d\x60\x06\x36\x4f"
-                          "\x14\x1c\xd8\x07\x1f\x78\xd0\xf8"
-                          "\x4f\x6c\x62\x7c\x15\xa5\x7c\x28"
-                          "\x7c\xcc\xeb\x1f\xd1\x07\x90\x93"
-                          "\x7e\xc2\xa8\x3a\x80\xc0\xf5\x30"
-                          "\xcc\x75\xcf\x16\x26\xa9\x26\x3b"
-                          "\xe7\x68\x2f\x15\x21\x5b\xe4\x00"
-                          "\xbd\x48\x50\xcd\x75\x70\xc4\x62"
-                          "\xbb\x41\xfb\x89\x4a\x88\x3b\x3b"
-                          "\x51\x66\x02\x69\x04\x97\x36\xd4"
-                          "\x75\xae\x0b\xa3\x42\xf8\xca\x79"
-                          "\x8f\x93\xe9\xcc\x38\xbd\xd6\xd2"
-                          "\xf9\x70\x4e\xc3\x6a\x8e\x25\xbd"
-                          "\xea\x15\x5a\xa0\x85\x7e\x81\x0d"
-                          "\x03\xe7\x05\x39\xf5\x05\x26\xee"
-                          "\xec\xaa\x1f\x3d\xc9\x98\x76\x01"
-                          "\x2c\xf4\xfc\xa3\x88\x77\x38\xc4"
-                          "\x50\x65\x50\x6d\x04\x1f\xdf\x5a"
-                          "\xaa\xf2\x01\xa9\xc1\x8d\xee\xca"
-                          "\x47\x26\xef\x39\xb8\xb4\xf2\xd1"
-                          "\xd6\xbb\x1b\x2a\xc1\x34\x14\xcf",
-                .len    = 512,
-        }, {
-                .key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
-                          "\x23\x53\x60\x28\x74\x71\x35\x26"
-                          "\x62\x49\x77\x57\x24\x70\x93\x69"
-                          "\x99\x59\x57\x49\x66\x96\x76\x27"
-                          "\x31\x41\x59\x26\x53\x58\x97\x93"
-                          "\x23\x84\x62\x64\x33\x83\x27\x95"
-                          "\x02\x88\x41\x97\x16\x93\x99\x37"
-                          "\x51\x05\x82\x09\x74\x94\x45\x92",
-                .klen   = 64,
-                .iv     = "\xff\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-                          "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-                .ctext  = "\xc5\x85\x2a\x4b\x73\xe4\xf6\xf1"
-                          "\x7e\xf9\xf6\xe9\xa3\x73\x36\xcb"
-                          "\xaa\xb6\x22\xb0\x24\x6e\x3d\x73"
-                          "\x92\x99\xde\xd3\x76\xed\xcd\x63"
-                          "\x64\x3a\x22\x57\xc1\x43\x49\xd4"
-                          "\x79\x36\x31\x19\x62\xae\x10\x7e"
-                          "\x7d\xcf\x7a\xe2\x6b\xce\x27\xfa"
-                          "\xdc\x3d\xd9\x83\xd3\x42\x4c\xe0"
-                          "\x1b\xd6\x1d\x1a\x6f\xd2\x03\x00"
-                          "\xfc\x81\x99\x8a\x14\x62\xf5\x7e"
-                          "\x0d\xe7\x12\xe8\x17\x9d\x0b\xec"
-                          "\xe2\xf7\xc9\xa7\x63\xd1\x79\xb6"
-                          "\x62\x62\x37\xfe\x0a\x4c\x4a\x37"
-                          "\x70\xc7\x5e\x96\x5f\xbc\x8e\x9e"
-                          "\x85\x3c\x4f\x26\x64\x85\xbc\x68"
-                          "\xb0\xe0\x86\x5e\x26\x41\xce\x11"
-                          "\x50\xda\x97\x14\xe9\x9e\xc7\x6d"
-                          "\x3b\xdc\x43\xde\x2b\x27\x69\x7d"
-                          "\xfc\xb0\x28\xbd\x8f\xb1\xc6\x31"
-                          "\x14\x4d\xf0\x74\x37\xfd\x07\x25"
-                          "\x96\x55\xe5\xfc\x9e\x27\x2a\x74"
-                          "\x1b\x83\x4d\x15\x83\xac\x57\xa0"
-                          "\xac\xa5\xd0\x38\xef\x19\x56\x53"
-                          "\x25\x4b\xfc\xce\x04\x23\xe5\x6b"
-                          "\xf6\xc6\x6c\x32\x0b\xb3\x12\xc5"
-                          "\xed\x22\x34\x1c\x5d\xed\x17\x06"
-                          "\x36\xa3\xe6\x77\xb9\x97\x46\xb8"
-                          "\xe9\x3f\x7e\xc7\xbc\x13\x5c\xdc"
-                          "\x6e\x3f\x04\x5e\xd1\x59\xa5\x82"
-                          "\x35\x91\x3d\x1b\xe4\x97\x9f\x92"
-                          "\x1c\x5e\x5f\x6f\x41\xd4\x62\xa1"
-                          "\x8d\x39\xfc\x42\xfb\x38\x80\xb9"
-                          "\x0a\xe3\xcc\x6a\x93\xd9\x7a\xb1"
-                          "\xe9\x69\xaf\x0a\x6b\x75\x38\xa7"
-                          "\xa1\xbf\xf7\xda\x95\x93\x4b\x78"
-                          "\x19\xf5\x94\xf9\xd2\x00\x33\x37"
-                          "\xcf\xf5\x9e\x9c\xf3\xcc\xa6\xee"
-                          "\x42\xb2\x9e\x2c\x5f\x48\x23\x26"
-                          "\x15\x25\x17\x03\x3d\xfe\x2c\xfc"
-                          "\xeb\xba\xda\xe0\x00\x05\xb6\xa6"
-                          "\x07\xb3\xe8\x36\x5b\xec\x5b\xbf"
-                          "\xd6\x5b\x00\x74\xc6\x97\xf1\x6a"
-                          "\x49\xa1\xc3\xfa\x10\x52\xb9\x14"
-                          "\xad\xb7\x73\xf8\x78\x12\xc8\x59"
-                          "\x17\x80\x4c\x57\x39\xf1\x6d\x80"
-                          "\x25\x77\x0f\x5e\x7d\xf0\xaf\x21"
-                          "\xec\xce\xb7\xc8\x02\x8a\xed\x53"
-                          "\x2c\x25\x68\x2e\x1f\x85\x5e\x67"
-                          "\xd1\x07\x7a\x3a\x89\x08\xe0\x34"
-                          "\xdc\xdb\x26\xb4\x6b\x77\xfc\x40"
-                          "\x31\x15\x72\xa0\xf0\x73\xd9\x3b"
-                          "\xd5\xdb\xfe\xfc\x8f\xa9\x44\xa2"
-                          "\x09\x9f\xc6\x33\xe5\xe2\x88\xe8"
-                          "\xf3\xf0\x1a\xf4\xce\x12\x0f\xd6"
-                          "\xf7\x36\xe6\xa4\xf4\x7a\x10\x58"
-                          "\xcc\x1f\x48\x49\x65\x47\x75\xe9"
-                          "\x28\xe1\x65\x7b\xf2\xc4\xb5\x07"
-                          "\xf2\xec\x76\xd8\x8f\x09\xf3\x16"
-                          "\xa1\x51\x89\x3b\xeb\x96\x42\xac"
-                          "\x65\xe0\x67\x63\x29\xdc\xb4\x7d"
-                          "\xf2\x41\x51\x6a\xcb\xde\x3c\xfb"
-                          "\x66\x8d\x13\xca\xe0\x59\x2a\x00"
-                          "\xc9\x53\x4c\xe6\x9e\xe2\x73\xd5"
-                          "\x67\x19\xb2\xbd\x9a\x63\xd7\x5c",
-                .len    = 512,
-                .also_non_np = 1,
-                .np     = 3,
-                .tap    = { 512 - 20, 4, 16 },
        }
 };
-static const struct cipher_testvec speck64_tv_template[] = {
+static const struct cipher_testvec sm4_ctr_tv_template[] = {
-        { /* Speck64/96 */
+        { /* A.2.5.1 SM4-CTR Example 1 */
-                .key    = "\x00\x01\x02\x03\x08\x09\x0a\x0b"
+                .key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-                          "\x10\x11\x12\x13",
+                          "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-                .klen   = 12,
-                .ptext  = "\x65\x61\x6e\x73\x20\x46\x61\x74",
-                .ctext  = "\x6c\x94\x75\x41\xec\x52\x79\x9f",
-                .len    = 8,
-        }, { /* Speck64/128 */
-                .key    = "\x00\x01\x02\x03\x08\x09\x0a\x0b"
-                          "\x10\x11\x12\x13\x18\x19\x1a\x1b",
                .klen   = 16,
-                .ptext  = "\x2d\x43\x75\x74\x74\x65\x72\x3b",
+                .ptext  = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-                .ctext  = "\x8b\x02\x4e\x45\x48\xa5\x6f\x8c",
+                          "\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb"
-                .len    = 8,
+                          "\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc"
-        },
+                          "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd"
-};
+                          "\xee\xee\xee\xee\xee\xee\xee\xee"
+                          "\xff\xff\xff\xff\xff\xff\xff\xff"
-/*
+                          "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
- * Speck64-XTS test vectors, taken from the AES-XTS test vectors with the
+                          "\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb",
- * ciphertext recomputed with Speck64 as the cipher, and key lengths adjusted
+                .iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
- */
+                          "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F",
-static const struct cipher_testvec speck64_xts_tv_template[] = {
+                .ctext  = "\xac\x32\x36\xcb\x97\x0c\xc2\x07"
-        {
+                          "\x91\x36\x4c\x39\x5a\x13\x42\xd1"
-                .key    = "\x00\x00\x00\x00\x00\x00\x00\x00"
+                          "\xa3\xcb\xc1\x87\x8c\x6f\x30\xcd"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
+                          "\x07\x4c\xce\x38\x5c\xdd\x70\xc7"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
+                          "\xf2\x34\xbc\x0e\x24\xc1\x19\x80"
-                .klen   = 24,
+                          "\xfd\x12\x86\x31\x0c\xe3\x7b\x92"
-                .iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+                          "\x6e\x02\xfc\xd0\xfa\xa0\xba\xf3"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
+                          "\x8b\x29\x33\x85\x1d\x82\x45\x14",
-                .ptext  = "\x00\x00\x00\x00\x00\x00\x00\x00"
+                .len    = 64,
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
+        }, { /* A.2.5.2 SM4-CTR Example 2 */
-                          "\x00\x00\x00\x00\x00\x00\x00\x00"
+                .key    = "\xFE\xDC\xBA\x98\x76\x54\x32\x10"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
+                          "\x01\x23\x45\x67\x89\xAB\xCD\xEF",
-                .ctext  = "\x84\xaf\x54\x07\x19\xd4\x7c\xa6"
+                .klen   = 16,
-                          "\xe4\xfe\xdf\xc4\x1f\x34\xc3\xc2"
+                .ptext  = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-                          "\x80\xf5\x72\xe7\xcd\xf0\x99\x22"
+                          "\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb"
-                          "\x35\xa7\x2f\x06\xef\xdc\x51\xaa",
+                          "\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc"
-                .len    = 32,
+                          "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd"
-        }, {
+                          "\xee\xee\xee\xee\xee\xee\xee\xee"
-                .key    = "\x11\x11\x11\x11\x11\x11\x11\x11"
+                          "\xff\xff\xff\xff\xff\xff\xff\xff"
-                          "\x11\x11\x11\x11\x11\x11\x11\x11"
+                          "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-                          "\x22\x22\x22\x22\x22\x22\x22\x22",
+                          "\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb",
-                .klen   = 24,
+                .iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                .iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
+                          "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F",
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
+                .ctext  = "\x5d\xcc\xcd\x25\xb9\x5a\xb0\x74"
-                .ptext  = "\x44\x44\x44\x44\x44\x44\x44\x44"
+                          "\x17\xa0\x85\x12\xee\x16\x0e\x2f"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
+                          "\x8f\x66\x15\x21\xcb\xba\xb4\x4c"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
+                          "\xc8\x71\x38\x44\x5b\xc2\x9e\x5c"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44",
+                          "\x0a\xe0\x29\x72\x05\xd6\x27\x04"
-                .ctext  = "\x12\x56\x73\xcd\x15\x87\xa8\x59"
+                          "\x17\x3b\x21\x23\x9b\x88\x7f\x6c"
-                          "\xcf\x84\xae\xd9\x1c\x66\xd6\x9f"
+                          "\x8c\xb5\xb8\x00\x91\x7a\x24\x88"
-                          "\xb3\x12\x69\x7e\x36\xeb\x52\xff"
+                          "\x28\x4b\xde\x9e\x16\xea\x29\x06",
-                          "\x62\xdd\xba\x90\xb3\xe1\xee\x99",
+                .len    = 64,
-                .len    = 32,
-        }, {
-                .key    = "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-                          "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-                          "\x22\x22\x22\x22\x22\x22\x22\x22",
-                .klen   = 24,
-                .iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x44\x44\x44\x44\x44\x44\x44\x44"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44"
-                          "\x44\x44\x44\x44\x44\x44\x44\x44",
-                .ctext  = "\x15\x1b\xe4\x2c\xa2\x5a\x2d\x2c"
-                          "\x27\x36\xc0\xbf\x5d\xea\x36\x37"
-                          "\x2d\x1a\x88\xbc\x66\xb5\xd0\x0b"
-                          "\xa1\xbc\x19\xb2\x0f\x3b\x75\x34",
-                .len    = 32,
-        }, {
-                .key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
-                          "\x23\x53\x60\x28\x74\x71\x35\x26"
-                          "\x31\x41\x59\x26\x53\x58\x97\x93",
-                .klen   = 24,
-                .iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-                          "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-                .ctext  = "\xaf\xa1\x81\xa6\x32\xbb\x15\x8e"
-                          "\xf8\x95\x2e\xd3\xe6\xee\x7e\x09"
-                          "\x0c\x1a\xf5\x02\x97\x8b\xe3\xb3"
-                          "\x11\xc7\x39\x96\xd0\x95\xf4\x56"
-                          "\xf4\xdd\x03\x38\x01\x44\x2c\xcf"
-                          "\x88\xae\x8e\x3c\xcd\xe7\xaa\x66"
-                          "\xfe\x3d\xc6\xfb\x01\x23\x51\x43"
-                          "\xd5\xd2\x13\x86\x94\x34\xe9\x62"
-                          "\xf9\x89\xe3\xd1\x7b\xbe\xf8\xef"
-                          "\x76\x35\x04\x3f\xdb\x23\x9d\x0b"
-                          "\x85\x42\xb9\x02\xd6\xcc\xdb\x96"
-                          "\xa7\x6b\x27\xb6\xd4\x45\x8f\x7d"
-                          "\xae\xd2\x04\xd5\xda\xc1\x7e\x24"
-                          "\x8c\x73\xbe\x48\x7e\xcf\x65\x28"
-                          "\x29\xe5\xbe\x54\x30\xcb\x46\x95"
-                          "\x4f\x2e\x8a\x36\xc8\x27\xc5\xbe"
-                          "\xd0\x1a\xaf\xab\x26\xcd\x9e\x69"
-                          "\xa1\x09\x95\x71\x26\xe9\xc4\xdf"
-                          "\xe6\x31\xc3\x46\xda\xaf\x0b\x41"
-                          "\x1f\xab\xb1\x8e\xd6\xfc\x0b\xb3"
-                          "\x82\xc0\x37\x27\xfc\x91\xa7\x05"
-                          "\xfb\xc5\xdc\x2b\x74\x96\x48\x43"
-                          "\x5d\x9c\x19\x0f\x60\x63\x3a\x1f"
-                          "\x6f\xf0\x03\xbe\x4d\xfd\xc8\x4a"
-                          "\xc6\xa4\x81\x6d\xc3\x12\x2a\x5c"
-                          "\x07\xff\xf3\x72\x74\x48\xb5\x40"
-                          "\x50\xb5\xdd\x90\x43\x31\x18\x15"
-                          "\x7b\xf2\xa6\xdb\x83\xc8\x4b\x4a"
-                          "\x29\x93\x90\x8b\xda\x07\xf0\x35"
-                          "\x6d\x90\x88\x09\x4e\x83\xf5\x5b"
-                          "\x94\x12\xbb\x33\x27\x1d\x3f\x23"
-                          "\x51\xa8\x7c\x07\xa2\xae\x77\xa6"
-                          "\x50\xfd\xcc\xc0\x4f\x80\x7a\x9f"
-                          "\x66\xdd\xcd\x75\x24\x8b\x33\xf7"
-                          "\x20\xdb\x83\x9b\x4f\x11\x63\x6e"
-                          "\xcf\x37\xef\xc9\x11\x01\x5c\x45"
-                          "\x32\x99\x7c\x3c\x9e\x42\x89\xe3"
-                          "\x70\x6d\x15\x9f\xb1\xe6\xb6\x05"
-                          "\xfe\x0c\xb9\x49\x2d\x90\x6d\xcc"
-                          "\x5d\x3f\xc1\xfe\x89\x0a\x2e\x2d"
-                          "\xa0\xa8\x89\x3b\x73\x39\xa5\x94"
-                          "\x4c\xa4\xa6\xbb\xa7\x14\x46\x89"
-                          "\x10\xff\xaf\xef\xca\xdd\x4f\x80"
-                          "\xb3\xdf\x3b\xab\xd4\xe5\x5a\xc7"
-                          "\x33\xca\x00\x8b\x8b\x3f\xea\xec"
-                          "\x68\x8a\xc2\x6d\xfd\xd4\x67\x0f"
-                          "\x22\x31\xe1\x0e\xfe\x5a\x04\xd5"
-                          "\x64\xa3\xf1\x1a\x76\x28\xcc\x35"
-                          "\x36\xa7\x0a\x74\xf7\x1c\x44\x9b"
-                          "\xc7\x1b\x53\x17\x02\xea\xd1\xad"
-                          "\x13\x51\x73\xc0\xa0\xb2\x05\x32"
-                          "\xa8\xa2\x37\x2e\xe1\x7a\x3a\x19"
-                          "\x26\xb4\x6c\x62\x5d\xb3\x1a\x1d"
-                          "\x59\xda\xee\x1a\x22\x18\xda\x0d"
-                          "\x88\x0f\x55\x8b\x72\x62\xfd\xc1"
-                          "\x69\x13\xcd\x0d\x5f\xc1\x09\x52"
-                          "\xee\xd6\xe3\x84\x4d\xee\xf6\x88"
-                          "\xaf\x83\xdc\x76\xf4\xc0\x93\x3f"
-                          "\x4a\x75\x2f\xb0\x0b\x3e\xc4\x54"
-                          "\x7d\x69\x8d\x00\x62\x77\x0d\x14"
-                          "\xbe\x7c\xa6\x7d\xc5\x24\x4f\xf3"
-                          "\x50\xf7\x5f\xf4\xc2\xca\x41\x97"
-                          "\x37\xbe\x75\x74\xcd\xf0\x75\x6e"
-                          "\x25\x23\x94\xbd\xda\x8d\xb0\xd4",
-                .len    = 512,
-        }, {
-                .key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
-                          "\x23\x53\x60\x28\x74\x71\x35\x26"
-                          "\x62\x49\x77\x57\x24\x70\x93\x69"
-                          "\x99\x59\x57\x49\x66\x96\x76\x27",
-                .klen   = 32,
-                .iv     = "\xff\x00\x00\x00\x00\x00\x00\x00"
-                          "\x00\x00\x00\x00\x00\x00\x00\x00",
-                .ptext  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-                          "\x00\x01\x02\x03\x04\x05\x06\x07"
-                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-                          "\x10\x11\x12\x13\x14\x15\x16\x17"
-                          "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                          "\x20\x21\x22\x23\x24\x25\x26\x27"
-                          "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-                          "\x30\x31\x32\x33\x34\x35\x36\x37"
-                          "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-                          "\x40\x41\x42\x43\x44\x45\x46\x47"
-                          "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-                          "\x50\x51\x52\x53\x54\x55\x56\x57"
-                          "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-                          "\x60\x61\x62\x63\x64\x65\x66\x67"
-                          "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-                          "\x70\x71\x72\x73\x74\x75\x76\x77"
-                          "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-                          "\x80\x81\x82\x83\x84\x85\x86\x87"
-                          "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-                          "\x90\x91\x92\x93\x94\x95\x96\x97"
-                          "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-                          "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-                          "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-                          "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-                          "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-                          "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-                          "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-                          "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-                          "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-                          "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-                          "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-                          "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-                          "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-                .ctext  = "\x55\xed\x71\xd3\x02\x8e\x15\x3b"
-                          "\xc6\x71\x29\x2d\x3e\x89\x9f\x59"
-                          "\x68\x6a\xcc\x8a\x56\x97\xf3\x95"
-                          "\x4e\x51\x08\xda\x2a\xf8\x6f\x3c"
-                          "\x78\x16\xea\x80\xdb\x33\x75\x94"
-                          "\xf9\x29\xc4\x2b\x76\x75\x97\xc7"
-                          "\xf2\x98\x2c\xf9\xff\xc8\xd5\x2b"
-                          "\x18\xf1\xaf\xcf\x7c\xc5\x0b\xee"
-                          "\xad\x3c\x76\x7c\xe6\x27\xa2\x2a"
-                          "\xe4\x66\xe1\xab\xa2\x39\xfc\x7c"
-                          "\xf5\xec\x32\x74\xa3\xb8\x03\x88"
-                          "\x52\xfc\x2e\x56\x3f\xa1\xf0\x9f"
-                          "\x84\x5e\x46\xed\x20\x89\xb6\x44"
-                          "\x8d\xd0\xed\x54\x47\x16\xbe\x95"
-                          "\x8a\xb3\x6b\x72\xc4\x32\x52\x13"
-                          "\x1b\xb0\x82\xbe\xac\xf9\x70\xa6"
-                          "\x44\x18\xdd\x8c\x6e\xca\x6e\x45"
-                          "\x8f\x1e\x10\x07\x57\x25\x98\x7b"
-                          "\x17\x8c\x78\xdd\x80\xa7\xd9\xd8"
-                          "\x63\xaf\xb9\x67\x57\xfd\xbc\xdb"
-                          "\x44\xe9\xc5\x65\xd1\xc7\x3b\xff"
-                          "\x20\xa0\x80\x1a\xc3\x9a\xad\x5e"
-                          "\x5d\x3b\xd3\x07\xd9\xf5\xfd\x3d"
-                          "\x4a\x8b\xa8\xd2\x6e\x7a\x51\x65"
-                          "\x6c\x8e\x95\xe0\x45\xc9\x5f\x4a"
-                          "\x09\x3c\x3d\x71\x7f\x0c\x84\x2a"
-                          "\xc8\x48\x52\x1a\xc2\xd5\xd6\x78"
-                          "\x92\x1e\xa0\x90\x2e\xea\xf0\xf3"
-                          "\xdc\x0f\xb1\xaf\x0d\x9b\x06\x2e"
-                          "\x35\x10\x30\x82\x0d\xe7\xc5\x9b"
-                          "\xde\x44\x18\xbd\x9f\xd1\x45\xa9"
-                          "\x7b\x7a\x4a\xad\x35\x65\x27\xca"
-                          "\xb2\xc3\xd4\x9b\x71\x86\x70\xee"
-                          "\xf1\x89\x3b\x85\x4b\x5b\xaa\xaf"
-                          "\xfc\x42\xc8\x31\x59\xbe\x16\x60"
-                          "\x4f\xf9\xfa\x12\xea\xd0\xa7\x14"
-                          "\xf0\x7a\xf3\xd5\x8d\xbd\x81\xef"
-                          "\x52\x7f\x29\x51\x94\x20\x67\x3c"
-                          "\xd1\xaf\x77\x9f\x22\x5a\x4e\x63"
-                          "\xe7\xff\x73\x25\xd1\xdd\x96\x8a"
-                          "\x98\x52\x6d\xf3\xac\x3e\xf2\x18"
-                          "\x6d\xf6\x0a\x29\xa6\x34\x3d\xed"
-                          "\xe3\x27\x0d\x9d\x0a\x02\x44\x7e"
-                          "\x5a\x7e\x67\x0f\x0a\x9e\xd6\xad"
-                          "\x91\xe6\x4d\x81\x8c\x5c\x59\xaa"
-                          "\xfb\xeb\x56\x53\xd2\x7d\x4c\x81"
-                          "\x65\x53\x0f\x41\x11\xbd\x98\x99"
-                          "\xf9\xc6\xfa\x51\x2e\xa3\xdd\x8d"
-                          "\x84\x98\xf9\x34\xed\x33\x2a\x1f"
-                          "\x82\xed\xc1\x73\x98\xd3\x02\xdc"
-                          "\xe6\xc2\x33\x1d\xa2\xb4\xca\x76"
-                          "\x63\x51\x34\x9d\x96\x12\xae\xce"
-                          "\x83\xc9\x76\x5e\xa4\x1b\x53\x37"
-                          "\x17\xd5\xc0\x80\x1d\x62\xf8\x3d"
-                          "\x54\x27\x74\xbb\x10\x86\x57\x46"
-                          "\x68\xe1\xed\x14\xe7\x9d\xfc\x84"
-                          "\x47\xbc\xc2\xf8\x19\x4b\x99\xcf"
-                          "\x7a\xe9\xc4\xb8\x8c\x82\x72\x4d"
-                          "\x7b\x4f\x38\x55\x36\x71\x64\xc1"
-                          "\xfc\x5c\x75\x52\x33\x02\x18\xf8"
-                          "\x17\xe1\x2b\xc2\x43\x39\xbd\x76"
-                          "\x9b\x63\x76\x32\x2f\x19\x72\x10"
-                          "\x9f\x21\x0c\xf1\x66\x50\x7f\xa5"
-                          "\x0d\x1f\x46\xe0\xba\xd3\x2f\x3c",
-                .len    = 512,
-                .also_non_np = 1,
-                .np     = 3,
-                .tap    = { 512 - 20, 4, 16 },
        }
 };
@@ -13883,6 +13251,27 @@ static const struct cipher_testvec aes_lrw_tv_template[] = {
                .ctext  = "\x5b\x90\x8e\xc1\xab\xdd\x67\x5f"
                          "\x3d\x69\x8a\x95\x53\xc8\x9c\xe5",
                .len    = 16,
+        }, { /* Test counter wrap-around, modified from LRW-32-AES 1 */
+                .key    = "\x45\x62\xac\x25\xf8\x28\x17\x6d"
+                          "\x4c\x26\x84\x14\xb5\x68\x01\x85"
+                          "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
+                          "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
+                .klen   = 32,
+                .iv     = "\xff\xff\xff\xff\xff\xff\xff\xff"
+                          "\xff\xff\xff\xff\xff\xff\xff\xff",
+                .ptext  = "\x30\x31\x32\x33\x34\x35\x36\x37"
+                          "\x38\x39\x41\x42\x43\x44\x45\x46"
+                          "\x30\x31\x32\x33\x34\x35\x36\x37"
+                          "\x38\x39\x41\x42\x43\x44\x45\x46"
+                          "\x30\x31\x32\x33\x34\x35\x36\x37"
+                          "\x38\x39\x41\x42\x43\x44\x45\x46",
+                .ctext  = "\x47\x90\x50\xf6\xf4\x8d\x5c\x7f"
+                          "\x84\xc7\x83\x95\x2d\xa2\x02\xc0"
+                          "\xda\x7f\xa3\xc0\x88\x2a\x0a\x50"
+                          "\xfb\xc1\x78\x03\x39\xfe\x1d\xe5"
+                          "\xf1\xb2\x73\xcd\x65\xa3\xdf\x5f"
+                          "\xe9\x5d\x48\x92\x54\x63\x4e\xb8",
+                .len    = 48,
        }, {
 /* http://www.mail-archive.com/stds-p1619@listserv.ieee.org/msg00173.html */
                .key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
diff --git a/crypto/xcbc.c b/crypto/xcbc.c
index 25c75af50d3f..c055f57fab11 100644
--- a/crypto/xcbc.c
+++ b/crypto/xcbc.c
@@ -57,15 +57,17 @@ struct xcbc_desc_ctx {
        u8 ctx[];
 };
+#define XCBC_BLOCKSIZE  16
 static int crypto_xcbc_digest_setkey(struct crypto_shash *parent,
                                     const u8 *inkey, unsigned int keylen)
 {
        unsigned long alignmask = crypto_shash_alignmask(parent);
        struct xcbc_tfm_ctx *ctx = crypto_shash_ctx(parent);
-        int bs = crypto_shash_blocksize(parent);
        u8 *consts = PTR_ALIGN(&ctx->ctx[0], alignmask + 1);
        int err = 0;
-        u8 key1[bs];
+        u8 key1[XCBC_BLOCKSIZE];
+        int bs = sizeof(key1);
        if ((err = crypto_cipher_setkey(ctx->child, inkey, keylen)))
                return err;
@@ -212,7 +214,7 @@ static int xcbc_create(struct crypto_template *tmpl, struct rtattr **tb)
                return PTR_ERR(alg);
        switch(alg->cra_blocksize) {
-        case 16:
+        case XCBC_BLOCKSIZE:
                break;
        default:
                goto out_put_alg;
diff --git a/crypto/xts.c b/crypto/xts.c
index ccf55fbb8bc2..847f54f76789 100644
--- a/crypto/xts.c
+++ b/crypto/xts.c
@@ -26,8 +26,6 @@
 #include <crypto/b128ops.h>
 #include <crypto/gf128mul.h>
-#define XTS_BUFFER_SIZE 128u
 struct priv {
        struct crypto_skcipher *child;
        struct crypto_cipher *tweak;
@@ -39,19 +37,7 @@ struct xts_instance_ctx {
 };
 struct rctx {
-        le128 buf[XTS_BUFFER_SIZE / sizeof(le128)];
        le128 t;
-        le128 *ext;
-        struct scatterlist srcbuf[2];
-        struct scatterlist dstbuf[2];
-        struct scatterlist *src;
-        struct scatterlist *dst;
-        unsigned int left;
        struct skcipher_request subreq;
 };
@@ -96,81 +82,27 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key,
        return err;
 }
-static int post_crypt(struct skcipher_request *req)
+/*
+ * We compute the tweak masks twice (both before and after the ECB encryption or
+ * decryption) to avoid having to allocate a temporary buffer and/or make
+ * mutliple calls to the 'ecb(..)' instance, which usually would be slower than
+ * just doing the gf128mul_x_ble() calls again.
+ */
+static int xor_tweak(struct skcipher_request *req, bool second_pass)
 {
        struct rctx *rctx = skcipher_request_ctx(req);
-        le128 *buf = rctx->ext ?: rctx->buf;
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        struct skcipher_request *subreq;
        const int bs = XTS_BLOCK_SIZE;
        struct skcipher_walk w;
-        struct scatterlist *sg;
+        le128 t = rctx->t;
-        unsigned offset;
        int err;
-        subreq = &rctx->subreq;
+        if (second_pass) {
-        err = skcipher_walk_virt(&w, subreq, false);
+                req = &rctx->subreq;
+                /* set to our TFM to enforce correct alignment: */
-        while (w.nbytes) {
+                skcipher_request_set_tfm(req, tfm);
-                unsigned int avail = w.nbytes;
-                le128 *wdst;
-                wdst = w.dst.virt.addr;
-                do {
-                        le128_xor(wdst, buf++, wdst);
-                        wdst++;
-                } while ((avail -= bs) >= bs);
-                err = skcipher_walk_done(&w, avail);
        }
+        err = skcipher_walk_virt(&w, req, false);
-        rctx->left -= subreq->cryptlen;
-        if (err || !rctx->left)
-                goto out;
-        rctx->dst = rctx->dstbuf;
-        scatterwalk_done(&w.out, 0, 1);
-        sg = w.out.sg;
-        offset = w.out.offset;
-        if (rctx->dst != sg) {
-                rctx->dst[0] = *sg;
-                sg_unmark_end(rctx->dst);
-                scatterwalk_crypto_chain(rctx->dst, sg_next(sg), 2);
-        }
-        rctx->dst[0].length -= offset - sg->offset;
-        rctx->dst[0].offset = offset;
-out:
-        return err;
-}
-static int pre_crypt(struct skcipher_request *req)
-{
-        struct rctx *rctx = skcipher_request_ctx(req);
-        le128 *buf = rctx->ext ?: rctx->buf;
-        struct skcipher_request *subreq;
-        const int bs = XTS_BLOCK_SIZE;
-        struct skcipher_walk w;
-        struct scatterlist *sg;
-        unsigned cryptlen;
-        unsigned offset;
-        bool more;
-        int err;
-        subreq = &rctx->subreq;
-        cryptlen = subreq->cryptlen;
-        more = rctx->left > cryptlen;
-        if (!more)
-                cryptlen = rctx->left;
-        skcipher_request_set_crypt(subreq, rctx->src, rctx->dst,
-                                   cryptlen, NULL);
-        err = skcipher_walk_virt(&w, subreq, false);
        while (w.nbytes) {
                unsigned int avail = w.nbytes;
@@ -181,180 +113,71 @@ static int pre_crypt(struct skcipher_request *req)
                wdst = w.dst.virt.addr;
                do {
-                        *buf++ = rctx->t;
+                        le128_xor(wdst++, &t, wsrc++);
-                        le128_xor(wdst++, &rctx->t, wsrc++);
+                        gf128mul_x_ble(&t, &t);
-                        gf128mul_x_ble(&rctx->t, &rctx->t);
                } while ((avail -= bs) >= bs);
                err = skcipher_walk_done(&w, avail);
        }
-        skcipher_request_set_crypt(subreq, rctx->dst, rctx->dst,
-                                   cryptlen, NULL);
-        if (err || !more)
-                goto out;
-        rctx->src = rctx->srcbuf;
-        scatterwalk_done(&w.in, 0, 1);
-        sg = w.in.sg;
-        offset = w.in.offset;
-        if (rctx->src != sg) {
-                rctx->src[0] = *sg;
-                sg_unmark_end(rctx->src);
-                scatterwalk_crypto_chain(rctx->src, sg_next(sg), 2);
-        }
-        rctx->src[0].length -= offset - sg->offset;
-        rctx->src[0].offset = offset;
-out:
        return err;
 }
-static int init_crypt(struct skcipher_request *req, crypto_completion_t done)
+static int xor_tweak_pre(struct skcipher_request *req)
 {
-        struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+        return xor_tweak(req, false);
-        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
-        gfp_t gfp;
-        subreq = &rctx->subreq;
-        skcipher_request_set_tfm(subreq, ctx->child);
-        skcipher_request_set_callback(subreq, req->base.flags, done, req);
-        gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
-                                                           GFP_ATOMIC;
-        rctx->ext = NULL;
-        subreq->cryptlen = XTS_BUFFER_SIZE;
-        if (req->cryptlen > XTS_BUFFER_SIZE) {
-                unsigned int n = min(req->cryptlen, (unsigned int)PAGE_SIZE);
-                rctx->ext = kmalloc(n, gfp);
-                if (rctx->ext)
-                        subreq->cryptlen = n;
-        }
-        rctx->src = req->src;
-        rctx->dst = req->dst;
-        rctx->left = req->cryptlen;
-        /* calculate first value of T */
-        crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);
-        return 0;
 }
-static void exit_crypt(struct skcipher_request *req)
+static int xor_tweak_post(struct skcipher_request *req)
 {
-        struct rctx *rctx = skcipher_request_ctx(req);
+        return xor_tweak(req, true);
-        rctx->left = 0;
-        if (rctx->ext)
-                kzfree(rctx->ext);
 }
-static int do_encrypt(struct skcipher_request *req, int err)
+static void crypt_done(struct crypto_async_request *areq, int err)
-{
-        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
-        subreq = &rctx->subreq;
-        while (!err && rctx->left) {
-                err = pre_crypt(req) ?:
-                      crypto_skcipher_encrypt(subreq) ?:
-                      post_crypt(req);
-                if (err == -EINPROGRESS || err == -EBUSY)
-                        return err;
-        }
-        exit_crypt(req);
-        return err;
-}
-static void encrypt_done(struct crypto_async_request *areq, int err)
 {
        struct skcipher_request *req = areq->data;
-        struct skcipher_request *subreq;
-        struct rctx *rctx;
-        rctx = skcipher_request_ctx(req);
-        if (err == -EINPROGRESS) {
-                if (rctx->left != req->cryptlen)
-                        return;
-                goto out;
-        }
-        subreq = &rctx->subreq;
-        subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
-        err = do_encrypt(req, err ?: post_crypt(req));
+        if (!err)
-        if (rctx->left)
+                err = xor_tweak_post(req);
-                return;
-out:
        skcipher_request_complete(req, err);
 }
-static int encrypt(struct skcipher_request *req)
+static void init_crypt(struct skcipher_request *req)
-{
-        return do_encrypt(req, init_crypt(req, encrypt_done));
-}
-static int do_decrypt(struct skcipher_request *req, int err)
 {
+        struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
+        struct skcipher_request *subreq = &rctx->subreq;
-        subreq = &rctx->subreq;
-        while (!err && rctx->left) {
+        skcipher_request_set_tfm(subreq, ctx->child);
-                err = pre_crypt(req) ?:
+        skcipher_request_set_callback(subreq, req->base.flags, crypt_done, req);
-                      crypto_skcipher_decrypt(subreq) ?:
+        skcipher_request_set_crypt(subreq, req->dst, req->dst,
-                      post_crypt(req);
+                                   req->cryptlen, NULL);
-                if (err == -EINPROGRESS || err == -EBUSY)
-                        return err;
-        }
-        exit_crypt(req);
+        /* calculate first value of T */
-        return err;
+        crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);
 }
-static void decrypt_done(struct crypto_async_request *areq, int err)
+static int encrypt(struct skcipher_request *req)
 {
-        struct skcipher_request *req = areq->data;
+        struct rctx *rctx = skcipher_request_ctx(req);
-        struct skcipher_request *subreq;
+        struct skcipher_request *subreq = &rctx->subreq;
-        struct rctx *rctx;
-        rctx = skcipher_request_ctx(req);
-        if (err == -EINPROGRESS) {
-                if (rctx->left != req->cryptlen)
-                        return;
-                goto out;
-        }
-        subreq = &rctx->subreq;
-        subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
-        err = do_decrypt(req, err ?: post_crypt(req));
-        if (rctx->left)
-                return;
-out:
+        init_crypt(req);
-        skcipher_request_complete(req, err);
+        return xor_tweak_pre(req) ?:
+                crypto_skcipher_encrypt(subreq) ?:
+                xor_tweak_post(req);
 }
 static int decrypt(struct skcipher_request *req)
 {
-        return do_decrypt(req, init_crypt(req, decrypt_done));
+        struct rctx *rctx = skcipher_request_ctx(req);
+        struct skcipher_request *subreq = &rctx->subreq;
+        init_crypt(req);
+        return xor_tweak_pre(req) ?:
+                crypto_skcipher_decrypt(subreq) ?:
+                xor_tweak_post(req);
 }
 static int init_tfm(struct crypto_skcipher *tfm)
diff --git a/drivers/block/cryptoloop.c b/drivers/block/cryptoloop.c
index 7033a4beda66..254ee7d54e91 100644
--- a/drivers/block/cryptoloop.c
+++ b/drivers/block/cryptoloop.c
@@ -45,7 +45,7 @@ cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info)
        char cms[LO_NAME_SIZE];                 /* cipher-mode string */
        char *mode;
        char *cmsp = cms;                       /* c-m string pointer */
-        struct crypto_skcipher *tfm;
+        struct crypto_sync_skcipher *tfm;
        /* encryption breaks for non sector aligned offsets */
@@ -80,13 +80,13 @@ cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info)
        *cmsp++ = ')';
        *cmsp = 0;
-        tfm = crypto_alloc_skcipher(cms, 0, CRYPTO_ALG_ASYNC);
+        tfm = crypto_alloc_sync_skcipher(cms, 0, 0);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);
-        err = crypto_skcipher_setkey(tfm, info->lo_encrypt_key,
+        err = crypto_sync_skcipher_setkey(tfm, info->lo_encrypt_key,
-                                     info->lo_encrypt_key_size);
+                                          info->lo_encrypt_key_size);
-        
        if (err != 0)
                goto out_free_tfm;
@@ -94,7 +94,7 @@ cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info)
        return 0;
 out_free_tfm:
-        crypto_free_skcipher(tfm);
+        crypto_free_sync_skcipher(tfm);
 out:
        return err;
@@ -109,8 +109,8 @@ cryptoloop_transfer(struct loop_device *lo, int cmd,
                    struct page *loop_page, unsigned loop_off,
                    int size, sector_t IV)
 {
-        struct crypto_skcipher *tfm = lo->key_data;
+        struct crypto_sync_skcipher *tfm = lo->key_data;
-        SKCIPHER_REQUEST_ON_STACK(req, tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
        struct scatterlist sg_out;
        struct scatterlist sg_in;
@@ -119,7 +119,7 @@ cryptoloop_transfer(struct loop_device *lo, int cmd,
        unsigned in_offs, out_offs;
        int err;
-        skcipher_request_set_tfm(req, tfm);
+        skcipher_request_set_sync_tfm(req, tfm);
        skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP,
                                      NULL, NULL);
@@ -175,9 +175,9 @@ cryptoloop_ioctl(struct loop_device *lo, int cmd, unsigned long arg)
 static int
 cryptoloop_release(struct loop_device *lo)
 {
-        struct crypto_skcipher *tfm = lo->key_data;
+        struct crypto_sync_skcipher *tfm = lo->key_data;
        if (tfm != NULL) {
-                crypto_free_skcipher(tfm);
+                crypto_free_sync_skcipher(tfm);
                lo->key_data = NULL;
                return 0;
        }
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index 5d8266c6571f..4552b06fe601 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -188,6 +188,10 @@ struct device_type fsl_mc_bus_dprtc_type = {
        .name = "fsl_mc_bus_dprtc"
 };
+struct device_type fsl_mc_bus_dpseci_type = {
+        .name = "fsl_mc_bus_dpseci"
+};
 static struct device_type *fsl_mc_get_device_type(const char *type)
 {
        static const struct {
@@ -203,6 +207,7 @@ static struct device_type *fsl_mc_get_device_type(const char *type)
                { &fsl_mc_bus_dpmcp_type, "dpmcp" },
                { &fsl_mc_bus_dpmac_type, "dpmac" },
                { &fsl_mc_bus_dprtc_type, "dprtc" },
+                { &fsl_mc_bus_dpseci_type, "dpseci" },
                { NULL, NULL }
        };
        int i;
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index aaf9e5afaad4..95be7228f327 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -44,10 +44,10 @@ static unsigned short default_quality; /* = 0; default to "off" */
 module_param(current_quality, ushort, 0644);
 MODULE_PARM_DESC(current_quality,
-                 "current hwrng entropy estimation per mill");
+                 "current hwrng entropy estimation per 1024 bits of input");
 module_param(default_quality, ushort, 0644);
 MODULE_PARM_DESC(default_quality,
-                 "default entropy content of hwrng per mill");
+                 "default entropy content of hwrng per 1024 bits of input");
 static void drop_current_rng(void);
 static int hwrng_init(struct hwrng *rng);
diff --git a/drivers/char/random.c b/drivers/char/random.c
index c75b6cdf0053..2eb70e76ed35 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -433,9 +433,9 @@ static int crng_init_cnt = 0;
 static unsigned long crng_global_init_time = 0;
 #define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE)
 static void _extract_crng(struct crng_state *crng,
-                          __u32 out[CHACHA20_BLOCK_WORDS]);
+                          __u8 out[CHACHA20_BLOCK_SIZE]);
 static void _crng_backtrack_protect(struct crng_state *crng,
-                                    __u32 tmp[CHACHA20_BLOCK_WORDS], int used);
+                                    __u8 tmp[CHACHA20_BLOCK_SIZE], int used);
 static void process_random_ready_list(void);
 static void _get_random_bytes(void *buf, int nbytes);
@@ -926,7 +926,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
        unsigned long   flags;
        int             i, num;
        union {
-                __u32   block[CHACHA20_BLOCK_WORDS];
+                __u8    block[CHACHA20_BLOCK_SIZE];
                __u32   key[8];
        } buf;
@@ -973,7 +973,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
 }
 static void _extract_crng(struct crng_state *crng,
-                          __u32 out[CHACHA20_BLOCK_WORDS])
+                          __u8 out[CHACHA20_BLOCK_SIZE])
 {
        unsigned long v, flags;
@@ -990,7 +990,7 @@ static void _extract_crng(struct crng_state *crng,
        spin_unlock_irqrestore(&crng->lock, flags);
 }
-static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS])
+static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE])
 {
        struct crng_state *crng = NULL;
@@ -1008,7 +1008,7 @@ static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS])
 * enough) to mutate the CRNG key to provide backtracking protection.
 */
 static void _crng_backtrack_protect(struct crng_state *crng,
-                                    __u32 tmp[CHACHA20_BLOCK_WORDS], int used)
+                                    __u8 tmp[CHACHA20_BLOCK_SIZE], int used)
 {
        unsigned long   flags;
        __u32           *s, *d;
@@ -1020,14 +1020,14 @@ static void _crng_backtrack_protect(struct crng_state *crng,
                used = 0;
        }
        spin_lock_irqsave(&crng->lock, flags);
-        s = &tmp[used / sizeof(__u32)];
+        s = (__u32 *) &tmp[used];
        d = &crng->state[4];
        for (i=0; i < 8; i++)
                *d++ ^= *s++;
        spin_unlock_irqrestore(&crng->lock, flags);
 }
-static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used)
+static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used)
 {
        struct crng_state *crng = NULL;
@@ -1043,7 +1043,7 @@ static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used)
 static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
 {
        ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE;
-        __u32 tmp[CHACHA20_BLOCK_WORDS];
+        __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4);
        int large_request = (nbytes > 256);
        while (nbytes) {
@@ -1622,7 +1622,7 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller,
 */
 static void _get_random_bytes(void *buf, int nbytes)
 {
-        __u32 tmp[CHACHA20_BLOCK_WORDS];
+        __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4);
        trace_get_random_bytes(nbytes, _RET_IP_);
@@ -2248,7 +2248,7 @@ u64 get_random_u64(void)
        if (use_lock)
                read_lock_irqsave(&batched_entropy_reset_lock, flags);
        if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
-                extract_crng((__u32 *)batch->entropy_u64);
+                extract_crng((u8 *)batch->entropy_u64);
                batch->position = 0;
        }
        ret = batch->entropy_u64[batch->position++];
@@ -2278,7 +2278,7 @@ u32 get_random_u32(void)
        if (use_lock)
                read_lock_irqsave(&batched_entropy_reset_lock, flags);
        if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
-                extract_crng(batch->entropy_u32);
+                extract_crng((u8 *)batch->entropy_u32);
                batch->position = 0;
        }
        ret = batch->entropy_u32[batch->position++];
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index c23396f32c8a..8e7e225d2446 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -10,7 +10,7 @@ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
 obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/
 obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/
 obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/
 obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
 obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
 obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 801aeab5ab1e..2b7af44c7b85 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
 * Cryptographic API.
 *
@@ -6,10 +7,6 @@
 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
 * Author: Nicolas Royer <nicolas@eukrea.com>
 *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
 * Some ideas are from omap-aes.c driver.
 */
diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h
index 2a60d1224143..cbd37a2edada 100644
--- a/drivers/crypto/atmel-authenc.h
+++ b/drivers/crypto/atmel-authenc.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
 * API for Atmel Secure Protocol Layers Improved Performances (SPLIP)
 *
@@ -5,18 +6,6 @@
 *
 * Author: Cyrille Pitchen <cyrille.pitchen@atmel.com>
 *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- *
 * This driver is based on drivers/mtd/spi-nor/fsl-quadspi.c from Freescale.
 */
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index 74f083f45e97..ba00e4563ca0 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -1,18 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
 * Microchip / Atmel ECC (I2C) driver.
 *
 * Copyright (c) 2017, Microchip Technology Inc.
 * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
 */
 #include <linux/bitrev.h>
diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h
index 25232c8abcc2..643a3b947338 100644
--- a/drivers/crypto/atmel-ecc.h
+++ b/drivers/crypto/atmel-ecc.h
@@ -1,19 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
 * Copyright (c) 2017, Microchip Technology Inc.
 * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- *
 */
 #ifndef __ATMEL_ECC_H__
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 8a19df2fba6a..ab0cfe748931 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
 * Cryptographic API.
 *
@@ -6,10 +7,6 @@
 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
 * Author: Nicolas Royer <nicolas@eukrea.com>
 *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
 * Some ideas are from omap-sham.c drivers.
 */
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index 97b0423efa7f..438e1ffb2ec0 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
 * Cryptographic API.
 *
@@ -6,10 +7,6 @@
 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
 * Author: Nicolas Royer <nicolas@eukrea.com>
 *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
 * Some ideas are from omap-aes.c drivers.
 */
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c
index 7f07a5085e9b..f3442c2bdbdc 100644
--- a/drivers/crypto/axis/artpec6_crypto.c
+++ b/drivers/crypto/axis/artpec6_crypto.c
@@ -330,7 +330,7 @@ struct artpec6_cryptotfm_context {
        size_t key_length;
        u32 key_md;
        int crypto_type;
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
 };
 struct artpec6_crypto_aead_hw_ctx {
@@ -1199,15 +1199,15 @@ artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt)
                pr_debug("counter %x will overflow (nblks %u), falling back\n",
                         counter, counter + nblks);
-                ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key,
+                ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key,
-                                             ctx->key_length);
+                                                  ctx->key_length);
                if (ret)
                        return ret;
                {
-                        SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                        skcipher_request_set_tfm(subreq, ctx->fallback);
+                        skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                        skcipher_request_set_callback(subreq, req->base.flags,
                                                      NULL, NULL);
                        skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -1561,10 +1561,9 @@ static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm)
 {
        struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
-        ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base),
+        ctx->fallback =
-                                              0,
+                crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base),
-                                              CRYPTO_ALG_ASYNC |
+                                           0, CRYPTO_ALG_NEED_FALLBACK);
-                                              CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(ctx->fallback))
                return PTR_ERR(ctx->fallback);
@@ -1605,7 +1604,7 @@ static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm)
 {
        struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
-        crypto_free_skcipher(ctx->fallback);
+        crypto_free_sync_skcipher(ctx->fallback);
        artpec6_crypto_aes_exit(tfm);
 }
@@ -3174,7 +3173,6 @@ static struct platform_driver artpec6_crypto_driver = {
        .remove  = artpec6_crypto_remove,
        .driver  = {
                .name  = "artpec6-crypto",
-                .owner = THIS_MODULE,
                .of_match_table = artpec6_crypto_of_match,
        },
 };
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index 1eb852765469..c4b1cade55c1 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -1,7 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+config CRYPTO_DEV_FSL_CAAM_COMMON
+        tristate
 config CRYPTO_DEV_FSL_CAAM
-        tristate "Freescale CAAM-Multicore driver backend"
+        tristate "Freescale CAAM-Multicore platform driver backend"
        depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
        select SOC_BUS
+        select CRYPTO_DEV_FSL_CAAM_COMMON
        help
          Enables the driver module for Freescale's Cryptographic Accelerator
          and Assurance Module (CAAM), also known as the SEC version 4 (SEC4).
@@ -12,9 +17,16 @@ config CRYPTO_DEV_FSL_CAAM
          To compile this driver as a module, choose M here: the module
          will be called caam.
+if CRYPTO_DEV_FSL_CAAM
+config CRYPTO_DEV_FSL_CAAM_DEBUG
+        bool "Enable debug output in CAAM driver"
+        help
+          Selecting this will enable printing of various debug
+          information in the CAAM driver.
 config CRYPTO_DEV_FSL_CAAM_JR
        tristate "Freescale CAAM Job Ring driver backend"
-        depends on CRYPTO_DEV_FSL_CAAM
        default y
        help
          Enables the driver module for Job Rings which are part of
@@ -25,9 +37,10 @@ config CRYPTO_DEV_FSL_CAAM_JR
          To compile this driver as a module, choose M here: the module
          will be called caam_jr.
+if CRYPTO_DEV_FSL_CAAM_JR
 config CRYPTO_DEV_FSL_CAAM_RINGSIZE
        int "Job Ring size"
-        depends on CRYPTO_DEV_FSL_CAAM_JR
        range 2 9
        default "9"
        help
@@ -45,7 +58,6 @@ config CRYPTO_DEV_FSL_CAAM_RINGSIZE
 config CRYPTO_DEV_FSL_CAAM_INTC
        bool "Job Ring interrupt coalescing"
-        depends on CRYPTO_DEV_FSL_CAAM_JR
        help
          Enable the Job Ring's interrupt coalescing feature.
@@ -75,7 +87,6 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
        tristate "Register algorithm implementations with the Crypto API"
-        depends on CRYPTO_DEV_FSL_CAAM_JR
        default y
        select CRYPTO_AEAD
        select CRYPTO_AUTHENC
@@ -90,7 +101,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
        tristate "Queue Interface as Crypto API backend"
-        depends on CRYPTO_DEV_FSL_CAAM_JR && FSL_DPAA && NET
+        depends on FSL_DPAA && NET
        default y
        select CRYPTO_AUTHENC
        select CRYPTO_BLKCIPHER
@@ -107,7 +118,6 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
 config CRYPTO_DEV_FSL_CAAM_AHASH_API
        tristate "Register hash algorithm implementations with Crypto API"
-        depends on CRYPTO_DEV_FSL_CAAM_JR
        default y
        select CRYPTO_HASH
        help
@@ -119,7 +129,6 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API
 config CRYPTO_DEV_FSL_CAAM_PKC_API
        tristate "Register public key cryptography implementations with Crypto API"
-        depends on CRYPTO_DEV_FSL_CAAM_JR
        default y
        select CRYPTO_RSA
        help
@@ -131,7 +140,6 @@ config CRYPTO_DEV_FSL_CAAM_PKC_API
 config CRYPTO_DEV_FSL_CAAM_RNG_API
        tristate "Register caam device for hwrng API"
-        depends on CRYPTO_DEV_FSL_CAAM_JR
        default y
        select CRYPTO_RNG
        select HW_RANDOM
@@ -142,13 +150,32 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API
          To compile this as a module, choose M here: the module
          will be called caamrng.
-config CRYPTO_DEV_FSL_CAAM_DEBUG
+endif # CRYPTO_DEV_FSL_CAAM_JR
-        bool "Enable debug output in CAAM driver"
-        depends on CRYPTO_DEV_FSL_CAAM
+endif # CRYPTO_DEV_FSL_CAAM
+config CRYPTO_DEV_FSL_DPAA2_CAAM
+        tristate "QorIQ DPAA2 CAAM (DPSECI) driver"
+        depends on FSL_MC_DPIO
+        depends on NETDEVICES
+        select CRYPTO_DEV_FSL_CAAM_COMMON
+        select CRYPTO_BLKCIPHER
+        select CRYPTO_AUTHENC
+        select CRYPTO_AEAD
+        select CRYPTO_HASH
        help
-          Selecting this will enable printing of various debug
+          CAAM driver for QorIQ Data Path Acceleration Architecture 2.
-          information in the CAAM driver.
+          It handles DPSECI DPAA2 objects that sit on the Management Complex
+          (MC) fsl-mc bus.
+          To compile this as a module, choose M here: the module
+          will be called dpaa2_caam.
 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
        def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \
-                      CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI)
+                      CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \
+                      CRYPTO_DEV_FSL_DPAA2_CAAM)
+config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
+        def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \
+                      CRYPTO_DEV_FSL_DPAA2_CAAM)
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
index cb652ee7dfc8..7bbfd06a11ff 100644
--- a/drivers/crypto/caam/Makefile
+++ b/drivers/crypto/caam/Makefile
@@ -6,19 +6,27 @@ ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y)
        ccflags-y := -DDEBUG
 endif
+ccflags-y += -DVERSION=\"\"
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
 caam-objs := ctrl.o
-caam_jr-objs := jr.o key_gen.o error.o
+caam_jr-objs := jr.o key_gen.o
 caam_pkc-y := caampkc.o pkc_desc.o
 ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),)
        ccflags-y += -DCONFIG_CAAM_QI
        caam-objs += qi.o
 endif
+obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
+dpaa2_caam-y    := caamalg_qi2.o dpseci.o
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index ec40f991e6c6..869f092432de 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -1,8 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
 * caam - Freescale FSL CAAM support for crypto API
 *
 * Copyright 2008-2011 Freescale Semiconductor, Inc.
- * Copyright 2016 NXP
+ * Copyright 2016-2018 NXP
 *
 * Based on talitos crypto API driver.
 *
@@ -81,8 +82,6 @@
 #define debug(format, arg...)
 #endif
-static struct list_head alg_list;
 struct caam_alg_entry {
        int class1_alg_type;
        int class2_alg_type;
@@ -96,17 +95,21 @@ struct caam_aead_alg {
        bool registered;
 };
+struct caam_skcipher_alg {
+        struct skcipher_alg skcipher;
+        struct caam_alg_entry caam;
+        bool registered;
+};
 /*
 * per-session context
 */
 struct caam_ctx {
        u32 sh_desc_enc[DESC_MAX_USED_LEN];
        u32 sh_desc_dec[DESC_MAX_USED_LEN];
-        u32 sh_desc_givenc[DESC_MAX_USED_LEN];
        u8 key[CAAM_MAX_KEY_SIZE];
        dma_addr_t sh_desc_enc_dma;
        dma_addr_t sh_desc_dec_dma;
-        dma_addr_t sh_desc_givenc_dma;
        dma_addr_t key_dma;
        enum dma_data_direction dir;
        struct device *jrdev;
@@ -648,20 +651,20 @@ static int rfc4543_setkey(struct crypto_aead *aead,
        return rfc4543_set_sh_desc(aead);
 }
-static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
+static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
-                             const u8 *key, unsigned int keylen)
+                           unsigned int keylen)
 {
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
-        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
+        struct caam_skcipher_alg *alg =
-        const char *alg_name = crypto_tfm_alg_name(tfm);
+                container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
+                             skcipher);
        struct device *jrdev = ctx->jrdev;
-        unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
        u32 *desc;
        u32 ctx1_iv_off = 0;
        const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
                               OP_ALG_AAI_CTR_MOD128);
-        const bool is_rfc3686 = (ctr_mode &&
+        const bool is_rfc3686 = alg->caam.rfc3686;
-                                 (strstr(alg_name, "rfc3686") != NULL));
 #ifdef DEBUG
        print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
@@ -689,40 +692,32 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
        ctx->cdata.key_virt = key;
        ctx->cdata.key_inline = true;
-        /* ablkcipher_encrypt shared descriptor */
+        /* skcipher_encrypt shared descriptor */
        desc = ctx->sh_desc_enc;
-        cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
+        cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
-                                     ctx1_iv_off);
+                                   ctx1_iv_off);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
                                   desc_bytes(desc), ctx->dir);
-        /* ablkcipher_decrypt shared descriptor */
+        /* skcipher_decrypt shared descriptor */
        desc = ctx->sh_desc_dec;
-        cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
+        cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
-                                     ctx1_iv_off);
+                                   ctx1_iv_off);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
                                   desc_bytes(desc), ctx->dir);
-        /* ablkcipher_givencrypt shared descriptor */
-        desc = ctx->sh_desc_givenc;
-        cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
-                                        ctx1_iv_off);
-        dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma,
-                                   desc_bytes(desc), ctx->dir);
        return 0;
 }
-static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
+static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
-                                 const u8 *key, unsigned int keylen)
+                               unsigned int keylen)
 {
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
        struct device *jrdev = ctx->jrdev;
        u32 *desc;
        if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
-                crypto_ablkcipher_set_flags(ablkcipher,
+                crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
-                                            CRYPTO_TFM_RES_BAD_KEY_LEN);
                dev_err(jrdev, "key size mismatch\n");
                return -EINVAL;
        }
@@ -731,15 +726,15 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
        ctx->cdata.key_virt = key;
        ctx->cdata.key_inline = true;
-        /* xts_ablkcipher_encrypt shared descriptor */
+        /* xts_skcipher_encrypt shared descriptor */
        desc = ctx->sh_desc_enc;
-        cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
+        cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
                                   desc_bytes(desc), ctx->dir);
-        /* xts_ablkcipher_decrypt shared descriptor */
+        /* xts_skcipher_decrypt shared descriptor */
        desc = ctx->sh_desc_dec;
-        cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
+        cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
                                   desc_bytes(desc), ctx->dir);
@@ -765,22 +760,20 @@ struct aead_edesc {
 };
 /*
- * ablkcipher_edesc - s/w-extended ablkcipher descriptor
+ * skcipher_edesc - s/w-extended skcipher descriptor
 * @src_nents: number of segments in input s/w scatterlist
 * @dst_nents: number of segments in output s/w scatterlist
 * @iv_dma: dma address of iv for checking continuity and link table
- * @iv_dir: DMA mapping direction for IV
 * @sec4_sg_bytes: length of dma mapped sec4_sg space
 * @sec4_sg_dma: bus physical mapped address of h/w link table
 * @sec4_sg: pointer to h/w link table
 * @hw_desc: the h/w job descriptor followed by any referenced link tables
 *           and IV
 */
-struct ablkcipher_edesc {
+struct skcipher_edesc {
        int src_nents;
        int dst_nents;
        dma_addr_t iv_dma;
-        enum dma_data_direction iv_dir;
        int sec4_sg_bytes;
        dma_addr_t sec4_sg_dma;
        struct sec4_sg_entry *sec4_sg;
@@ -790,8 +783,7 @@ struct ablkcipher_edesc {
 static void caam_unmap(struct device *dev, struct scatterlist *src,
                       struct scatterlist *dst, int src_nents,
                       int dst_nents,
-                       dma_addr_t iv_dma, int ivsize,
+                       dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
-                       enum dma_data_direction iv_dir, dma_addr_t sec4_sg_dma,
                       int sec4_sg_bytes)
 {
        if (dst != src) {
@@ -803,7 +795,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
        }
        if (iv_dma)
-                dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
+                dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
        if (sec4_sg_bytes)
                dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
                                 DMA_TO_DEVICE);
@@ -814,20 +806,19 @@ static void aead_unmap(struct device *dev,
                       struct aead_request *req)
 {
        caam_unmap(dev, req->src, req->dst,
-                   edesc->src_nents, edesc->dst_nents, 0, 0, DMA_NONE,
+                   edesc->src_nents, edesc->dst_nents, 0, 0,
                   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 }
-static void ablkcipher_unmap(struct device *dev,
+static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
-                             struct ablkcipher_edesc *edesc,
+                           struct skcipher_request *req)
-                             struct ablkcipher_request *req)
 {
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        caam_unmap(dev, req->src, req->dst,
                   edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->iv_dir,
+                   edesc->iv_dma, ivsize,
                   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 }
@@ -881,87 +872,74 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
        aead_request_complete(req, err);
 }
-static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
+static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
-                                   void *context)
+                                  void *context)
 {
-        struct ablkcipher_request *req = context;
+        struct skcipher_request *req = context;
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
 #ifdef DEBUG
        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
-        edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
+        edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
        if (err)
                caam_jr_strstatus(jrdev, err);
 #ifdef DEBUG
        print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
                       edesc->src_nents > 1 ? 100 : ivsize, 1);
 #endif
        caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-                     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
-        ablkcipher_unmap(jrdev, edesc, req);
+        skcipher_unmap(jrdev, edesc, req);
        /*
-         * The crypto API expects us to set the IV (req->info) to the last
+         * The crypto API expects us to set the IV (req->iv) to the last
         * ciphertext block. This is used e.g. by the CTS mode.
         */
-        scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize,
+        scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
                                 ivsize, 0);
-        /* In case initial IV was generated, copy it in GIVCIPHER request */
-        if (edesc->iv_dir == DMA_FROM_DEVICE) {
-                u8 *iv;
-                struct skcipher_givcrypt_request *greq;
-                greq = container_of(req, struct skcipher_givcrypt_request,
-                                    creq);
-                iv = (u8 *)edesc->hw_desc + desc_bytes(edesc->hw_desc) +
-                     edesc->sec4_sg_bytes;
-                memcpy(greq->giv, iv, ivsize);
-        }
        kfree(edesc);
-        ablkcipher_request_complete(req, err);
+        skcipher_request_complete(req, err);
 }
-static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
+static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
-                                    void *context)
+                                  void *context)
 {
-        struct ablkcipher_request *req = context;
+        struct skcipher_request *req = context;
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
 #ifdef DEBUG
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
-        edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
+        edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
        if (err)
                caam_jr_strstatus(jrdev, err);
 #ifdef DEBUG
        print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
-                       ivsize, 1);
 #endif
        caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-                     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
-        ablkcipher_unmap(jrdev, edesc, req);
+        skcipher_unmap(jrdev, edesc, req);
        kfree(edesc);
-        ablkcipher_request_complete(req, err);
+        skcipher_request_complete(req, err);
 }
 /*
@@ -1103,34 +1081,38 @@ static void init_authenc_job(struct aead_request *req,
 }
 /*
- * Fill in ablkcipher job descriptor
+ * Fill in skcipher job descriptor
 */
-static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
+static void init_skcipher_job(struct skcipher_request *req,
-                                struct ablkcipher_edesc *edesc,
+                              struct skcipher_edesc *edesc,
-                                struct ablkcipher_request *req)
+                              const bool encrypt)
 {
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        u32 *desc = edesc->hw_desc;
+        u32 *sh_desc;
        u32 out_options = 0;
-        dma_addr_t dst_dma;
+        dma_addr_t dst_dma, ptr;
        int len;
 #ifdef DEBUG
        print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
-                       ivsize, 1);
+        pr_err("asked=%d, cryptlen%d\n",
-        pr_err("asked=%d, nbytes%d\n",
+               (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
-               (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes);
 #endif
        caam_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__)": ",
                     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-                     edesc->src_nents > 1 ? 100 : req->nbytes, 1);
+                     edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
+        sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
+        ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
        len = desc_len(sh_desc);
        init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
-        append_seq_in_ptr(desc, edesc->sec4_sg_dma, req->nbytes + ivsize,
+        append_seq_in_ptr(desc, edesc->sec4_sg_dma, req->cryptlen + ivsize,
                          LDST_SGF);
        if (likely(req->src == req->dst)) {
@@ -1145,48 +1127,7 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
                        out_options = LDST_SGF;
                }
        }
-        append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
+        append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
-}
-/*
- * Fill in ablkcipher givencrypt job descriptor
- */
-static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
-                                    struct ablkcipher_edesc *edesc,
-                                    struct ablkcipher_request *req)
-{
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
-        u32 *desc = edesc->hw_desc;
-        u32 in_options;
-        dma_addr_t dst_dma, src_dma;
-        int len, sec4_sg_index = 0;
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
-                       ivsize, 1);
-#endif
-        caam_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
-                     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-                     edesc->src_nents > 1 ? 100 : req->nbytes, 1);
-        len = desc_len(sh_desc);
-        init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
-        if (edesc->src_nents == 1) {
-                src_dma = sg_dma_address(req->src);
-                in_options = 0;
-        } else {
-                src_dma = edesc->sec4_sg_dma;
-                sec4_sg_index += edesc->src_nents;
-                in_options = LDST_SGF;
-        }
-        append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
-        dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
-                  sizeof(struct sec4_sg_entry);
-        append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, LDST_SGF);
 }
 /*
@@ -1275,7 +1216,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                        GFP_DMA | flags);
        if (!edesc) {
                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, DMA_NONE, 0, 0);
+                           0, 0, 0);
                return ERR_PTR(-ENOMEM);
        }
@@ -1476,35 +1417,35 @@ static int aead_decrypt(struct aead_request *req)
 }
 /*
- * allocate and map the ablkcipher extended descriptor for ablkcipher
+ * allocate and map the skcipher extended descriptor for skcipher
 */
-static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
+static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
-                                                       *req, int desc_bytes)
+                                                   int desc_bytes)
 {
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
        struct device *jrdev = ctx->jrdev;
        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                       GFP_KERNEL : GFP_ATOMIC;
        int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
        dma_addr_t iv_dma;
        u8 *iv;
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
-        src_nents = sg_nents_for_len(req->src, req->nbytes);
+        src_nents = sg_nents_for_len(req->src, req->cryptlen);
        if (unlikely(src_nents < 0)) {
                dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
-                        req->nbytes);
+                        req->cryptlen);
                return ERR_PTR(src_nents);
        }
        if (req->dst != req->src) {
-                dst_nents = sg_nents_for_len(req->dst, req->nbytes);
+                dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
                if (unlikely(dst_nents < 0)) {
                        dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->nbytes);
+                                req->cryptlen);
                        return ERR_PTR(dst_nents);
                }
        }
@@ -1546,7 +1487,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        if (!edesc) {
                dev_err(jrdev, "could not allocate extended descriptor\n");
                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, DMA_NONE, 0, 0);
+                           0, 0, 0);
                return ERR_PTR(-ENOMEM);
        }
@@ -1555,17 +1496,16 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        edesc->sec4_sg_bytes = sec4_sg_bytes;
        edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
                                                  desc_bytes);
-        edesc->iv_dir = DMA_TO_DEVICE;
        /* Make sure IV is located in a DMAable area */
        iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
-        memcpy(iv, req->info, ivsize);
+        memcpy(iv, req->iv, ivsize);
        iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
        if (dma_mapping_error(jrdev, iv_dma)) {
                dev_err(jrdev, "unable to map IV\n");
                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, DMA_NONE, 0, 0);
+                           0, 0, 0);
                kfree(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1583,7 +1523,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
                dev_err(jrdev, "unable to map S/G table\n");
                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
+                           iv_dma, ivsize, 0, 0);
                kfree(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1591,7 +1531,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        edesc->iv_dma = iv_dma;
 #ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
+        print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
                       sec4_sg_bytes, 1);
 #endif
@@ -1599,362 +1539,187 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        return edesc;
 }
-static int ablkcipher_encrypt(struct ablkcipher_request *req)
+static int skcipher_encrypt(struct skcipher_request *req)
 {
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
        struct device *jrdev = ctx->jrdev;
        u32 *desc;
        int ret = 0;
        /* allocate extended descriptor */
-        edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
+        edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
        if (IS_ERR(edesc))
                return PTR_ERR(edesc);
        /* Create and submit job descriptor*/
-        init_ablkcipher_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req);
+        init_skcipher_job(req, edesc, true);
 #ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
+        print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
                       desc_bytes(edesc->hw_desc), 1);
 #endif
        desc = edesc->hw_desc;
-        ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
+        ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req);
        if (!ret) {
                ret = -EINPROGRESS;
        } else {
-                ablkcipher_unmap(jrdev, edesc, req);
+                skcipher_unmap(jrdev, edesc, req);
                kfree(edesc);
        }
        return ret;
 }
-static int ablkcipher_decrypt(struct ablkcipher_request *req)
+static int skcipher_decrypt(struct skcipher_request *req)
 {
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        struct device *jrdev = ctx->jrdev;
        u32 *desc;
        int ret = 0;
        /* allocate extended descriptor */
-        edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
+        edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
        if (IS_ERR(edesc))
                return PTR_ERR(edesc);
        /*
-         * The crypto API expects us to set the IV (req->info) to the last
+         * The crypto API expects us to set the IV (req->iv) to the last
         * ciphertext block.
         */
-        scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize,
+        scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
                                 ivsize, 0);
        /* Create and submit job descriptor*/
-        init_ablkcipher_job(ctx->sh_desc_dec, ctx->sh_desc_dec_dma, edesc, req);
+        init_skcipher_job(req, edesc, false);
        desc = edesc->hw_desc;
 #ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
+        print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
                       desc_bytes(edesc->hw_desc), 1);
 #endif
-        ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
+        ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req);
        if (!ret) {
                ret = -EINPROGRESS;
        } else {
-                ablkcipher_unmap(jrdev, edesc, req);
+                skcipher_unmap(jrdev, edesc, req);
                kfree(edesc);
        }
        return ret;
 }
-/*
+static struct caam_skcipher_alg driver_algs[] = {
- * allocate and map the ablkcipher extended descriptor
- * for ablkcipher givencrypt
- */
-static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
-                                struct skcipher_givcrypt_request *greq,
-                                int desc_bytes)
-{
-        struct ablkcipher_request *req = &greq->creq;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
-        struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags &  CRYPTO_TFM_REQ_MAY_SLEEP) ?
-                       GFP_KERNEL : GFP_ATOMIC;
-        int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
-        struct ablkcipher_edesc *edesc;
-        dma_addr_t iv_dma;
-        u8 *iv;
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
-        int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
-        src_nents = sg_nents_for_len(req->src, req->nbytes);
-        if (unlikely(src_nents < 0)) {
-                dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
-                        req->nbytes);
-                return ERR_PTR(src_nents);
-        }
-        if (likely(req->src == req->dst)) {
-                mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
-                                              DMA_BIDIRECTIONAL);
-                if (unlikely(!mapped_src_nents)) {
-                        dev_err(jrdev, "unable to map source\n");
-                        return ERR_PTR(-ENOMEM);
-                }
-                dst_nents = src_nents;
-                mapped_dst_nents = src_nents;
-        } else {
-                mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
-                                              DMA_TO_DEVICE);
-                if (unlikely(!mapped_src_nents)) {
-                        dev_err(jrdev, "unable to map source\n");
-                        return ERR_PTR(-ENOMEM);
-                }
-                dst_nents = sg_nents_for_len(req->dst, req->nbytes);
-                if (unlikely(dst_nents < 0)) {
-                        dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->nbytes);
-                        return ERR_PTR(dst_nents);
-                }
-                mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
-                                              DMA_FROM_DEVICE);
-                if (unlikely(!mapped_dst_nents)) {
-                        dev_err(jrdev, "unable to map destination\n");
-                        dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
-                        return ERR_PTR(-ENOMEM);
-                }
-        }
-        sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0;
-        dst_sg_idx = sec4_sg_ents;
-        sec4_sg_ents += 1 + mapped_dst_nents;
-        /*
-         * allocate space for base edesc and hw desc commands, link tables, IV
-         */
-        sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
-        edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
-                        GFP_DMA | flags);
-        if (!edesc) {
-                dev_err(jrdev, "could not allocate extended descriptor\n");
-                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, DMA_NONE, 0, 0);
-                return ERR_PTR(-ENOMEM);
-        }
-        edesc->src_nents = src_nents;
-        edesc->dst_nents = dst_nents;
-        edesc->sec4_sg_bytes = sec4_sg_bytes;
-        edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
-                                                  desc_bytes);
-        edesc->iv_dir = DMA_FROM_DEVICE;
-        /* Make sure IV is located in a DMAable area */
-        iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
-        iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_FROM_DEVICE);
-        if (dma_mapping_error(jrdev, iv_dma)) {
-                dev_err(jrdev, "unable to map IV\n");
-                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, DMA_NONE, 0, 0);
-                kfree(edesc);
-                return ERR_PTR(-ENOMEM);
-        }
-        if (mapped_src_nents > 1)
-                sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg,
-                                   0);
-        dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx, iv_dma, ivsize, 0);
-        sg_to_sec4_sg_last(req->dst, mapped_dst_nents, edesc->sec4_sg +
-                           dst_sg_idx + 1, 0);
-        edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
-                                            sec4_sg_bytes, DMA_TO_DEVICE);
-        if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
-                dev_err(jrdev, "unable to map S/G table\n");
-                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, DMA_FROM_DEVICE, 0, 0);
-                kfree(edesc);
-                return ERR_PTR(-ENOMEM);
-        }
-        edesc->iv_dma = iv_dma;
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
-                       sec4_sg_bytes, 1);
-#endif
-        return edesc;
-}
-static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
-{
-        struct ablkcipher_request *req = &creq->creq;
-        struct ablkcipher_edesc *edesc;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
-        struct device *jrdev = ctx->jrdev;
-        u32 *desc;
-        int ret = 0;
-        /* allocate extended descriptor */
-        edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
-        if (IS_ERR(edesc))
-                return PTR_ERR(edesc);
-        /* Create and submit job descriptor*/
-        init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
-                                edesc, req);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
-        desc = edesc->hw_desc;
-        ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
-        if (!ret) {
-                ret = -EINPROGRESS;
-        } else {
-                ablkcipher_unmap(jrdev, edesc, req);
-                kfree(edesc);
-        }
-        return ret;
-}
-#define template_aead           template_u.aead
-#define template_ablkcipher     template_u.ablkcipher
-struct caam_alg_template {
-        char name[CRYPTO_MAX_ALG_NAME];
-        char driver_name[CRYPTO_MAX_ALG_NAME];
-        unsigned int blocksize;
-        u32 type;
-        union {
-                struct ablkcipher_alg ablkcipher;
-        } template_u;
-        u32 class1_alg_type;
-        u32 class2_alg_type;
-};
-static struct caam_alg_template driver_algs[] = {
-        /* ablkcipher descriptor */
        {
-                .name = "cbc(aes)",
+                .skcipher = {
-                .driver_name = "cbc-aes-caam",
+                        .base = {
-                .blocksize = AES_BLOCK_SIZE,
+                                .cra_name = "cbc(aes)",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "cbc-aes-caam",
-                .template_ablkcipher = {
+                                .cra_blocksize = AES_BLOCK_SIZE,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = AES_MIN_KEY_SIZE,
                        .max_keysize = AES_MAX_KEY_SIZE,
                        .ivsize = AES_BLOCK_SIZE,
-                        },
+                },
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
        },
        {
-                .name = "cbc(des3_ede)",
+                .skcipher = {
-                .driver_name = "cbc-3des-caam",
+                        .base = {
-                .blocksize = DES3_EDE_BLOCK_SIZE,
+                                .cra_name = "cbc(des3_ede)",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "cbc-3des-caam",
-                .template_ablkcipher = {
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = DES3_EDE_KEY_SIZE,
                        .max_keysize = DES3_EDE_KEY_SIZE,
                        .ivsize = DES3_EDE_BLOCK_SIZE,
-                        },
+                },
-                .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
        },
        {
-                .name = "cbc(des)",
+                .skcipher = {
-                .driver_name = "cbc-des-caam",
+                        .base = {
-                .blocksize = DES_BLOCK_SIZE,
+                                .cra_name = "cbc(des)",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "cbc-des-caam",
-                .template_ablkcipher = {
+                                .cra_blocksize = DES_BLOCK_SIZE,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = DES_KEY_SIZE,
                        .max_keysize = DES_KEY_SIZE,
                        .ivsize = DES_BLOCK_SIZE,
-                        },
+                },
-                .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
        },
        {
-                .name = "ctr(aes)",
+                .skcipher = {
-                .driver_name = "ctr-aes-caam",
+                        .base = {
-                .blocksize = 1,
+                                .cra_name = "ctr(aes)",
-                .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+                                .cra_driver_name = "ctr-aes-caam",
-                .template_ablkcipher = {
+                                .cra_blocksize = 1,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .geniv = "chainiv",
+                        .decrypt = skcipher_decrypt,
                        .min_keysize = AES_MIN_KEY_SIZE,
                        .max_keysize = AES_MAX_KEY_SIZE,
                        .ivsize = AES_BLOCK_SIZE,
-                        },
+                        .chunksize = AES_BLOCK_SIZE,
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+                },
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
+                                        OP_ALG_AAI_CTR_MOD128,
        },
        {
-                .name = "rfc3686(ctr(aes))",
+                .skcipher = {
-                .driver_name = "rfc3686-ctr-aes-caam",
+                        .base = {
-                .blocksize = 1,
+                                .cra_name = "rfc3686(ctr(aes))",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "rfc3686-ctr-aes-caam",
-                .template_ablkcipher = {
+                                .cra_blocksize = 1,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = AES_MIN_KEY_SIZE +
                                       CTR_RFC3686_NONCE_SIZE,
                        .max_keysize = AES_MAX_KEY_SIZE +
                                       CTR_RFC3686_NONCE_SIZE,
                        .ivsize = CTR_RFC3686_IV_SIZE,
-                        },
+                        .chunksize = AES_BLOCK_SIZE,
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .rfc3686 = true,
+                },
        },
        {
-                .name = "xts(aes)",
+                .skcipher = {
-                .driver_name = "xts-aes-caam",
+                        .base = {
-                .blocksize = AES_BLOCK_SIZE,
+                                .cra_name = "xts(aes)",
-                .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+                                .cra_driver_name = "xts-aes-caam",
-                .template_ablkcipher = {
+                                .cra_blocksize = AES_BLOCK_SIZE,
-                        .setkey = xts_ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = xts_skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .geniv = "eseqiv",
+                        .decrypt = skcipher_decrypt,
                        .min_keysize = 2 * AES_MIN_KEY_SIZE,
                        .max_keysize = 2 * AES_MAX_KEY_SIZE,
                        .ivsize = AES_BLOCK_SIZE,
-                        },
+                },
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
        },
 };
@@ -3239,12 +3004,6 @@ static struct caam_aead_alg driver_aeads[] = {
        },
 };
-struct caam_crypto_alg {
-        struct crypto_alg crypto_alg;
-        struct list_head entry;
-        struct caam_alg_entry caam;
-};
 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
                            bool uses_dkp)
 {
@@ -3276,8 +3035,6 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
        ctx->sh_desc_enc_dma = dma_addr;
        ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
                                                   sh_desc_dec);
-        ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx,
-                                                      sh_desc_givenc);
        ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key);
        /* copy descriptor header template value */
@@ -3287,14 +3044,14 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
        return 0;
 }
-static int caam_cra_init(struct crypto_tfm *tfm)
+static int caam_cra_init(struct crypto_skcipher *tfm)
 {
-        struct crypto_alg *alg = tfm->__crt_alg;
+        struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
-        struct caam_crypto_alg *caam_alg =
+        struct caam_skcipher_alg *caam_alg =
-                 container_of(alg, struct caam_crypto_alg, crypto_alg);
+                container_of(alg, typeof(*caam_alg), skcipher);
-        struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
-        return caam_init_common(ctx, &caam_alg->caam, false);
+        return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
+                                false);
 }
 static int caam_aead_init(struct crypto_aead *tfm)
@@ -3316,9 +3073,9 @@ static void caam_exit_common(struct caam_ctx *ctx)
        caam_jr_free(ctx->jrdev);
 }
-static void caam_cra_exit(struct crypto_tfm *tfm)
+static void caam_cra_exit(struct crypto_skcipher *tfm)
 {
-        caam_exit_common(crypto_tfm_ctx(tfm));
+        caam_exit_common(crypto_skcipher_ctx(tfm));
 }
 static void caam_aead_exit(struct crypto_aead *tfm)
@@ -3328,8 +3085,6 @@ static void caam_aead_exit(struct crypto_aead *tfm)
 static void __exit caam_algapi_exit(void)
 {
-        struct caam_crypto_alg *t_alg, *n;
        int i;
        for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
@@ -3339,57 +3094,25 @@ static void __exit caam_algapi_exit(void)
                        crypto_unregister_aead(&t_alg->aead);
        }
-        if (!alg_list.next)
+        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
-                return;
+                struct caam_skcipher_alg *t_alg = driver_algs + i;
-        list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
+                if (t_alg->registered)
-                crypto_unregister_alg(&t_alg->crypto_alg);
+                        crypto_unregister_skcipher(&t_alg->skcipher);
-                list_del(&t_alg->entry);
-                kfree(t_alg);
        }
 }
-static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
+static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
-                                              *template)
 {
-        struct caam_crypto_alg *t_alg;
+        struct skcipher_alg *alg = &t_alg->skcipher;
-        struct crypto_alg *alg;
-        t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
+        alg->base.cra_module = THIS_MODULE;
-        if (!t_alg) {
+        alg->base.cra_priority = CAAM_CRA_PRIORITY;
-                pr_err("failed to allocate t_alg\n");
+        alg->base.cra_ctxsize = sizeof(struct caam_ctx);
-                return ERR_PTR(-ENOMEM);
+        alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
-        }
-        alg = &t_alg->crypto_alg;
-        snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
-        snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-                 template->driver_name);
-        alg->cra_module = THIS_MODULE;
-        alg->cra_init = caam_cra_init;
-        alg->cra_exit = caam_cra_exit;
-        alg->cra_priority = CAAM_CRA_PRIORITY;
-        alg->cra_blocksize = template->blocksize;
-        alg->cra_alignmask = 0;
-        alg->cra_ctxsize = sizeof(struct caam_ctx);
-        alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
-                         template->type;
-        switch (template->type) {
-        case CRYPTO_ALG_TYPE_GIVCIPHER:
-                alg->cra_type = &crypto_givcipher_type;
-                alg->cra_ablkcipher = template->template_ablkcipher;
-                break;
-        case CRYPTO_ALG_TYPE_ABLKCIPHER:
-                alg->cra_type = &crypto_ablkcipher_type;
-                alg->cra_ablkcipher = template->template_ablkcipher;
-                break;
-        }
-        t_alg->caam.class1_alg_type = template->class1_alg_type;
-        t_alg->caam.class2_alg_type = template->class2_alg_type;
-        return t_alg;
+        alg->init = caam_cra_init;
+        alg->exit = caam_cra_exit;
 }
 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
@@ -3441,8 +3164,6 @@ static int __init caam_algapi_init(void)
                return -ENODEV;
-        INIT_LIST_HEAD(&alg_list);
        /*
         * Register crypto algorithms the device supports.
         * First, detect presence and attributes of DES, AES, and MD blocks.
@@ -3458,9 +3179,8 @@ static int __init caam_algapi_init(void)
                md_limit = SHA256_DIGEST_SIZE;
        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
-                struct caam_crypto_alg *t_alg;
+                struct caam_skcipher_alg *t_alg = driver_algs + i;
-                struct caam_alg_template *alg = driver_algs + i;
+                u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
-                u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
                /* Skip DES algorithms if not supported by device */
                if (!des_inst &&
@@ -3477,26 +3197,20 @@ static int __init caam_algapi_init(void)
                 * on LP devices.
                 */
                if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
-                        if ((alg->class1_alg_type & OP_ALG_AAI_MASK) ==
+                        if ((t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
                             OP_ALG_AAI_XTS)
                                continue;
-                t_alg = caam_alg_alloc(alg);
+                caam_skcipher_alg_init(t_alg);
-                if (IS_ERR(t_alg)) {
-                        err = PTR_ERR(t_alg);
-                        pr_warn("%s alg allocation failed\n", alg->driver_name);
-                        continue;
-                }
-                err = crypto_register_alg(&t_alg->crypto_alg);
+                err = crypto_register_skcipher(&t_alg->skcipher);
                if (err) {
                        pr_warn("%s alg registration failed\n",
-                                t_alg->crypto_alg.cra_driver_name);
+                                t_alg->skcipher.base.cra_driver_name);
-                        kfree(t_alg);
                        continue;
                }
-                list_add_tail(&t_alg->entry, &alg_list);
+                t_alg->registered = true;
                registered = true;
        }
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index a408edd84f34..1a6f0da14106 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
- * Shared descriptors for aead, ablkcipher algorithms
+ * Shared descriptors for aead, skcipher algorithms
 *
- * Copyright 2016 NXP
+ * Copyright 2016-2018 NXP
 */
 #include "compat.h"
@@ -1212,11 +1213,8 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
 }
 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
-/*
+/* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
- * For ablkcipher encrypt and decrypt, read from req->src and
+static inline void skcipher_append_src_dst(u32 *desc)
- * write to req->dst
- */
-static inline void ablkcipher_append_src_dst(u32 *desc)
 {
        append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
        append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
@@ -1226,7 +1224,7 @@ static inline void ablkcipher_append_src_dst(u32 *desc)
 }
 /**
- * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
+ * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
 * @desc: pointer to buffer used for descriptor construction
 * @cdata: pointer to block cipher transform definitions
 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
@@ -1235,9 +1233,9 @@ static inline void ablkcipher_append_src_dst(u32 *desc)
 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 * @ctx1_iv_off: IV offset in CONTEXT1 register
 */
-void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
+void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
-                                  unsigned int ivsize, const bool is_rfc3686,
+                                unsigned int ivsize, const bool is_rfc3686,
-                                  const u32 ctx1_iv_off)
+                                const u32 ctx1_iv_off)
 {
        u32 *key_jump_cmd;
@@ -1280,18 +1278,18 @@ void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
                         OP_ALG_ENCRYPT);
        /* Perform operation */
-        ablkcipher_append_src_dst(desc);
+        skcipher_append_src_dst(desc);
 #ifdef DEBUG
        print_hex_dump(KERN_ERR,
-                       "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
+                       "skcipher enc shdesc@" __stringify(__LINE__)": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 #endif
 }
-EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
+EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
 /**
- * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
+ * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
 * @desc: pointer to buffer used for descriptor construction
 * @cdata: pointer to block cipher transform definitions
 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
@@ -1300,9 +1298,9 @@ EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 * @ctx1_iv_off: IV offset in CONTEXT1 register
 */
-void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
+void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
-                                  unsigned int ivsize, const bool is_rfc3686,
+                                unsigned int ivsize, const bool is_rfc3686,
-                                  const u32 ctx1_iv_off)
+                                const u32 ctx1_iv_off)
 {
        u32 *key_jump_cmd;
@@ -1348,105 +1346,23 @@ void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
                append_dec_op1(desc, cdata->algtype);
        /* Perform operation */
-        ablkcipher_append_src_dst(desc);
+        skcipher_append_src_dst(desc);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
-}
-EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
-/**
- * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
- *                                   with HW-generated initialization vector.
- * @desc: pointer to buffer used for descriptor construction
- * @cdata: pointer to block cipher transform definitions
- *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
- *         with OP_ALG_AAI_CBC.
- * @ivsize: initialization vector size
- * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
- * @ctx1_iv_off: IV offset in CONTEXT1 register
- */
-void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
-                                     unsigned int ivsize, const bool is_rfc3686,
-                                     const u32 ctx1_iv_off)
-{
-        u32 *key_jump_cmd, geniv;
-        init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-        /* Skip if already shared */
-        key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-                                   JUMP_COND_SHRD);
-        /* Load class1 key only */
-        append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
-                          cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
-        /* Load Nonce into CONTEXT1 reg */
-        if (is_rfc3686) {
-                const u8 *nonce = cdata->key_virt + cdata->keylen;
-                append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
-                                   LDST_CLASS_IND_CCB |
-                                   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
-                append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
-                            MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
-                            (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
-        }
-        set_jump_tgt_here(desc, key_jump_cmd);
-        /* Generate IV */
-        geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
-                NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
-                (ivsize << NFIFOENTRY_DLEN_SHIFT);
-        append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
-                            LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
-        append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-        append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
-                    MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
-                    (ctx1_iv_off << MOVE_OFFSET_SHIFT));
-        append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
-        /* Copy generated IV to memory */
-        append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
-                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
-        /* Load Counter into CONTEXT1 reg */
-        if (is_rfc3686)
-                append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-                                     LDST_SRCDST_BYTE_CONTEXT |
-                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-                                      LDST_OFFSET_SHIFT));
-        if (ctx1_iv_off)
-                append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
-                            (1 << JUMP_OFFSET_SHIFT));
-        /* Load operation */
-        append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
-                         OP_ALG_ENCRYPT);
-        /* Perform operation */
-        ablkcipher_append_src_dst(desc);
 #ifdef DEBUG
        print_hex_dump(KERN_ERR,
-                       "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
+                       "skcipher dec shdesc@" __stringify(__LINE__)": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 #endif
 }
-EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
+EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
 /**
- * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
+ * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
- *                                    descriptor
 * @desc: pointer to buffer used for descriptor construction
 * @cdata: pointer to block cipher transform definitions
 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
 */
-void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
+void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
 {
        __be64 sector_size = cpu_to_be64(512);
        u32 *key_jump_cmd;
@@ -1481,24 +1397,23 @@ void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
                         OP_ALG_ENCRYPT);
        /* Perform operation */
-        ablkcipher_append_src_dst(desc);
+        skcipher_append_src_dst(desc);
 #ifdef DEBUG
        print_hex_dump(KERN_ERR,
-                       "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
+                       "xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 #endif
 }
-EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
+EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
 /**
- * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
+ * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
- *                                    descriptor
 * @desc: pointer to buffer used for descriptor construction
 * @cdata: pointer to block cipher transform definitions
 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
 */
-void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
+void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
 {
        __be64 sector_size = cpu_to_be64(512);
        u32 *key_jump_cmd;
@@ -1532,15 +1447,15 @@ void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
        append_dec_op1(desc, cdata->algtype);
        /* Perform operation */
-        ablkcipher_append_src_dst(desc);
+        skcipher_append_src_dst(desc);
 #ifdef DEBUG
        print_hex_dump(KERN_ERR,
-                       "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
+                       "xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 #endif
 }
-EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
+EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("FSL CAAM descriptor support");
diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h
index a917af5776ce..1315c8f6f951 100644
--- a/drivers/crypto/caam/caamalg_desc.h
+++ b/drivers/crypto/caam/caamalg_desc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Shared descriptors for aead, ablkcipher algorithms
+ * Shared descriptors for aead, skcipher algorithms
 *
 * Copyright 2016 NXP
 */
@@ -42,10 +42,10 @@
 #define DESC_QI_RFC4543_ENC_LEN         (DESC_RFC4543_ENC_LEN + 4 * CAAM_CMD_SZ)
 #define DESC_QI_RFC4543_DEC_LEN         (DESC_RFC4543_DEC_LEN + 4 * CAAM_CMD_SZ)
-#define DESC_ABLKCIPHER_BASE            (3 * CAAM_CMD_SZ)
+#define DESC_SKCIPHER_BASE              (3 * CAAM_CMD_SZ)
-#define DESC_ABLKCIPHER_ENC_LEN         (DESC_ABLKCIPHER_BASE + \
+#define DESC_SKCIPHER_ENC_LEN           (DESC_SKCIPHER_BASE + \
                                         20 * CAAM_CMD_SZ)
-#define DESC_ABLKCIPHER_DEC_LEN         (DESC_ABLKCIPHER_BASE + \
+#define DESC_SKCIPHER_DEC_LEN           (DESC_SKCIPHER_BASE + \
                                         15 * CAAM_CMD_SZ)
 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
@@ -96,20 +96,16 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
                               unsigned int ivsize, unsigned int icvsize,
                               const bool is_qi);
-void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
+void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
-                                  unsigned int ivsize, const bool is_rfc3686,
+                                unsigned int ivsize, const bool is_rfc3686,
-                                  const u32 ctx1_iv_off);
+                                const u32 ctx1_iv_off);
-void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
+void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
-                                  unsigned int ivsize, const bool is_rfc3686,
+                                unsigned int ivsize, const bool is_rfc3686,
-                                  const u32 ctx1_iv_off);
+                                const u32 ctx1_iv_off);
-void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
+void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata);
-                                     unsigned int ivsize, const bool is_rfc3686,
-                                     const u32 ctx1_iv_off);
-void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata);
+void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata);
-void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata);
 #endif /* _CAAMALG_DESC_H_ */
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index d7aa7d7ff102..23c9fc4975f8 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -1,9 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
 * Freescale FSL CAAM support for crypto API over QI backend.
 * Based on caamalg.c
 *
 * Copyright 2013-2016 Freescale Semiconductor, Inc.
- * Copyright 2016-2017 NXP
+ * Copyright 2016-2018 NXP
 */
 #include "compat.h"
@@ -43,6 +44,12 @@ struct caam_aead_alg {
        bool registered;
 };
+struct caam_skcipher_alg {
+        struct skcipher_alg skcipher;
+        struct caam_alg_entry caam;
+        bool registered;
+};
 /*
 * per-session context
 */
@@ -50,7 +57,6 @@ struct caam_ctx {
        struct device *jrdev;
        u32 sh_desc_enc[DESC_MAX_USED_LEN];
        u32 sh_desc_dec[DESC_MAX_USED_LEN];
-        u32 sh_desc_givenc[DESC_MAX_USED_LEN];
        u8 key[CAAM_MAX_KEY_SIZE];
        dma_addr_t key_dma;
        enum dma_data_direction dir;
@@ -589,18 +595,19 @@ static int rfc4543_setkey(struct crypto_aead *aead,
        return 0;
 }
-static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
+static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
-                             const u8 *key, unsigned int keylen)
+                           unsigned int keylen)
 {
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
-        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
+        struct caam_skcipher_alg *alg =
-        const char *alg_name = crypto_tfm_alg_name(tfm);
+                container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
+                             skcipher);
        struct device *jrdev = ctx->jrdev;
-        unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
        u32 ctx1_iv_off = 0;
        const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
                               OP_ALG_AAI_CTR_MOD128);
-        const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686"));
+        const bool is_rfc3686 = alg->caam.rfc3686;
        int ret = 0;
 #ifdef DEBUG
@@ -629,13 +636,11 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
        ctx->cdata.key_virt = key;
        ctx->cdata.key_inline = true;
-        /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */
+        /* skcipher encrypt, decrypt shared descriptors */
-        cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
+        cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
-                                     is_rfc3686, ctx1_iv_off);
+                                   is_rfc3686, ctx1_iv_off);
-        cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
+        cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
-                                     is_rfc3686, ctx1_iv_off);
+                                   is_rfc3686, ctx1_iv_off);
-        cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata,
-                                        ivsize, is_rfc3686, ctx1_iv_off);
        /* Now update the driver contexts with the new shared descriptor */
        if (ctx->drv_ctx[ENCRYPT]) {
@@ -656,25 +661,16 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
                }
        }
-        if (ctx->drv_ctx[GIVENCRYPT]) {
-                ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT],
-                                          ctx->sh_desc_givenc);
-                if (ret) {
-                        dev_err(jrdev, "driver givenc context update failed\n");
-                        goto badkey;
-                }
-        }
        return ret;
 badkey:
-        crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
        return -EINVAL;
 }
-static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
+static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
-                                 const u8 *key, unsigned int keylen)
+                               unsigned int keylen)
 {
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
        struct device *jrdev = ctx->jrdev;
        int ret = 0;
@@ -687,9 +683,9 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
        ctx->cdata.key_virt = key;
        ctx->cdata.key_inline = true;
-        /* xts ablkcipher encrypt, decrypt shared descriptors */
+        /* xts skcipher encrypt, decrypt shared descriptors */
-        cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
+        cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
-        cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
+        cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
        /* Now update the driver contexts with the new shared descriptor */
        if (ctx->drv_ctx[ENCRYPT]) {
@@ -712,7 +708,7 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
        return ret;
 badkey:
-        crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
        return -EINVAL;
 }
@@ -741,7 +737,7 @@ struct aead_edesc {
 };
 /*
- * ablkcipher_edesc - s/w-extended ablkcipher descriptor
+ * skcipher_edesc - s/w-extended skcipher descriptor
 * @src_nents: number of segments in input scatterlist
 * @dst_nents: number of segments in output scatterlist
 * @iv_dma: dma address of iv for checking continuity and link table
@@ -750,7 +746,7 @@ struct aead_edesc {
 * @drv_req: driver-specific request structure
 * @sgt: the h/w link table, followed by IV
 */
-struct ablkcipher_edesc {
+struct skcipher_edesc {
        int src_nents;
        int dst_nents;
        dma_addr_t iv_dma;
@@ -781,10 +777,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
                        if (type == ENCRYPT)
                                desc = ctx->sh_desc_enc;
-                        else if (type == DECRYPT)
+                        else /* (type == DECRYPT) */
                                desc = ctx->sh_desc_dec;
-                        else /* (type == GIVENCRYPT) */
-                                desc = ctx->sh_desc_givenc;
                        cpu = smp_processor_id();
                        drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
@@ -803,8 +797,7 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
 static void caam_unmap(struct device *dev, struct scatterlist *src,
                       struct scatterlist *dst, int src_nents,
                       int dst_nents, dma_addr_t iv_dma, int ivsize,
-                       enum optype op_type, dma_addr_t qm_sg_dma,
+                       dma_addr_t qm_sg_dma, int qm_sg_bytes)
-                       int qm_sg_bytes)
 {
        if (dst != src) {
                if (src_nents)
@@ -815,9 +808,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
        }
        if (iv_dma)
-                dma_unmap_single(dev, iv_dma, ivsize,
+                dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
-                                 op_type == GIVENCRYPT ? DMA_FROM_DEVICE :
-                                                         DMA_TO_DEVICE);
        if (qm_sg_bytes)
                dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 }
@@ -830,21 +821,18 @@ static void aead_unmap(struct device *dev,
        int ivsize = crypto_aead_ivsize(aead);
        caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type,
+                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
-                   edesc->qm_sg_dma, edesc->qm_sg_bytes);
        dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 }
-static void ablkcipher_unmap(struct device *dev,
+static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
-                             struct ablkcipher_edesc *edesc,
+                           struct skcipher_request *req)
-                             struct ablkcipher_request *req)
 {
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type,
+                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
-                   edesc->qm_sg_dma, edesc->qm_sg_bytes);
 }
 static void aead_done(struct caam_drv_req *drv_req, u32 status)
@@ -902,9 +890,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
        int in_len, out_len;
        struct qm_sg_entry *sg_table, *fd_sgt;
        struct caam_drv_ctx *drv_ctx;
-        enum optype op_type = encrypt ? ENCRYPT : DECRYPT;
-        drv_ctx = get_drv_ctx(ctx, op_type);
+        drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
        if (unlikely(IS_ERR_OR_NULL(drv_ctx)))
                return (struct aead_edesc *)drv_ctx;
@@ -994,7 +981,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
                        qm_sg_ents, ivsize);
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
+                           0, 0, 0);
                qi_cache_free(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1009,7 +996,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                if (dma_mapping_error(qidev, iv_dma)) {
                        dev_err(qidev, "unable to map IV\n");
                        caam_unmap(qidev, req->src, req->dst, src_nents,
-                                   dst_nents, 0, 0, 0, 0, 0);
+                                   dst_nents, 0, 0, 0, 0);
                        qi_cache_free(edesc);
                        return ERR_PTR(-ENOMEM);
                }
@@ -1028,7 +1015,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
        if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
                dev_err(qidev, "unable to map assoclen\n");
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, op_type, 0, 0);
+                           iv_dma, ivsize, 0, 0);
                qi_cache_free(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1051,7 +1038,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                dev_err(qidev, "unable to map S/G table\n");
                dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, op_type, 0, 0);
+                           iv_dma, ivsize, 0, 0);
                qi_cache_free(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1138,14 +1125,14 @@ static int ipsec_gcm_decrypt(struct aead_request *req)
        return aead_crypt(req, false);
 }
-static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
+static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
 {
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
-        struct ablkcipher_request *req = drv_req->app_ctx;
+        struct skcipher_request *req = drv_req->app_ctx;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
        struct device *qidev = caam_ctx->qidev;
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
 #ifdef DEBUG
        dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
@@ -1158,72 +1145,60 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
 #ifdef DEBUG
        print_hex_dump(KERN_ERR, "dstiv  @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
                       edesc->src_nents > 1 ? 100 : ivsize, 1);
        caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-                     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
 #endif
-        ablkcipher_unmap(qidev, edesc, req);
+        skcipher_unmap(qidev, edesc, req);
-        /* In case initial IV was generated, copy it in GIVCIPHER request */
-        if (edesc->drv_req.drv_ctx->op_type == GIVENCRYPT) {
-                u8 *iv;
-                struct skcipher_givcrypt_request *greq;
-                greq = container_of(req, struct skcipher_givcrypt_request,
-                                    creq);
-                iv = (u8 *)edesc->sgt + edesc->qm_sg_bytes;
-                memcpy(greq->giv, iv, ivsize);
-        }
        /*
-         * The crypto API expects us to set the IV (req->info) to the last
+         * The crypto API expects us to set the IV (req->iv) to the last
         * ciphertext block. This is used e.g. by the CTS mode.
         */
-        if (edesc->drv_req.drv_ctx->op_type != DECRYPT)
+        if (edesc->drv_req.drv_ctx->op_type == ENCRYPT)
-                scatterwalk_map_and_copy(req->info, req->dst, req->nbytes -
+                scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
                                         ivsize, ivsize, 0);
        qi_cache_free(edesc);
-        ablkcipher_request_complete(req, status);
+        skcipher_request_complete(req, status);
 }
-static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
+static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
-                                                       *req, bool encrypt)
+                                                   bool encrypt)
 {
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
        struct device *qidev = ctx->qidev;
        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                       GFP_KERNEL : GFP_ATOMIC;
        int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
        dma_addr_t iv_dma;
        u8 *iv;
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
        struct qm_sg_entry *sg_table, *fd_sgt;
        struct caam_drv_ctx *drv_ctx;
-        enum optype op_type = encrypt ? ENCRYPT : DECRYPT;
-        drv_ctx = get_drv_ctx(ctx, op_type);
+        drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
        if (unlikely(IS_ERR_OR_NULL(drv_ctx)))
-                return (struct ablkcipher_edesc *)drv_ctx;
+                return (struct skcipher_edesc *)drv_ctx;
-        src_nents = sg_nents_for_len(req->src, req->nbytes);
+        src_nents = sg_nents_for_len(req->src, req->cryptlen);
        if (unlikely(src_nents < 0)) {
                dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
-                        req->nbytes);
+                        req->cryptlen);
                return ERR_PTR(src_nents);
        }
        if (unlikely(req->src != req->dst)) {
-                dst_nents = sg_nents_for_len(req->dst, req->nbytes);
+                dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
                if (unlikely(dst_nents < 0)) {
                        dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->nbytes);
+                                req->cryptlen);
                        return ERR_PTR(dst_nents);
                }
@@ -1255,12 +1230,12 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
        qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
-        if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes +
+        if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
                     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
                dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
                        qm_sg_ents, ivsize);
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
+                           0, 0, 0);
                return ERR_PTR(-ENOMEM);
        }
@@ -1269,20 +1244,20 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        if (unlikely(!edesc)) {
                dev_err(qidev, "could not allocate extended descriptor\n");
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
+                           0, 0, 0);
                return ERR_PTR(-ENOMEM);
        }
        /* Make sure IV is located in a DMAable area */
        sg_table = &edesc->sgt[0];
        iv = (u8 *)(sg_table + qm_sg_ents);
-        memcpy(iv, req->info, ivsize);
+        memcpy(iv, req->iv, ivsize);
        iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
        if (dma_mapping_error(qidev, iv_dma)) {
                dev_err(qidev, "unable to map IV\n");
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
+                           0, 0, 0);
                qi_cache_free(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1292,7 +1267,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        edesc->iv_dma = iv_dma;
        edesc->qm_sg_bytes = qm_sg_bytes;
        edesc->drv_req.app_ctx = req;
-        edesc->drv_req.cbk = ablkcipher_done;
+        edesc->drv_req.cbk = skcipher_done;
        edesc->drv_req.drv_ctx = drv_ctx;
        dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
@@ -1307,7 +1282,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
                dev_err(qidev, "unable to map S/G table\n");
                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, op_type, 0, 0);
+                           iv_dma, ivsize, 0, 0);
                qi_cache_free(edesc);
                return ERR_PTR(-ENOMEM);
        }
@@ -1315,348 +1290,172 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
        fd_sgt = &edesc->drv_req.fd_sgt[0];
        dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
-                                  ivsize + req->nbytes, 0);
+                                  ivsize + req->cryptlen, 0);
        if (req->src == req->dst) {
                dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
-                                     sizeof(*sg_table), req->nbytes, 0);
+                                     sizeof(*sg_table), req->cryptlen, 0);
        } else if (mapped_dst_nents > 1) {
                dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
-                                     sizeof(*sg_table), req->nbytes, 0);
+                                     sizeof(*sg_table), req->cryptlen, 0);
        } else {
                dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst),
-                                 req->nbytes, 0);
+                                 req->cryptlen, 0);
-        }
-        return edesc;
-}
-static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
-        struct skcipher_givcrypt_request *creq)
-{
-        struct ablkcipher_request *req = &creq->creq;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
-        struct device *qidev = ctx->qidev;
-        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
-                       GFP_KERNEL : GFP_ATOMIC;
-        int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
-        struct ablkcipher_edesc *edesc;
-        dma_addr_t iv_dma;
-        u8 *iv;
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
-        struct qm_sg_entry *sg_table, *fd_sgt;
-        int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
-        struct caam_drv_ctx *drv_ctx;
-        drv_ctx = get_drv_ctx(ctx, GIVENCRYPT);
-        if (unlikely(IS_ERR_OR_NULL(drv_ctx)))
-                return (struct ablkcipher_edesc *)drv_ctx;
-        src_nents = sg_nents_for_len(req->src, req->nbytes);
-        if (unlikely(src_nents < 0)) {
-                dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
-                        req->nbytes);
-                return ERR_PTR(src_nents);
-        }
-        if (unlikely(req->src != req->dst)) {
-                dst_nents = sg_nents_for_len(req->dst, req->nbytes);
-                if (unlikely(dst_nents < 0)) {
-                        dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->nbytes);
-                        return ERR_PTR(dst_nents);
-                }
-                mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
-                                              DMA_TO_DEVICE);
-                if (unlikely(!mapped_src_nents)) {
-                        dev_err(qidev, "unable to map source\n");
-                        return ERR_PTR(-ENOMEM);
-                }
-                mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
-                                              DMA_FROM_DEVICE);
-                if (unlikely(!mapped_dst_nents)) {
-                        dev_err(qidev, "unable to map destination\n");
-                        dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
-                        return ERR_PTR(-ENOMEM);
-                }
-        } else {
-                mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
-                                              DMA_BIDIRECTIONAL);
-                if (unlikely(!mapped_src_nents)) {
-                        dev_err(qidev, "unable to map source\n");
-                        return ERR_PTR(-ENOMEM);
-                }
-                dst_nents = src_nents;
-                mapped_dst_nents = src_nents;
-        }
-        qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0;
-        dst_sg_idx = qm_sg_ents;
-        qm_sg_ents += 1 + mapped_dst_nents;
-        qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
-        if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes +
-                     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
-                dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
-                        qm_sg_ents, ivsize);
-                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
-                return ERR_PTR(-ENOMEM);
-        }
-        /* allocate space for base edesc, link tables and IV */
-        edesc = qi_cache_alloc(GFP_DMA | flags);
-        if (!edesc) {
-                dev_err(qidev, "could not allocate extended descriptor\n");
-                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
-                return ERR_PTR(-ENOMEM);
-        }
-        /* Make sure IV is located in a DMAable area */
-        sg_table = &edesc->sgt[0];
-        iv = (u8 *)(sg_table + qm_sg_ents);
-        iv_dma = dma_map_single(qidev, iv, ivsize, DMA_FROM_DEVICE);
-        if (dma_mapping_error(qidev, iv_dma)) {
-                dev_err(qidev, "unable to map IV\n");
-                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0, 0);
-                qi_cache_free(edesc);
-                return ERR_PTR(-ENOMEM);
-        }
-        edesc->src_nents = src_nents;
-        edesc->dst_nents = dst_nents;
-        edesc->iv_dma = iv_dma;
-        edesc->qm_sg_bytes = qm_sg_bytes;
-        edesc->drv_req.app_ctx = req;
-        edesc->drv_req.cbk = ablkcipher_done;
-        edesc->drv_req.drv_ctx = drv_ctx;
-        if (mapped_src_nents > 1)
-                sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0);
-        dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0);
-        sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + dst_sg_idx + 1,
-                         0);
-        edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
-                                          DMA_TO_DEVICE);
-        if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
-                dev_err(qidev, "unable to map S/G table\n");
-                caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, GIVENCRYPT, 0, 0);
-                qi_cache_free(edesc);
-                return ERR_PTR(-ENOMEM);
        }
-        fd_sgt = &edesc->drv_req.fd_sgt[0];
-        if (mapped_src_nents > 1)
-                dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes,
-                                     0);
-        else
-                dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src),
-                                 req->nbytes, 0);
-        dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
-                             sizeof(*sg_table), ivsize + req->nbytes, 0);
        return edesc;
 }
-static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt)
+static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
 {
-        struct ablkcipher_edesc *edesc;
+        struct skcipher_edesc *edesc;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
-        int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
        int ret;
        if (unlikely(caam_congested))
                return -EAGAIN;
        /* allocate extended descriptor */
-        edesc = ablkcipher_edesc_alloc(req, encrypt);
+        edesc = skcipher_edesc_alloc(req, encrypt);
        if (IS_ERR(edesc))
                return PTR_ERR(edesc);
        /*
-         * The crypto API expects us to set the IV (req->info) to the last
+         * The crypto API expects us to set the IV (req->iv) to the last
         * ciphertext block.
         */
        if (!encrypt)
-                scatterwalk_map_and_copy(req->info, req->src, req->nbytes -
+                scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
                                         ivsize, ivsize, 0);
        ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
        if (!ret) {
                ret = -EINPROGRESS;
        } else {
-                ablkcipher_unmap(ctx->qidev, edesc, req);
+                skcipher_unmap(ctx->qidev, edesc, req);
                qi_cache_free(edesc);
        }
        return ret;
 }
-static int ablkcipher_encrypt(struct ablkcipher_request *req)
+static int skcipher_encrypt(struct skcipher_request *req)
 {
-        return ablkcipher_crypt(req, true);
+        return skcipher_crypt(req, true);
 }
-static int ablkcipher_decrypt(struct ablkcipher_request *req)
+static int skcipher_decrypt(struct skcipher_request *req)
 {
-        return ablkcipher_crypt(req, false);
+        return skcipher_crypt(req, false);
-}
-static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
-{
-        struct ablkcipher_request *req = &creq->creq;
-        struct ablkcipher_edesc *edesc;
-        struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
-        struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
-        int ret;
-        if (unlikely(caam_congested))
-                return -EAGAIN;
-        /* allocate extended descriptor */
-        edesc = ablkcipher_giv_edesc_alloc(creq);
-        if (IS_ERR(edesc))
-                return PTR_ERR(edesc);
-        ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
-        if (!ret) {
-                ret = -EINPROGRESS;
-        } else {
-                ablkcipher_unmap(ctx->qidev, edesc, req);
-                qi_cache_free(edesc);
-        }
-        return ret;
 }
-#define template_ablkcipher     template_u.ablkcipher
+static struct caam_skcipher_alg driver_algs[] = {
-struct caam_alg_template {
-        char name[CRYPTO_MAX_ALG_NAME];
-        char driver_name[CRYPTO_MAX_ALG_NAME];
-        unsigned int blocksize;
-        u32 type;
-        union {
-                struct ablkcipher_alg ablkcipher;
-        } template_u;
-        u32 class1_alg_type;
-        u32 class2_alg_type;
-};
-static struct caam_alg_template driver_algs[] = {
-        /* ablkcipher descriptor */
        {
-                .name = "cbc(aes)",
+                .skcipher = {
-                .driver_name = "cbc-aes-caam-qi",
+                        .base = {
-                .blocksize = AES_BLOCK_SIZE,
+                                .cra_name = "cbc(aes)",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "cbc-aes-caam-qi",
-                .template_ablkcipher = {
+                                .cra_blocksize = AES_BLOCK_SIZE,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = AES_MIN_KEY_SIZE,
                        .max_keysize = AES_MAX_KEY_SIZE,
                        .ivsize = AES_BLOCK_SIZE,
                },
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
        },
        {
-                .name = "cbc(des3_ede)",
+                .skcipher = {
-                .driver_name = "cbc-3des-caam-qi",
+                        .base = {
-                .blocksize = DES3_EDE_BLOCK_SIZE,
+                                .cra_name = "cbc(des3_ede)",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "cbc-3des-caam-qi",
-                .template_ablkcipher = {
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = DES3_EDE_KEY_SIZE,
                        .max_keysize = DES3_EDE_KEY_SIZE,
                        .ivsize = DES3_EDE_BLOCK_SIZE,
                },
-                .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
        },
        {
-                .name = "cbc(des)",
+                .skcipher = {
-                .driver_name = "cbc-des-caam-qi",
+                        .base = {
-                .blocksize = DES_BLOCK_SIZE,
+                                .cra_name = "cbc(des)",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "cbc-des-caam-qi",
-                .template_ablkcipher = {
+                                .cra_blocksize = DES_BLOCK_SIZE,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = DES_KEY_SIZE,
                        .max_keysize = DES_KEY_SIZE,
                        .ivsize = DES_BLOCK_SIZE,
                },
-                .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
        },
        {
-                .name = "ctr(aes)",
+                .skcipher = {
-                .driver_name = "ctr-aes-caam-qi",
+                        .base = {
-                .blocksize = 1,
+                                .cra_name = "ctr(aes)",
-                .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+                                .cra_driver_name = "ctr-aes-caam-qi",
-                .template_ablkcipher = {
+                                .cra_blocksize = 1,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .geniv = "chainiv",
+                        .decrypt = skcipher_decrypt,
                        .min_keysize = AES_MIN_KEY_SIZE,
                        .max_keysize = AES_MAX_KEY_SIZE,
                        .ivsize = AES_BLOCK_SIZE,
+                        .chunksize = AES_BLOCK_SIZE,
                },
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
+                                        OP_ALG_AAI_CTR_MOD128,
        },
        {
-                .name = "rfc3686(ctr(aes))",
+                .skcipher = {
-                .driver_name = "rfc3686-ctr-aes-caam-qi",
+                        .base = {
-                .blocksize = 1,
+                                .cra_name = "rfc3686(ctr(aes))",
-                .type = CRYPTO_ALG_TYPE_GIVCIPHER,
+                                .cra_driver_name = "rfc3686-ctr-aes-caam-qi",
-                .template_ablkcipher = {
+                                .cra_blocksize = 1,
-                        .setkey = ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .givencrypt = ablkcipher_givencrypt,
+                        .decrypt = skcipher_decrypt,
-                        .geniv = "<built-in>",
                        .min_keysize = AES_MIN_KEY_SIZE +
                                       CTR_RFC3686_NONCE_SIZE,
                        .max_keysize = AES_MAX_KEY_SIZE +
                                       CTR_RFC3686_NONCE_SIZE,
                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .chunksize = AES_BLOCK_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .rfc3686 = true,
                },
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
        },
        {
-                .name = "xts(aes)",
+                .skcipher = {
-                .driver_name = "xts-aes-caam-qi",
+                        .base = {
-                .blocksize = AES_BLOCK_SIZE,
+                                .cra_name = "xts(aes)",
-                .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+                                .cra_driver_name = "xts-aes-caam-qi",
-                .template_ablkcipher = {
+                                .cra_blocksize = AES_BLOCK_SIZE,
-                        .setkey = xts_ablkcipher_setkey,
+                        },
-                        .encrypt = ablkcipher_encrypt,
+                        .setkey = xts_skcipher_setkey,
-                        .decrypt = ablkcipher_decrypt,
+                        .encrypt = skcipher_encrypt,
-                        .geniv = "eseqiv",
+                        .decrypt = skcipher_decrypt,
                        .min_keysize = 2 * AES_MIN_KEY_SIZE,
                        .max_keysize = 2 * AES_MAX_KEY_SIZE,
                        .ivsize = AES_BLOCK_SIZE,
                },
-                .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
        },
 };
@@ -2528,12 +2327,6 @@ static struct caam_aead_alg driver_aeads[] = {
        },
 };
-struct caam_crypto_alg {
-        struct list_head entry;
-        struct crypto_alg crypto_alg;
-        struct caam_alg_entry caam;
-};
 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
                            bool uses_dkp)
 {
@@ -2572,19 +2365,18 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
        spin_lock_init(&ctx->lock);
        ctx->drv_ctx[ENCRYPT] = NULL;
        ctx->drv_ctx[DECRYPT] = NULL;
-        ctx->drv_ctx[GIVENCRYPT] = NULL;
        return 0;
 }
-static int caam_cra_init(struct crypto_tfm *tfm)
+static int caam_cra_init(struct crypto_skcipher *tfm)
 {
-        struct crypto_alg *alg = tfm->__crt_alg;
+        struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
-        struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg),
+        struct caam_skcipher_alg *caam_alg =
-                                                        crypto_alg);
+                container_of(alg, typeof(*caam_alg), skcipher);
-        struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
-        return caam_init_common(ctx, &caam_alg->caam, false);
+        return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
+                                false);
 }
 static int caam_aead_init(struct crypto_aead *tfm)
@@ -2602,16 +2394,15 @@ static void caam_exit_common(struct caam_ctx *ctx)
 {
        caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
        caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
-        caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]);
        dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
        caam_jr_free(ctx->jrdev);
 }
-static void caam_cra_exit(struct crypto_tfm *tfm)
+static void caam_cra_exit(struct crypto_skcipher *tfm)
 {
-        caam_exit_common(crypto_tfm_ctx(tfm));
+        caam_exit_common(crypto_skcipher_ctx(tfm));
 }
 static void caam_aead_exit(struct crypto_aead *tfm)
@@ -2619,10 +2410,8 @@ static void caam_aead_exit(struct crypto_aead *tfm)
        caam_exit_common(crypto_aead_ctx(tfm));
 }
-static struct list_head alg_list;
 static void __exit caam_qi_algapi_exit(void)
 {
-        struct caam_crypto_alg *t_alg, *n;
        int i;
        for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
@@ -2632,55 +2421,25 @@ static void __exit caam_qi_algapi_exit(void)
                        crypto_unregister_aead(&t_alg->aead);
        }
-        if (!alg_list.next)
+        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
-                return;
+                struct caam_skcipher_alg *t_alg = driver_algs + i;
-        list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
+                if (t_alg->registered)
-                crypto_unregister_alg(&t_alg->crypto_alg);
+                        crypto_unregister_skcipher(&t_alg->skcipher);
-                list_del(&t_alg->entry);
-                kfree(t_alg);
        }
 }
-static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
+static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
-                                              *template)
 {
-        struct caam_crypto_alg *t_alg;
+        struct skcipher_alg *alg = &t_alg->skcipher;
-        struct crypto_alg *alg;
-        t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
-        if (!t_alg)
-                return ERR_PTR(-ENOMEM);
-        alg = &t_alg->crypto_alg;
+        alg->base.cra_module = THIS_MODULE;
+        alg->base.cra_priority = CAAM_CRA_PRIORITY;
-        snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
+        alg->base.cra_ctxsize = sizeof(struct caam_ctx);
-        snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+        alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
-                 template->driver_name);
-        alg->cra_module = THIS_MODULE;
-        alg->cra_init = caam_cra_init;
-        alg->cra_exit = caam_cra_exit;
-        alg->cra_priority = CAAM_CRA_PRIORITY;
-        alg->cra_blocksize = template->blocksize;
-        alg->cra_alignmask = 0;
-        alg->cra_ctxsize = sizeof(struct caam_ctx);
-        alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
-                         template->type;
-        switch (template->type) {
-        case CRYPTO_ALG_TYPE_GIVCIPHER:
-                alg->cra_type = &crypto_givcipher_type;
-                alg->cra_ablkcipher = template->template_ablkcipher;
-                break;
-        case CRYPTO_ALG_TYPE_ABLKCIPHER:
-                alg->cra_type = &crypto_ablkcipher_type;
-                alg->cra_ablkcipher = template->template_ablkcipher;
-                break;
-        }
-        t_alg->caam.class1_alg_type = template->class1_alg_type;
-        t_alg->caam.class2_alg_type = template->class2_alg_type;
-        return t_alg;
+        alg->init = caam_cra_init;
+        alg->exit = caam_cra_exit;
 }
 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
@@ -2734,8 +2493,6 @@ static int __init caam_qi_algapi_init(void)
                return -ENODEV;
        }
-        INIT_LIST_HEAD(&alg_list);
        /*
         * Register crypto algorithms the device supports.
         * First, detect presence and attributes of DES, AES, and MD blocks.
@@ -2751,9 +2508,8 @@ static int __init caam_qi_algapi_init(void)
                md_limit = SHA256_DIGEST_SIZE;
        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
-                struct caam_crypto_alg *t_alg;
+                struct caam_skcipher_alg *t_alg = driver_algs + i;
-                struct caam_alg_template *alg = driver_algs + i;
+                u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
-                u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
                /* Skip DES algorithms if not supported by device */
                if (!des_inst &&
@@ -2765,23 +2521,16 @@ static int __init caam_qi_algapi_init(void)
                if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
                        continue;
-                t_alg = caam_alg_alloc(alg);
+                caam_skcipher_alg_init(t_alg);
-                if (IS_ERR(t_alg)) {
-                        err = PTR_ERR(t_alg);
-                        dev_warn(priv->qidev, "%s alg allocation failed\n",
-                                 alg->driver_name);
-                        continue;
-                }
-                err = crypto_register_alg(&t_alg->crypto_alg);
+                err = crypto_register_skcipher(&t_alg->skcipher);
                if (err) {
                        dev_warn(priv->qidev, "%s alg registration failed\n",
-                                 t_alg->crypto_alg.cra_driver_name);
+                                 t_alg->skcipher.base.cra_driver_name);
-                        kfree(t_alg);
                        continue;
                }
-                list_add_tail(&t_alg->entry, &alg_list);
+                t_alg->registered = true;
                registered = true;
        }
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
new file mode 100644
index 000000000000..7d8ac0222fa3
--- /dev/null
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -0,0 +1,5165 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright 2015-2016 Freescale Semiconductor Inc.
+ * Copyright 2017-2018 NXP
+ */
+#include "compat.h"
+#include "regs.h"
+#include "caamalg_qi2.h"
+#include "dpseci_cmd.h"
+#include "desc_constr.h"
+#include "error.h"
+#include "sg_sw_sec4.h"
+#include "sg_sw_qm2.h"
+#include "key_gen.h"
+#include "caamalg_desc.h"
+#include "caamhash_desc.h"
+#include <linux/fsl/mc.h>
+#include <soc/fsl/dpaa2-io.h>
+#include <soc/fsl/dpaa2-fd.h>
+#define CAAM_CRA_PRIORITY       2000
+/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
+#define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
+                                 SHA512_DIGEST_SIZE * 2)
+#if !IS_ENABLED(CONFIG_CRYPTO_DEV_FSL_CAAM)
+bool caam_little_end;
+EXPORT_SYMBOL(caam_little_end);
+bool caam_imx;
+EXPORT_SYMBOL(caam_imx);
+#endif
+/*
+ * This is a a cache of buffers, from which the users of CAAM QI driver
+ * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
+ * NOTE: A more elegant solution would be to have some headroom in the frames
+ *       being processed. This can be added by the dpaa2-eth driver. This would
+ *       pose a problem for userspace application processing which cannot
+ *       know of this limitation. So for now, this will work.
+ * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
+ */
+static struct kmem_cache *qi_cache;
+struct caam_alg_entry {
+        struct device *dev;
+        int class1_alg_type;
+        int class2_alg_type;
+        bool rfc3686;
+        bool geniv;
+};
+struct caam_aead_alg {
+        struct aead_alg aead;
+        struct caam_alg_entry caam;
+        bool registered;
+};
+struct caam_skcipher_alg {
+        struct skcipher_alg skcipher;
+        struct caam_alg_entry caam;
+        bool registered;
+};
+/**
+ * caam_ctx - per-session context
+ * @flc: Flow Contexts array
+ * @key:  [authentication key], encryption key
+ * @flc_dma: I/O virtual addresses of the Flow Contexts
+ * @key_dma: I/O virtual address of the key
+ * @dir: DMA direction for mapping key and Flow Contexts
+ * @dev: dpseci device
+ * @adata: authentication algorithm details
+ * @cdata: encryption algorithm details
+ * @authsize: authentication tag (a.k.a. ICV / MAC) size
+ */
+struct caam_ctx {
+        struct caam_flc flc[NUM_OP];
+        u8 key[CAAM_MAX_KEY_SIZE];
+        dma_addr_t flc_dma[NUM_OP];
+        dma_addr_t key_dma;
+        enum dma_data_direction dir;
+        struct device *dev;
+        struct alginfo adata;
+        struct alginfo cdata;
+        unsigned int authsize;
+};
+static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
+                                     dma_addr_t iova_addr)
+{
+        phys_addr_t phys_addr;
+        phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
+                                   iova_addr;
+        return phys_to_virt(phys_addr);
+}
+/*
+ * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
+ *
+ * Allocate data on the hotpath. Instead of using kzalloc, one can use the
+ * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
+ * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
+ * hosting 16 SG entries.
+ *
+ * @flags - flags that would be used for the equivalent kmalloc(..) call
+ *
+ * Returns a pointer to a retrieved buffer on success or NULL on failure.
+ */
+static inline void *qi_cache_zalloc(gfp_t flags)
+{
+        return kmem_cache_zalloc(qi_cache, flags);
+}
+/*
+ * qi_cache_free - Frees buffers allocated from CAAM-QI cache
+ *
+ * @obj - buffer previously allocated by qi_cache_zalloc
+ *
+ * No checking is being done, the call is a passthrough call to
+ * kmem_cache_free(...)
+ */
+static inline void qi_cache_free(void *obj)
+{
+        kmem_cache_free(qi_cache, obj);
+}
+static struct caam_request *to_caam_req(struct crypto_async_request *areq)
+{
+        switch (crypto_tfm_alg_type(areq->tfm)) {
+        case CRYPTO_ALG_TYPE_SKCIPHER:
+                return skcipher_request_ctx(skcipher_request_cast(areq));
+        case CRYPTO_ALG_TYPE_AEAD:
+                return aead_request_ctx(container_of(areq, struct aead_request,
+                                                     base));
+        case CRYPTO_ALG_TYPE_AHASH:
+                return ahash_request_ctx(ahash_request_cast(areq));
+        default:
+                return ERR_PTR(-EINVAL);
+        }
+}
+static void caam_unmap(struct device *dev, struct scatterlist *src,
+                       struct scatterlist *dst, int src_nents,
+                       int dst_nents, dma_addr_t iv_dma, int ivsize,
+                       dma_addr_t qm_sg_dma, int qm_sg_bytes)
+{
+        if (dst != src) {
+                if (src_nents)
+                        dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
+                dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
+        } else {
+                dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
+        }
+        if (iv_dma)
+                dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+        if (qm_sg_bytes)
+                dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
+}
+static int aead_set_sh_desc(struct crypto_aead *aead)
+{
+        struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
+                                                 typeof(*alg), aead);
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        unsigned int ivsize = crypto_aead_ivsize(aead);
+        struct device *dev = ctx->dev;
+        struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
+        struct caam_flc *flc;
+        u32 *desc;
+        u32 ctx1_iv_off = 0;
+        u32 *nonce = NULL;
+        unsigned int data_len[2];
+        u32 inl_mask;
+        const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
+                               OP_ALG_AAI_CTR_MOD128);
+        const bool is_rfc3686 = alg->caam.rfc3686;
+        if (!ctx->cdata.keylen || !ctx->authsize)
+                return 0;
+        /*
+         * AES-CTR needs to load IV in CONTEXT1 reg
+         * at an offset of 128bits (16bytes)
+         * CONTEXT1[255:128] = IV
+         */
+        if (ctr_mode)
+                ctx1_iv_off = 16;
+        /*
+         * RFC3686 specific:
+         *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
+         */
+        if (is_rfc3686) {
+                ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+                nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
+                                ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
+        }
+        data_len[0] = ctx->adata.keylen_pad;
+        data_len[1] = ctx->cdata.keylen;
+        /* aead_encrypt shared descriptor */
+        if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
+                                                 DESC_QI_AEAD_ENC_LEN) +
+                              (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
+                              DESC_JOB_IO_LEN, data_len, &inl_mask,
+                              ARRAY_SIZE(data_len)) < 0)
+                return -EINVAL;
+        if (inl_mask & 1)
+                ctx->adata.key_virt = ctx->key;
+        else
+                ctx->adata.key_dma = ctx->key_dma;
+        if (inl_mask & 2)
+                ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+        else
+                ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
+        ctx->adata.key_inline = !!(inl_mask & 1);
+        ctx->cdata.key_inline = !!(inl_mask & 2);
+        flc = &ctx->flc[ENCRYPT];
+        desc = flc->sh_desc;
+        if (alg->caam.geniv)
+                cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
+                                          ivsize, ctx->authsize, is_rfc3686,
+                                          nonce, ctx1_iv_off, true,
+                                          priv->sec_attr.era);
+        else
+                cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
+                                       ivsize, ctx->authsize, is_rfc3686, nonce,
+                                       ctx1_iv_off, true, priv->sec_attr.era);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        /* aead_decrypt shared descriptor */
+        if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
+                              (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
+                              DESC_JOB_IO_LEN, data_len, &inl_mask,
+                              ARRAY_SIZE(data_len)) < 0)
+                return -EINVAL;
+        if (inl_mask & 1)
+                ctx->adata.key_virt = ctx->key;
+        else
+                ctx->adata.key_dma = ctx->key_dma;
+        if (inl_mask & 2)
+                ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+        else
+                ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
+        ctx->adata.key_inline = !!(inl_mask & 1);
+        ctx->cdata.key_inline = !!(inl_mask & 2);
+        flc = &ctx->flc[DECRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
+                               ivsize, ctx->authsize, alg->caam.geniv,
+                               is_rfc3686, nonce, ctx1_iv_off, true,
+                               priv->sec_attr.era);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        return 0;
+}
+static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+        ctx->authsize = authsize;
+        aead_set_sh_desc(authenc);
+        return 0;
+}
+static int aead_setkey(struct crypto_aead *aead, const u8 *key,
+                       unsigned int keylen)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        struct crypto_authenc_keys keys;
+        if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
+                goto badkey;
+        dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
+                keys.authkeylen + keys.enckeylen, keys.enckeylen,
+                keys.authkeylen);
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+        ctx->adata.keylen = keys.authkeylen;
+        ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
+                                              OP_ALG_ALGSEL_MASK);
+        if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
+                goto badkey;
+        memcpy(ctx->key, keys.authkey, keys.authkeylen);
+        memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
+        dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
+                                   keys.enckeylen, ctx->dir);
+        print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+                             ctx->adata.keylen_pad + keys.enckeylen, 1);
+        ctx->cdata.keylen = keys.enckeylen;
+        memzero_explicit(&keys, sizeof(keys));
+        return aead_set_sh_desc(aead);
+badkey:
+        crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        memzero_explicit(&keys, sizeof(keys));
+        return -EINVAL;
+}
+static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
+                                           bool encrypt)
+{
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct caam_request *req_ctx = aead_request_ctx(req);
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
+                                                 typeof(*alg), aead);
+        struct device *dev = ctx->dev;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+        struct aead_edesc *edesc;
+        dma_addr_t qm_sg_dma, iv_dma = 0;
+        int ivsize = 0;
+        unsigned int authsize = ctx->authsize;
+        int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
+        int in_len, out_len;
+        struct dpaa2_sg_entry *sg_table;
+        /* allocate space for base edesc, link tables and IV */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (unlikely(!edesc)) {
+                dev_err(dev, "could not allocate extended descriptor\n");
+                return ERR_PTR(-ENOMEM);
+        }
+        if (unlikely(req->dst != req->src)) {
+                src_nents = sg_nents_for_len(req->src, req->assoclen +
+                                             req->cryptlen);
+                if (unlikely(src_nents < 0)) {
+                        dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
+                                req->assoclen + req->cryptlen);
+                        qi_cache_free(edesc);
+                        return ERR_PTR(src_nents);
+                }
+                dst_nents = sg_nents_for_len(req->dst, req->assoclen +
+                                             req->cryptlen +
+                                             (encrypt ? authsize :
+                                                        (-authsize)));
+                if (unlikely(dst_nents < 0)) {
+                        dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
+                                req->assoclen + req->cryptlen +
+                                (encrypt ? authsize : (-authsize)));
+                        qi_cache_free(edesc);
+                        return ERR_PTR(dst_nents);
+                }
+                if (src_nents) {
+                        mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
+                                                      DMA_TO_DEVICE);
+                        if (unlikely(!mapped_src_nents)) {
+                                dev_err(dev, "unable to map source\n");
+                                qi_cache_free(edesc);
+                                return ERR_PTR(-ENOMEM);
+                        }
+                } else {
+                        mapped_src_nents = 0;
+                }
+                mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
+                                              DMA_FROM_DEVICE);
+                if (unlikely(!mapped_dst_nents)) {
+                        dev_err(dev, "unable to map destination\n");
+                        dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
+                        qi_cache_free(edesc);
+                        return ERR_PTR(-ENOMEM);
+                }
+        } else {
+                src_nents = sg_nents_for_len(req->src, req->assoclen +
+                                             req->cryptlen +
+                                                (encrypt ? authsize : 0));
+                if (unlikely(src_nents < 0)) {
+                        dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
+                                req->assoclen + req->cryptlen +
+                                (encrypt ? authsize : 0));
+                        qi_cache_free(edesc);
+                        return ERR_PTR(src_nents);
+                }
+                mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
+                                              DMA_BIDIRECTIONAL);
+                if (unlikely(!mapped_src_nents)) {
+                        dev_err(dev, "unable to map source\n");
+                        qi_cache_free(edesc);
+                        return ERR_PTR(-ENOMEM);
+                }
+        }
+        if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
+                ivsize = crypto_aead_ivsize(aead);
+        /*
+         * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
+         * Input is not contiguous.
+         */
+        qm_sg_nents = 1 + !!ivsize + mapped_src_nents +
+                      (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
+        sg_table = &edesc->sgt[0];
+        qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
+        if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
+                     CAAM_QI_MEMCACHE_SIZE)) {
+                dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
+                        qm_sg_nents, ivsize);
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
+                           0, 0, 0);
+                qi_cache_free(edesc);
+                return ERR_PTR(-ENOMEM);
+        }
+        if (ivsize) {
+                u8 *iv = (u8 *)(sg_table + qm_sg_nents);
+                /* Make sure IV is located in a DMAable area */
+                memcpy(iv, req->iv, ivsize);
+                iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
+                if (dma_mapping_error(dev, iv_dma)) {
+                        dev_err(dev, "unable to map IV\n");
+                        caam_unmap(dev, req->src, req->dst, src_nents,
+                                   dst_nents, 0, 0, 0, 0);
+                        qi_cache_free(edesc);
+                        return ERR_PTR(-ENOMEM);
+                }
+        }
+        edesc->src_nents = src_nents;
+        edesc->dst_nents = dst_nents;
+        edesc->iv_dma = iv_dma;
+        edesc->assoclen = cpu_to_caam32(req->assoclen);
+        edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
+                                             DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, edesc->assoclen_dma)) {
+                dev_err(dev, "unable to map assoclen\n");
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
+                           iv_dma, ivsize, 0, 0);
+                qi_cache_free(edesc);
+                return ERR_PTR(-ENOMEM);
+        }
+        dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
+        qm_sg_index++;
+        if (ivsize) {
+                dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
+                qm_sg_index++;
+        }
+        sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
+        qm_sg_index += mapped_src_nents;
+        if (mapped_dst_nents > 1)
+                sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
+                                 qm_sg_index, 0);
+        qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, qm_sg_dma)) {
+                dev_err(dev, "unable to map S/G table\n");
+                dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
+                           iv_dma, ivsize, 0, 0);
+                qi_cache_free(edesc);
+                return ERR_PTR(-ENOMEM);
+        }
+        edesc->qm_sg_dma = qm_sg_dma;
+        edesc->qm_sg_bytes = qm_sg_bytes;
+        out_len = req->assoclen + req->cryptlen +
+                  (encrypt ? ctx->authsize : (-ctx->authsize));
+        in_len = 4 + ivsize + req->assoclen + req->cryptlen;
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+        dpaa2_fl_set_addr(in_fle, qm_sg_dma);
+        dpaa2_fl_set_len(in_fle, in_len);
+        if (req->dst == req->src) {
+                if (mapped_src_nents == 1) {
+                        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+                        dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
+                } else {
+                        dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+                        dpaa2_fl_set_addr(out_fle, qm_sg_dma +
+                                          (1 + !!ivsize) * sizeof(*sg_table));
+                }
+        } else if (mapped_dst_nents == 1) {
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+                dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
+        } else {
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+                dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
+                                  sizeof(*sg_table));
+        }
+        dpaa2_fl_set_len(out_fle, out_len);
+        return edesc;
+}
+static int gcm_set_sh_desc(struct crypto_aead *aead)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        unsigned int ivsize = crypto_aead_ivsize(aead);
+        struct caam_flc *flc;
+        u32 *desc;
+        int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
+                        ctx->cdata.keylen;
+        if (!ctx->cdata.keylen || !ctx->authsize)
+                return 0;
+        /*
+         * AES GCM encrypt shared descriptor
+         * Job Descriptor and Shared Descriptor
+         * must fit into the 64-word Descriptor h/w Buffer
+         */
+        if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
+                ctx->cdata.key_inline = true;
+                ctx->cdata.key_virt = ctx->key;
+        } else {
+                ctx->cdata.key_inline = false;
+                ctx->cdata.key_dma = ctx->key_dma;
+        }
+        flc = &ctx->flc[ENCRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        /*
+         * Job Descriptor and Shared Descriptors
+         * must all fit into the 64-word Descriptor h/w Buffer
+         */
+        if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
+                ctx->cdata.key_inline = true;
+                ctx->cdata.key_virt = ctx->key;
+        } else {
+                ctx->cdata.key_inline = false;
+                ctx->cdata.key_dma = ctx->key_dma;
+        }
+        flc = &ctx->flc[DECRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        return 0;
+}
+static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+        ctx->authsize = authsize;
+        gcm_set_sh_desc(authenc);
+        return 0;
+}
+static int gcm_setkey(struct crypto_aead *aead,
+                      const u8 *key, unsigned int keylen)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+        memcpy(ctx->key, key, keylen);
+        dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
+        ctx->cdata.keylen = keylen;
+        return gcm_set_sh_desc(aead);
+}
+static int rfc4106_set_sh_desc(struct crypto_aead *aead)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        unsigned int ivsize = crypto_aead_ivsize(aead);
+        struct caam_flc *flc;
+        u32 *desc;
+        int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
+                        ctx->cdata.keylen;
+        if (!ctx->cdata.keylen || !ctx->authsize)
+                return 0;
+        ctx->cdata.key_virt = ctx->key;
+        /*
+         * RFC4106 encrypt shared descriptor
+         * Job Descriptor and Shared Descriptor
+         * must fit into the 64-word Descriptor h/w Buffer
+         */
+        if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
+                ctx->cdata.key_inline = true;
+        } else {
+                ctx->cdata.key_inline = false;
+                ctx->cdata.key_dma = ctx->key_dma;
+        }
+        flc = &ctx->flc[ENCRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
+                                  true);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        /*
+         * Job Descriptor and Shared Descriptors
+         * must all fit into the 64-word Descriptor h/w Buffer
+         */
+        if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
+                ctx->cdata.key_inline = true;
+        } else {
+                ctx->cdata.key_inline = false;
+                ctx->cdata.key_dma = ctx->key_dma;
+        }
+        flc = &ctx->flc[DECRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
+                                  true);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        return 0;
+}
+static int rfc4106_setauthsize(struct crypto_aead *authenc,
+                               unsigned int authsize)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+        ctx->authsize = authsize;
+        rfc4106_set_sh_desc(authenc);
+        return 0;
+}
+static int rfc4106_setkey(struct crypto_aead *aead,
+                          const u8 *key, unsigned int keylen)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        if (keylen < 4)
+                return -EINVAL;
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+        memcpy(ctx->key, key, keylen);
+        /*
+         * The last four bytes of the key material are used as the salt value
+         * in the nonce. Update the AES key length.
+         */
+        ctx->cdata.keylen = keylen - 4;
+        dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
+                                   ctx->dir);
+        return rfc4106_set_sh_desc(aead);
+}
+static int rfc4543_set_sh_desc(struct crypto_aead *aead)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        unsigned int ivsize = crypto_aead_ivsize(aead);
+        struct caam_flc *flc;
+        u32 *desc;
+        int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
+                        ctx->cdata.keylen;
+        if (!ctx->cdata.keylen || !ctx->authsize)
+                return 0;
+        ctx->cdata.key_virt = ctx->key;
+        /*
+         * RFC4543 encrypt shared descriptor
+         * Job Descriptor and Shared Descriptor
+         * must fit into the 64-word Descriptor h/w Buffer
+         */
+        if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
+                ctx->cdata.key_inline = true;
+        } else {
+                ctx->cdata.key_inline = false;
+                ctx->cdata.key_dma = ctx->key_dma;
+        }
+        flc = &ctx->flc[ENCRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
+                                  true);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        /*
+         * Job Descriptor and Shared Descriptors
+         * must all fit into the 64-word Descriptor h/w Buffer
+         */
+        if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
+                ctx->cdata.key_inline = true;
+        } else {
+                ctx->cdata.key_inline = false;
+                ctx->cdata.key_dma = ctx->key_dma;
+        }
+        flc = &ctx->flc[DECRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
+                                  true);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        return 0;
+}
+static int rfc4543_setauthsize(struct crypto_aead *authenc,
+                               unsigned int authsize)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+        ctx->authsize = authsize;
+        rfc4543_set_sh_desc(authenc);
+        return 0;
+}
+static int rfc4543_setkey(struct crypto_aead *aead,
+                          const u8 *key, unsigned int keylen)
+{
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct device *dev = ctx->dev;
+        if (keylen < 4)
+                return -EINVAL;
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+        memcpy(ctx->key, key, keylen);
+        /*
+         * The last four bytes of the key material are used as the salt value
+         * in the nonce. Update the AES key length.
+         */
+        ctx->cdata.keylen = keylen - 4;
+        dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
+                                   ctx->dir);
+        return rfc4543_set_sh_desc(aead);
+}
+static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
+                           unsigned int keylen)
+{
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct caam_skcipher_alg *alg =
+                container_of(crypto_skcipher_alg(skcipher),
+                             struct caam_skcipher_alg, skcipher);
+        struct device *dev = ctx->dev;
+        struct caam_flc *flc;
+        unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
+        u32 *desc;
+        u32 ctx1_iv_off = 0;
+        const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
+                               OP_ALG_AAI_CTR_MOD128);
+        const bool is_rfc3686 = alg->caam.rfc3686;
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+        /*
+         * AES-CTR needs to load IV in CONTEXT1 reg
+         * at an offset of 128bits (16bytes)
+         * CONTEXT1[255:128] = IV
+         */
+        if (ctr_mode)
+                ctx1_iv_off = 16;
+        /*
+         * RFC3686 specific:
+         *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
+         *      | *key = {KEY, NONCE}
+         */
+        if (is_rfc3686) {
+                ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+                keylen -= CTR_RFC3686_NONCE_SIZE;
+        }
+        ctx->cdata.keylen = keylen;
+        ctx->cdata.key_virt = key;
+        ctx->cdata.key_inline = true;
+        /* skcipher_encrypt shared descriptor */
+        flc = &ctx->flc[ENCRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
+                                   ctx1_iv_off);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        /* skcipher_decrypt shared descriptor */
+        flc = &ctx->flc[DECRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
+                                   ctx1_iv_off);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        return 0;
+}
+static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
+                               unsigned int keylen)
+{
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct device *dev = ctx->dev;
+        struct caam_flc *flc;
+        u32 *desc;
+        if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
+                dev_err(dev, "key size mismatch\n");
+                crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        ctx->cdata.keylen = keylen;
+        ctx->cdata.key_virt = key;
+        ctx->cdata.key_inline = true;
+        /* xts_skcipher_encrypt shared descriptor */
+        flc = &ctx->flc[ENCRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        /* xts_skcipher_decrypt shared descriptor */
+        flc = &ctx->flc[DECRYPT];
+        desc = flc->sh_desc;
+        cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
+                                   sizeof(flc->flc) + desc_bytes(desc),
+                                   ctx->dir);
+        return 0;
+}
+static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
+{
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+        struct caam_request *req_ctx = skcipher_request_ctx(req);
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct device *dev = ctx->dev;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
+        int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+        struct skcipher_edesc *edesc;
+        dma_addr_t iv_dma;
+        u8 *iv;
+        int ivsize = crypto_skcipher_ivsize(skcipher);
+        int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
+        struct dpaa2_sg_entry *sg_table;
+        src_nents = sg_nents_for_len(req->src, req->cryptlen);
+        if (unlikely(src_nents < 0)) {
+                dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
+                        req->cryptlen);
+                return ERR_PTR(src_nents);
+        }
+        if (unlikely(req->dst != req->src)) {
+                dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
+                if (unlikely(dst_nents < 0)) {
+                        dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
+                                req->cryptlen);
+                        return ERR_PTR(dst_nents);
+                }
+                mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
+                                              DMA_TO_DEVICE);
+                if (unlikely(!mapped_src_nents)) {
+                        dev_err(dev, "unable to map source\n");
+                        return ERR_PTR(-ENOMEM);
+                }
+                mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
+                                              DMA_FROM_DEVICE);
+                if (unlikely(!mapped_dst_nents)) {
+                        dev_err(dev, "unable to map destination\n");
+                        dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
+                        return ERR_PTR(-ENOMEM);
+                }
+        } else {
+                mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
+                                              DMA_BIDIRECTIONAL);
+                if (unlikely(!mapped_src_nents)) {
+                        dev_err(dev, "unable to map source\n");
+                        return ERR_PTR(-ENOMEM);
+                }
+        }
+        qm_sg_ents = 1 + mapped_src_nents;
+        dst_sg_idx = qm_sg_ents;
+        qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+        qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
+        if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
+                     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
+                dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
+                        qm_sg_ents, ivsize);
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
+                           0, 0, 0);
+                return ERR_PTR(-ENOMEM);
+        }
+        /* allocate space for base edesc, link tables and IV */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (unlikely(!edesc)) {
+                dev_err(dev, "could not allocate extended descriptor\n");
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
+                           0, 0, 0);
+                return ERR_PTR(-ENOMEM);
+        }
+        /* Make sure IV is located in a DMAable area */
+        sg_table = &edesc->sgt[0];
+        iv = (u8 *)(sg_table + qm_sg_ents);
+        memcpy(iv, req->iv, ivsize);
+        iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, iv_dma)) {
+                dev_err(dev, "unable to map IV\n");
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
+                           0, 0, 0);
+                qi_cache_free(edesc);
+                return ERR_PTR(-ENOMEM);
+        }
+        edesc->src_nents = src_nents;
+        edesc->dst_nents = dst_nents;
+        edesc->iv_dma = iv_dma;
+        edesc->qm_sg_bytes = qm_sg_bytes;
+        dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
+        sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
+        if (mapped_dst_nents > 1)
+                sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
+                                 dst_sg_idx, 0);
+        edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
+                                          DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
+                dev_err(dev, "unable to map S/G table\n");
+                caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
+                           iv_dma, ivsize, 0, 0);
+                qi_cache_free(edesc);
+                return ERR_PTR(-ENOMEM);
+        }
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
+        dpaa2_fl_set_len(out_fle, req->cryptlen);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+        dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+        if (req->src == req->dst) {
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+                dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
+                                  sizeof(*sg_table));
+        } else if (mapped_dst_nents > 1) {
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+                dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
+                                  sizeof(*sg_table));
+        } else {
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+                dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
+        }
+        return edesc;
+}
+static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
+                       struct aead_request *req)
+{
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        int ivsize = crypto_aead_ivsize(aead);
+        caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
+                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+        dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
+}
+static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
+                           struct skcipher_request *req)
+{
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
+        caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
+                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+}
+static void aead_encrypt_done(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct aead_request *req = container_of(areq, struct aead_request,
+                                                base);
+        struct caam_request *req_ctx = to_caam_req(areq);
+        struct aead_edesc *edesc = req_ctx->edesc;
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        int ecode = 0;
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        aead_unmap(ctx->dev, edesc, req);
+        qi_cache_free(edesc);
+        aead_request_complete(req, ecode);
+}
+static void aead_decrypt_done(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct aead_request *req = container_of(areq, struct aead_request,
+                                                base);
+        struct caam_request *req_ctx = to_caam_req(areq);
+        struct aead_edesc *edesc = req_ctx->edesc;
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        int ecode = 0;
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                /*
+                 * verify hw auth check passed else return -EBADMSG
+                 */
+                if ((status & JRSTA_CCBERR_ERRID_MASK) ==
+                     JRSTA_CCBERR_ERRID_ICVCHK)
+                        ecode = -EBADMSG;
+                else
+                        ecode = -EIO;
+        }
+        aead_unmap(ctx->dev, edesc, req);
+        qi_cache_free(edesc);
+        aead_request_complete(req, ecode);
+}
+static int aead_encrypt(struct aead_request *req)
+{
+        struct aead_edesc *edesc;
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct caam_request *caam_req = aead_request_ctx(req);
+        int ret;
+        /* allocate extended descriptor */
+        edesc = aead_edesc_alloc(req, true);
+        if (IS_ERR(edesc))
+                return PTR_ERR(edesc);
+        caam_req->flc = &ctx->flc[ENCRYPT];
+        caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
+        caam_req->cbk = aead_encrypt_done;
+        caam_req->ctx = &req->base;
+        caam_req->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
+        if (ret != -EINPROGRESS &&
+            !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+                aead_unmap(ctx->dev, edesc, req);
+                qi_cache_free(edesc);
+        }
+        return ret;
+}
+static int aead_decrypt(struct aead_request *req)
+{
+        struct aead_edesc *edesc;
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct caam_ctx *ctx = crypto_aead_ctx(aead);
+        struct caam_request *caam_req = aead_request_ctx(req);
+        int ret;
+        /* allocate extended descriptor */
+        edesc = aead_edesc_alloc(req, false);
+        if (IS_ERR(edesc))
+                return PTR_ERR(edesc);
+        caam_req->flc = &ctx->flc[DECRYPT];
+        caam_req->flc_dma = ctx->flc_dma[DECRYPT];
+        caam_req->cbk = aead_decrypt_done;
+        caam_req->ctx = &req->base;
+        caam_req->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
+        if (ret != -EINPROGRESS &&
+            !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+                aead_unmap(ctx->dev, edesc, req);
+                qi_cache_free(edesc);
+        }
+        return ret;
+}
+static int ipsec_gcm_encrypt(struct aead_request *req)
+{
+        if (req->assoclen < 8)
+                return -EINVAL;
+        return aead_encrypt(req);
+}
+static int ipsec_gcm_decrypt(struct aead_request *req)
+{
+        if (req->assoclen < 8)
+                return -EINVAL;
+        return aead_decrypt(req);
+}
+static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct skcipher_request *req = skcipher_request_cast(areq);
+        struct caam_request *req_ctx = to_caam_req(areq);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct skcipher_edesc *edesc = req_ctx->edesc;
+        int ecode = 0;
+        int ivsize = crypto_skcipher_ivsize(skcipher);
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+                             edesc->src_nents > 1 ? 100 : ivsize, 1);
+        caam_dump_sg(KERN_DEBUG, "dst    @" __stringify(__LINE__)": ",
+                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
+        skcipher_unmap(ctx->dev, edesc, req);
+        /*
+         * The crypto API expects us to set the IV (req->iv) to the last
+         * ciphertext block. This is used e.g. by the CTS mode.
+         */
+        scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
+                                 ivsize, 0);
+        qi_cache_free(edesc);
+        skcipher_request_complete(req, ecode);
+}
+static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct skcipher_request *req = skcipher_request_cast(areq);
+        struct caam_request *req_ctx = to_caam_req(areq);
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct skcipher_edesc *edesc = req_ctx->edesc;
+        int ecode = 0;
+        int ivsize = crypto_skcipher_ivsize(skcipher);
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+                             edesc->src_nents > 1 ? 100 : ivsize, 1);
+        caam_dump_sg(KERN_DEBUG, "dst    @" __stringify(__LINE__)": ",
+                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
+        skcipher_unmap(ctx->dev, edesc, req);
+        qi_cache_free(edesc);
+        skcipher_request_complete(req, ecode);
+}
+static int skcipher_encrypt(struct skcipher_request *req)
+{
+        struct skcipher_edesc *edesc;
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct caam_request *caam_req = skcipher_request_ctx(req);
+        int ret;
+        /* allocate extended descriptor */
+        edesc = skcipher_edesc_alloc(req);
+        if (IS_ERR(edesc))
+                return PTR_ERR(edesc);
+        caam_req->flc = &ctx->flc[ENCRYPT];
+        caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
+        caam_req->cbk = skcipher_encrypt_done;
+        caam_req->ctx = &req->base;
+        caam_req->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
+        if (ret != -EINPROGRESS &&
+            !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+                skcipher_unmap(ctx->dev, edesc, req);
+                qi_cache_free(edesc);
+        }
+        return ret;
+}
+static int skcipher_decrypt(struct skcipher_request *req)
+{
+        struct skcipher_edesc *edesc;
+        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct caam_request *caam_req = skcipher_request_ctx(req);
+        int ivsize = crypto_skcipher_ivsize(skcipher);
+        int ret;
+        /* allocate extended descriptor */
+        edesc = skcipher_edesc_alloc(req);
+        if (IS_ERR(edesc))
+                return PTR_ERR(edesc);
+        /*
+         * The crypto API expects us to set the IV (req->iv) to the last
+         * ciphertext block.
+         */
+        scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
+                                 ivsize, 0);
+        caam_req->flc = &ctx->flc[DECRYPT];
+        caam_req->flc_dma = ctx->flc_dma[DECRYPT];
+        caam_req->cbk = skcipher_decrypt_done;
+        caam_req->ctx = &req->base;
+        caam_req->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
+        if (ret != -EINPROGRESS &&
+            !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+                skcipher_unmap(ctx->dev, edesc, req);
+                qi_cache_free(edesc);
+        }
+        return ret;
+}
+static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
+                         bool uses_dkp)
+{
+        dma_addr_t dma_addr;
+        int i;
+        /* copy descriptor header template value */
+        ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
+        ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
+        ctx->dev = caam->dev;
+        ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+        dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
+                                        offsetof(struct caam_ctx, flc_dma),
+                                        ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
+        if (dma_mapping_error(ctx->dev, dma_addr)) {
+                dev_err(ctx->dev, "unable to map key, shared descriptors\n");
+                return -ENOMEM;
+        }
+        for (i = 0; i < NUM_OP; i++)
+                ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
+        ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
+        return 0;
+}
+static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
+{
+        struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+        struct caam_skcipher_alg *caam_alg =
+                container_of(alg, typeof(*caam_alg), skcipher);
+        crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
+        return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
+}
+static int caam_cra_init_aead(struct crypto_aead *tfm)
+{
+        struct aead_alg *alg = crypto_aead_alg(tfm);
+        struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
+                                                      aead);
+        crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
+        return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
+                             alg->setkey == aead_setkey);
+}
+static void caam_exit_common(struct caam_ctx *ctx)
+{
+        dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
+                               offsetof(struct caam_ctx, flc_dma), ctx->dir,
+                               DMA_ATTR_SKIP_CPU_SYNC);
+}
+static void caam_cra_exit(struct crypto_skcipher *tfm)
+{
+        caam_exit_common(crypto_skcipher_ctx(tfm));
+}
+static void caam_cra_exit_aead(struct crypto_aead *tfm)
+{
+        caam_exit_common(crypto_aead_ctx(tfm));
+}
+static struct caam_skcipher_alg driver_algs[] = {
+        {
+                .skcipher = {
+                        .base = {
+                                .cra_name = "cbc(aes)",
+                                .cra_driver_name = "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = skcipher_setkey,
+                        .encrypt = skcipher_encrypt,
+                        .decrypt = skcipher_decrypt,
+                        .min_keysize = AES_MIN_KEY_SIZE,
+                        .max_keysize = AES_MAX_KEY_SIZE,
+                        .ivsize = AES_BLOCK_SIZE,
+                },
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+        },
+        {
+                .skcipher = {
+                        .base = {
+                                .cra_name = "cbc(des3_ede)",
+                                .cra_driver_name = "cbc-3des-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = skcipher_setkey,
+                        .encrypt = skcipher_encrypt,
+                        .decrypt = skcipher_decrypt,
+                        .min_keysize = DES3_EDE_KEY_SIZE,
+                        .max_keysize = DES3_EDE_KEY_SIZE,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                },
+                .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+        },
+        {
+                .skcipher = {
+                        .base = {
+                                .cra_name = "cbc(des)",
+                                .cra_driver_name = "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = skcipher_setkey,
+                        .encrypt = skcipher_encrypt,
+                        .decrypt = skcipher_decrypt,
+                        .min_keysize = DES_KEY_SIZE,
+                        .max_keysize = DES_KEY_SIZE,
+                        .ivsize = DES_BLOCK_SIZE,
+                },
+                .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+        },
+        {
+                .skcipher = {
+                        .base = {
+                                .cra_name = "ctr(aes)",
+                                .cra_driver_name = "ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = skcipher_setkey,
+                        .encrypt = skcipher_encrypt,
+                        .decrypt = skcipher_decrypt,
+                        .min_keysize = AES_MIN_KEY_SIZE,
+                        .max_keysize = AES_MAX_KEY_SIZE,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .chunksize = AES_BLOCK_SIZE,
+                },
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
+                                        OP_ALG_AAI_CTR_MOD128,
+        },
+        {
+                .skcipher = {
+                        .base = {
+                                .cra_name = "rfc3686(ctr(aes))",
+                                .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = skcipher_setkey,
+                        .encrypt = skcipher_encrypt,
+                        .decrypt = skcipher_decrypt,
+                        .min_keysize = AES_MIN_KEY_SIZE +
+                                       CTR_RFC3686_NONCE_SIZE,
+                        .max_keysize = AES_MAX_KEY_SIZE +
+                                       CTR_RFC3686_NONCE_SIZE,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .chunksize = AES_BLOCK_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .skcipher = {
+                        .base = {
+                                .cra_name = "xts(aes)",
+                                .cra_driver_name = "xts-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = xts_skcipher_setkey,
+                        .encrypt = skcipher_encrypt,
+                        .decrypt = skcipher_decrypt,
+                        .min_keysize = 2 * AES_MIN_KEY_SIZE,
+                        .max_keysize = 2 * AES_MAX_KEY_SIZE,
+                        .ivsize = AES_BLOCK_SIZE,
+                },
+                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
+        }
+};
+static struct caam_aead_alg driver_aeads[] = {
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "rfc4106(gcm(aes))",
+                                .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = rfc4106_setkey,
+                        .setauthsize = rfc4106_setauthsize,
+                        .encrypt = ipsec_gcm_encrypt,
+                        .decrypt = ipsec_gcm_decrypt,
+                        .ivsize = 8,
+                        .maxauthsize = AES_BLOCK_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "rfc4543(gcm(aes))",
+                                .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = rfc4543_setkey,
+                        .setauthsize = rfc4543_setauthsize,
+                        .encrypt = ipsec_gcm_encrypt,
+                        .decrypt = ipsec_gcm_decrypt,
+                        .ivsize = 8,
+                        .maxauthsize = AES_BLOCK_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
+                },
+        },
+        /* Galois Counter Mode */
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "gcm(aes)",
+                                .cra_driver_name = "gcm-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = gcm_setkey,
+                        .setauthsize = gcm_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = 12,
+                        .maxauthsize = AES_BLOCK_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
+                }
+        },
+        /* single-pass ipsec_esp descriptor */
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(md5),cbc(aes))",
+                                .cra_driver_name = "authenc-hmac-md5-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(md5),"
+                                            "cbc(aes)))",
+                                .cra_driver_name = "echainiv-authenc-hmac-md5-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha1),cbc(aes))",
+                                .cra_driver_name = "authenc-hmac-sha1-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha1),"
+                                            "cbc(aes)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha1-cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha224),cbc(aes))",
+                                .cra_driver_name = "authenc-hmac-sha224-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha224),"
+                                            "cbc(aes)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha224-cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha256),cbc(aes))",
+                                .cra_driver_name = "authenc-hmac-sha256-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha256),"
+                                            "cbc(aes)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha256-cbc-aes-"
+                                                   "caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha384),cbc(aes))",
+                                .cra_driver_name = "authenc-hmac-sha384-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha384),"
+                                            "cbc(aes)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha384-cbc-aes-"
+                                                   "caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha512),cbc(aes))",
+                                .cra_driver_name = "authenc-hmac-sha512-"
+                                                   "cbc-aes-caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha512),"
+                                            "cbc(aes)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha512-cbc-aes-"
+                                                   "caam-qi2",
+                                .cra_blocksize = AES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = AES_BLOCK_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
+                                .cra_driver_name = "authenc-hmac-md5-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(md5),"
+                                            "cbc(des3_ede)))",
+                                .cra_driver_name = "echainiv-authenc-hmac-md5-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha1),"
+                                            "cbc(des3_ede))",
+                                .cra_driver_name = "authenc-hmac-sha1-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha1),"
+                                            "cbc(des3_ede)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha1-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha224),"
+                                            "cbc(des3_ede))",
+                                .cra_driver_name = "authenc-hmac-sha224-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha224),"
+                                            "cbc(des3_ede)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha224-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha256),"
+                                            "cbc(des3_ede))",
+                                .cra_driver_name = "authenc-hmac-sha256-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha256),"
+                                            "cbc(des3_ede)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha256-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha384),"
+                                            "cbc(des3_ede))",
+                                .cra_driver_name = "authenc-hmac-sha384-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha384),"
+                                            "cbc(des3_ede)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha384-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha512),"
+                                            "cbc(des3_ede))",
+                                .cra_driver_name = "authenc-hmac-sha512-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha512),"
+                                            "cbc(des3_ede)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha512-"
+                                                   "cbc-des3_ede-caam-qi2",
+                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES3_EDE_BLOCK_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(md5),cbc(des))",
+                                .cra_driver_name = "authenc-hmac-md5-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(md5),"
+                                            "cbc(des)))",
+                                .cra_driver_name = "echainiv-authenc-hmac-md5-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha1),cbc(des))",
+                                .cra_driver_name = "authenc-hmac-sha1-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha1),"
+                                            "cbc(des)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha1-cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha224),cbc(des))",
+                                .cra_driver_name = "authenc-hmac-sha224-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha224),"
+                                            "cbc(des)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha224-cbc-des-"
+                                                   "caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha256),cbc(des))",
+                                .cra_driver_name = "authenc-hmac-sha256-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha256),"
+                                            "cbc(des)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha256-cbc-desi-"
+                                                   "caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha384),cbc(des))",
+                                .cra_driver_name = "authenc-hmac-sha384-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha384),"
+                                            "cbc(des)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha384-cbc-des-"
+                                                   "caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha512),cbc(des))",
+                                .cra_driver_name = "authenc-hmac-sha512-"
+                                                   "cbc-des-caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "echainiv(authenc(hmac(sha512),"
+                                            "cbc(des)))",
+                                .cra_driver_name = "echainiv-authenc-"
+                                                   "hmac-sha512-cbc-des-"
+                                                   "caam-qi2",
+                                .cra_blocksize = DES_BLOCK_SIZE,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .geniv = true,
+                }
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(md5),"
+                                            "rfc3686(ctr(aes)))",
+                                .cra_driver_name = "authenc-hmac-md5-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "seqiv(authenc("
+                                            "hmac(md5),rfc3686(ctr(aes))))",
+                                .cra_driver_name = "seqiv-authenc-hmac-md5-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = MD5_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha1),"
+                                            "rfc3686(ctr(aes)))",
+                                .cra_driver_name = "authenc-hmac-sha1-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "seqiv(authenc("
+                                            "hmac(sha1),rfc3686(ctr(aes))))",
+                                .cra_driver_name = "seqiv-authenc-hmac-sha1-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA1_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha224),"
+                                            "rfc3686(ctr(aes)))",
+                                .cra_driver_name = "authenc-hmac-sha224-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "seqiv(authenc("
+                                            "hmac(sha224),rfc3686(ctr(aes))))",
+                                .cra_driver_name = "seqiv-authenc-hmac-sha224-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA224_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha256),"
+                                            "rfc3686(ctr(aes)))",
+                                .cra_driver_name = "authenc-hmac-sha256-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "seqiv(authenc(hmac(sha256),"
+                                            "rfc3686(ctr(aes))))",
+                                .cra_driver_name = "seqiv-authenc-hmac-sha256-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA256_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha384),"
+                                            "rfc3686(ctr(aes)))",
+                                .cra_driver_name = "authenc-hmac-sha384-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "seqiv(authenc(hmac(sha384),"
+                                            "rfc3686(ctr(aes))))",
+                                .cra_driver_name = "seqiv-authenc-hmac-sha384-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA384_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                        .geniv = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "authenc(hmac(sha512),"
+                                            "rfc3686(ctr(aes)))",
+                                .cra_driver_name = "authenc-hmac-sha512-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                },
+        },
+        {
+                .aead = {
+                        .base = {
+                                .cra_name = "seqiv(authenc(hmac(sha512),"
+                                            "rfc3686(ctr(aes))))",
+                                .cra_driver_name = "seqiv-authenc-hmac-sha512-"
+                                                   "rfc3686-ctr-aes-caam-qi2",
+                                .cra_blocksize = 1,
+                        },
+                        .setkey = aead_setkey,
+                        .setauthsize = aead_setauthsize,
+                        .encrypt = aead_encrypt,
+                        .decrypt = aead_decrypt,
+                        .ivsize = CTR_RFC3686_IV_SIZE,
+                        .maxauthsize = SHA512_DIGEST_SIZE,
+                },
+                .caam = {
+                        .class1_alg_type = OP_ALG_ALGSEL_AES |
+                                           OP_ALG_AAI_CTR_MOD128,
+                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+                                           OP_ALG_AAI_HMAC_PRECOMP,
+                        .rfc3686 = true,
+                        .geniv = true,
+                },
+        },
+};
+static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
+{
+        struct skcipher_alg *alg = &t_alg->skcipher;
+        alg->base.cra_module = THIS_MODULE;
+        alg->base.cra_priority = CAAM_CRA_PRIORITY;
+        alg->base.cra_ctxsize = sizeof(struct caam_ctx);
+        alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
+        alg->init = caam_cra_init_skcipher;
+        alg->exit = caam_cra_exit;
+}
+static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
+{
+        struct aead_alg *alg = &t_alg->aead;
+        alg->base.cra_module = THIS_MODULE;
+        alg->base.cra_priority = CAAM_CRA_PRIORITY;
+        alg->base.cra_ctxsize = sizeof(struct caam_ctx);
+        alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
+        alg->init = caam_cra_init_aead;
+        alg->exit = caam_cra_exit_aead;
+}
+/* max hash key is max split key size */
+#define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
+#define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
+/* caam context sizes for hashes: running digest + 8 */
+#define HASH_MSG_LEN                    8
+#define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
+enum hash_optype {
+        UPDATE = 0,
+        UPDATE_FIRST,
+        FINALIZE,
+        DIGEST,
+        HASH_NUM_OP
+};
+/**
+ * caam_hash_ctx - ahash per-session context
+ * @flc: Flow Contexts array
+ * @flc_dma: I/O virtual addresses of the Flow Contexts
+ * @dev: dpseci device
+ * @ctx_len: size of Context Register
+ * @adata: hashing algorithm details
+ */
+struct caam_hash_ctx {
+        struct caam_flc flc[HASH_NUM_OP];
+        dma_addr_t flc_dma[HASH_NUM_OP];
+        struct device *dev;
+        int ctx_len;
+        struct alginfo adata;
+};
+/* ahash state */
+struct caam_hash_state {
+        struct caam_request caam_req;
+        dma_addr_t buf_dma;
+        dma_addr_t ctx_dma;
+        u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
+        int buflen_0;
+        u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
+        int buflen_1;
+        u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
+        int (*update)(struct ahash_request *req);
+        int (*final)(struct ahash_request *req);
+        int (*finup)(struct ahash_request *req);
+        int current_buf;
+};
+struct caam_export_state {
+        u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
+        u8 caam_ctx[MAX_CTX_LEN];
+        int buflen;
+        int (*update)(struct ahash_request *req);
+        int (*final)(struct ahash_request *req);
+        int (*finup)(struct ahash_request *req);
+};
+static inline void switch_buf(struct caam_hash_state *state)
+{
+        state->current_buf ^= 1;
+}
+static inline u8 *current_buf(struct caam_hash_state *state)
+{
+        return state->current_buf ? state->buf_1 : state->buf_0;
+}
+static inline u8 *alt_buf(struct caam_hash_state *state)
+{
+        return state->current_buf ? state->buf_0 : state->buf_1;
+}
+static inline int *current_buflen(struct caam_hash_state *state)
+{
+        return state->current_buf ? &state->buflen_1 : &state->buflen_0;
+}
+static inline int *alt_buflen(struct caam_hash_state *state)
+{
+        return state->current_buf ? &state->buflen_0 : &state->buflen_1;
+}
+/* Map current buffer in state (if length > 0) and put it in link table */
+static inline int buf_map_to_qm_sg(struct device *dev,
+                                   struct dpaa2_sg_entry *qm_sg,
+                                   struct caam_hash_state *state)
+{
+        int buflen = *current_buflen(state);
+        if (!buflen)
+                return 0;
+        state->buf_dma = dma_map_single(dev, current_buf(state), buflen,
+                                        DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, state->buf_dma)) {
+                dev_err(dev, "unable to map buf\n");
+                state->buf_dma = 0;
+                return -ENOMEM;
+        }
+        dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
+        return 0;
+}
+/* Map state->caam_ctx, and add it to link table */
+static inline int ctx_map_to_qm_sg(struct device *dev,
+                                   struct caam_hash_state *state, int ctx_len,
+                                   struct dpaa2_sg_entry *qm_sg, u32 flag)
+{
+        state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
+        if (dma_mapping_error(dev, state->ctx_dma)) {
+                dev_err(dev, "unable to map ctx\n");
+                state->ctx_dma = 0;
+                return -ENOMEM;
+        }
+        dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
+        return 0;
+}
+static int ahash_set_sh_desc(struct crypto_ahash *ahash)
+{
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        int digestsize = crypto_ahash_digestsize(ahash);
+        struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
+        struct caam_flc *flc;
+        u32 *desc;
+        /* ahash_update shared descriptor */
+        flc = &ctx->flc[UPDATE];
+        desc = flc->sh_desc;
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
+                          ctx->ctx_len, true, priv->sec_attr.era);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
+                                   desc_bytes(desc), DMA_BIDIRECTIONAL);
+        print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
+        /* ahash_update_first shared descriptor */
+        flc = &ctx->flc[UPDATE_FIRST];
+        desc = flc->sh_desc;
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
+                          ctx->ctx_len, false, priv->sec_attr.era);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
+                                   desc_bytes(desc), DMA_BIDIRECTIONAL);
+        print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
+        /* ahash_final shared descriptor */
+        flc = &ctx->flc[FINALIZE];
+        desc = flc->sh_desc;
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
+                          ctx->ctx_len, true, priv->sec_attr.era);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
+                                   desc_bytes(desc), DMA_BIDIRECTIONAL);
+        print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
+        /* ahash_digest shared descriptor */
+        flc = &ctx->flc[DIGEST];
+        desc = flc->sh_desc;
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
+                          ctx->ctx_len, false, priv->sec_attr.era);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
+                                   desc_bytes(desc), DMA_BIDIRECTIONAL);
+        print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
+        return 0;
+}
+struct split_key_sh_result {
+        struct completion completion;
+        int err;
+        struct device *dev;
+};
+static void split_key_sh_done(void *cbk_ctx, u32 err)
+{
+        struct split_key_sh_result *res = cbk_ctx;
+        dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+        if (err)
+                caam_qi2_strstatus(res->dev, err);
+        res->err = err;
+        complete(&res->completion);
+}
+/* Digest hash size if it is too large */
+static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
+                           u32 *keylen, u8 *key_out, u32 digestsize)
+{
+        struct caam_request *req_ctx;
+        u32 *desc;
+        struct split_key_sh_result result;
+        dma_addr_t src_dma, dst_dma;
+        struct caam_flc *flc;
+        dma_addr_t flc_dma;
+        int ret = -ENOMEM;
+        struct dpaa2_fl_entry *in_fle, *out_fle;
+        req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
+        if (!req_ctx)
+                return -ENOMEM;
+        in_fle = &req_ctx->fd_flt[1];
+        out_fle = &req_ctx->fd_flt[0];
+        flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
+        if (!flc)
+                goto err_flc;
+        src_dma = dma_map_single(ctx->dev, (void *)key_in, *keylen,
+                                 DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, src_dma)) {
+                dev_err(ctx->dev, "unable to map key input memory\n");
+                goto err_src_dma;
+        }
+        dst_dma = dma_map_single(ctx->dev, (void *)key_out, digestsize,
+                                 DMA_FROM_DEVICE);
+        if (dma_mapping_error(ctx->dev, dst_dma)) {
+                dev_err(ctx->dev, "unable to map key output memory\n");
+                goto err_dst_dma;
+        }
+        desc = flc->sh_desc;
+        init_sh_desc(desc, 0);
+        /* descriptor to perform unkeyed hash on key_in */
+        append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
+                         OP_ALG_AS_INITFINAL);
+        append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
+                             FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
+        append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
+                         LDST_SRCDST_BYTE_CONTEXT);
+        flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
+        flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
+                                 desc_bytes(desc), DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, flc_dma)) {
+                dev_err(ctx->dev, "unable to map shared descriptor\n");
+                goto err_flc_dma;
+        }
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(in_fle, src_dma);
+        dpaa2_fl_set_len(in_fle, *keylen);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(out_fle, dst_dma);
+        dpaa2_fl_set_len(out_fle, digestsize);
+        print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
+        print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
+        result.err = 0;
+        init_completion(&result.completion);
+        result.dev = ctx->dev;
+        req_ctx->flc = flc;
+        req_ctx->flc_dma = flc_dma;
+        req_ctx->cbk = split_key_sh_done;
+        req_ctx->ctx = &result;
+        ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+        if (ret == -EINPROGRESS) {
+                /* in progress */
+                wait_for_completion(&result.completion);
+                ret = result.err;
+                print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, key_in,
+                                     digestsize, 1);
+        }
+        dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
+                         DMA_TO_DEVICE);
+err_flc_dma:
+        dma_unmap_single(ctx->dev, dst_dma, digestsize, DMA_FROM_DEVICE);
+err_dst_dma:
+        dma_unmap_single(ctx->dev, src_dma, *keylen, DMA_TO_DEVICE);
+err_src_dma:
+        kfree(flc);
+err_flc:
+        kfree(req_ctx);
+        *keylen = digestsize;
+        return ret;
+}
+static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
+                        unsigned int keylen)
+{
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
+        unsigned int digestsize = crypto_ahash_digestsize(ahash);
+        int ret;
+        u8 *hashed_key = NULL;
+        dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
+        if (keylen > blocksize) {
+                hashed_key = kmalloc_array(digestsize, sizeof(*hashed_key),
+                                           GFP_KERNEL | GFP_DMA);
+                if (!hashed_key)
+                        return -ENOMEM;
+                ret = hash_digest_key(ctx, key, &keylen, hashed_key,
+                                      digestsize);
+                if (ret)
+                        goto bad_free_key;
+                key = hashed_key;
+        }
+        ctx->adata.keylen = keylen;
+        ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
+                                              OP_ALG_ALGSEL_MASK);
+        if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
+                goto bad_free_key;
+        ctx->adata.key_virt = key;
+        ctx->adata.key_inline = true;
+        ret = ahash_set_sh_desc(ahash);
+        kfree(hashed_key);
+        return ret;
+bad_free_key:
+        kfree(hashed_key);
+        crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        return -EINVAL;
+}
+static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
+                               struct ahash_request *req, int dst_len)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        if (edesc->src_nents)
+                dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
+        if (edesc->dst_dma)
+                dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
+        if (edesc->qm_sg_bytes)
+                dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
+                                 DMA_TO_DEVICE);
+        if (state->buf_dma) {
+                dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
+                                 DMA_TO_DEVICE);
+                state->buf_dma = 0;
+        }
+}
+static inline void ahash_unmap_ctx(struct device *dev,
+                                   struct ahash_edesc *edesc,
+                                   struct ahash_request *req, int dst_len,
+                                   u32 flag)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        if (state->ctx_dma) {
+                dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
+                state->ctx_dma = 0;
+        }
+        ahash_unmap(dev, edesc, req, dst_len);
+}
+static void ahash_done(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct ahash_request *req = ahash_request_cast(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct ahash_edesc *edesc = state->caam_req.edesc;
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        int digestsize = crypto_ahash_digestsize(ahash);
+        int ecode = 0;
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        ahash_unmap(ctx->dev, edesc, req, digestsize);
+        qi_cache_free(edesc);
+        print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
+        if (req->result)
+                print_hex_dump_debug("result@" __stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+                                     digestsize, 1);
+        req->base.complete(&req->base, ecode);
+}
+static void ahash_done_bi(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct ahash_request *req = ahash_request_cast(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct ahash_edesc *edesc = state->caam_req.edesc;
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        int ecode = 0;
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
+        switch_buf(state);
+        qi_cache_free(edesc);
+        print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
+        if (req->result)
+                print_hex_dump_debug("result@" __stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+                                     crypto_ahash_digestsize(ahash), 1);
+        req->base.complete(&req->base, ecode);
+}
+static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct ahash_request *req = ahash_request_cast(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct ahash_edesc *edesc = state->caam_req.edesc;
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        int digestsize = crypto_ahash_digestsize(ahash);
+        int ecode = 0;
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_TO_DEVICE);
+        qi_cache_free(edesc);
+        print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
+        if (req->result)
+                print_hex_dump_debug("result@" __stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+                                     digestsize, 1);
+        req->base.complete(&req->base, ecode);
+}
+static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
+{
+        struct crypto_async_request *areq = cbk_ctx;
+        struct ahash_request *req = ahash_request_cast(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct ahash_edesc *edesc = state->caam_req.edesc;
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        int ecode = 0;
+        dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
+        if (unlikely(status)) {
+                caam_qi2_strstatus(ctx->dev, status);
+                ecode = -EIO;
+        }
+        ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
+        switch_buf(state);
+        qi_cache_free(edesc);
+        print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
+        if (req->result)
+                print_hex_dump_debug("result@" __stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+                                     crypto_ahash_digestsize(ahash), 1);
+        req->base.complete(&req->base, ecode);
+}
+static int ahash_update_ctx(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        u8 *buf = current_buf(state);
+        int *buflen = current_buflen(state);
+        u8 *next_buf = alt_buf(state);
+        int *next_buflen = alt_buflen(state), last_buflen;
+        int in_len = *buflen + req->nbytes, to_hash;
+        int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
+        struct ahash_edesc *edesc;
+        int ret = 0;
+        last_buflen = *next_buflen;
+        *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
+        to_hash = in_len - *next_buflen;
+        if (to_hash) {
+                struct dpaa2_sg_entry *sg_table;
+                src_nents = sg_nents_for_len(req->src,
+                                             req->nbytes - (*next_buflen));
+                if (src_nents < 0) {
+                        dev_err(ctx->dev, "Invalid number of src SG.\n");
+                        return src_nents;
+                }
+                if (src_nents) {
+                        mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
+                                                  DMA_TO_DEVICE);
+                        if (!mapped_nents) {
+                                dev_err(ctx->dev, "unable to DMA map source\n");
+                                return -ENOMEM;
+                        }
+                } else {
+                        mapped_nents = 0;
+                }
+                /* allocate space for base edesc and link tables */
+                edesc = qi_cache_zalloc(GFP_DMA | flags);
+                if (!edesc) {
+                        dma_unmap_sg(ctx->dev, req->src, src_nents,
+                                     DMA_TO_DEVICE);
+                        return -ENOMEM;
+                }
+                edesc->src_nents = src_nents;
+                qm_sg_src_index = 1 + (*buflen ? 1 : 0);
+                qm_sg_bytes = (qm_sg_src_index + mapped_nents) *
+                              sizeof(*sg_table);
+                sg_table = &edesc->sgt[0];
+                ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
+                                       DMA_BIDIRECTIONAL);
+                if (ret)
+                        goto unmap_ctx;
+                ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
+                if (ret)
+                        goto unmap_ctx;
+                if (mapped_nents) {
+                        sg_to_qm_sg_last(req->src, mapped_nents,
+                                         sg_table + qm_sg_src_index, 0);
+                        if (*next_buflen)
+                                scatterwalk_map_and_copy(next_buf, req->src,
+                                                         to_hash - *buflen,
+                                                         *next_buflen, 0);
+                } else {
+                        dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
+                                           true);
+                }
+                edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
+                                                  qm_sg_bytes, DMA_TO_DEVICE);
+                if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                        dev_err(ctx->dev, "unable to map S/G table\n");
+                        ret = -ENOMEM;
+                        goto unmap_ctx;
+                }
+                edesc->qm_sg_bytes = qm_sg_bytes;
+                memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+                dpaa2_fl_set_final(in_fle, true);
+                dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+                dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+                dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+                dpaa2_fl_set_addr(out_fle, state->ctx_dma);
+                dpaa2_fl_set_len(out_fle, ctx->ctx_len);
+                req_ctx->flc = &ctx->flc[UPDATE];
+                req_ctx->flc_dma = ctx->flc_dma[UPDATE];
+                req_ctx->cbk = ahash_done_bi;
+                req_ctx->ctx = &req->base;
+                req_ctx->edesc = edesc;
+                ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+                if (ret != -EINPROGRESS &&
+                    !(ret == -EBUSY &&
+                      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                        goto unmap_ctx;
+        } else if (*next_buflen) {
+                scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
+                                         req->nbytes, 0);
+                *buflen = *next_buflen;
+                *next_buflen = last_buflen;
+        }
+        print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
+        print_hex_dump_debug("next buf@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+                             1);
+        return ret;
+unmap_ctx:
+        ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_final_ctx(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        int buflen = *current_buflen(state);
+        int qm_sg_bytes, qm_sg_src_index;
+        int digestsize = crypto_ahash_digestsize(ahash);
+        struct ahash_edesc *edesc;
+        struct dpaa2_sg_entry *sg_table;
+        int ret;
+        /* allocate space for base edesc and link tables */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (!edesc)
+                return -ENOMEM;
+        qm_sg_src_index = 1 + (buflen ? 1 : 0);
+        qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table);
+        sg_table = &edesc->sgt[0];
+        ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
+                               DMA_TO_DEVICE);
+        if (ret)
+                goto unmap_ctx;
+        ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
+        if (ret)
+                goto unmap_ctx;
+        dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true);
+        edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
+                                          DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                dev_err(ctx->dev, "unable to map S/G table\n");
+                ret = -ENOMEM;
+                goto unmap_ctx;
+        }
+        edesc->qm_sg_bytes = qm_sg_bytes;
+        edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
+                                        DMA_FROM_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
+                dev_err(ctx->dev, "unable to map dst\n");
+                edesc->dst_dma = 0;
+                ret = -ENOMEM;
+                goto unmap_ctx;
+        }
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+        dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+        dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
+        dpaa2_fl_set_len(out_fle, digestsize);
+        req_ctx->flc = &ctx->flc[FINALIZE];
+        req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
+        req_ctx->cbk = ahash_done_ctx_src;
+        req_ctx->ctx = &req->base;
+        req_ctx->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+        if (ret == -EINPROGRESS ||
+            (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                return ret;
+unmap_ctx:
+        ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_finup_ctx(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        int buflen = *current_buflen(state);
+        int qm_sg_bytes, qm_sg_src_index;
+        int src_nents, mapped_nents;
+        int digestsize = crypto_ahash_digestsize(ahash);
+        struct ahash_edesc *edesc;
+        struct dpaa2_sg_entry *sg_table;
+        int ret;
+        src_nents = sg_nents_for_len(req->src, req->nbytes);
+        if (src_nents < 0) {
+                dev_err(ctx->dev, "Invalid number of src SG.\n");
+                return src_nents;
+        }
+        if (src_nents) {
+                mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
+                                          DMA_TO_DEVICE);
+                if (!mapped_nents) {
+                        dev_err(ctx->dev, "unable to DMA map source\n");
+                        return -ENOMEM;
+                }
+        } else {
+                mapped_nents = 0;
+        }
+        /* allocate space for base edesc and link tables */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (!edesc) {
+                dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
+                return -ENOMEM;
+        }
+        edesc->src_nents = src_nents;
+        qm_sg_src_index = 1 + (buflen ? 1 : 0);
+        qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table);
+        sg_table = &edesc->sgt[0];
+        ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
+                               DMA_TO_DEVICE);
+        if (ret)
+                goto unmap_ctx;
+        ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
+        if (ret)
+                goto unmap_ctx;
+        sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0);
+        edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
+                                          DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                dev_err(ctx->dev, "unable to map S/G table\n");
+                ret = -ENOMEM;
+                goto unmap_ctx;
+        }
+        edesc->qm_sg_bytes = qm_sg_bytes;
+        edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
+                                        DMA_FROM_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
+                dev_err(ctx->dev, "unable to map dst\n");
+                edesc->dst_dma = 0;
+                ret = -ENOMEM;
+                goto unmap_ctx;
+        }
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+        dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+        dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
+        dpaa2_fl_set_len(out_fle, digestsize);
+        req_ctx->flc = &ctx->flc[FINALIZE];
+        req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
+        req_ctx->cbk = ahash_done_ctx_src;
+        req_ctx->ctx = &req->base;
+        req_ctx->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+        if (ret == -EINPROGRESS ||
+            (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                return ret;
+unmap_ctx:
+        ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_digest(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        int digestsize = crypto_ahash_digestsize(ahash);
+        int src_nents, mapped_nents;
+        struct ahash_edesc *edesc;
+        int ret = -ENOMEM;
+        state->buf_dma = 0;
+        src_nents = sg_nents_for_len(req->src, req->nbytes);
+        if (src_nents < 0) {
+                dev_err(ctx->dev, "Invalid number of src SG.\n");
+                return src_nents;
+        }
+        if (src_nents) {
+                mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
+                                          DMA_TO_DEVICE);
+                if (!mapped_nents) {
+                        dev_err(ctx->dev, "unable to map source for DMA\n");
+                        return ret;
+                }
+        } else {
+                mapped_nents = 0;
+        }
+        /* allocate space for base edesc and link tables */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (!edesc) {
+                dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
+                return ret;
+        }
+        edesc->src_nents = src_nents;
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        if (mapped_nents > 1) {
+                int qm_sg_bytes;
+                struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
+                qm_sg_bytes = mapped_nents * sizeof(*sg_table);
+                sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
+                edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
+                                                  qm_sg_bytes, DMA_TO_DEVICE);
+                if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                        dev_err(ctx->dev, "unable to map S/G table\n");
+                        goto unmap;
+                }
+                edesc->qm_sg_bytes = qm_sg_bytes;
+                dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+                dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+        } else {
+                dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
+                dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
+        }
+        edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
+                                        DMA_FROM_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
+                dev_err(ctx->dev, "unable to map dst\n");
+                edesc->dst_dma = 0;
+                goto unmap;
+        }
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_len(in_fle, req->nbytes);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
+        dpaa2_fl_set_len(out_fle, digestsize);
+        req_ctx->flc = &ctx->flc[DIGEST];
+        req_ctx->flc_dma = ctx->flc_dma[DIGEST];
+        req_ctx->cbk = ahash_done;
+        req_ctx->ctx = &req->base;
+        req_ctx->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+        if (ret == -EINPROGRESS ||
+            (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                return ret;
+unmap:
+        ahash_unmap(ctx->dev, edesc, req, digestsize);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_final_no_ctx(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        u8 *buf = current_buf(state);
+        int buflen = *current_buflen(state);
+        int digestsize = crypto_ahash_digestsize(ahash);
+        struct ahash_edesc *edesc;
+        int ret = -ENOMEM;
+        /* allocate space for base edesc and link tables */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (!edesc)
+                return ret;
+        state->buf_dma = dma_map_single(ctx->dev, buf, buflen, DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, state->buf_dma)) {
+                dev_err(ctx->dev, "unable to map src\n");
+                goto unmap;
+        }
+        edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
+                                        DMA_FROM_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
+                dev_err(ctx->dev, "unable to map dst\n");
+                edesc->dst_dma = 0;
+                goto unmap;
+        }
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(in_fle, state->buf_dma);
+        dpaa2_fl_set_len(in_fle, buflen);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
+        dpaa2_fl_set_len(out_fle, digestsize);
+        req_ctx->flc = &ctx->flc[DIGEST];
+        req_ctx->flc_dma = ctx->flc_dma[DIGEST];
+        req_ctx->cbk = ahash_done;
+        req_ctx->ctx = &req->base;
+        req_ctx->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+        if (ret == -EINPROGRESS ||
+            (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                return ret;
+unmap:
+        ahash_unmap(ctx->dev, edesc, req, digestsize);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_update_no_ctx(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        u8 *buf = current_buf(state);
+        int *buflen = current_buflen(state);
+        u8 *next_buf = alt_buf(state);
+        int *next_buflen = alt_buflen(state);
+        int in_len = *buflen + req->nbytes, to_hash;
+        int qm_sg_bytes, src_nents, mapped_nents;
+        struct ahash_edesc *edesc;
+        int ret = 0;
+        *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
+        to_hash = in_len - *next_buflen;
+        if (to_hash) {
+                struct dpaa2_sg_entry *sg_table;
+                src_nents = sg_nents_for_len(req->src,
+                                             req->nbytes - *next_buflen);
+                if (src_nents < 0) {
+                        dev_err(ctx->dev, "Invalid number of src SG.\n");
+                        return src_nents;
+                }
+                if (src_nents) {
+                        mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
+                                                  DMA_TO_DEVICE);
+                        if (!mapped_nents) {
+                                dev_err(ctx->dev, "unable to DMA map source\n");
+                                return -ENOMEM;
+                        }
+                } else {
+                        mapped_nents = 0;
+                }
+                /* allocate space for base edesc and link tables */
+                edesc = qi_cache_zalloc(GFP_DMA | flags);
+                if (!edesc) {
+                        dma_unmap_sg(ctx->dev, req->src, src_nents,
+                                     DMA_TO_DEVICE);
+                        return -ENOMEM;
+                }
+                edesc->src_nents = src_nents;
+                qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table);
+                sg_table = &edesc->sgt[0];
+                ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
+                if (ret)
+                        goto unmap_ctx;
+                sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
+                if (*next_buflen)
+                        scatterwalk_map_and_copy(next_buf, req->src,
+                                                 to_hash - *buflen,
+                                                 *next_buflen, 0);
+                edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
+                                                  qm_sg_bytes, DMA_TO_DEVICE);
+                if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                        dev_err(ctx->dev, "unable to map S/G table\n");
+                        ret = -ENOMEM;
+                        goto unmap_ctx;
+                }
+                edesc->qm_sg_bytes = qm_sg_bytes;
+                state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
+                                                ctx->ctx_len, DMA_FROM_DEVICE);
+                if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
+                        dev_err(ctx->dev, "unable to map ctx\n");
+                        state->ctx_dma = 0;
+                        ret = -ENOMEM;
+                        goto unmap_ctx;
+                }
+                memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+                dpaa2_fl_set_final(in_fle, true);
+                dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+                dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+                dpaa2_fl_set_len(in_fle, to_hash);
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+                dpaa2_fl_set_addr(out_fle, state->ctx_dma);
+                dpaa2_fl_set_len(out_fle, ctx->ctx_len);
+                req_ctx->flc = &ctx->flc[UPDATE_FIRST];
+                req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
+                req_ctx->cbk = ahash_done_ctx_dst;
+                req_ctx->ctx = &req->base;
+                req_ctx->edesc = edesc;
+                ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+                if (ret != -EINPROGRESS &&
+                    !(ret == -EBUSY &&
+                      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                        goto unmap_ctx;
+                state->update = ahash_update_ctx;
+                state->finup = ahash_finup_ctx;
+                state->final = ahash_final_ctx;
+        } else if (*next_buflen) {
+                scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
+                                         req->nbytes, 0);
+                *buflen = *next_buflen;
+                *next_buflen = 0;
+        }
+        print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
+        print_hex_dump_debug("next buf@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+                             1);
+        return ret;
+unmap_ctx:
+        ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_finup_no_ctx(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        int buflen = *current_buflen(state);
+        int qm_sg_bytes, src_nents, mapped_nents;
+        int digestsize = crypto_ahash_digestsize(ahash);
+        struct ahash_edesc *edesc;
+        struct dpaa2_sg_entry *sg_table;
+        int ret;
+        src_nents = sg_nents_for_len(req->src, req->nbytes);
+        if (src_nents < 0) {
+                dev_err(ctx->dev, "Invalid number of src SG.\n");
+                return src_nents;
+        }
+        if (src_nents) {
+                mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
+                                          DMA_TO_DEVICE);
+                if (!mapped_nents) {
+                        dev_err(ctx->dev, "unable to DMA map source\n");
+                        return -ENOMEM;
+                }
+        } else {
+                mapped_nents = 0;
+        }
+        /* allocate space for base edesc and link tables */
+        edesc = qi_cache_zalloc(GFP_DMA | flags);
+        if (!edesc) {
+                dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
+                return -ENOMEM;
+        }
+        edesc->src_nents = src_nents;
+        qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table);
+        sg_table = &edesc->sgt[0];
+        ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
+        if (ret)
+                goto unmap;
+        sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
+        edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
+                                          DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                dev_err(ctx->dev, "unable to map S/G table\n");
+                ret = -ENOMEM;
+                goto unmap;
+        }
+        edesc->qm_sg_bytes = qm_sg_bytes;
+        edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
+                                        DMA_FROM_DEVICE);
+        if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
+                dev_err(ctx->dev, "unable to map dst\n");
+                edesc->dst_dma = 0;
+                ret = -ENOMEM;
+                goto unmap;
+        }
+        memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+        dpaa2_fl_set_final(in_fle, true);
+        dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+        dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+        dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+        dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
+        dpaa2_fl_set_len(out_fle, digestsize);
+        req_ctx->flc = &ctx->flc[DIGEST];
+        req_ctx->flc_dma = ctx->flc_dma[DIGEST];
+        req_ctx->cbk = ahash_done;
+        req_ctx->ctx = &req->base;
+        req_ctx->edesc = edesc;
+        ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+        if (ret != -EINPROGRESS &&
+            !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                goto unmap;
+        return ret;
+unmap:
+        ahash_unmap(ctx->dev, edesc, req, digestsize);
+        qi_cache_free(edesc);
+        return -ENOMEM;
+}
+static int ahash_update_first(struct ahash_request *req)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
+        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_request *req_ctx = &state->caam_req;
+        struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
+        struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                      GFP_KERNEL : GFP_ATOMIC;
+        u8 *next_buf = alt_buf(state);
+        int *next_buflen = alt_buflen(state);
+        int to_hash;
+        int src_nents, mapped_nents;
+        struct ahash_edesc *edesc;
+        int ret = 0;
+        *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
+                                      1);
+        to_hash = req->nbytes - *next_buflen;
+        if (to_hash) {
+                struct dpaa2_sg_entry *sg_table;
+                src_nents = sg_nents_for_len(req->src,
+                                             req->nbytes - (*next_buflen));
+                if (src_nents < 0) {
+                        dev_err(ctx->dev, "Invalid number of src SG.\n");
+                        return src_nents;
+                }
+                if (src_nents) {
+                        mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
+                                                  DMA_TO_DEVICE);
+                        if (!mapped_nents) {
+                                dev_err(ctx->dev, "unable to map source for DMA\n");
+                                return -ENOMEM;
+                        }
+                } else {
+                        mapped_nents = 0;
+                }
+                /* allocate space for base edesc and link tables */
+                edesc = qi_cache_zalloc(GFP_DMA | flags);
+                if (!edesc) {
+                        dma_unmap_sg(ctx->dev, req->src, src_nents,
+                                     DMA_TO_DEVICE);
+                        return -ENOMEM;
+                }
+                edesc->src_nents = src_nents;
+                sg_table = &edesc->sgt[0];
+                memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
+                dpaa2_fl_set_final(in_fle, true);
+                dpaa2_fl_set_len(in_fle, to_hash);
+                if (mapped_nents > 1) {
+                        int qm_sg_bytes;
+                        sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
+                        qm_sg_bytes = mapped_nents * sizeof(*sg_table);
+                        edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
+                                                          qm_sg_bytes,
+                                                          DMA_TO_DEVICE);
+                        if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
+                                dev_err(ctx->dev, "unable to map S/G table\n");
+                                ret = -ENOMEM;
+                                goto unmap_ctx;
+                        }
+                        edesc->qm_sg_bytes = qm_sg_bytes;
+                        dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
+                        dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
+                } else {
+                        dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
+                        dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
+                }
+                if (*next_buflen)
+                        scatterwalk_map_and_copy(next_buf, req->src, to_hash,
+                                                 *next_buflen, 0);
+                state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
+                                                ctx->ctx_len, DMA_FROM_DEVICE);
+                if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
+                        dev_err(ctx->dev, "unable to map ctx\n");
+                        state->ctx_dma = 0;
+                        ret = -ENOMEM;
+                        goto unmap_ctx;
+                }
+                dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
+                dpaa2_fl_set_addr(out_fle, state->ctx_dma);
+                dpaa2_fl_set_len(out_fle, ctx->ctx_len);
+                req_ctx->flc = &ctx->flc[UPDATE_FIRST];
+                req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
+                req_ctx->cbk = ahash_done_ctx_dst;
+                req_ctx->ctx = &req->base;
+                req_ctx->edesc = edesc;
+                ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
+                if (ret != -EINPROGRESS &&
+                    !(ret == -EBUSY && req->base.flags &
+                      CRYPTO_TFM_REQ_MAY_BACKLOG))
+                        goto unmap_ctx;
+                state->update = ahash_update_ctx;
+                state->finup = ahash_finup_ctx;
+                state->final = ahash_final_ctx;
+        } else if (*next_buflen) {
+                state->update = ahash_update_no_ctx;
+                state->finup = ahash_finup_no_ctx;
+                state->final = ahash_final_no_ctx;
+                scatterwalk_map_and_copy(next_buf, req->src, 0,
+                                         req->nbytes, 0);
+                switch_buf(state);
+        }
+        print_hex_dump_debug("next buf@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+                             1);
+        return ret;
+unmap_ctx:
+        ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
+        qi_cache_free(edesc);
+        return ret;
+}
+static int ahash_finup_first(struct ahash_request *req)
+{
+        return ahash_digest(req);
+}
+static int ahash_init(struct ahash_request *req)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        state->update = ahash_update_first;
+        state->finup = ahash_finup_first;
+        state->final = ahash_final_no_ctx;
+        state->ctx_dma = 0;
+        state->current_buf = 0;
+        state->buf_dma = 0;
+        state->buflen_0 = 0;
+        state->buflen_1 = 0;
+        return 0;
+}
+static int ahash_update(struct ahash_request *req)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        return state->update(req);
+}
+static int ahash_finup(struct ahash_request *req)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        return state->finup(req);
+}
+static int ahash_final(struct ahash_request *req)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        return state->final(req);
+}
+static int ahash_export(struct ahash_request *req, void *out)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        struct caam_export_state *export = out;
+        int len;
+        u8 *buf;
+        if (state->current_buf) {
+                buf = state->buf_1;
+                len = state->buflen_1;
+        } else {
+                buf = state->buf_0;
+                len = state->buflen_0;
+        }
+        memcpy(export->buf, buf, len);
+        memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
+        export->buflen = len;
+        export->update = state->update;
+        export->final = state->final;
+        export->finup = state->finup;
+        return 0;
+}
+static int ahash_import(struct ahash_request *req, const void *in)
+{
+        struct caam_hash_state *state = ahash_request_ctx(req);
+        const struct caam_export_state *export = in;
+        memset(state, 0, sizeof(*state));
+        memcpy(state->buf_0, export->buf, export->buflen);
+        memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
+        state->buflen_0 = export->buflen;
+        state->update = export->update;
+        state->final = export->final;
+        state->finup = export->finup;
+        return 0;
+}
+struct caam_hash_template {
+        char name[CRYPTO_MAX_ALG_NAME];
+        char driver_name[CRYPTO_MAX_ALG_NAME];
+        char hmac_name[CRYPTO_MAX_ALG_NAME];
+        char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
+        unsigned int blocksize;
+        struct ahash_alg template_ahash;
+        u32 alg_type;
+};
+/* ahash descriptors */
+static struct caam_hash_template driver_hash[] = {
+        {
+                .name = "sha1",
+                .driver_name = "sha1-caam-qi2",
+                .hmac_name = "hmac(sha1)",
+                .hmac_driver_name = "hmac-sha1-caam-qi2",
+                .blocksize = SHA1_BLOCK_SIZE,
+                .template_ahash = {
+                        .init = ahash_init,
+                        .update = ahash_update,
+                        .final = ahash_final,
+                        .finup = ahash_finup,
+                        .digest = ahash_digest,
+                        .export = ahash_export,
+                        .import = ahash_import,
+                        .setkey = ahash_setkey,
+                        .halg = {
+                                .digestsize = SHA1_DIGEST_SIZE,
+                                .statesize = sizeof(struct caam_export_state),
+                        },
+                },
+                .alg_type = OP_ALG_ALGSEL_SHA1,
+        }, {
+                .name = "sha224",
+                .driver_name = "sha224-caam-qi2",
+                .hmac_name = "hmac(sha224)",
+                .hmac_driver_name = "hmac-sha224-caam-qi2",
+                .blocksize = SHA224_BLOCK_SIZE,
+                .template_ahash = {
+                        .init = ahash_init,
+                        .update = ahash_update,
+                        .final = ahash_final,
+                        .finup = ahash_finup,
+                        .digest = ahash_digest,
+                        .export = ahash_export,
+                        .import = ahash_import,
+                        .setkey = ahash_setkey,
+                        .halg = {
+                                .digestsize = SHA224_DIGEST_SIZE,
+                                .statesize = sizeof(struct caam_export_state),
+                        },
+                },
+                .alg_type = OP_ALG_ALGSEL_SHA224,
+        }, {
+                .name = "sha256",
+                .driver_name = "sha256-caam-qi2",
+                .hmac_name = "hmac(sha256)",
+                .hmac_driver_name = "hmac-sha256-caam-qi2",
+                .blocksize = SHA256_BLOCK_SIZE,
+                .template_ahash = {
+                        .init = ahash_init,
+                        .update = ahash_update,
+                        .final = ahash_final,
+                        .finup = ahash_finup,
+                        .digest = ahash_digest,
+                        .export = ahash_export,
+                        .import = ahash_import,
+                        .setkey = ahash_setkey,
+                        .halg = {
+                                .digestsize = SHA256_DIGEST_SIZE,
+                                .statesize = sizeof(struct caam_export_state),
+                        },
+                },
+                .alg_type = OP_ALG_ALGSEL_SHA256,
+        }, {
+                .name = "sha384",
+                .driver_name = "sha384-caam-qi2",
+                .hmac_name = "hmac(sha384)",
+                .hmac_driver_name = "hmac-sha384-caam-qi2",
+                .blocksize = SHA384_BLOCK_SIZE,
+                .template_ahash = {
+                        .init = ahash_init,
+                        .update = ahash_update,
+                        .final = ahash_final,
+                        .finup = ahash_finup,
+                        .digest = ahash_digest,
+                        .export = ahash_export,
+                        .import = ahash_import,
+                        .setkey = ahash_setkey,
+                        .halg = {
+                                .digestsize = SHA384_DIGEST_SIZE,
+                                .statesize = sizeof(struct caam_export_state),
+                        },
+                },
+                .alg_type = OP_ALG_ALGSEL_SHA384,
+        }, {
+                .name = "sha512",
+                .driver_name = "sha512-caam-qi2",
+                .hmac_name = "hmac(sha512)",
+                .hmac_driver_name = "hmac-sha512-caam-qi2",
+                .blocksize = SHA512_BLOCK_SIZE,
+                .template_ahash = {
+                        .init = ahash_init,
+                        .update = ahash_update,
+                        .final = ahash_final,
+                        .finup = ahash_finup,
+                        .digest = ahash_digest,
+                        .export = ahash_export,
+                        .import = ahash_import,
+                        .setkey = ahash_setkey,
+                        .halg = {
+                                .digestsize = SHA512_DIGEST_SIZE,
+                                .statesize = sizeof(struct caam_export_state),
+                        },
+                },
+                .alg_type = OP_ALG_ALGSEL_SHA512,
+        }, {
+                .name = "md5",
+                .driver_name = "md5-caam-qi2",
+                .hmac_name = "hmac(md5)",
+                .hmac_driver_name = "hmac-md5-caam-qi2",
+                .blocksize = MD5_BLOCK_WORDS * 4,
+                .template_ahash = {
+                        .init = ahash_init,
+                        .update = ahash_update,
+                        .final = ahash_final,
+                        .finup = ahash_finup,
+                        .digest = ahash_digest,
+                        .export = ahash_export,
+                        .import = ahash_import,
+                        .setkey = ahash_setkey,
+                        .halg = {
+                                .digestsize = MD5_DIGEST_SIZE,
+                                .statesize = sizeof(struct caam_export_state),
+                        },
+                },
+                .alg_type = OP_ALG_ALGSEL_MD5,
+        }
+};
+struct caam_hash_alg {
+        struct list_head entry;
+        struct device *dev;
+        int alg_type;
+        struct ahash_alg ahash_alg;
+};
+static int caam_hash_cra_init(struct crypto_tfm *tfm)
+{
+        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
+        struct crypto_alg *base = tfm->__crt_alg;
+        struct hash_alg_common *halg =
+                 container_of(base, struct hash_alg_common, base);
+        struct ahash_alg *alg =
+                 container_of(halg, struct ahash_alg, halg);
+        struct caam_hash_alg *caam_hash =
+                 container_of(alg, struct caam_hash_alg, ahash_alg);
+        struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+        /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
+        static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
+                                         HASH_MSG_LEN + SHA1_DIGEST_SIZE,
+                                         HASH_MSG_LEN + 32,
+                                         HASH_MSG_LEN + SHA256_DIGEST_SIZE,
+                                         HASH_MSG_LEN + 64,
+                                         HASH_MSG_LEN + SHA512_DIGEST_SIZE };
+        dma_addr_t dma_addr;
+        int i;
+        ctx->dev = caam_hash->dev;
+        dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
+                                        DMA_BIDIRECTIONAL,
+                                        DMA_ATTR_SKIP_CPU_SYNC);
+        if (dma_mapping_error(ctx->dev, dma_addr)) {
+                dev_err(ctx->dev, "unable to map shared descriptors\n");
+                return -ENOMEM;
+        }
+        for (i = 0; i < HASH_NUM_OP; i++)
+                ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
+        /* copy descriptor header template value */
+        ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
+        ctx->ctx_len = runninglen[(ctx->adata.algtype &
+                                   OP_ALG_ALGSEL_SUBMASK) >>
+                                  OP_ALG_ALGSEL_SHIFT];
+        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+                                 sizeof(struct caam_hash_state));
+        return ahash_set_sh_desc(ahash);
+}
+static void caam_hash_cra_exit(struct crypto_tfm *tfm)
+{
+        struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+        dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
+                               DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
+}
+static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
+        struct caam_hash_template *template, bool keyed)
+{
+        struct caam_hash_alg *t_alg;
+        struct ahash_alg *halg;
+        struct crypto_alg *alg;
+        t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
+        if (!t_alg)
+                return ERR_PTR(-ENOMEM);
+        t_alg->ahash_alg = template->template_ahash;
+        halg = &t_alg->ahash_alg;
+        alg = &halg->halg.base;
+        if (keyed) {
+                snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
+                         template->hmac_name);
+                snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+                         template->hmac_driver_name);
+        } else {
+                snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
+                         template->name);
+                snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+                         template->driver_name);
+                t_alg->ahash_alg.setkey = NULL;
+        }
+        alg->cra_module = THIS_MODULE;
+        alg->cra_init = caam_hash_cra_init;
+        alg->cra_exit = caam_hash_cra_exit;
+        alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
+        alg->cra_priority = CAAM_CRA_PRIORITY;
+        alg->cra_blocksize = template->blocksize;
+        alg->cra_alignmask = 0;
+        alg->cra_flags = CRYPTO_ALG_ASYNC;
+        t_alg->alg_type = template->alg_type;
+        t_alg->dev = dev;
+        return t_alg;
+}
+static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
+{
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
+        napi_schedule_irqoff(&ppriv->napi);
+}
+static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
+{
+        struct device *dev = priv->dev;
+        struct dpaa2_io_notification_ctx *nctx;
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        int err, i = 0, cpu;
+        for_each_online_cpu(cpu) {
+                ppriv = per_cpu_ptr(priv->ppriv, cpu);
+                ppriv->priv = priv;
+                nctx = &ppriv->nctx;
+                nctx->is_cdan = 0;
+                nctx->id = ppriv->rsp_fqid;
+                nctx->desired_cpu = cpu;
+                nctx->cb = dpaa2_caam_fqdan_cb;
+                /* Register notification callbacks */
+                err = dpaa2_io_service_register(NULL, nctx);
+                if (unlikely(err)) {
+                        dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
+                        nctx->cb = NULL;
+                        /*
+                         * If no affine DPIO for this core, there's probably
+                         * none available for next cores either. Signal we want
+                         * to retry later, in case the DPIO devices weren't
+                         * probed yet.
+                         */
+                        err = -EPROBE_DEFER;
+                        goto err;
+                }
+                ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
+                                                     dev);
+                if (unlikely(!ppriv->store)) {
+                        dev_err(dev, "dpaa2_io_store_create() failed\n");
+                        err = -ENOMEM;
+                        goto err;
+                }
+                if (++i == priv->num_pairs)
+                        break;
+        }
+        return 0;
+err:
+        for_each_online_cpu(cpu) {
+                ppriv = per_cpu_ptr(priv->ppriv, cpu);
+                if (!ppriv->nctx.cb)
+                        break;
+                dpaa2_io_service_deregister(NULL, &ppriv->nctx);
+        }
+        for_each_online_cpu(cpu) {
+                ppriv = per_cpu_ptr(priv->ppriv, cpu);
+                if (!ppriv->store)
+                        break;
+                dpaa2_io_store_destroy(ppriv->store);
+        }
+        return err;
+}
+static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
+{
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        int i = 0, cpu;
+        for_each_online_cpu(cpu) {
+                ppriv = per_cpu_ptr(priv->ppriv, cpu);
+                dpaa2_io_service_deregister(NULL, &ppriv->nctx);
+                dpaa2_io_store_destroy(ppriv->store);
+                if (++i == priv->num_pairs)
+                        return;
+        }
+}
+static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
+{
+        struct dpseci_rx_queue_cfg rx_queue_cfg;
+        struct device *dev = priv->dev;
+        struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        int err = 0, i = 0, cpu;
+        /* Configure Rx queues */
+        for_each_online_cpu(cpu) {
+                ppriv = per_cpu_ptr(priv->ppriv, cpu);
+                rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
+                                       DPSECI_QUEUE_OPT_USER_CTX;
+                rx_queue_cfg.order_preservation_en = 0;
+                rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
+                rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
+                /*
+                 * Rx priority (WQ) doesn't really matter, since we use
+                 * pull mode, i.e. volatile dequeues from specific FQs
+                 */
+                rx_queue_cfg.dest_cfg.priority = 0;
+                rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
+                err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
+                                          &rx_queue_cfg);
+                if (err) {
+                        dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
+                                err);
+                        return err;
+                }
+                if (++i == priv->num_pairs)
+                        break;
+        }
+        return err;
+}
+static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
+{
+        struct device *dev = priv->dev;
+        if (!priv->cscn_mem)
+                return;
+        dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
+        kfree(priv->cscn_mem);
+}
+static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
+{
+        struct device *dev = priv->dev;
+        struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
+        dpaa2_dpseci_congestion_free(priv);
+        dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
+}
+static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
+                                  const struct dpaa2_fd *fd)
+{
+        struct caam_request *req;
+        u32 fd_err;
+        if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
+                dev_err(priv->dev, "Only Frame List FD format is supported!\n");
+                return;
+        }
+        fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
+        if (unlikely(fd_err))
+                dev_err(priv->dev, "FD error: %08x\n", fd_err);
+        /*
+         * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
+         * in FD[ERR] or FD[FRC].
+         */
+        req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
+        dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
+                         DMA_BIDIRECTIONAL);
+        req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
+}
+static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
+{
+        int err;
+        /* Retry while portal is busy */
+        do {
+                err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
+                                               ppriv->store);
+        } while (err == -EBUSY);
+        if (unlikely(err))
+                dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
+        return err;
+}
+static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
+{
+        struct dpaa2_dq *dq;
+        int cleaned = 0, is_last;
+        do {
+                dq = dpaa2_io_store_next(ppriv->store, &is_last);
+                if (unlikely(!dq)) {
+                        if (unlikely(!is_last)) {
+                                dev_dbg(ppriv->priv->dev,
+                                        "FQ %d returned no valid frames\n",
+                                        ppriv->rsp_fqid);
+                                /*
+                                 * MUST retry until we get some sort of
+                                 * valid response token (be it "empty dequeue"
+                                 * or a valid frame).
+                                 */
+                                continue;
+                        }
+                        break;
+                }
+                /* Process FD */
+                dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
+                cleaned++;
+        } while (!is_last);
+        return cleaned;
+}
+static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
+{
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        struct dpaa2_caam_priv *priv;
+        int err, cleaned = 0, store_cleaned;
+        ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
+        priv = ppriv->priv;
+        if (unlikely(dpaa2_caam_pull_fq(ppriv)))
+                return 0;
+        do {
+                store_cleaned = dpaa2_caam_store_consume(ppriv);
+                cleaned += store_cleaned;
+                if (store_cleaned == 0 ||
+                    cleaned > budget - DPAA2_CAAM_STORE_SIZE)
+                        break;
+                /* Try to dequeue some more */
+                err = dpaa2_caam_pull_fq(ppriv);
+                if (unlikely(err))
+                        break;
+        } while (1);
+        if (cleaned < budget) {
+                napi_complete_done(napi, cleaned);
+                err = dpaa2_io_service_rearm(NULL, &ppriv->nctx);
+                if (unlikely(err))
+                        dev_err(priv->dev, "Notification rearm failed: %d\n",
+                                err);
+        }
+        return cleaned;
+}
+static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
+                                         u16 token)
+{
+        struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
+        struct device *dev = priv->dev;
+        int err;
+        /*
+         * Congestion group feature supported starting with DPSECI API v5.1
+         * and only when object has been created with this capability.
+         */
+        if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
+            !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
+                return 0;
+        priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
+                                 GFP_KERNEL | GFP_DMA);
+        if (!priv->cscn_mem)
+                return -ENOMEM;
+        priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
+        priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
+                                        DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
+        if (dma_mapping_error(dev, priv->cscn_dma)) {
+                dev_err(dev, "Error mapping CSCN memory area\n");
+                err = -ENOMEM;
+                goto err_dma_map;
+        }
+        cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
+        cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
+        cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
+        cong_notif_cfg.message_ctx = (uintptr_t)priv;
+        cong_notif_cfg.message_iova = priv->cscn_dma;
+        cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
+                                        DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
+                                        DPSECI_CGN_MODE_COHERENT_WRITE;
+        err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
+                                                 &cong_notif_cfg);
+        if (err) {
+                dev_err(dev, "dpseci_set_congestion_notification failed\n");
+                goto err_set_cong;
+        }
+        return 0;
+err_set_cong:
+        dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
+err_dma_map:
+        kfree(priv->cscn_mem);
+        return err;
+}
+static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
+{
+        struct device *dev = &ls_dev->dev;
+        struct dpaa2_caam_priv *priv;
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        int err, cpu;
+        u8 i;
+        priv = dev_get_drvdata(dev);
+        priv->dev = dev;
+        priv->dpsec_id = ls_dev->obj_desc.id;
+        /* Get a handle for the DPSECI this interface is associate with */
+        err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
+        if (err) {
+                dev_err(dev, "dpseci_open() failed: %d\n", err);
+                goto err_open;
+        }
+        err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
+                                     &priv->minor_ver);
+        if (err) {
+                dev_err(dev, "dpseci_get_api_version() failed\n");
+                goto err_get_vers;
+        }
+        dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
+        err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
+                                    &priv->dpseci_attr);
+        if (err) {
+                dev_err(dev, "dpseci_get_attributes() failed\n");
+                goto err_get_vers;
+        }
+        err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
+                                  &priv->sec_attr);
+        if (err) {
+                dev_err(dev, "dpseci_get_sec_attr() failed\n");
+                goto err_get_vers;
+        }
+        err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
+        if (err) {
+                dev_err(dev, "setup_congestion() failed\n");
+                goto err_get_vers;
+        }
+        priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
+                              priv->dpseci_attr.num_tx_queues);
+        if (priv->num_pairs > num_online_cpus()) {
+                dev_warn(dev, "%d queues won't be used\n",
+                         priv->num_pairs - num_online_cpus());
+                priv->num_pairs = num_online_cpus();
+        }
+        for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
+                err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
+                                          &priv->rx_queue_attr[i]);
+                if (err) {
+                        dev_err(dev, "dpseci_get_rx_queue() failed\n");
+                        goto err_get_rx_queue;
+                }
+        }
+        for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
+                err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
+                                          &priv->tx_queue_attr[i]);
+                if (err) {
+                        dev_err(dev, "dpseci_get_tx_queue() failed\n");
+                        goto err_get_rx_queue;
+                }
+        }
+        i = 0;
+        for_each_online_cpu(cpu) {
+                dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", i,
+                        priv->rx_queue_attr[i].fqid,
+                        priv->tx_queue_attr[i].fqid);
+                ppriv = per_cpu_ptr(priv->ppriv, cpu);
+                ppriv->req_fqid = priv->tx_queue_attr[i].fqid;
+                ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
+                ppriv->prio = i;
+                ppriv->net_dev.dev = *dev;
+                INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
+                netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
+                               DPAA2_CAAM_NAPI_WEIGHT);
+                if (++i == priv->num_pairs)
+                        break;
+        }
+        return 0;
+err_get_rx_queue:
+        dpaa2_dpseci_congestion_free(priv);
+err_get_vers:
+        dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
+err_open:
+        return err;
+}
+static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
+{
+        struct device *dev = priv->dev;
+        struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        int i;
+        for (i = 0; i < priv->num_pairs; i++) {
+                ppriv = per_cpu_ptr(priv->ppriv, i);
+                napi_enable(&ppriv->napi);
+        }
+        return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
+}
+static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
+{
+        struct device *dev = priv->dev;
+        struct dpaa2_caam_priv_per_cpu *ppriv;
+        struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
+        int i, err = 0, enabled;
+        err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
+        if (err) {
+                dev_err(dev, "dpseci_disable() failed\n");
+                return err;
+        }
+        err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
+        if (err) {
+                dev_err(dev, "dpseci_is_enabled() failed\n");
+                return err;
+        }
+        dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
+        for (i = 0; i < priv->num_pairs; i++) {
+                ppriv = per_cpu_ptr(priv->ppriv, i);
+                napi_disable(&ppriv->napi);
+                netif_napi_del(&ppriv->napi);
+        }
+        return 0;
+}
+static struct list_head hash_list;
+static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
+{
+        struct device *dev;
+        struct dpaa2_caam_priv *priv;
+        int i, err = 0;
+        bool registered = false;
+        /*
+         * There is no way to get CAAM endianness - there is no direct register
+         * space access and MC f/w does not provide this attribute.
+         * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
+         * property.
+         */
+        caam_little_end = true;
+        caam_imx = false;
+        dev = &dpseci_dev->dev;
+        priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        dev_set_drvdata(dev, priv);
+        priv->domain = iommu_get_domain_for_dev(dev);
+        qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
+                                     0, SLAB_CACHE_DMA, NULL);
+        if (!qi_cache) {
+                dev_err(dev, "Can't allocate SEC cache\n");
+                return -ENOMEM;
+        }
+        err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
+        if (err) {
+                dev_err(dev, "dma_set_mask_and_coherent() failed\n");
+                goto err_dma_mask;
+        }
+        /* Obtain a MC portal */
+        err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
+        if (err) {
+                if (err == -ENXIO)
+                        err = -EPROBE_DEFER;
+                else
+                        dev_err(dev, "MC portal allocation failed\n");
+                goto err_dma_mask;
+        }
+        priv->ppriv = alloc_percpu(*priv->ppriv);
+        if (!priv->ppriv) {
+                dev_err(dev, "alloc_percpu() failed\n");
+                err = -ENOMEM;
+                goto err_alloc_ppriv;
+        }
+        /* DPSECI initialization */
+        err = dpaa2_dpseci_setup(dpseci_dev);
+        if (err) {
+                dev_err(dev, "dpaa2_dpseci_setup() failed\n");
+                goto err_dpseci_setup;
+        }
+        /* DPIO */
+        err = dpaa2_dpseci_dpio_setup(priv);
+        if (err) {
+                if (err != -EPROBE_DEFER)
+                        dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
+                goto err_dpio_setup;
+        }
+        /* DPSECI binding to DPIO */
+        err = dpaa2_dpseci_bind(priv);
+        if (err) {
+                dev_err(dev, "dpaa2_dpseci_bind() failed\n");
+                goto err_bind;
+        }
+        /* DPSECI enable */
+        err = dpaa2_dpseci_enable(priv);
+        if (err) {
+                dev_err(dev, "dpaa2_dpseci_enable() failed\n");
+                goto err_bind;
+        }
+        /* register crypto algorithms the device supports */
+        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+                struct caam_skcipher_alg *t_alg = driver_algs + i;
+                u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
+                /* Skip DES algorithms if not supported by device */
+                if (!priv->sec_attr.des_acc_num &&
+                    (alg_sel == OP_ALG_ALGSEL_3DES ||
+                     alg_sel == OP_ALG_ALGSEL_DES))
+                        continue;
+                /* Skip AES algorithms if not supported by device */
+                if (!priv->sec_attr.aes_acc_num &&
+                    alg_sel == OP_ALG_ALGSEL_AES)
+                        continue;
+                t_alg->caam.dev = dev;
+                caam_skcipher_alg_init(t_alg);
+                err = crypto_register_skcipher(&t_alg->skcipher);
+                if (err) {
+                        dev_warn(dev, "%s alg registration failed: %d\n",
+                                 t_alg->skcipher.base.cra_driver_name, err);
+                        continue;
+                }
+                t_alg->registered = true;
+                registered = true;
+        }
+        for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
+                struct caam_aead_alg *t_alg = driver_aeads + i;
+                u32 c1_alg_sel = t_alg->caam.class1_alg_type &
+                                 OP_ALG_ALGSEL_MASK;
+                u32 c2_alg_sel = t_alg->caam.class2_alg_type &
+                                 OP_ALG_ALGSEL_MASK;
+                /* Skip DES algorithms if not supported by device */
+                if (!priv->sec_attr.des_acc_num &&
+                    (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
+                     c1_alg_sel == OP_ALG_ALGSEL_DES))
+                        continue;
+                /* Skip AES algorithms if not supported by device */
+                if (!priv->sec_attr.aes_acc_num &&
+                    c1_alg_sel == OP_ALG_ALGSEL_AES)
+                        continue;
+                /*
+                 * Skip algorithms requiring message digests
+                 * if MD not supported by device.
+                 */
+                if (!priv->sec_attr.md_acc_num && c2_alg_sel)
+                        continue;
+                t_alg->caam.dev = dev;
+                caam_aead_alg_init(t_alg);
+                err = crypto_register_aead(&t_alg->aead);
+                if (err) {
+                        dev_warn(dev, "%s alg registration failed: %d\n",
+                                 t_alg->aead.base.cra_driver_name, err);
+                        continue;
+                }
+                t_alg->registered = true;
+                registered = true;
+        }
+        if (registered)
+                dev_info(dev, "algorithms registered in /proc/crypto\n");
+        /* register hash algorithms the device supports */
+        INIT_LIST_HEAD(&hash_list);
+        /*
+         * Skip registration of any hashing algorithms if MD block
+         * is not present.
+         */
+        if (!priv->sec_attr.md_acc_num)
+                return 0;
+        for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
+                struct caam_hash_alg *t_alg;
+                struct caam_hash_template *alg = driver_hash + i;
+                /* register hmac version */
+                t_alg = caam_hash_alloc(dev, alg, true);
+                if (IS_ERR(t_alg)) {
+                        err = PTR_ERR(t_alg);
+                        dev_warn(dev, "%s hash alg allocation failed: %d\n",
+                                 alg->driver_name, err);
+                        continue;
+                }
+                err = crypto_register_ahash(&t_alg->ahash_alg);
+                if (err) {
+                        dev_warn(dev, "%s alg registration failed: %d\n",
+                                 t_alg->ahash_alg.halg.base.cra_driver_name,
+                                 err);
+                        kfree(t_alg);
+                } else {
+                        list_add_tail(&t_alg->entry, &hash_list);
+                }
+                /* register unkeyed version */
+                t_alg = caam_hash_alloc(dev, alg, false);
+                if (IS_ERR(t_alg)) {
+                        err = PTR_ERR(t_alg);
+                        dev_warn(dev, "%s alg allocation failed: %d\n",
+                                 alg->driver_name, err);
+                        continue;
+                }
+                err = crypto_register_ahash(&t_alg->ahash_alg);
+                if (err) {
+                        dev_warn(dev, "%s alg registration failed: %d\n",
+                                 t_alg->ahash_alg.halg.base.cra_driver_name,
+                                 err);
+                        kfree(t_alg);
+                } else {
+                        list_add_tail(&t_alg->entry, &hash_list);
+                }
+        }
+        if (!list_empty(&hash_list))
+                dev_info(dev, "hash algorithms registered in /proc/crypto\n");
+        return err;
+err_bind:
+        dpaa2_dpseci_dpio_free(priv);
+err_dpio_setup:
+        dpaa2_dpseci_free(priv);
+err_dpseci_setup:
+        free_percpu(priv->ppriv);
+err_alloc_ppriv:
+        fsl_mc_portal_free(priv->mc_io);
+err_dma_mask:
+        kmem_cache_destroy(qi_cache);
+        return err;
+}
+static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
+{
+        struct device *dev;
+        struct dpaa2_caam_priv *priv;
+        int i;
+        dev = &ls_dev->dev;
+        priv = dev_get_drvdata(dev);
+        for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
+                struct caam_aead_alg *t_alg = driver_aeads + i;
+                if (t_alg->registered)
+                        crypto_unregister_aead(&t_alg->aead);
+        }
+        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+                struct caam_skcipher_alg *t_alg = driver_algs + i;
+                if (t_alg->registered)
+                        crypto_unregister_skcipher(&t_alg->skcipher);
+        }
+        if (hash_list.next) {
+                struct caam_hash_alg *t_hash_alg, *p;
+                list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
+                        crypto_unregister_ahash(&t_hash_alg->ahash_alg);
+                        list_del(&t_hash_alg->entry);
+                        kfree(t_hash_alg);
+                }
+        }
+        dpaa2_dpseci_disable(priv);
+        dpaa2_dpseci_dpio_free(priv);
+        dpaa2_dpseci_free(priv);
+        free_percpu(priv->ppriv);
+        fsl_mc_portal_free(priv->mc_io);
+        kmem_cache_destroy(qi_cache);
+        return 0;
+}
+int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
+{
+        struct dpaa2_fd fd;
+        struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
+        int err = 0, i, id;
+        if (IS_ERR(req))
+                return PTR_ERR(req);
+        if (priv->cscn_mem) {
+                dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
+                                        DPAA2_CSCN_SIZE,
+                                        DMA_FROM_DEVICE);
+                if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
+                        dev_dbg_ratelimited(dev, "Dropping request\n");
+                        return -EBUSY;
+                }
+        }
+        dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
+        req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
+                                         DMA_BIDIRECTIONAL);
+        if (dma_mapping_error(dev, req->fd_flt_dma)) {
+                dev_err(dev, "DMA mapping error for QI enqueue request\n");
+                goto err_out;
+        }
+        memset(&fd, 0, sizeof(fd));
+        dpaa2_fd_set_format(&fd, dpaa2_fd_list);
+        dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
+        dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
+        dpaa2_fd_set_flc(&fd, req->flc_dma);
+        /*
+         * There is no guarantee that preemption is disabled here,
+         * thus take action.
+         */
+        preempt_disable();
+        id = smp_processor_id() % priv->dpseci_attr.num_tx_queues;
+        for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
+                err = dpaa2_io_service_enqueue_fq(NULL,
+                                                  priv->tx_queue_attr[id].fqid,
+                                                  &fd);
+                if (err != -EBUSY)
+                        break;
+        }
+        preempt_enable();
+        if (unlikely(err)) {
+                dev_err(dev, "Error enqueuing frame: %d\n", err);
+                goto err_out;
+        }
+        return -EINPROGRESS;
+err_out:
+        dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
+                         DMA_BIDIRECTIONAL);
+        return -EIO;
+}
+EXPORT_SYMBOL(dpaa2_caam_enqueue);
+static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
+        {
+                .vendor = FSL_MC_VENDOR_FREESCALE,
+                .obj_type = "dpseci",
+        },
+        { .vendor = 0x0 }
+};
+static struct fsl_mc_driver dpaa2_caam_driver = {
+        .driver = {
+                .name           = KBUILD_MODNAME,
+                .owner          = THIS_MODULE,
+        },
+        .probe          = dpaa2_caam_probe,
+        .remove         = dpaa2_caam_remove,
+        .match_id_table = dpaa2_caam_match_id_table
+};
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Freescale Semiconductor, Inc");
+MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
+module_fsl_mc_driver(dpaa2_caam_driver);
diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h
new file mode 100644
index 000000000000..9823bdefd029
--- /dev/null
+++ b/drivers/crypto/caam/caamalg_qi2.h
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/*
+ * Copyright 2015-2016 Freescale Semiconductor Inc.
+ * Copyright 2017-2018 NXP
+ */
+#ifndef _CAAMALG_QI2_H_
+#define _CAAMALG_QI2_H_
+#include <soc/fsl/dpaa2-io.h>
+#include <soc/fsl/dpaa2-fd.h>
+#include <linux/threads.h>
+#include "dpseci.h"
+#include "desc_constr.h"
+#define DPAA2_CAAM_STORE_SIZE   16
+/* NAPI weight *must* be a multiple of the store size. */
+#define DPAA2_CAAM_NAPI_WEIGHT  64
+/* The congestion entrance threshold was chosen so that on LS2088
+ * we support the maximum throughput for the available memory
+ */
+#define DPAA2_SEC_CONG_ENTRY_THRESH     (128 * 1024 * 1024)
+#define DPAA2_SEC_CONG_EXIT_THRESH      (DPAA2_SEC_CONG_ENTRY_THRESH * 9 / 10)
+/**
+ * dpaa2_caam_priv - driver private data
+ * @dpseci_id: DPSECI object unique ID
+ * @major_ver: DPSECI major version
+ * @minor_ver: DPSECI minor version
+ * @dpseci_attr: DPSECI attributes
+ * @sec_attr: SEC engine attributes
+ * @rx_queue_attr: array of Rx queue attributes
+ * @tx_queue_attr: array of Tx queue attributes
+ * @cscn_mem: pointer to memory region containing the congestion SCN
+ *      it's size is larger than to accommodate alignment
+ * @cscn_mem_aligned: pointer to congestion SCN; it is computed as
+ *      PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN)
+ * @cscn_dma: dma address used by the QMAN to write CSCN messages
+ * @dev: device associated with the DPSECI object
+ * @mc_io: pointer to MC portal's I/O object
+ * @domain: IOMMU domain
+ * @ppriv: per CPU pointers to privata data
+ */
+struct dpaa2_caam_priv {
+        int dpsec_id;
+        u16 major_ver;
+        u16 minor_ver;
+        struct dpseci_attr dpseci_attr;
+        struct dpseci_sec_attr sec_attr;
+        struct dpseci_rx_queue_attr rx_queue_attr[DPSECI_MAX_QUEUE_NUM];
+        struct dpseci_tx_queue_attr tx_queue_attr[DPSECI_MAX_QUEUE_NUM];
+        int num_pairs;
+        /* congestion */
+        void *cscn_mem;
+        void *cscn_mem_aligned;
+        dma_addr_t cscn_dma;
+        struct device *dev;
+        struct fsl_mc_io *mc_io;
+        struct iommu_domain *domain;
+        struct dpaa2_caam_priv_per_cpu __percpu *ppriv;
+};
+/**
+ * dpaa2_caam_priv_per_cpu - per CPU private data
+ * @napi: napi structure
+ * @net_dev: netdev used by napi
+ * @req_fqid: (virtual) request (Tx / enqueue) FQID
+ * @rsp_fqid: (virtual) response (Rx / dequeue) FQID
+ * @prio: internal queue number - index for dpaa2_caam_priv.*_queue_attr
+ * @nctx: notification context of response FQ
+ * @store: where dequeued frames are stored
+ * @priv: backpointer to dpaa2_caam_priv
+ */
+struct dpaa2_caam_priv_per_cpu {
+        struct napi_struct napi;
+        struct net_device net_dev;
+        int req_fqid;
+        int rsp_fqid;
+        int prio;
+        struct dpaa2_io_notification_ctx nctx;
+        struct dpaa2_io_store *store;
+        struct dpaa2_caam_priv *priv;
+};
+/*
+ * The CAAM QI hardware constructs a job descriptor which points
+ * to shared descriptor (as pointed by context_a of FQ to CAAM).
+ * When the job descriptor is executed by deco, the whole job
+ * descriptor together with shared descriptor gets loaded in
+ * deco buffer which is 64 words long (each 32-bit).
+ *
+ * The job descriptor constructed by QI hardware has layout:
+ *
+ *      HEADER          (1 word)
+ *      Shdesc ptr      (1 or 2 words)
+ *      SEQ_OUT_PTR     (1 word)
+ *      Out ptr         (1 or 2 words)
+ *      Out length      (1 word)
+ *      SEQ_IN_PTR      (1 word)
+ *      In ptr          (1 or 2 words)
+ *      In length       (1 word)
+ *
+ * The shdesc ptr is used to fetch shared descriptor contents
+ * into deco buffer.
+ *
+ * Apart from shdesc contents, the total number of words that
+ * get loaded in deco buffer are '8' or '11'. The remaining words
+ * in deco buffer can be used for storing shared descriptor.
+ */
+#define MAX_SDLEN       ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
+/* Length of a single buffer in the QI driver memory cache */
+#define CAAM_QI_MEMCACHE_SIZE   512
+/*
+ * aead_edesc - s/w-extended aead descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @iv_dma: dma address of iv for checking continuity and link table
+ * @qm_sg_bytes: length of dma mapped h/w link table
+ * @qm_sg_dma: bus physical mapped address of h/w link table
+ * @assoclen: associated data length, in CAAM endianness
+ * @assoclen_dma: bus physical mapped address of req->assoclen
+ * @sgt: the h/w link table, followed by IV
+ */
+struct aead_edesc {
+        int src_nents;
+        int dst_nents;
+        dma_addr_t iv_dma;
+        int qm_sg_bytes;
+        dma_addr_t qm_sg_dma;
+        unsigned int assoclen;
+        dma_addr_t assoclen_dma;
+        struct dpaa2_sg_entry sgt[0];
+};
+/*
+ * skcipher_edesc - s/w-extended skcipher descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @iv_dma: dma address of iv for checking continuity and link table
+ * @qm_sg_bytes: length of dma mapped qm_sg space
+ * @qm_sg_dma: I/O virtual address of h/w link table
+ * @sgt: the h/w link table, followed by IV
+ */
+struct skcipher_edesc {
+        int src_nents;
+        int dst_nents;
+        dma_addr_t iv_dma;
+        int qm_sg_bytes;
+        dma_addr_t qm_sg_dma;
+        struct dpaa2_sg_entry sgt[0];
+};
+/*
+ * ahash_edesc - s/w-extended ahash descriptor
+ * @dst_dma: I/O virtual address of req->result
+ * @qm_sg_dma: I/O virtual address of h/w link table
+ * @src_nents: number of segments in input scatterlist
+ * @qm_sg_bytes: length of dma mapped qm_sg space
+ * @sgt: pointer to h/w link table
+ */
+struct ahash_edesc {
+        dma_addr_t dst_dma;
+        dma_addr_t qm_sg_dma;
+        int src_nents;
+        int qm_sg_bytes;
+        struct dpaa2_sg_entry sgt[0];
+};
+/**
+ * caam_flc - Flow Context (FLC)
+ * @flc: Flow Context options
+ * @sh_desc: Shared Descriptor
+ */
+struct caam_flc {
+        u32 flc[16];
+        u32 sh_desc[MAX_SDLEN];
+} ____cacheline_aligned;
+enum optype {
+        ENCRYPT = 0,
+        DECRYPT,
+        NUM_OP
+};
+/**
+ * caam_request - the request structure the driver application should fill while
+ *                submitting a job to driver.
+ * @fd_flt: Frame list table defining input and output
+ *          fd_flt[0] - FLE pointing to output buffer
+ *          fd_flt[1] - FLE pointing to input buffer
+ * @fd_flt_dma: DMA address for the frame list table
+ * @flc: Flow Context
+ * @flc_dma: I/O virtual address of Flow Context
+ * @cbk: Callback function to invoke when job is completed
+ * @ctx: arbit context attached with request by the application
+ * @edesc: extended descriptor; points to one of {skcipher,aead}_edesc
+ */
+struct caam_request {
+        struct dpaa2_fl_entry fd_flt[2];
+        dma_addr_t fd_flt_dma;
+        struct caam_flc *flc;
+        dma_addr_t flc_dma;
+        void (*cbk)(void *ctx, u32 err);
+        void *ctx;
+        void *edesc;
+};
+/**
+ * dpaa2_caam_enqueue() - enqueue a crypto request
+ * @dev: device associated with the DPSECI object
+ * @req: pointer to caam_request
+ */
+int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req);
+#endif  /* _CAAMALG_QI2_H_ */
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 43975ab5f09c..46924affa0bd 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
 * caam - Freescale FSL CAAM support for ahash functions of crypto API
 *
@@ -62,6 +63,7 @@
 #include "error.h"
 #include "sg_sw_sec4.h"
 #include "key_gen.h"
+#include "caamhash_desc.h"
 #define CAAM_CRA_PRIORITY               3000
@@ -71,14 +73,6 @@
 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
 #define CAAM_MAX_HASH_DIGEST_SIZE       SHA512_DIGEST_SIZE
-/* length of descriptors text */
-#define DESC_AHASH_BASE                 (3 * CAAM_CMD_SZ)
-#define DESC_AHASH_UPDATE_LEN           (6 * CAAM_CMD_SZ)
-#define DESC_AHASH_UPDATE_FIRST_LEN     (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
-#define DESC_AHASH_FINAL_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
-#define DESC_AHASH_FINUP_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
-#define DESC_AHASH_DIGEST_LEN           (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
 #define DESC_HASH_MAX_USED_BYTES        (DESC_AHASH_FINAL_LEN + \
                                         CAAM_MAX_HASH_KEY_SIZE)
 #define DESC_HASH_MAX_USED_LEN          (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
@@ -235,60 +229,6 @@ static inline int ctx_map_to_sec4_sg(struct device *jrdev,
        return 0;
 }
-/*
- * For ahash update, final and finup (import_ctx = true)
- *     import context, read and write to seqout
- * For ahash firsts and digest (import_ctx = false)
- *     read and write to seqout
- */
-static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
-                                     struct caam_hash_ctx *ctx, bool import_ctx,
-                                     int era)
-{
-        u32 op = ctx->adata.algtype;
-        u32 *skip_key_load;
-        init_sh_desc(desc, HDR_SHARE_SERIAL);
-        /* Append key if it has been set; ahash update excluded */
-        if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
-                /* Skip key loading if already shared */
-                skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-                                            JUMP_COND_SHRD);
-                if (era < 6)
-                        append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
-                                          ctx->adata.keylen, CLASS_2 |
-                                          KEY_DEST_MDHA_SPLIT | KEY_ENC);
-                else
-                        append_proto_dkp(desc, &ctx->adata);
-                set_jump_tgt_here(desc, skip_key_load);
-                op |= OP_ALG_AAI_HMAC_PRECOMP;
-        }
-        /* If needed, import context from software */
-        if (import_ctx)
-                append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
-                                LDST_SRCDST_BYTE_CONTEXT);
-        /* Class 2 operation */
-        append_operation(desc, op | state | OP_ALG_ENCRYPT);
-        /*
-         * Load from buf and/or src and write to req->result or state->context
-         * Calculate remaining bytes to read
-         */
-        append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-        /* Read remaining bytes */
-        append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
-                             FIFOLD_TYPE_MSG | KEY_VLF);
-        /* Store class2 context bytes */
-        append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
-                         LDST_SRCDST_BYTE_CONTEXT);
-}
 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 {
        struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
@@ -301,8 +241,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
        /* ahash_update shared descriptor */
        desc = ctx->sh_desc_update;
-        ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true,
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
-                          ctrlpriv->era);
+                          ctx->ctx_len, true, ctrlpriv->era);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
                                   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
@@ -313,8 +253,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
        /* ahash_update_first shared descriptor */
        desc = ctx->sh_desc_update_first;
-        ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false,
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
-                          ctrlpriv->era);
+                          ctx->ctx_len, false, ctrlpriv->era);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
                                   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
@@ -325,8 +265,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
        /* ahash_final shared descriptor */
        desc = ctx->sh_desc_fin;
-        ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true,
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
-                          ctrlpriv->era);
+                          ctx->ctx_len, true, ctrlpriv->era);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
                                   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
@@ -337,8 +277,8 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
        /* ahash_digest shared descriptor */
        desc = ctx->sh_desc_digest;
-        ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false,
+        cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
-                          ctrlpriv->era);
+                          ctx->ctx_len, false, ctrlpriv->era);
        dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
                                   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
diff --git a/drivers/crypto/caam/caamhash_desc.c b/drivers/crypto/caam/caamhash_desc.c
new file mode 100644
index 000000000000..a12f7959a2c3
--- /dev/null
+++ b/drivers/crypto/caam/caamhash_desc.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Shared descriptors for ahash algorithms
+ *
+ * Copyright 2017 NXP
+ */
+#include "compat.h"
+#include "desc_constr.h"
+#include "caamhash_desc.h"
+/**
+ * cnstr_shdsc_ahash - ahash shared descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @adata: pointer to authentication transform definitions.
+ *         A split key is required for SEC Era < 6; the size of the split key
+ *         is specified in this case.
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224,
+ *         SHA256, SHA384, SHA512}.
+ * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE}
+ * @digestsize: algorithm's digest size
+ * @ctx_len: size of Context Register
+ * @import_ctx: true if previous Context Register needs to be restored
+ *              must be true for ahash update and final
+ *              must be false for for ahash first and digest
+ * @era: SEC Era
+ */
+void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
+                       int digestsize, int ctx_len, bool import_ctx, int era)
+{
+        u32 op = adata->algtype;
+        init_sh_desc(desc, HDR_SHARE_SERIAL);
+        /* Append key if it has been set; ahash update excluded */
+        if (state != OP_ALG_AS_UPDATE && adata->keylen) {
+                u32 *skip_key_load;
+                /* Skip key loading if already shared */
+                skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+                                            JUMP_COND_SHRD);
+                if (era < 6)
+                        append_key_as_imm(desc, adata->key_virt,
+                                          adata->keylen_pad,
+                                          adata->keylen, CLASS_2 |
+                                          KEY_DEST_MDHA_SPLIT | KEY_ENC);
+                else
+                        append_proto_dkp(desc, adata);
+                set_jump_tgt_here(desc, skip_key_load);
+                op |= OP_ALG_AAI_HMAC_PRECOMP;
+        }
+        /* If needed, import context from software */
+        if (import_ctx)
+                append_seq_load(desc, ctx_len, LDST_CLASS_2_CCB |
+                                LDST_SRCDST_BYTE_CONTEXT);
+        /* Class 2 operation */
+        append_operation(desc, op | state | OP_ALG_ENCRYPT);
+        /*
+         * Load from buf and/or src and write to req->result or state->context
+         * Calculate remaining bytes to read
+         */
+        append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+        /* Read remaining bytes */
+        append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
+                             FIFOLD_TYPE_MSG | KEY_VLF);
+        /* Store class2 context bytes */
+        append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
+                         LDST_SRCDST_BYTE_CONTEXT);
+}
+EXPORT_SYMBOL(cnstr_shdsc_ahash);
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("FSL CAAM ahash descriptors support");
+MODULE_AUTHOR("NXP Semiconductors");
diff --git a/drivers/crypto/caam/caamhash_desc.h b/drivers/crypto/caam/caamhash_desc.h
new file mode 100644
index 000000000000..631fc1ac312c
--- /dev/null
+++ b/drivers/crypto/caam/caamhash_desc.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/*
+ * Shared descriptors for ahash algorithms
+ *
+ * Copyright 2017 NXP
+ */
+#ifndef _CAAMHASH_DESC_H_
+#define _CAAMHASH_DESC_H_
+/* length of descriptors text */
+#define DESC_AHASH_BASE                 (3 * CAAM_CMD_SZ)
+#define DESC_AHASH_UPDATE_LEN           (6 * CAAM_CMD_SZ)
+#define DESC_AHASH_UPDATE_FIRST_LEN     (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
+#define DESC_AHASH_FINAL_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
+#define DESC_AHASH_DIGEST_LEN           (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
+void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
+                       int digestsize, int ctx_len, bool import_ctx, int era);
+#endif /* _CAAMHASH_DESC_H_ */
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index f26d62e5533a..4fc209cbbeab 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
 /*
 * caam - Freescale FSL CAAM support for Public Key Cryptography
 *
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index fde07d4ff019..4318b0aa6fb9 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
 * caam - Freescale FSL CAAM support for hw_random
 *
diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h
index 1c71e0cd5098..9604ff7a335e 100644
--- a/drivers/crypto/caam/compat.h
+++ b/drivers/crypto/caam/compat.h
@@ -17,6 +17,7 @@
 #include <linux/of_platform.h>
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
+#include <linux/iommu.h>
 #include <linux/spinlock.h>
 #include <linux/rtnetlink.h>
 #include <linux/in.h>
@@ -39,6 +40,7 @@
 #include <crypto/authenc.h>
 #include <crypto/akcipher.h>
 #include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/rsa.h>
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 538c01f428c1..3fc793193821 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /* * CAAM control-plane driver backend
 * Controller-level driver, kernel property detection, initialization
 *
diff --git a/drivers/crypto/caam/dpseci.c b/drivers/crypto/caam/dpseci.c
new file mode 100644
index 000000000000..8a68531ded0b
--- /dev/null
+++ b/drivers/crypto/caam/dpseci.c
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2017-2018 NXP
+ */
+#include <linux/fsl/mc.h>
+#include "dpseci.h"
+#include "dpseci_cmd.h"
+/**
+ * dpseci_open() - Open a control session for the specified object
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @dpseci_id:  DPSECI unique ID
+ * @token:      Returned token; use in subsequent API calls
+ *
+ * This function can be used to open a control session for an already created
+ * object; an object may have been declared statically in the DPL
+ * or created dynamically.
+ * This function returns a unique authentication token, associated with the
+ * specific object ID and the specific MC portal; this token must be used in all
+ * subsequent commands for this specific object.
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id,
+                u16 *token)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_cmd_open *cmd_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN,
+                                          cmd_flags,
+                                          0);
+        cmd_params = (struct dpseci_cmd_open *)cmd.params;
+        cmd_params->dpseci_id = cpu_to_le32(dpseci_id);
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        *token = mc_cmd_hdr_read_token(&cmd);
+        return 0;
+}
+/**
+ * dpseci_close() - Close the control session of the object
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ *
+ * After this function is called, no further operations are allowed on the
+ * object without opening a new control session.
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
+{
+        struct fsl_mc_command cmd = { 0 };
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLOSE,
+                                          cmd_flags,
+                                          token);
+        return mc_send_command(mc_io, &cmd);
+}
+/**
+ * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
+{
+        struct fsl_mc_command cmd = { 0 };
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE,
+                                          cmd_flags,
+                                          token);
+        return mc_send_command(mc_io, &cmd);
+}
+/**
+ * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
+{
+        struct fsl_mc_command cmd = { 0 };
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE,
+                                          cmd_flags,
+                                          token);
+        return mc_send_command(mc_io, &cmd);
+}
+/**
+ * dpseci_is_enabled() - Check if the DPSECI is enabled.
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @en:         Returns '1' if object is enabled; '0' otherwise
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                      int *en)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_rsp_is_enabled *rsp_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED,
+                                          cmd_flags,
+                                          token);
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params;
+        *en = dpseci_get_field(rsp_params->is_enabled, ENABLE);
+        return 0;
+}
+/**
+ * dpseci_get_attributes() - Retrieve DPSECI attributes
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @attr:       Returned object's attributes
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                          struct dpseci_attr *attr)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_rsp_get_attributes *rsp_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR,
+                                          cmd_flags,
+                                          token);
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        rsp_params = (struct dpseci_rsp_get_attributes *)cmd.params;
+        attr->id = le32_to_cpu(rsp_params->id);
+        attr->num_tx_queues = rsp_params->num_tx_queues;
+        attr->num_rx_queues = rsp_params->num_rx_queues;
+        attr->options = le32_to_cpu(rsp_params->options);
+        return 0;
+}
+/**
+ * dpseci_set_rx_queue() - Set Rx queue configuration
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @queue:      Select the queue relative to number of priorities configured at
+ *              DPSECI creation; use DPSECI_ALL_QUEUES to configure all
+ *              Rx queues identically.
+ * @cfg:        Rx queue configuration
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        u8 queue, const struct dpseci_rx_queue_cfg *cfg)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_cmd_queue *cmd_params;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE,
+                                          cmd_flags,
+                                          token);
+        cmd_params = (struct dpseci_cmd_queue *)cmd.params;
+        cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
+        cmd_params->priority = cfg->dest_cfg.priority;
+        cmd_params->queue = queue;
+        dpseci_set_field(cmd_params->dest_type, DEST_TYPE,
+                         cfg->dest_cfg.dest_type);
+        cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx);
+        cmd_params->options = cpu_to_le32(cfg->options);
+        dpseci_set_field(cmd_params->order_preservation_en, ORDER_PRESERVATION,
+                         cfg->order_preservation_en);
+        return mc_send_command(mc_io, &cmd);
+}
+/**
+ * dpseci_get_rx_queue() - Retrieve Rx queue attributes
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @queue:      Select the queue relative to number of priorities configured at
+ *              DPSECI creation
+ * @attr:       Returned Rx queue attributes
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        u8 queue, struct dpseci_rx_queue_attr *attr)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_cmd_queue *cmd_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE,
+                                          cmd_flags,
+                                          token);
+        cmd_params = (struct dpseci_cmd_queue *)cmd.params;
+        cmd_params->queue = queue;
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id);
+        attr->dest_cfg.priority = cmd_params->priority;
+        attr->dest_cfg.dest_type = dpseci_get_field(cmd_params->dest_type,
+                                                    DEST_TYPE);
+        attr->user_ctx = le64_to_cpu(cmd_params->user_ctx);
+        attr->fqid = le32_to_cpu(cmd_params->fqid);
+        attr->order_preservation_en =
+                dpseci_get_field(cmd_params->order_preservation_en,
+                                 ORDER_PRESERVATION);
+        return 0;
+}
+/**
+ * dpseci_get_tx_queue() - Retrieve Tx queue attributes
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @queue:      Select the queue relative to number of priorities configured at
+ *              DPSECI creation
+ * @attr:       Returned Tx queue attributes
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        u8 queue, struct dpseci_tx_queue_attr *attr)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_cmd_queue *cmd_params;
+        struct dpseci_rsp_get_tx_queue *rsp_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE,
+                                          cmd_flags,
+                                          token);
+        cmd_params = (struct dpseci_cmd_queue *)cmd.params;
+        cmd_params->queue = queue;
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params;
+        attr->fqid = le32_to_cpu(rsp_params->fqid);
+        attr->priority = rsp_params->priority;
+        return 0;
+}
+/**
+ * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @attr:       Returned SEC attributes
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        struct dpseci_sec_attr *attr)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_rsp_get_sec_attr *rsp_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR,
+                                          cmd_flags,
+                                          token);
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params;
+        attr->ip_id = le16_to_cpu(rsp_params->ip_id);
+        attr->major_rev = rsp_params->major_rev;
+        attr->minor_rev = rsp_params->minor_rev;
+        attr->era = rsp_params->era;
+        attr->deco_num = rsp_params->deco_num;
+        attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num;
+        attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num;
+        attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num;
+        attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num;
+        attr->crc_acc_num = rsp_params->crc_acc_num;
+        attr->pk_acc_num = rsp_params->pk_acc_num;
+        attr->kasumi_acc_num = rsp_params->kasumi_acc_num;
+        attr->rng_acc_num = rsp_params->rng_acc_num;
+        attr->md_acc_num = rsp_params->md_acc_num;
+        attr->arc4_acc_num = rsp_params->arc4_acc_num;
+        attr->des_acc_num = rsp_params->des_acc_num;
+        attr->aes_acc_num = rsp_params->aes_acc_num;
+        attr->ccha_acc_num = rsp_params->ccha_acc_num;
+        attr->ptha_acc_num = rsp_params->ptha_acc_num;
+        return 0;
+}
+/**
+ * dpseci_get_api_version() - Get Data Path SEC Interface API version
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @major_ver:  Major version of data path sec API
+ * @minor_ver:  Minor version of data path sec API
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags,
+                           u16 *major_ver, u16 *minor_ver)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_rsp_get_api_version *rsp_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION,
+                                          cmd_flags, 0);
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params;
+        *major_ver = le16_to_cpu(rsp_params->major);
+        *minor_ver = le16_to_cpu(rsp_params->minor);
+        return 0;
+}
+/**
+ * dpseci_set_congestion_notification() - Set congestion group
+ *      notification configuration
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @cfg:        congestion notification configuration
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
+        u16 token, const struct dpseci_congestion_notification_cfg *cfg)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_cmd_congestion_notification *cmd_params;
+        cmd.header = mc_encode_cmd_header(
+                        DPSECI_CMDID_SET_CONGESTION_NOTIFICATION,
+                        cmd_flags,
+                        token);
+        cmd_params = (struct dpseci_cmd_congestion_notification *)cmd.params;
+        cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
+        cmd_params->notification_mode = cpu_to_le16(cfg->notification_mode);
+        cmd_params->priority = cfg->dest_cfg.priority;
+        dpseci_set_field(cmd_params->options, CGN_DEST_TYPE,
+                         cfg->dest_cfg.dest_type);
+        dpseci_set_field(cmd_params->options, CGN_UNITS, cfg->units);
+        cmd_params->message_iova = cpu_to_le64(cfg->message_iova);
+        cmd_params->message_ctx = cpu_to_le64(cfg->message_ctx);
+        cmd_params->threshold_entry = cpu_to_le32(cfg->threshold_entry);
+        cmd_params->threshold_exit = cpu_to_le32(cfg->threshold_exit);
+        return mc_send_command(mc_io, &cmd);
+}
+/**
+ * dpseci_get_congestion_notification() - Get congestion group notification
+ *      configuration
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSECI object
+ * @cfg:        congestion notification configuration
+ *
+ * Return:      '0' on success, error code otherwise
+ */
+int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
+        u16 token, struct dpseci_congestion_notification_cfg *cfg)
+{
+        struct fsl_mc_command cmd = { 0 };
+        struct dpseci_cmd_congestion_notification *rsp_params;
+        int err;
+        cmd.header = mc_encode_cmd_header(
+                        DPSECI_CMDID_GET_CONGESTION_NOTIFICATION,
+                        cmd_flags,
+                        token);
+        err = mc_send_command(mc_io, &cmd);
+        if (err)
+                return err;
+        rsp_params = (struct dpseci_cmd_congestion_notification *)cmd.params;
+        cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id);
+        cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode);
+        cfg->dest_cfg.priority = rsp_params->priority;
+        cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->options,
+                                                   CGN_DEST_TYPE);
+        cfg->units = dpseci_get_field(rsp_params->options, CGN_UNITS);
+        cfg->message_iova = le64_to_cpu(rsp_params->message_iova);
+        cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx);
+        cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry);
+        cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit);
+        return 0;
+}
diff --git a/drivers/crypto/caam/dpseci.h b/drivers/crypto/caam/dpseci.h
new file mode 100644
index 000000000000..4550e134d166
--- /dev/null
+++ b/drivers/crypto/caam/dpseci.h
@@ -0,0 +1,333 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2017-2018 NXP
+ */
+#ifndef _DPSECI_H_
+#define _DPSECI_H_
+/*
+ * Data Path SEC Interface API
+ * Contains initialization APIs and runtime control APIs for DPSECI
+ */
+struct fsl_mc_io;
+/**
+ * General DPSECI macros
+ */
+/**
+ * Maximum number of Tx/Rx queues per DPSECI object
+ */
+#define DPSECI_MAX_QUEUE_NUM            16
+/**
+ * All queues considered; see dpseci_set_rx_queue()
+ */
+#define DPSECI_ALL_QUEUES       (u8)(-1)
+int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id,
+                u16 *token);
+int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
+/**
+ * Enable the Congestion Group support
+ */
+#define DPSECI_OPT_HAS_CG               0x000020
+/**
+ * struct dpseci_cfg - Structure representing DPSECI configuration
+ * @options: Any combination of the following flags:
+ *              DPSECI_OPT_HAS_CG
+ * @num_tx_queues: num of queues towards the SEC
+ * @num_rx_queues: num of queues back from the SEC
+ * @priorities: Priorities for the SEC hardware processing;
+ *              each place in the array is the priority of the tx queue
+ *              towards the SEC;
+ *              valid priorities are configured with values 1-8;
+ */
+struct dpseci_cfg {
+        u32 options;
+        u8 num_tx_queues;
+        u8 num_rx_queues;
+        u8 priorities[DPSECI_MAX_QUEUE_NUM];
+};
+int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
+int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
+int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                      int *en);
+/**
+ * struct dpseci_attr - Structure representing DPSECI attributes
+ * @id: DPSECI object ID
+ * @num_tx_queues: number of queues towards the SEC
+ * @num_rx_queues: number of queues back from the SEC
+ * @options: any combination of the following flags:
+ *              DPSECI_OPT_HAS_CG
+ */
+struct dpseci_attr {
+        int id;
+        u8 num_tx_queues;
+        u8 num_rx_queues;
+        u32 options;
+};
+int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                          struct dpseci_attr *attr);
+/**
+ * enum dpseci_dest - DPSECI destination types
+ * @DPSECI_DEST_NONE: Unassigned destination; The queue is set in parked mode
+ *      and does not generate FQDAN notifications; user is expected to dequeue
+ *      from the queue based on polling or other user-defined method
+ * @DPSECI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN
+ *      notifications to the specified DPIO; user is expected to dequeue from
+ *      the queue only after notification is received
+ * @DPSECI_DEST_DPCON: The queue is set in schedule mode and does not generate
+ *      FQDAN notifications, but is connected to the specified DPCON object;
+ *      user is expected to dequeue from the DPCON channel
+ */
+enum dpseci_dest {
+        DPSECI_DEST_NONE = 0,
+        DPSECI_DEST_DPIO,
+        DPSECI_DEST_DPCON
+};
+/**
+ * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters
+ * @dest_type: Destination type
+ * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type
+ * @priority: Priority selection within the DPIO or DPCON channel; valid values
+ *      are 0-1 or 0-7, depending on the number of priorities in that channel;
+ *      not relevant for 'DPSECI_DEST_NONE' option
+ */
+struct dpseci_dest_cfg {
+        enum dpseci_dest dest_type;
+        int dest_id;
+        u8 priority;
+};
+/**
+ * DPSECI queue modification options
+ */
+/**
+ * Select to modify the user's context associated with the queue
+ */
+#define DPSECI_QUEUE_OPT_USER_CTX               0x00000001
+/**
+ * Select to modify the queue's destination
+ */
+#define DPSECI_QUEUE_OPT_DEST                   0x00000002
+/**
+ * Select to modify the queue's order preservation
+ */
+#define DPSECI_QUEUE_OPT_ORDER_PRESERVATION     0x00000004
+/**
+ * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration
+ * @options: Flags representing the suggested modifications to the queue;
+ *      Use any combination of 'DPSECI_QUEUE_OPT_<X>' flags
+ * @order_preservation_en: order preservation configuration for the rx queue
+ * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options'
+ * @user_ctx: User context value provided in the frame descriptor of each
+ *      dequeued frame; valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained
+ *      in 'options'
+ * @dest_cfg: Queue destination parameters; valid only if
+ *      'DPSECI_QUEUE_OPT_DEST' is contained in 'options'
+ */
+struct dpseci_rx_queue_cfg {
+        u32 options;
+        int order_preservation_en;
+        u64 user_ctx;
+        struct dpseci_dest_cfg dest_cfg;
+};
+int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        u8 queue, const struct dpseci_rx_queue_cfg *cfg);
+/**
+ * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues
+ * @user_ctx: User context value provided in the frame descriptor of each
+ *      dequeued frame
+ * @order_preservation_en: Status of the order preservation configuration on the
+ *      queue
+ * @dest_cfg: Queue destination configuration
+ * @fqid: Virtual FQID value to be used for dequeue operations
+ */
+struct dpseci_rx_queue_attr {
+        u64 user_ctx;
+        int order_preservation_en;
+        struct dpseci_dest_cfg dest_cfg;
+        u32 fqid;
+};
+int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        u8 queue, struct dpseci_rx_queue_attr *attr);
+/**
+ * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues
+ * @fqid: Virtual FQID to be used for sending frames to SEC hardware
+ * @priority: SEC hardware processing priority for the queue
+ */
+struct dpseci_tx_queue_attr {
+        u32 fqid;
+        u8 priority;
+};
+int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        u8 queue, struct dpseci_tx_queue_attr *attr);
+/**
+ * struct dpseci_sec_attr - Structure representing attributes of the SEC
+ *      hardware accelerator
+ * @ip_id: ID for SEC
+ * @major_rev: Major revision number for SEC
+ * @minor_rev: Minor revision number for SEC
+ * @era: SEC Era
+ * @deco_num: The number of copies of the DECO that are implemented in this
+ *      version of SEC
+ * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented in this
+ *      version of SEC
+ * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented in this
+ *      version of SEC
+ * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are
+ *      implemented in this version of SEC
+ * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are
+ *      implemented in this version of SEC
+ * @crc_acc_num: The number of copies of the CRC module that are implemented in
+ *      this version of SEC
+ * @pk_acc_num:  The number of copies of the Public Key module that are
+ *      implemented in this version of SEC
+ * @kasumi_acc_num: The number of copies of the Kasumi module that are
+ *      implemented in this version of SEC
+ * @rng_acc_num: The number of copies of the Random Number Generator that are
+ *      implemented in this version of SEC
+ * @md_acc_num: The number of copies of the MDHA (Hashing module) that are
+ *      implemented in this version of SEC
+ * @arc4_acc_num: The number of copies of the ARC4 module that are implemented
+ *      in this version of SEC
+ * @des_acc_num: The number of copies of the DES module that are implemented in
+ *      this version of SEC
+ * @aes_acc_num: The number of copies of the AES module that are implemented in
+ *      this version of SEC
+ * @ccha_acc_num: The number of copies of the ChaCha20 module that are
+ *      implemented in this version of SEC.
+ * @ptha_acc_num: The number of copies of the Poly1305 module that are
+ *      implemented in this version of SEC.
+ **/
+struct dpseci_sec_attr {
+        u16 ip_id;
+        u8 major_rev;
+        u8 minor_rev;
+        u8 era;
+        u8 deco_num;
+        u8 zuc_auth_acc_num;
+        u8 zuc_enc_acc_num;
+        u8 snow_f8_acc_num;
+        u8 snow_f9_acc_num;
+        u8 crc_acc_num;
+        u8 pk_acc_num;
+        u8 kasumi_acc_num;
+        u8 rng_acc_num;
+        u8 md_acc_num;
+        u8 arc4_acc_num;
+        u8 des_acc_num;
+        u8 aes_acc_num;
+        u8 ccha_acc_num;
+        u8 ptha_acc_num;
+};
+int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+                        struct dpseci_sec_attr *attr);
+int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags,
+                           u16 *major_ver, u16 *minor_ver);
+/**
+ * enum dpseci_congestion_unit - DPSECI congestion units
+ * @DPSECI_CONGESTION_UNIT_BYTES: bytes units
+ * @DPSECI_CONGESTION_UNIT_FRAMES: frames units
+ */
+enum dpseci_congestion_unit {
+        DPSECI_CONGESTION_UNIT_BYTES = 0,
+        DPSECI_CONGESTION_UNIT_FRAMES
+};
+/**
+ * CSCN message is written to message_iova once entering a
+ * congestion state (see 'threshold_entry')
+ */
+#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER              0x00000001
+/**
+ * CSCN message is written to message_iova once exiting a
+ * congestion state (see 'threshold_exit')
+ */
+#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT               0x00000002
+/**
+ * CSCN write will attempt to allocate into a cache (coherent write);
+ * valid only if 'DPSECI_CGN_MODE_WRITE_MEM_<X>' is selected
+ */
+#define DPSECI_CGN_MODE_COHERENT_WRITE                  0x00000004
+/**
+ * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to
+ * DPIO/DPCON's WQ channel once entering a congestion state
+ * (see 'threshold_entry')
+ */
+#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER            0x00000008
+/**
+ * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to
+ * DPIO/DPCON's WQ channel once exiting a congestion state
+ * (see 'threshold_exit')
+ */
+#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT             0x00000010
+/**
+ * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' when the CSCN is written
+ * to the sw-portal's DQRR, the DQRI interrupt is asserted immediately
+ * (if enabled)
+ */
+#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED        0x00000020
+/**
+ * struct dpseci_congestion_notification_cfg - congestion notification
+ *      configuration
+ * @units: units type
+ * @threshold_entry: above this threshold we enter a congestion state.
+ *      set it to '0' to disable it
+ * @threshold_exit: below this threshold we exit the congestion state.
+ * @message_ctx: The context that will be part of the CSCN message
+ * @message_iova: I/O virtual address (must be in DMA-able memory),
+ *      must be 16B aligned;
+ * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel
+ * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_<X>'
+ *      values
+ */
+struct dpseci_congestion_notification_cfg {
+        enum dpseci_congestion_unit units;
+        u32 threshold_entry;
+        u32 threshold_exit;
+        u64 message_ctx;
+        u64 message_iova;
+        struct dpseci_dest_cfg dest_cfg;
+        u16 notification_mode;
+};
+int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
+        u16 token, const struct dpseci_congestion_notification_cfg *cfg);
+int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
+        u16 token, struct dpseci_congestion_notification_cfg *cfg);
+#endif /* _DPSECI_H_ */
diff --git a/drivers/crypto/caam/dpseci_cmd.h b/drivers/crypto/caam/dpseci_cmd.h
new file mode 100644
index 000000000000..6ab77ead6e3d
--- /dev/null
+++ b/drivers/crypto/caam/dpseci_cmd.h
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2017-2018 NXP
+ */
+#ifndef _DPSECI_CMD_H_
+#define _DPSECI_CMD_H_
+/* DPSECI Version */
+#define DPSECI_VER_MAJOR                                5
+#define DPSECI_VER_MINOR                                3
+#define DPSECI_VER(maj, min)    (((maj) << 16) | (min))
+#define DPSECI_VERSION          DPSECI_VER(DPSECI_VER_MAJOR, DPSECI_VER_MINOR)
+/* Command versioning */
+#define DPSECI_CMD_BASE_VERSION         1
+#define DPSECI_CMD_BASE_VERSION_V2      2
+#define DPSECI_CMD_ID_OFFSET            4
+#define DPSECI_CMD_V1(id)       (((id) << DPSECI_CMD_ID_OFFSET) | \
+                                 DPSECI_CMD_BASE_VERSION)
+#define DPSECI_CMD_V2(id)       (((id) << DPSECI_CMD_ID_OFFSET) | \
+                                 DPSECI_CMD_BASE_VERSION_V2)
+/* Command IDs */
+#define DPSECI_CMDID_CLOSE                              DPSECI_CMD_V1(0x800)
+#define DPSECI_CMDID_OPEN                               DPSECI_CMD_V1(0x809)
+#define DPSECI_CMDID_GET_API_VERSION                    DPSECI_CMD_V1(0xa09)
+#define DPSECI_CMDID_ENABLE                             DPSECI_CMD_V1(0x002)
+#define DPSECI_CMDID_DISABLE                            DPSECI_CMD_V1(0x003)
+#define DPSECI_CMDID_GET_ATTR                           DPSECI_CMD_V1(0x004)
+#define DPSECI_CMDID_IS_ENABLED                         DPSECI_CMD_V1(0x006)
+#define DPSECI_CMDID_SET_RX_QUEUE                       DPSECI_CMD_V1(0x194)
+#define DPSECI_CMDID_GET_RX_QUEUE                       DPSECI_CMD_V1(0x196)
+#define DPSECI_CMDID_GET_TX_QUEUE                       DPSECI_CMD_V1(0x197)
+#define DPSECI_CMDID_GET_SEC_ATTR                       DPSECI_CMD_V2(0x198)
+#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION        DPSECI_CMD_V1(0x170)
+#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION        DPSECI_CMD_V1(0x171)
+/* Macros for accessing command fields smaller than 1 byte */
+#define DPSECI_MASK(field)      \
+        GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1,     \
+                DPSECI_##field##_SHIFT)
+#define dpseci_set_field(var, field, val)       \
+        ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field)))
+#define dpseci_get_field(var, field)    \
+        (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT)
+struct dpseci_cmd_open {
+        __le32 dpseci_id;
+};
+#define DPSECI_ENABLE_SHIFT     0
+#define DPSECI_ENABLE_SIZE      1
+struct dpseci_rsp_is_enabled {
+        u8 is_enabled;
+};
+struct dpseci_rsp_get_attributes {
+        __le32 id;
+        __le32 pad0;
+        u8 num_tx_queues;
+        u8 num_rx_queues;
+        u8 pad1[6];
+        __le32 options;
+};
+#define DPSECI_DEST_TYPE_SHIFT  0
+#define DPSECI_DEST_TYPE_SIZE   4
+#define DPSECI_ORDER_PRESERVATION_SHIFT 0
+#define DPSECI_ORDER_PRESERVATION_SIZE  1
+struct dpseci_cmd_queue {
+        __le32 dest_id;
+        u8 priority;
+        u8 queue;
+        u8 dest_type;
+        u8 pad;
+        __le64 user_ctx;
+        union {
+                __le32 options;
+                __le32 fqid;
+        };
+        u8 order_preservation_en;
+};
+struct dpseci_rsp_get_tx_queue {
+        __le32 pad;
+        __le32 fqid;
+        u8 priority;
+};
+struct dpseci_rsp_get_sec_attr {
+        __le16 ip_id;
+        u8 major_rev;
+        u8 minor_rev;
+        u8 era;
+        u8 pad0[3];
+        u8 deco_num;
+        u8 zuc_auth_acc_num;
+        u8 zuc_enc_acc_num;
+        u8 pad1;
+        u8 snow_f8_acc_num;
+        u8 snow_f9_acc_num;
+        u8 crc_acc_num;
+        u8 pad2;
+        u8 pk_acc_num;
+        u8 kasumi_acc_num;
+        u8 rng_acc_num;
+        u8 pad3;
+        u8 md_acc_num;
+        u8 arc4_acc_num;
+        u8 des_acc_num;
+        u8 aes_acc_num;
+        u8 ccha_acc_num;
+        u8 ptha_acc_num;
+};
+struct dpseci_rsp_get_api_version {
+        __le16 major;
+        __le16 minor;
+};
+#define DPSECI_CGN_DEST_TYPE_SHIFT      0
+#define DPSECI_CGN_DEST_TYPE_SIZE       4
+#define DPSECI_CGN_UNITS_SHIFT          4
+#define DPSECI_CGN_UNITS_SIZE           2
+struct dpseci_cmd_congestion_notification {
+        __le32 dest_id;
+        __le16 notification_mode;
+        u8 priority;
+        u8 options;
+        __le64 message_iova;
+        __le64 message_ctx;
+        __le32 threshold_entry;
+        __le32 threshold_exit;
+};
+#endif /* _DPSECI_CMD_H_ */
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 8da88beb1abb..7e8d690f2827 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -108,6 +108,54 @@ static const struct {
        { 0xF1, "3GPP HFN matches or exceeds the Threshold" },
 };
+static const struct {
+        u8 value;
+        const char *error_text;
+} qi_error_list[] = {
+        { 0x1F, "Job terminated by FQ or ICID flush" },
+        { 0x20, "FD format error"},
+        { 0x21, "FD command format error"},
+        { 0x23, "FL format error"},
+        { 0x25, "CRJD specified in FD, but not enabled in FLC"},
+        { 0x30, "Max. buffer size too small"},
+        { 0x31, "DHR exceeds max. buffer size (allocate mode, S/G format)"},
+        { 0x32, "SGT exceeds max. buffer size (allocate mode, S/G format"},
+        { 0x33, "Size over/underflow (allocate mode)"},
+        { 0x34, "Size over/underflow (reuse mode)"},
+        { 0x35, "Length exceeds max. short length (allocate mode, S/G/ format)"},
+        { 0x36, "Memory footprint exceeds max. value (allocate mode, S/G/ format)"},
+        { 0x41, "SBC frame format not supported (allocate mode)"},
+        { 0x42, "Pool 0 invalid / pool 1 size < pool 0 size (allocate mode)"},
+        { 0x43, "Annotation output enabled but ASAR = 0 (allocate mode)"},
+        { 0x44, "Unsupported or reserved frame format or SGHR = 1 (reuse mode)"},
+        { 0x45, "DHR correction underflow (reuse mode, single buffer format)"},
+        { 0x46, "Annotation length exceeds offset (reuse mode)"},
+        { 0x48, "Annotation output enabled but ASA limited by ASAR (reuse mode)"},
+        { 0x49, "Data offset correction exceeds input frame data length (reuse mode)"},
+        { 0x4B, "Annotation output enabled but ASA cannote be expanded (frame list)"},
+        { 0x51, "Unsupported IF reuse mode"},
+        { 0x52, "Unsupported FL use mode"},
+        { 0x53, "Unsupported RJD use mode"},
+        { 0x54, "Unsupported inline descriptor use mode"},
+        { 0xC0, "Table buffer pool 0 depletion"},
+        { 0xC1, "Table buffer pool 1 depletion"},
+        { 0xC2, "Data buffer pool 0 depletion, no OF allocated"},
+        { 0xC3, "Data buffer pool 1 depletion, no OF allocated"},
+        { 0xC4, "Data buffer pool 0 depletion, partial OF allocated"},
+        { 0xC5, "Data buffer pool 1 depletion, partial OF allocated"},
+        { 0xD0, "FLC read error"},
+        { 0xD1, "FL read error"},
+        { 0xD2, "FL write error"},
+        { 0xD3, "OF SGT write error"},
+        { 0xD4, "PTA read error"},
+        { 0xD5, "PTA write error"},
+        { 0xD6, "OF SGT F-bit write error"},
+        { 0xD7, "ASA write error"},
+        { 0xE1, "FLC[ICR]=0 ICID error"},
+        { 0xE2, "FLC[ICR]=1 ICID error"},
+        { 0xE4, "source of ICID flush not trusted (BDI = 0)"},
+};
 static const char * const cha_id_list[] = {
        "",
        "AES",
@@ -236,6 +284,27 @@ static void report_deco_status(struct device *jrdev, const u32 status,
                status, error, idx_str, idx, err_str, err_err_code);
 }
+static void report_qi_status(struct device *qidev, const u32 status,
+                             const char *error)
+{
+        u8 err_id = status & JRSTA_QIERR_ERROR_MASK;
+        const char *err_str = "unidentified error value 0x";
+        char err_err_code[3] = { 0 };
+        int i;
+        for (i = 0; i < ARRAY_SIZE(qi_error_list); i++)
+                if (qi_error_list[i].value == err_id)
+                        break;
+        if (i != ARRAY_SIZE(qi_error_list) && qi_error_list[i].error_text)
+                err_str = qi_error_list[i].error_text;
+        else
+                snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
+        dev_err(qidev, "%08x: %s: %s%s\n",
+                status, error, err_str, err_err_code);
+}
 static void report_jr_status(struct device *jrdev, const u32 status,
                             const char *error)
 {
@@ -250,7 +319,7 @@ static void report_cond_code_status(struct device *jrdev, const u32 status,
                status, error, __func__);
 }
-void caam_jr_strstatus(struct device *jrdev, u32 status)
+void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
 {
        static const struct stat_src {
                void (*report_ssed)(struct device *jrdev, const u32 status,
@@ -262,7 +331,7 @@ void caam_jr_strstatus(struct device *jrdev, u32 status)
                { report_ccb_status, "CCB" },
                { report_jump_status, "Jump" },
                { report_deco_status, "DECO" },
-                { NULL, "Queue Manager Interface" },
+                { report_qi_status, "Queue Manager Interface" },
                { report_jr_status, "Job Ring" },
                { report_cond_code_status, "Condition Code" },
                { NULL, NULL },
@@ -288,4 +357,8 @@ void caam_jr_strstatus(struct device *jrdev, u32 status)
        else
                dev_err(jrdev, "%d: unknown error source\n", ssrc);
 }
-EXPORT_SYMBOL(caam_jr_strstatus);
+EXPORT_SYMBOL(caam_strstatus);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("FSL CAAM error reporting");
+MODULE_AUTHOR("Freescale Semiconductor");
diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h
index 5aa332bac4b0..67ea94079837 100644
--- a/drivers/crypto/caam/error.h
+++ b/drivers/crypto/caam/error.h
@@ -8,7 +8,11 @@
 #ifndef CAAM_ERROR_H
 #define CAAM_ERROR_H
 #define CAAM_ERROR_STR_MAX 302
-void caam_jr_strstatus(struct device *jrdev, u32 status);
+void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
+#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false)
+#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true)
 void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
                  int rowsize, int groupsize, struct scatterlist *sg,
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index acdd72016ffe..d50085a03597 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
 * CAAM/SEC 4.x transport/backend driver
 * JobR backend functionality
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 67f7f8c42c93..b84e6c8b1e13 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -84,13 +84,6 @@ static u64 times_congested;
 #endif
 /*
- * CPU from where the module initialised. This is required because QMan driver
- * requires CGRs to be removed from same CPU from where they were originally
- * allocated.
- */
-static int mod_init_cpu;
-/*
 * This is a a cache of buffers, from which the users of CAAM QI driver
 * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
 * doing malloc on the hotpath.
@@ -492,12 +485,11 @@ void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
 }
 EXPORT_SYMBOL(caam_drv_ctx_rel);
-int caam_qi_shutdown(struct device *qidev)
+void caam_qi_shutdown(struct device *qidev)
 {
-        int i, ret;
+        int i;
        struct caam_qi_priv *priv = dev_get_drvdata(qidev);
        const cpumask_t *cpus = qman_affine_cpus();
-        struct cpumask old_cpumask = current->cpus_allowed;
        for_each_cpu(i, cpus) {
                struct napi_struct *irqtask;
@@ -510,26 +502,12 @@ int caam_qi_shutdown(struct device *qidev)
                        dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
        }
-        /*
+        qman_delete_cgr_safe(&priv->cgr);
-         * QMan driver requires CGRs to be deleted from same CPU from where they
+        qman_release_cgrid(priv->cgr.cgrid);
-         * were instantiated. Hence we get the module removal execute from the
-         * same CPU from where it was originally inserted.
-         */
-        set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu));
-        ret = qman_delete_cgr(&priv->cgr);
-        if (ret)
-                dev_err(qidev, "Deletion of CGR failed: %d\n", ret);
-        else
-                qman_release_cgrid(priv->cgr.cgrid);
        kmem_cache_destroy(qi_cache);
-        /* Now that we're done with the CGRs, restore the cpus allowed mask */
-        set_cpus_allowed_ptr(current, &old_cpumask);
        platform_device_unregister(priv->qi_pdev);
-        return ret;
 }
 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
@@ -718,22 +696,11 @@ int caam_qi_init(struct platform_device *caam_pdev)
        struct device *ctrldev = &caam_pdev->dev, *qidev;
        struct caam_drv_private *ctrlpriv;
        const cpumask_t *cpus = qman_affine_cpus();
-        struct cpumask old_cpumask = current->cpus_allowed;
        static struct platform_device_info qi_pdev_info = {
                .name = "caam_qi",
                .id = PLATFORM_DEVID_NONE
        };
-        /*
-         * QMAN requires CGRs to be removed from same CPU+portal from where it
-         * was originally allocated. Hence we need to note down the
-         * initialisation CPU and use the same CPU for module exit.
-         * We select the first CPU to from the list of portal owning CPUs.
-         * Then we pin module init to this CPU.
-         */
-        mod_init_cpu = cpumask_first(cpus);
-        set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu));
        qi_pdev_info.parent = ctrldev;
        qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
        qi_pdev = platform_device_register_full(&qi_pdev_info);
@@ -795,8 +762,6 @@ int caam_qi_init(struct platform_device *caam_pdev)
                return -ENOMEM;
        }
-        /* Done with the CGRs; restore the cpus allowed mask */
-        set_cpus_allowed_ptr(current, &old_cpumask);
 #ifdef CONFIG_DEBUG_FS
        debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
                            &times_congested, &caam_fops_u64_ro);
diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h
index 357b69f57072..f93c9c7ed430 100644
--- a/drivers/crypto/caam/qi.h
+++ b/drivers/crypto/caam/qi.h
@@ -62,7 +62,6 @@ typedef void (*caam_qi_cbk)(struct caam_drv_req *drv_req, u32 status);
 enum optype {
        ENCRYPT,
        DECRYPT,
-        GIVENCRYPT,
        NUM_OP
 };
@@ -174,7 +173,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc);
 void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx);
 int caam_qi_init(struct platform_device *pdev);
-int caam_qi_shutdown(struct device *dev);
+void caam_qi_shutdown(struct device *dev);
 /**
 * qi_cache_alloc - Allocate buffers from CAAM-QI cache
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index 4fb91ba39c36..457815f965c0 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -70,22 +70,22 @@
 extern bool caam_little_end;
 extern bool caam_imx;
-#define caam_to_cpu(len)                                \
+#define caam_to_cpu(len)                                                \
-static inline u##len caam##len ## _to_cpu(u##len val)   \
+static inline u##len caam##len ## _to_cpu(u##len val)                   \
-{                                                       \
+{                                                                       \
-        if (caam_little_end)                            \
+        if (caam_little_end)                                            \
-                return le##len ## _to_cpu(val);         \
+                return le##len ## _to_cpu((__force __le##len)val);      \
-        else                                            \
+        else                                                            \
-                return be##len ## _to_cpu(val);         \
+                return be##len ## _to_cpu((__force __be##len)val);      \
 }
-#define cpu_to_caam(len)                                \
+#define cpu_to_caam(len)                                        \
-static inline u##len cpu_to_caam##len(u##len val)       \
+static inline u##len cpu_to_caam##len(u##len val)               \
-{                                                       \
+{                                                               \
-        if (caam_little_end)                            \
+        if (caam_little_end)                                    \
-                return cpu_to_le##len(val);             \
+                return (__force u##len)cpu_to_le##len(val);     \
-        else                                            \
+        else                                                    \
-                return cpu_to_be##len(val);             \
+                return (__force u##len)cpu_to_be##len(val);     \
 }
 caam_to_cpu(16)
@@ -633,6 +633,8 @@ struct caam_job_ring {
 #define JRSTA_DECOERR_INVSIGN       0x86
 #define JRSTA_DECOERR_DSASIGN       0x87
+#define JRSTA_QIERR_ERROR_MASK      0x00ff
 #define JRSTA_CCBERR_JUMP           0x08000000
 #define JRSTA_CCBERR_INDEX_MASK     0xff00
 #define JRSTA_CCBERR_INDEX_SHIFT    8
diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h
index d000b4df745f..b3e1aaaeffea 100644
--- a/drivers/crypto/caam/sg_sw_qm.h
+++ b/drivers/crypto/caam/sg_sw_qm.h
@@ -1,34 +1,7 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /*
 * Copyright 2013-2016 Freescale Semiconductor, Inc.
 * Copyright 2016-2017 NXP
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 #ifndef __SG_SW_QM_H
diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h
index b5b4c12179df..c9378402a5f8 100644
--- a/drivers/crypto/caam/sg_sw_qm2.h
+++ b/drivers/crypto/caam/sg_sw_qm2.h
@@ -1,35 +1,7 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /*
 * Copyright 2015-2016 Freescale Semiconductor, Inc.
 * Copyright 2017 NXP
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the names of the above-listed copyright holders nor the
- *       names of any contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
 */
 #ifndef _SG_SW_QM2_H_
diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
index b0ba4331944b..ca549c5dc08e 100644
--- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
+++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
@@ -308,21 +308,11 @@ void do_request_cleanup(struct cpt_vf *cptvf,
                }
        }
-        if (info->scatter_components)
+        kzfree(info->scatter_components);
-                kzfree(info->scatter_components);
+        kzfree(info->gather_components);
+        kzfree(info->out_buffer);
-        if (info->gather_components)
+        kzfree(info->in_buffer);
-                kzfree(info->gather_components);
+        kzfree((void *)info->completion_addr);
-        if (info->out_buffer)
-                kzfree(info->out_buffer);
-        if (info->in_buffer)
-                kzfree(info->in_buffer);
-        if (info->completion_addr)
-                kzfree((void *)info->completion_addr);
        kzfree(info);
 }
diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile
index 45b7379e8e30..e12954791673 100644
--- a/drivers/crypto/cavium/nitrox/Makefile
+++ b/drivers/crypto/cavium/nitrox/Makefile
@@ -7,3 +7,6 @@ n5pf-objs := nitrox_main.o \
        nitrox_hal.o \
        nitrox_reqmgr.o \
        nitrox_algs.o
+n5pf-$(CONFIG_PCI_IOV) += nitrox_sriov.o
+n5pf-$(CONFIG_DEBUG_FS) += nitrox_debugfs.o
diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h
index 312f72801af6..863143a8336b 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_common.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_common.h
@@ -12,32 +12,15 @@ void crypto_free_context(void *ctx);
 struct nitrox_device *nitrox_get_first_device(void);
 void nitrox_put_device(struct nitrox_device *ndev);
-void nitrox_pf_cleanup_isr(struct nitrox_device *ndev);
-int nitrox_pf_init_isr(struct nitrox_device *ndev);
 int nitrox_common_sw_init(struct nitrox_device *ndev);
 void nitrox_common_sw_cleanup(struct nitrox_device *ndev);
-void pkt_slc_resp_handler(unsigned long data);
+void pkt_slc_resp_tasklet(unsigned long data);
 int nitrox_process_se_request(struct nitrox_device *ndev,
                              struct se_crypto_request *req,
                              completion_t cb,
                              struct skcipher_request *skreq);
 void backlog_qflush_work(struct work_struct *work);
-void nitrox_config_emu_unit(struct nitrox_device *ndev);
-void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
-void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
-void nitrox_config_vfmode(struct nitrox_device *ndev, int mode);
-void nitrox_config_nps_unit(struct nitrox_device *ndev);
-void nitrox_config_pom_unit(struct nitrox_device *ndev);
-void nitrox_config_rand_unit(struct nitrox_device *ndev);
-void nitrox_config_efl_unit(struct nitrox_device *ndev);
-void nitrox_config_bmi_unit(struct nitrox_device *ndev);
-void nitrox_config_bmo_unit(struct nitrox_device *ndev);
-void nitrox_config_lbc_unit(struct nitrox_device *ndev);
-void invalidate_lbc(struct nitrox_device *ndev);
-void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
-void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
 #endif /* __NITROX_COMMON_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h
index 9dcb7fdbe0a7..1ad27b1a87c5 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_csr.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h
@@ -7,9 +7,16 @@
 /* EMU clusters */
 #define NR_CLUSTERS             4
+/* Maximum cores per cluster,
+ * varies based on partname
+ */
 #define AE_CORES_PER_CLUSTER    20
 #define SE_CORES_PER_CLUSTER    16
+#define AE_MAX_CORES    (AE_CORES_PER_CLUSTER * NR_CLUSTERS)
+#define SE_MAX_CORES    (SE_CORES_PER_CLUSTER * NR_CLUSTERS)
+#define ZIP_MAX_CORES   5
 /* BIST registers */
 #define EMU_BIST_STATUSX(_i)    (0x1402700 + ((_i) * 0x40000))
 #define UCD_BIST_STATUS         0x12C0070
@@ -111,6 +118,9 @@
 #define LBC_ELM_VF65_128_INT            0x120C000
 #define LBC_ELM_VF65_128_INT_ENA_W1S    0x120F000
+#define RST_BOOT        0x10C1600
+#define FUS_DAT1        0x10C1408
 /* PEM registers */
 #define PEM0_INT 0x1080428
@@ -1082,4 +1092,105 @@ union lbc_inval_status {
        } s;
 };
+/**
+ * struct rst_boot: RST Boot Register
+ * @jtcsrdis: when set, internal CSR access via JTAG TAP controller
+ *   is disabled
+ * @jt_tst_mode: JTAG test mode
+ * @io_supply: I/O power supply setting based on IO_VDD_SELECT pin:
+ *    0x1 = 1.8V
+ *    0x2 = 2.5V
+ *    0x4 = 3.3V
+ *    All other values are reserved
+ * @pnr_mul: clock multiplier
+ * @lboot: last boot cause mask, resets only with PLL_DC_OK
+ * @rboot: determines whether core 0 remains in reset after
+ *    chip cold or warm or soft reset
+ * @rboot_pin: read only access to REMOTE_BOOT pin
+ */
+union rst_boot {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_63 : 1;
+                u64 jtcsrdis : 1;
+                u64 raz_59_61 : 3;
+                u64 jt_tst_mode : 1;
+                u64 raz_40_57 : 18;
+                u64 io_supply : 3;
+                u64 raz_30_36 : 7;
+                u64 pnr_mul : 6;
+                u64 raz_12_23 : 12;
+                u64 lboot : 10;
+                u64 rboot : 1;
+                u64 rboot_pin : 1;
+#else
+                u64 rboot_pin : 1;
+                u64 rboot : 1;
+                u64 lboot : 10;
+                u64 raz_12_23 : 12;
+                u64 pnr_mul : 6;
+                u64 raz_30_36 : 7;
+                u64 io_supply : 3;
+                u64 raz_40_57 : 18;
+                u64 jt_tst_mode : 1;
+                u64 raz_59_61 : 3;
+                u64 jtcsrdis : 1;
+                u64 raz_63 : 1;
+#endif
+        };
+};
+/**
+ * struct fus_dat1: Fuse Data 1 Register
+ * @pll_mul: main clock PLL multiplier hardware limit
+ * @pll_half_dis: main clock PLL control
+ * @efus_lck: efuse lockdown
+ * @zip_info: ZIP information
+ * @bar2_sz_conf: when zero, BAR2 size conforms to
+ *    PCIe specification
+ * @efus_ign: efuse ignore
+ * @nozip: ZIP disable
+ * @pll_alt_matrix: select alternate PLL matrix
+ * @pll_bwadj_denom: select CLKF denominator for
+ *    BWADJ value
+ * @chip_id: chip ID
+ */
+union fus_dat1 {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_57_63 : 7;
+                u64 pll_mul : 3;
+                u64 pll_half_dis : 1;
+                u64 raz_43_52 : 10;
+                u64 efus_lck : 3;
+                u64 raz_26_39 : 14;
+                u64 zip_info : 5;
+                u64 bar2_sz_conf : 1;
+                u64 efus_ign : 1;
+                u64 nozip : 1;
+                u64 raz_11_17 : 7;
+                u64 pll_alt_matrix : 1;
+                u64 pll_bwadj_denom : 2;
+                u64 chip_id : 8;
+#else
+                u64 chip_id : 8;
+                u64 pll_bwadj_denom : 2;
+                u64 pll_alt_matrix : 1;
+                u64 raz_11_17 : 7;
+                u64 nozip : 1;
+                u64 efus_ign : 1;
+                u64 bar2_sz_conf : 1;
+                u64 zip_info : 5;
+                u64 raz_26_39 : 14;
+                u64 efus_lck : 3;
+                u64 raz_43_52 : 10;
+                u64 pll_half_dis : 1;
+                u64 pll_mul : 3;
+                u64 raz_57_63 : 7;
+#endif
+        };
+};
 #endif /* __NITROX_CSR_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
new file mode 100644
index 000000000000..5f3cd5fafe04
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+#include "nitrox_csr.h"
+#include "nitrox_dev.h"
+static int firmware_show(struct seq_file *s, void *v)
+{
+        struct nitrox_device *ndev = s->private;
+        seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
+        return 0;
+}
+static int firmware_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, firmware_show, inode->i_private);
+}
+static const struct file_operations firmware_fops = {
+        .owner = THIS_MODULE,
+        .open = firmware_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+static int device_show(struct seq_file *s, void *v)
+{
+        struct nitrox_device *ndev = s->private;
+        seq_printf(s, "NITROX [%d]\n", ndev->idx);
+        seq_printf(s, "  Part Name: %s\n", ndev->hw.partname);
+        seq_printf(s, "  Frequency: %d MHz\n", ndev->hw.freq);
+        seq_printf(s, "  Device ID: 0x%0x\n", ndev->hw.device_id);
+        seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
+        seq_printf(s, "  Cores: [AE=%u  SE=%u  ZIP=%u]\n",
+                   ndev->hw.ae_cores, ndev->hw.se_cores, ndev->hw.zip_cores);
+        return 0;
+}
+static int nitrox_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, device_show, inode->i_private);
+}
+static const struct file_operations nitrox_fops = {
+        .owner = THIS_MODULE,
+        .open = nitrox_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+static int stats_show(struct seq_file *s, void *v)
+{
+        struct nitrox_device *ndev = s->private;
+        seq_printf(s, "NITROX [%d] Request Statistics\n", ndev->idx);
+        seq_printf(s, "  Posted: %llu\n",
+                   (u64)atomic64_read(&ndev->stats.posted));
+        seq_printf(s, "  Completed: %llu\n",
+                   (u64)atomic64_read(&ndev->stats.completed));
+        seq_printf(s, "  Dropped: %llu\n",
+                   (u64)atomic64_read(&ndev->stats.dropped));
+        return 0;
+}
+static int nitrox_stats_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, stats_show, inode->i_private);
+}
+static const struct file_operations nitrox_stats_fops = {
+        .owner = THIS_MODULE,
+        .open = nitrox_stats_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+void nitrox_debugfs_exit(struct nitrox_device *ndev)
+{
+        debugfs_remove_recursive(ndev->debugfs_dir);
+        ndev->debugfs_dir = NULL;
+}
+int nitrox_debugfs_init(struct nitrox_device *ndev)
+{
+        struct dentry *dir, *f;
+        dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+        if (!dir)
+                return -ENOMEM;
+        ndev->debugfs_dir = dir;
+        f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
+        if (!f)
+                goto err;
+        f = debugfs_create_file("device", 0400, dir, ndev, &nitrox_fops);
+        if (!f)
+                goto err;
+        f = debugfs_create_file("stats", 0400, dir, ndev, &nitrox_stats_fops);
+        if (!f)
+                goto err;
+        return 0;
+err:
+        nitrox_debugfs_exit(ndev);
+        return -ENODEV;
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h
index af596455b420..283e252385fb 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_dev.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h
@@ -5,92 +5,123 @@
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/pci.h>
+#include <linux/if.h>
 #define VERSION_LEN 32
+/**
+ * struct nitrox_cmdq - NITROX command queue
+ * @cmd_qlock: command queue lock
+ * @resp_qlock: response queue lock
+ * @backlog_qlock: backlog queue lock
+ * @ndev: NITROX device
+ * @response_head: submitted request list
+ * @backlog_head: backlog queue
+ * @dbell_csr_addr: doorbell register address for this queue
+ * @compl_cnt_csr_addr: completion count register address of the slc port
+ * @base: command queue base address
+ * @dma: dma address of the base
+ * @pending_count: request pending at device
+ * @backlog_count: backlog request count
+ * @write_idx: next write index for the command
+ * @instr_size: command size
+ * @qno: command queue number
+ * @qsize: command queue size
+ * @unalign_base: unaligned base address
+ * @unalign_dma: unaligned dma address
+ */
 struct nitrox_cmdq {
-        /* command queue lock */
+        spinlock_t cmd_qlock;
-        spinlock_t cmdq_lock;
+        spinlock_t resp_qlock;
-        /* response list lock */
+        spinlock_t backlog_qlock;
-        spinlock_t response_lock;
-        /* backlog list lock */
+        struct nitrox_device *ndev;
-        spinlock_t backlog_lock;
-        /* request submitted to chip, in progress */
        struct list_head response_head;
-        /* hw queue full, hold in backlog list */
        struct list_head backlog_head;
-        /* doorbell address */
        u8 __iomem *dbell_csr_addr;
-        /* base address of the queue */
+        u8 __iomem *compl_cnt_csr_addr;
-        u8 *head;
+        u8 *base;
+        dma_addr_t dma;
-        struct nitrox_device *ndev;
-        /* flush pending backlog commands */
        struct work_struct backlog_qflush;
-        /* requests posted waiting for completion */
        atomic_t pending_count;
-        /* requests in backlog queues */
        atomic_t backlog_count;
        int write_idx;
-        /* command size 32B/64B */
        u8 instr_size;
        u8 qno;
        u32 qsize;
-        /* unaligned addresses */
+        u8 *unalign_base;
-        u8 *head_unaligned;
+        dma_addr_t unalign_dma;
-        dma_addr_t dma_unaligned;
-        /* dma address of the base */
-        dma_addr_t dma;
 };
+/**
+ * struct nitrox_hw - NITROX hardware information
+ * @partname: partname ex: CNN55xxx-xxx
+ * @fw_name: firmware version
+ * @freq: NITROX frequency
+ * @vendor_id: vendor ID
+ * @device_id: device ID
+ * @revision_id: revision ID
+ * @se_cores: number of symmetric cores
+ * @ae_cores: number of asymmetric cores
+ * @zip_cores: number of zip cores
+ */
 struct nitrox_hw {
-        /* firmware version */
+        char partname[IFNAMSIZ * 2];
        char fw_name[VERSION_LEN];
+        int freq;
        u16 vendor_id;
        u16 device_id;
        u8 revision_id;
-        /* CNN55XX cores */
        u8 se_cores;
        u8 ae_cores;
        u8 zip_cores;
 };
-#define MAX_MSIX_VECTOR_NAME    20
+struct nitrox_stats {
-/**
+        atomic64_t posted;
- * vectors for queues (64 AE, 64 SE and 64 ZIP) and
+        atomic64_t completed;
- * error condition/mailbox.
+        atomic64_t dropped;
- */
-#define MAX_MSIX_VECTORS        192
-struct nitrox_msix {
-        struct msix_entry *entries;
-        char **names;
-        DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS);
-        u32 nr_entries;
 };
-struct bh_data {
+#define IRQ_NAMESZ      32
-        /* slc port completion count address */
-        u8 __iomem *completion_cnt_csr_addr;
+struct nitrox_q_vector {
+        char name[IRQ_NAMESZ];
+        bool valid;
+        int ring;
+        struct tasklet_struct resp_tasklet;
+        union {
+                struct nitrox_cmdq *cmdq;
+                struct nitrox_device *ndev;
+        };
+};
-        struct nitrox_cmdq *cmdq;
+/*
-        struct tasklet_struct resp_handler;
+ * NITROX Device states
+ */
+enum ndev_state {
+        __NDEV_NOT_READY,
+        __NDEV_READY,
+        __NDEV_IN_RESET,
 };
-struct nitrox_bh {
+/* NITROX support modes for VF(s) */
-        struct bh_data *slc;
+enum vf_mode {
+        __NDEV_MODE_PF,
+        __NDEV_MODE_VF16,
+        __NDEV_MODE_VF32,
+        __NDEV_MODE_VF64,
+        __NDEV_MODE_VF128,
 };
-/* NITROX-V driver state */
+#define __NDEV_SRIOV_BIT 0
-#define NITROX_UCODE_LOADED     0
-#define NITROX_READY            1
 /* command queue size */
 #define DEFAULT_CMD_QLEN 2048
@@ -98,7 +129,6 @@ struct nitrox_bh {
 #define CMD_TIMEOUT 2000
 #define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev))
-#define PF_MODE 0
 #define NITROX_CSR_ADDR(ndev, offset) \
        ((ndev)->bar_addr + (offset))
@@ -108,17 +138,18 @@ struct nitrox_bh {
 * @list: pointer to linked list of devices
 * @bar_addr: iomap address
 * @pdev: PCI device information
- * @status: NITROX status
+ * @state: NITROX device state
+ * @flags: flags to indicate device the features
 * @timeout: Request timeout in jiffies
 * @refcnt: Device usage count
 * @idx: device index (0..N)
 * @node: NUMA node id attached
 * @qlen: Command queue length
 * @nr_queues: Number of command queues
+ * @mode: Device mode PF/VF
 * @ctx_pool: DMA pool for crypto context
- * @pkt_cmdqs: SE Command queues
+ * @pkt_inq: Packet input rings
- * @msix: MSI-X information
+ * @qvec: MSI-X queue vectors information
- * @bh: post processing work
 * @hw: hardware information
 * @debugfs_dir: debugfs directory
 */
@@ -128,7 +159,8 @@ struct nitrox_device {
        u8 __iomem *bar_addr;
        struct pci_dev *pdev;
-        unsigned long status;
+        atomic_t state;
+        unsigned long flags;
        unsigned long timeout;
        refcount_t refcnt;
@@ -136,13 +168,16 @@ struct nitrox_device {
        int node;
        u16 qlen;
        u16 nr_queues;
+        int num_vfs;
+        enum vf_mode mode;
        struct dma_pool *ctx_pool;
-        struct nitrox_cmdq *pkt_cmdqs;
+        struct nitrox_cmdq *pkt_inq;
-        struct nitrox_msix msix;
+        struct nitrox_q_vector *qvec;
-        struct nitrox_bh bh;
+        int num_vecs;
+        struct nitrox_stats stats;
        struct nitrox_hw hw;
 #if IS_ENABLED(CONFIG_DEBUG_FS)
        struct dentry *debugfs_dir;
@@ -173,9 +208,22 @@ static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset,
        writeq(value, (ndev->bar_addr + offset));
 }
-static inline int nitrox_ready(struct nitrox_device *ndev)
+static inline bool nitrox_ready(struct nitrox_device *ndev)
 {
-        return test_bit(NITROX_READY, &ndev->status);
+        return atomic_read(&ndev->state) == __NDEV_READY;
 }
+#ifdef CONFIG_DEBUG_FS
+int nitrox_debugfs_init(struct nitrox_device *ndev);
+void nitrox_debugfs_exit(struct nitrox_device *ndev);
+#else
+static inline int nitrox_debugfs_init(struct nitrox_device *ndev)
+{
+        return 0;
+}
+static inline void nitrox_debugfs_exit(struct nitrox_device *ndev)
+{ }
+#endif
 #endif /* __NITROX_DEV_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c
index ab4ccf2f9e77..a9b82387cf53 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_hal.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c
@@ -4,6 +4,8 @@
 #include "nitrox_dev.h"
 #include "nitrox_csr.h"
+#define PLL_REF_CLK 50
 /**
 * emu_enable_cores - Enable EMU cluster cores.
 * @ndev: N5 device
@@ -117,7 +119,7 @@ void nitrox_config_pkt_input_rings(struct nitrox_device *ndev)
        int i;
        for (i = 0; i < ndev->nr_queues; i++) {
-                struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i];
+                struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
                union nps_pkt_in_instr_rsize pkt_in_rsize;
                u64 offset;
@@ -256,7 +258,7 @@ void nitrox_config_nps_unit(struct nitrox_device *ndev)
        /* disable ILK interface */
        core_gbl_vfcfg.value = 0;
        core_gbl_vfcfg.s.ilk_disable = 1;
-        core_gbl_vfcfg.s.cfg = PF_MODE;
+        core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF;
        nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
        /* config input and solicit ports */
        nitrox_config_pkt_input_rings(ndev);
@@ -400,3 +402,68 @@ void nitrox_config_lbc_unit(struct nitrox_device *ndev)
        offset = LBC_ELM_VF65_128_INT_ENA_W1S;
        nitrox_write_csr(ndev, offset, (~0ULL));
 }
+void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode)
+{
+        union nps_core_gbl_vfcfg vfcfg;
+        vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG);
+        vfcfg.s.cfg = mode & 0x7;
+        nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value);
+}
+void nitrox_get_hwinfo(struct nitrox_device *ndev)
+{
+        union emu_fuse_map emu_fuse;
+        union rst_boot rst_boot;
+        union fus_dat1 fus_dat1;
+        unsigned char name[IFNAMSIZ * 2] = {};
+        int i, dead_cores;
+        u64 offset;
+        /* get core frequency */
+        offset = RST_BOOT;
+        rst_boot.value = nitrox_read_csr(ndev, offset);
+        ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK;
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                offset = EMU_FUSE_MAPX(i);
+                emu_fuse.value = nitrox_read_csr(ndev, offset);
+                if (emu_fuse.s.valid) {
+                        dead_cores = hweight32(emu_fuse.s.ae_fuse);
+                        ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
+                        dead_cores = hweight16(emu_fuse.s.se_fuse);
+                        ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
+                }
+        }
+        /* find zip hardware availability */
+        offset = FUS_DAT1;
+        fus_dat1.value = nitrox_read_csr(ndev, offset);
+        if (!fus_dat1.nozip) {
+                dead_cores = hweight8(fus_dat1.zip_info);
+                ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores;
+        }
+        /* determine the partname CNN55<cores>-<freq><pincount>-<rev>*/
+        if (ndev->hw.ae_cores == AE_MAX_CORES) {
+                switch (ndev->hw.se_cores) {
+                case SE_MAX_CORES:
+                        i = snprintf(name, sizeof(name), "CNN5560");
+                        break;
+                case 40:
+                        i = snprintf(name, sizeof(name), "CNN5560s");
+                        break;
+                }
+        } else if (ndev->hw.ae_cores == (AE_MAX_CORES / 2)) {
+                i = snprintf(name, sizeof(name), "CNN5530");
+        } else {
+                i = snprintf(name, sizeof(name), "CNN5560i");
+        }
+        snprintf(name + i, sizeof(name) - i, "-%3dBG676-1.%u",
+                 ndev->hw.freq, ndev->hw.revision_id);
+        /* copy partname */
+        strncpy(ndev->hw.partname, name, sizeof(ndev->hw.partname));
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.h b/drivers/crypto/cavium/nitrox/nitrox_hal.h
new file mode 100644
index 000000000000..489ee64c119e
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_HAL_H
+#define __NITROX_HAL_H
+#include "nitrox_dev.h"
+void nitrox_config_emu_unit(struct nitrox_device *ndev);
+void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
+void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
+void nitrox_config_nps_unit(struct nitrox_device *ndev);
+void nitrox_config_pom_unit(struct nitrox_device *ndev);
+void nitrox_config_rand_unit(struct nitrox_device *ndev);
+void nitrox_config_efl_unit(struct nitrox_device *ndev);
+void nitrox_config_bmi_unit(struct nitrox_device *ndev);
+void nitrox_config_bmo_unit(struct nitrox_device *ndev);
+void nitrox_config_lbc_unit(struct nitrox_device *ndev);
+void invalidate_lbc(struct nitrox_device *ndev);
+void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
+void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
+void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode);
+void nitrox_get_hwinfo(struct nitrox_device *ndev);
+#endif /* __NITROX_HAL_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c
index ee0d70ba25d5..88a77b8fb3fb 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_isr.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c
@@ -6,9 +6,16 @@
 #include "nitrox_dev.h"
 #include "nitrox_csr.h"
 #include "nitrox_common.h"
+#include "nitrox_hal.h"
+/**
+ * One vector for each type of ring
+ *  - NPS packet ring, AQMQ ring and ZQMQ ring
+ */
 #define NR_RING_VECTORS 3
-#define NPS_CORE_INT_ACTIVE_ENTRY 192
+/* base entry for packet ring/port */
+#define PKT_RING_MSIX_BASE 0
+#define NON_RING_MSIX_BASE 192
 /**
 * nps_pkt_slc_isr - IRQ handler for NPS solicit port
@@ -17,13 +24,14 @@
 */
 static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
 {
-        struct bh_data *slc = data;
+        struct nitrox_q_vector *qvec = data;
-        union nps_pkt_slc_cnts pkt_slc_cnts;
+        union nps_pkt_slc_cnts slc_cnts;
+        struct nitrox_cmdq *cmdq = qvec->cmdq;
-        pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr);
+        slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
        /* New packet on SLC output port */
-        if (pkt_slc_cnts.s.slc_int)
+        if (slc_cnts.s.slc_int)
-                tasklet_hi_schedule(&slc->resp_handler);
+                tasklet_hi_schedule(&qvec->resp_tasklet);
        return IRQ_HANDLED;
 }
@@ -190,165 +198,92 @@ static void clear_bmi_err_intr(struct nitrox_device *ndev)
        dev_err_ratelimited(DEV(ndev), "BMI_INT  0x%016llx\n", value);
 }
+static void nps_core_int_tasklet(unsigned long data)
+{
+        struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
+        struct nitrox_device *ndev = qvec->ndev;
+        /* if pf mode do queue recovery */
+        if (ndev->mode == __NDEV_MODE_PF) {
+        } else {
+                /**
+                 * if VF(s) enabled communicate the error information
+                 * to VF(s)
+                 */
+        }
+}
 /**
- * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts
+ * nps_core_int_isr - interrupt handler for NITROX errors and
- * @ndev: NITROX device
+ *   mailbox communication
 */
-static void clear_nps_core_int_active(struct nitrox_device *ndev)
+static irqreturn_t nps_core_int_isr(int irq, void *data)
 {
-        union nps_core_int_active core_int_active;
+        struct nitrox_device *ndev = data;
+        union nps_core_int_active core_int;
-        core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
+        core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
-        if (core_int_active.s.nps_core)
+        if (core_int.s.nps_core)
                clear_nps_core_err_intr(ndev);
-        if (core_int_active.s.nps_pkt)
+        if (core_int.s.nps_pkt)
                clear_nps_pkt_err_intr(ndev);
-        if (core_int_active.s.pom)
+        if (core_int.s.pom)
                clear_pom_err_intr(ndev);
-        if (core_int_active.s.pem)
+        if (core_int.s.pem)
                clear_pem_err_intr(ndev);
-        if (core_int_active.s.lbc)
+        if (core_int.s.lbc)
                clear_lbc_err_intr(ndev);
-        if (core_int_active.s.efl)
+        if (core_int.s.efl)
                clear_efl_err_intr(ndev);
-        if (core_int_active.s.bmi)
+        if (core_int.s.bmi)
                clear_bmi_err_intr(ndev);
        /* If more work callback the ISR, set resend */
-        core_int_active.s.resend = 1;
+        core_int.s.resend = 1;
-        nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value);
+        nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value);
-}
-static irqreturn_t nps_core_int_isr(int irq, void *data)
-{
-        struct nitrox_device *ndev = data;
-        clear_nps_core_int_active(ndev);
        return IRQ_HANDLED;
 }
-static int nitrox_enable_msix(struct nitrox_device *ndev)
+void nitrox_unregister_interrupts(struct nitrox_device *ndev)
 {
-        struct msix_entry *entries;
+        struct pci_dev *pdev = ndev->pdev;
-        char **names;
-        int i, nr_entries, ret;
-        /*
-         * PF MSI-X vectors
-         *
-         * Entry 0: NPS PKT ring 0
-         * Entry 1: AQMQ ring 0
-         * Entry 2: ZQM ring 0
-         * Entry 3: NPS PKT ring 1
-         * Entry 4: AQMQ ring 1
-         * Entry 5: ZQM ring 1
-         * ....
-         * Entry 192: NPS_CORE_INT_ACTIVE
-         */
-        nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1;
-        entries = kcalloc_node(nr_entries, sizeof(struct msix_entry),
-                               GFP_KERNEL, ndev->node);
-        if (!entries)
-                return -ENOMEM;
-        names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL);
-        if (!names) {
-                kfree(entries);
-                return -ENOMEM;
-        }
-        /* fill entires */
-        for (i = 0; i < (nr_entries - 1); i++)
-                entries[i].entry = i;
-        entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY;
-        for (i = 0; i < nr_entries; i++) {
-                *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
-                if (!(*(names + i))) {
-                        ret = -ENOMEM;
-                        goto msix_fail;
-                }
-        }
-        ndev->msix.entries = entries;
-        ndev->msix.names = names;
-        ndev->msix.nr_entries = nr_entries;
-        ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries,
-                                    ndev->msix.nr_entries);
-        if (ret) {
-                dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n",
-                        ret);
-                goto msix_fail;
-        }
-        return 0;
-msix_fail:
-        for (i = 0; i < nr_entries; i++)
-                kfree(*(names + i));
-        kfree(entries);
-        kfree(names);
-        return ret;
-}
-static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev)
-{
-        int i;
-        if (!ndev->bh.slc)
-                return;
-        for (i = 0; i < ndev->nr_queues; i++) {
-                struct bh_data *bh = &ndev->bh.slc[i];
-                tasklet_disable(&bh->resp_handler);
-                tasklet_kill(&bh->resp_handler);
-        }
-        kfree(ndev->bh.slc);
-        ndev->bh.slc = NULL;
-}
-static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev)
-{
-        u32 size;
        int i;
-        size = ndev->nr_queues * sizeof(struct bh_data);
+        for (i = 0; i < ndev->num_vecs; i++) {
-        ndev->bh.slc = kzalloc(size, GFP_KERNEL);
+                struct nitrox_q_vector *qvec;
-        if (!ndev->bh.slc)
+                int vec;
-                return -ENOMEM;
-        for (i = 0; i < ndev->nr_queues; i++) {
+                qvec = ndev->qvec + i;
-                struct bh_data *bh = &ndev->bh.slc[i];
+                if (!qvec->valid)
-                u64 offset;
+                        continue;
-                offset = NPS_PKT_SLC_CNTSX(i);
+                /* get the vector number */
-                /* pre calculate completion count address */
+                vec = pci_irq_vector(pdev, i);
-                bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+                irq_set_affinity_hint(vec, NULL);
-                bh->cmdq = &ndev->pkt_cmdqs[i];
+                free_irq(vec, qvec);
-                tasklet_init(&bh->resp_handler, pkt_slc_resp_handler,
+                tasklet_disable(&qvec->resp_tasklet);
-                             (unsigned long)bh);
+                tasklet_kill(&qvec->resp_tasklet);
+                qvec->valid = false;
        }
+        kfree(ndev->qvec);
-        return 0;
+        pci_free_irq_vectors(pdev);
 }
-static int nitrox_request_irqs(struct nitrox_device *ndev)
+int nitrox_register_interrupts(struct nitrox_device *ndev)
 {
        struct pci_dev *pdev = ndev->pdev;
-        struct msix_entry *msix_ent = ndev->msix.entries;
+        struct nitrox_q_vector *qvec;
-        int nr_ring_vectors, i = 0, ring, cpu, ret;
+        int nr_vecs, vec, cpu;
-        char *name;
+        int ret, i;
        /*
         * PF MSI-X vectors
@@ -357,112 +292,76 @@ static int nitrox_request_irqs(struct nitrox_device *ndev)
         * Entry 1: AQMQ ring 0
         * Entry 2: ZQM ring 0
         * Entry 3: NPS PKT ring 1
+         * Entry 4: AQMQ ring 1
+         * Entry 5: ZQM ring 1
         * ....
         * Entry 192: NPS_CORE_INT_ACTIVE
         */
-        nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS;
+        nr_vecs = pci_msix_vec_count(pdev);
-        /* request irq for pkt ring/ports only */
-        while (i < nr_ring_vectors) {
-                name = *(ndev->msix.names + i);
-                ring = (i / NR_RING_VECTORS);
-                snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d",
-                         ndev->idx, ring);
-                ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0,
+        /* Enable MSI-X */
-                                  name, &ndev->bh.slc[ring]);
+        ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
-                if (ret) {
+        if (ret < 0) {
-                        dev_err(&pdev->dev, "failed to get irq %d for %s\n",
+                dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
-                                msix_ent[i].vector, name);
-                        return ret;
-                }
-                cpu = ring % num_online_cpus();
-                irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu));
-                set_bit(i, ndev->msix.irqs);
-                i += NR_RING_VECTORS;
-        }
-        /* Request IRQ for NPS_CORE_INT_ACTIVE */
-        name = *(ndev->msix.names + i);
-        snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx);
-        ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev);
-        if (ret) {
-                dev_err(&pdev->dev, "failed to get irq %d for %s\n",
-                        msix_ent[i].vector, name);
                return ret;
        }
-        set_bit(i, ndev->msix.irqs);
+        ndev->num_vecs = nr_vecs;
-        return 0;
+        ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL);
-}
+        if (!ndev->qvec) {
+                pci_free_irq_vectors(pdev);
-static void nitrox_disable_msix(struct nitrox_device *ndev)
+                return -ENOMEM;
-{
-        struct msix_entry *msix_ent = ndev->msix.entries;
-        char **names = ndev->msix.names;
-        int i = 0, ring, nr_ring_vectors;
-        nr_ring_vectors = ndev->msix.nr_entries - 1;
-        /* clear pkt ring irqs */
-        while (i < nr_ring_vectors) {
-                if (test_and_clear_bit(i, ndev->msix.irqs)) {
-                        ring = (i / NR_RING_VECTORS);
-                        irq_set_affinity_hint(msix_ent[i].vector, NULL);
-                        free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]);
-                }
-                i += NR_RING_VECTORS;
        }
-        irq_set_affinity_hint(msix_ent[i].vector, NULL);
-        free_irq(msix_ent[i].vector, ndev);
-        clear_bit(i, ndev->msix.irqs);
-        kfree(ndev->msix.entries);
+        /* request irqs for packet rings/ports */
-        for (i = 0; i < ndev->msix.nr_entries; i++)
+        for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
-                kfree(*(names + i));
+                qvec = &ndev->qvec[i];
-        kfree(names);
+                qvec->ring = i / NR_RING_VECTORS;
-        pci_disable_msix(ndev->pdev);
+                if (qvec->ring >= ndev->nr_queues)
-}
+                        break;
-/**
- * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ
- * @ndev: NITROX device
- */
-void nitrox_pf_cleanup_isr(struct nitrox_device *ndev)
-{
-        nitrox_disable_msix(ndev);
-        nitrox_cleanup_pkt_slc_bh(ndev);
-}
-/**
+                snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring);
- * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ
+                /* get the vector number */
- * @ndev: NITROX device
+                vec = pci_irq_vector(pdev, i);
- *
+                ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec);
- * Return: 0 on success, a negative value on failure.
+                if (ret) {
- */
+                        dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
-int nitrox_pf_init_isr(struct nitrox_device *ndev)
+                                qvec->ring);
-{
+                        goto irq_fail;
-        int err;
+                }
+                cpu = qvec->ring % num_online_cpus();
+                irq_set_affinity_hint(vec, get_cpu_mask(cpu));
-        err = nitrox_setup_pkt_slc_bh(ndev);
+                tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet,
-        if (err)
+                             (unsigned long)qvec);
-                return err;
+                qvec->cmdq = &ndev->pkt_inq[qvec->ring];
+                qvec->valid = true;
+        }
-        err = nitrox_enable_msix(ndev);
+        /* request irqs for non ring vectors */
-        if (err)
+        i = NON_RING_MSIX_BASE;
-                goto msix_fail;
+        qvec = &ndev->qvec[i];
-        err = nitrox_request_irqs(ndev);
+        snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i);
-        if (err)
+        /* get the vector number */
+        vec = pci_irq_vector(pdev, i);
+        ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
+        if (ret) {
+                dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
                goto irq_fail;
+        }
+        cpu = num_online_cpus();
+        irq_set_affinity_hint(vec, get_cpu_mask(cpu));
+        tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
+                     (unsigned long)qvec);
+        qvec->ndev = ndev;
+        qvec->valid = true;
        return 0;
 irq_fail:
-        nitrox_disable_msix(ndev);
+        nitrox_unregister_interrupts(ndev);
-msix_fail:
+        return ret;
-        nitrox_cleanup_pkt_slc_bh(ndev);
-        return err;
 }
diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h
new file mode 100644
index 000000000000..63418a6cc52c
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NITROX_ISR_H
+#define __NITROX_ISR_H
+#include "nitrox_dev.h"
+int nitrox_register_interrupts(struct nitrox_device *ndev);
+void nitrox_unregister_interrupts(struct nitrox_device *ndev);
+#endif /* __NITROX_ISR_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c
index 4d31df07777f..2260efa42308 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_lib.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c
@@ -17,30 +17,27 @@
 #define CRYPTO_CTX_SIZE 256
-/* command queue alignments */
+/* packet inuput ring alignments */
-#define PKT_IN_ALIGN    16
+#define PKTIN_Q_ALIGN_BYTES 16
-static int cmdq_common_init(struct nitrox_cmdq *cmdq)
+static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
 {
        struct nitrox_device *ndev = cmdq->ndev;
-        u32 qsize;
+        cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes;
-        qsize = (ndev->qlen) * cmdq->instr_size;
+        cmdq->unalign_base = dma_zalloc_coherent(DEV(ndev), cmdq->qsize,
-        cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev),
+                                                 &cmdq->unalign_dma,
-                                                   (qsize + PKT_IN_ALIGN),
+                                                 GFP_KERNEL);
-                                                   &cmdq->dma_unaligned,
+        if (!cmdq->unalign_base)
-                                                   GFP_KERNEL);
-        if (!cmdq->head_unaligned)
                return -ENOMEM;
-        cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN);
+        cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes);
-        cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN);
+        cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma);
-        cmdq->qsize = (qsize + PKT_IN_ALIGN);
        cmdq->write_idx = 0;
-        spin_lock_init(&cmdq->response_lock);
+        spin_lock_init(&cmdq->cmd_qlock);
-        spin_lock_init(&cmdq->cmdq_lock);
+        spin_lock_init(&cmdq->resp_qlock);
-        spin_lock_init(&cmdq->backlog_lock);
+        spin_lock_init(&cmdq->backlog_qlock);
        INIT_LIST_HEAD(&cmdq->response_head);
        INIT_LIST_HEAD(&cmdq->backlog_head);
@@ -51,68 +48,83 @@ static int cmdq_common_init(struct nitrox_cmdq *cmdq)
        return 0;
 }
-static void cmdq_common_cleanup(struct nitrox_cmdq *cmdq)
+static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
+{
+        cmdq->write_idx = 0;
+        atomic_set(&cmdq->pending_count, 0);
+        atomic_set(&cmdq->backlog_count, 0);
+}
+static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
 {
        struct nitrox_device *ndev = cmdq->ndev;
+        if (!cmdq->unalign_base)
+                return;
        cancel_work_sync(&cmdq->backlog_qflush);
        dma_free_coherent(DEV(ndev), cmdq->qsize,
-                          cmdq->head_unaligned, cmdq->dma_unaligned);
+                          cmdq->unalign_base, cmdq->unalign_dma);
+        nitrox_cmdq_reset(cmdq);
-        atomic_set(&cmdq->pending_count, 0);
-        atomic_set(&cmdq->backlog_count, 0);
        cmdq->dbell_csr_addr = NULL;
-        cmdq->head = NULL;
+        cmdq->compl_cnt_csr_addr = NULL;
+        cmdq->unalign_base = NULL;
+        cmdq->base = NULL;
+        cmdq->unalign_dma = 0;
        cmdq->dma = 0;
        cmdq->qsize = 0;
        cmdq->instr_size = 0;
 }
-static void nitrox_cleanup_pkt_cmdqs(struct nitrox_device *ndev)
+static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
 {
        int i;
        for (i = 0; i < ndev->nr_queues; i++) {
-                struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i];
+                struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
-                cmdq_common_cleanup(cmdq);
+                nitrox_cmdq_cleanup(cmdq);
        }
-        kfree(ndev->pkt_cmdqs);
+        kfree(ndev->pkt_inq);
-        ndev->pkt_cmdqs = NULL;
+        ndev->pkt_inq = NULL;
 }
-static int nitrox_init_pkt_cmdqs(struct nitrox_device *ndev)
+static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev)
 {
-        int i, err, size;
+        int i, err;
-        size = ndev->nr_queues * sizeof(struct nitrox_cmdq);
+        ndev->pkt_inq = kcalloc_node(ndev->nr_queues,
-        ndev->pkt_cmdqs = kzalloc(size, GFP_KERNEL);
+                                     sizeof(struct nitrox_cmdq),
-        if (!ndev->pkt_cmdqs)
+                                     GFP_KERNEL, ndev->node);
+        if (!ndev->pkt_inq)
                return -ENOMEM;
        for (i = 0; i < ndev->nr_queues; i++) {
                struct nitrox_cmdq *cmdq;
                u64 offset;
-                cmdq = &ndev->pkt_cmdqs[i];
+                cmdq = &ndev->pkt_inq[i];
                cmdq->ndev = ndev;
                cmdq->qno = i;
                cmdq->instr_size = sizeof(struct nps_pkt_instr);
+                /* packet input ring doorbell address */
                offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
-                /* SE ring doorbell address for this queue */
                cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+                /* packet solicit port completion count address */
+                offset = NPS_PKT_SLC_CNTSX(i);
+                cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
-                err = cmdq_common_init(cmdq);
+                err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES);
                if (err)
-                        goto pkt_cmdq_fail;
+                        goto pktq_fail;
        }
        return 0;
-pkt_cmdq_fail:
+pktq_fail:
-        nitrox_cleanup_pkt_cmdqs(ndev);
+        nitrox_free_pktin_queues(ndev);
        return err;
 }
@@ -122,7 +134,7 @@ static int create_crypto_dma_pool(struct nitrox_device *ndev)
        /* Crypto context pool, 16 byte aligned */
        size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
-        ndev->ctx_pool = dma_pool_create("crypto-context",
+        ndev->ctx_pool = dma_pool_create("nitrox-context",
                                         DEV(ndev), size, 16, 0);
        if (!ndev->ctx_pool)
                return -ENOMEM;
@@ -149,7 +161,7 @@ void *crypto_alloc_context(struct nitrox_device *ndev)
        void *vaddr;
        dma_addr_t dma;
-        vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_KERNEL | __GFP_ZERO), &dma);
+        vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma);
        if (!vaddr)
                return NULL;
@@ -194,7 +206,7 @@ int nitrox_common_sw_init(struct nitrox_device *ndev)
        if (err)
                return err;
-        err = nitrox_init_pkt_cmdqs(ndev);
+        err = nitrox_alloc_pktin_queues(ndev);
        if (err)
                destroy_crypto_dma_pool(ndev);
@@ -207,6 +219,6 @@ int nitrox_common_sw_init(struct nitrox_device *ndev)
 */
 void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
 {
-        nitrox_cleanup_pkt_cmdqs(ndev);
+        nitrox_free_pktin_queues(ndev);
        destroy_crypto_dma_pool(ndev);
 }
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index fee7cb2ce747..6595c95af9f1 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -11,13 +11,15 @@
 #include "nitrox_dev.h"
 #include "nitrox_common.h"
 #include "nitrox_csr.h"
+#include "nitrox_hal.h"
+#include "nitrox_isr.h"
 #define CNN55XX_DEV_ID  0x12
 #define MAX_PF_QUEUES   64
 #define UCODE_HLEN 48
 #define SE_GROUP 0
-#define DRIVER_VERSION "1.0"
+#define DRIVER_VERSION "1.1"
 #define FW_DIR "cavium/"
 /* SE microcode */
 #define SE_FW   FW_DIR "cnn55xx_se.fw"
@@ -42,6 +44,15 @@ static unsigned int qlen = DEFAULT_CMD_QLEN;
 module_param(qlen, uint, 0644);
 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
+#ifdef CONFIG_PCI_IOV
+int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
+#else
+int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+        return 0;
+}
+#endif
 /**
 * struct ucode - Firmware Header
 * @id: microcode ID
@@ -136,9 +147,6 @@ static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
        write_to_ucd_unit(ndev, ucode);
        release_firmware(fw);
-        set_bit(NITROX_UCODE_LOADED, &ndev->status);
-        /* barrier to sync with other cpus */
-        smp_mb__after_atomic();
        return 0;
 }
@@ -210,7 +218,7 @@ void nitrox_put_device(struct nitrox_device *ndev)
        smp_mb__after_atomic();
 }
-static int nitrox_reset_device(struct pci_dev *pdev)
+static int nitrox_device_flr(struct pci_dev *pdev)
 {
        int pos = 0;
@@ -220,15 +228,10 @@ static int nitrox_reset_device(struct pci_dev *pdev)
                return -ENOMEM;
        }
-        pos = pci_pcie_cap(pdev);
+        /* check flr support */
-        if (!pos)
+        if (pcie_has_flr(pdev))
-                return -ENOTTY;
+                pcie_flr(pdev);
-        if (!pci_wait_for_pending_transaction(pdev))
-                dev_err(&pdev->dev, "waiting for pending transaction\n");
-        pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
-        msleep(100);
        pci_restore_state(pdev);
        return 0;
@@ -242,7 +245,7 @@ static int nitrox_pf_sw_init(struct nitrox_device *ndev)
        if (err)
                return err;
-        err = nitrox_pf_init_isr(ndev);
+        err = nitrox_register_interrupts(ndev);
        if (err)
                nitrox_common_sw_cleanup(ndev);
@@ -251,7 +254,7 @@ static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 {
-        nitrox_pf_cleanup_isr(ndev);
+        nitrox_unregister_interrupts(ndev);
        nitrox_common_sw_cleanup(ndev);
 }
@@ -284,26 +287,6 @@ static int nitrox_bist_check(struct nitrox_device *ndev)
        return 0;
 }
-static void nitrox_get_hwinfo(struct nitrox_device *ndev)
-{
-        union emu_fuse_map emu_fuse;
-        u64 offset;
-        int i;
-        for (i = 0; i < NR_CLUSTERS; i++) {
-                u8 dead_cores;
-                offset = EMU_FUSE_MAPX(i);
-                emu_fuse.value = nitrox_read_csr(ndev, offset);
-                if (emu_fuse.s.valid) {
-                        dead_cores = hweight32(emu_fuse.s.ae_fuse);
-                        ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
-                        dead_cores = hweight16(emu_fuse.s.se_fuse);
-                        ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
-                }
-        }
-}
 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 {
        int err;
@@ -336,135 +319,6 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev)
        return 0;
 }
-#if IS_ENABLED(CONFIG_DEBUG_FS)
-static int registers_show(struct seq_file *s, void *v)
-{
-        struct nitrox_device *ndev = s->private;
-        u64 offset;
-        /* NPS DMA stats */
-        offset = NPS_STATS_PKT_DMA_RD_CNT;
-        seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
-                   nitrox_read_csr(ndev, offset));
-        offset = NPS_STATS_PKT_DMA_WR_CNT;
-        seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
-                   nitrox_read_csr(ndev, offset));
-        /* BMI/BMO stats */
-        offset = BMI_NPS_PKT_CNT;
-        seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
-                   nitrox_read_csr(ndev, offset));
-        offset = BMO_NPS_SLC_PKT_CNT;
-        seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
-                   nitrox_read_csr(ndev, offset));
-        return 0;
-}
-static int registers_open(struct inode *inode, struct file *file)
-{
-        return single_open(file, registers_show, inode->i_private);
-}
-static const struct file_operations register_fops = {
-        .owner = THIS_MODULE,
-        .open = registers_open,
-        .read = seq_read,
-        .llseek = seq_lseek,
-        .release = single_release,
-};
-static int firmware_show(struct seq_file *s, void *v)
-{
-        struct nitrox_device *ndev = s->private;
-        seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
-        return 0;
-}
-static int firmware_open(struct inode *inode, struct file *file)
-{
-        return single_open(file, firmware_show, inode->i_private);
-}
-static const struct file_operations firmware_fops = {
-        .owner = THIS_MODULE,
-        .open = firmware_open,
-        .read = seq_read,
-        .llseek = seq_lseek,
-        .release = single_release,
-};
-static int nitrox_show(struct seq_file *s, void *v)
-{
-        struct nitrox_device *ndev = s->private;
-        seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
-        seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
-        seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
-                   ndev->hw.ae_cores, ndev->hw.se_cores);
-        seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
-        seq_printf(s, "  Queue length: %u\n", ndev->qlen);
-        seq_printf(s, "  Node: %u\n", ndev->node);
-        return 0;
-}
-static int nitrox_open(struct inode *inode, struct file *file)
-{
-        return single_open(file, nitrox_show, inode->i_private);
-}
-static const struct file_operations nitrox_fops = {
-        .owner = THIS_MODULE,
-        .open = nitrox_open,
-        .read = seq_read,
-        .llseek = seq_lseek,
-        .release = single_release,
-};
-static void nitrox_debugfs_exit(struct nitrox_device *ndev)
-{
-        debugfs_remove_recursive(ndev->debugfs_dir);
-        ndev->debugfs_dir = NULL;
-}
-static int nitrox_debugfs_init(struct nitrox_device *ndev)
-{
-        struct dentry *dir, *f;
-        dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
-        if (!dir)
-                return -ENOMEM;
-        ndev->debugfs_dir = dir;
-        f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
-        if (!f)
-                goto err;
-        f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
-        if (!f)
-                goto err;
-        f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
-        if (!f)
-                goto err;
-        return 0;
-err:
-        nitrox_debugfs_exit(ndev);
-        return -ENODEV;
-}
-#else
-static int nitrox_debugfs_init(struct nitrox_device *ndev)
-{
-        return 0;
-}
-static void nitrox_debugfs_exit(struct nitrox_device *ndev)
-{
-}
-#endif
 /**
 * nitrox_probe - NITROX Initialization function.
 * @pdev: PCI device information struct
@@ -487,7 +341,7 @@ static int nitrox_probe(struct pci_dev *pdev,
                return err;
        /* do FLR */
-        err = nitrox_reset_device(pdev);
+        err = nitrox_device_flr(pdev);
        if (err) {
                dev_err(&pdev->dev, "FLR failed\n");
                pci_disable_device(pdev);
@@ -555,7 +409,12 @@ static int nitrox_probe(struct pci_dev *pdev,
        if (err)
                goto pf_hw_fail;
-        set_bit(NITROX_READY, &ndev->status);
+        /* clear the statistics */
+        atomic64_set(&ndev->stats.posted, 0);
+        atomic64_set(&ndev->stats.completed, 0);
+        atomic64_set(&ndev->stats.dropped, 0);
+        atomic_set(&ndev->state, __NDEV_READY);
        /* barrier to sync with other cpus */
        smp_mb__after_atomic();
@@ -567,7 +426,7 @@ static int nitrox_probe(struct pci_dev *pdev,
 crypto_fail:
        nitrox_debugfs_exit(ndev);
-        clear_bit(NITROX_READY, &ndev->status);
+        atomic_set(&ndev->state, __NDEV_NOT_READY);
        /* barrier to sync with other cpus */
        smp_mb__after_atomic();
 pf_hw_fail:
@@ -602,11 +461,16 @@ static void nitrox_remove(struct pci_dev *pdev)
        dev_info(DEV(ndev), "Removing Device %x:%x\n",
                 ndev->hw.vendor_id, ndev->hw.device_id);
-        clear_bit(NITROX_READY, &ndev->status);
+        atomic_set(&ndev->state, __NDEV_NOT_READY);
        /* barrier to sync with other cpus */
        smp_mb__after_atomic();
        nitrox_remove_from_devlist(ndev);
+#ifdef CONFIG_PCI_IOV
+        /* disable SR-IOV */
+        nitrox_sriov_configure(pdev, 0);
+#endif
        nitrox_crypto_unregister();
        nitrox_debugfs_exit(ndev);
        nitrox_pf_sw_cleanup(ndev);
@@ -632,6 +496,9 @@ static struct pci_driver nitrox_driver = {
        .probe = nitrox_probe,
        .remove = nitrox_remove,
        .shutdown = nitrox_shutdown,
+#ifdef CONFIG_PCI_IOV
+        .sriov_configure = nitrox_sriov_configure,
+#endif
 };
 module_pci_driver(nitrox_driver);
diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
index 4a362fc22f62..3987cd84c033 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
@@ -382,11 +382,11 @@ static inline void backlog_list_add(struct nitrox_softreq *sr,
 {
        INIT_LIST_HEAD(&sr->backlog);
-        spin_lock_bh(&cmdq->backlog_lock);
+        spin_lock_bh(&cmdq->backlog_qlock);
        list_add_tail(&sr->backlog, &cmdq->backlog_head);
        atomic_inc(&cmdq->backlog_count);
        atomic_set(&sr->status, REQ_BACKLOG);
-        spin_unlock_bh(&cmdq->backlog_lock);
+        spin_unlock_bh(&cmdq->backlog_qlock);
 }
 static inline void response_list_add(struct nitrox_softreq *sr,
@@ -394,17 +394,17 @@ static inline void response_list_add(struct nitrox_softreq *sr,
 {
        INIT_LIST_HEAD(&sr->response);
-        spin_lock_bh(&cmdq->response_lock);
+        spin_lock_bh(&cmdq->resp_qlock);
        list_add_tail(&sr->response, &cmdq->response_head);
-        spin_unlock_bh(&cmdq->response_lock);
+        spin_unlock_bh(&cmdq->resp_qlock);
 }
 static inline void response_list_del(struct nitrox_softreq *sr,
                                     struct nitrox_cmdq *cmdq)
 {
-        spin_lock_bh(&cmdq->response_lock);
+        spin_lock_bh(&cmdq->resp_qlock);
        list_del(&sr->response);
-        spin_unlock_bh(&cmdq->response_lock);
+        spin_unlock_bh(&cmdq->resp_qlock);
 }
 static struct nitrox_softreq *
@@ -439,11 +439,11 @@ static void post_se_instr(struct nitrox_softreq *sr,
        int idx;
        u8 *ent;
-        spin_lock_bh(&cmdq->cmdq_lock);
+        spin_lock_bh(&cmdq->cmd_qlock);
        idx = cmdq->write_idx;
        /* copy the instruction */
-        ent = cmdq->head + (idx * cmdq->instr_size);
+        ent = cmdq->base + (idx * cmdq->instr_size);
        memcpy(ent, &sr->instr, cmdq->instr_size);
        atomic_set(&sr->status, REQ_POSTED);
@@ -459,7 +459,10 @@ static void post_se_instr(struct nitrox_softreq *sr,
        cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
-        spin_unlock_bh(&cmdq->cmdq_lock);
+        spin_unlock_bh(&cmdq->cmd_qlock);
+        /* increment the posted command count */
+        atomic64_inc(&ndev->stats.posted);
 }
 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
@@ -471,7 +474,7 @@ static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
        if (!atomic_read(&cmdq->backlog_count))
                return 0;
-        spin_lock_bh(&cmdq->backlog_lock);
+        spin_lock_bh(&cmdq->backlog_qlock);
        list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
                struct skcipher_request *skreq;
@@ -494,7 +497,7 @@ static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
                /* backlog requests are posted, wakeup with -EINPROGRESS */
                skcipher_request_complete(skreq, -EINPROGRESS);
        }
-        spin_unlock_bh(&cmdq->backlog_lock);
+        spin_unlock_bh(&cmdq->backlog_qlock);
        return ret;
 }
@@ -508,8 +511,11 @@ static int nitrox_enqueue_request(struct nitrox_softreq *sr)
        post_backlog_cmds(cmdq);
        if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
-                if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+                        /* increment drop count */
+                        atomic64_inc(&ndev->stats.dropped);
                        return -ENOSPC;
+                }
                /* add to backlog list */
                backlog_list_add(sr, cmdq);
                return -EBUSY;
@@ -572,7 +578,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev,
        /* select the queue */
        qno = smp_processor_id() % ndev->nr_queues;
-        sr->cmdq = &ndev->pkt_cmdqs[qno];
+        sr->cmdq = &ndev->pkt_inq[qno];
        /*
         * 64-Byte Instruction Format
@@ -694,6 +700,7 @@ static void process_response_list(struct nitrox_cmdq *cmdq)
                                            READ_ONCE(sr->resp.orh));
                }
                atomic_dec(&cmdq->pending_count);
+                atomic64_inc(&ndev->stats.completed);
                /* sync with other cpus */
                smp_mb__after_atomic();
                /* remove from response list */
@@ -714,18 +721,18 @@ static void process_response_list(struct nitrox_cmdq *cmdq)
 }
 /**
- * pkt_slc_resp_handler - post processing of SE responses
+ * pkt_slc_resp_tasklet - post processing of SE responses
 */
-void pkt_slc_resp_handler(unsigned long data)
+void pkt_slc_resp_tasklet(unsigned long data)
 {
-        struct bh_data *bh = (void *)(uintptr_t)(data);
+        struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
-        struct nitrox_cmdq *cmdq = bh->cmdq;
+        struct nitrox_cmdq *cmdq = qvec->cmdq;
-        union nps_pkt_slc_cnts pkt_slc_cnts;
+        union nps_pkt_slc_cnts slc_cnts;
        /* read completion count */
-        pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
+        slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
        /* resend the interrupt if more work to do */
-        pkt_slc_cnts.s.resend = 1;
+        slc_cnts.s.resend = 1;
        process_response_list(cmdq);
@@ -733,7 +740,7 @@ void pkt_slc_resp_handler(unsigned long data)
         * clear the interrupt with resend bit enabled,
         * MSI-X interrupt generates if Completion count > Threshold
         */
-        writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
+        writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
        /* order the writes */
        mmiowb();
diff --git a/drivers/crypto/cavium/nitrox/nitrox_sriov.c b/drivers/crypto/cavium/nitrox/nitrox_sriov.c
new file mode 100644
index 000000000000..30c0aa874583
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_sriov.c
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include "nitrox_dev.h"
+#include "nitrox_hal.h"
+#include "nitrox_common.h"
+#include "nitrox_isr.h"
+static inline bool num_vfs_valid(int num_vfs)
+{
+        bool valid = false;
+        switch (num_vfs) {
+        case 16:
+        case 32:
+        case 64:
+        case 128:
+                valid = true;
+                break;
+        }
+        return valid;
+}
+static inline enum vf_mode num_vfs_to_mode(int num_vfs)
+{
+        enum vf_mode mode = 0;
+        switch (num_vfs) {
+        case 0:
+                mode = __NDEV_MODE_PF;
+                break;
+        case 16:
+                mode = __NDEV_MODE_VF16;
+                break;
+        case 32:
+                mode = __NDEV_MODE_VF32;
+                break;
+        case 64:
+                mode = __NDEV_MODE_VF64;
+                break;
+        case 128:
+                mode = __NDEV_MODE_VF128;
+                break;
+        }
+        return mode;
+}
+static void pf_sriov_cleanup(struct nitrox_device *ndev)
+{
+         /* PF has no queues in SR-IOV mode */
+        atomic_set(&ndev->state, __NDEV_NOT_READY);
+        /* unregister crypto algorithms */
+        nitrox_crypto_unregister();
+        /* cleanup PF resources */
+        nitrox_unregister_interrupts(ndev);
+        nitrox_common_sw_cleanup(ndev);
+}
+static int pf_sriov_init(struct nitrox_device *ndev)
+{
+        int err;
+        /* allocate resources for PF */
+        err = nitrox_common_sw_init(ndev);
+        if (err)
+                return err;
+        err = nitrox_register_interrupts(ndev);
+        if (err) {
+                nitrox_common_sw_cleanup(ndev);
+                return err;
+        }
+        /* configure the packet queues */
+        nitrox_config_pkt_input_rings(ndev);
+        nitrox_config_pkt_solicit_ports(ndev);
+        /* set device to ready state */
+        atomic_set(&ndev->state, __NDEV_READY);
+        /* register crypto algorithms */
+        return nitrox_crypto_register();
+}
+static int nitrox_sriov_enable(struct pci_dev *pdev, int num_vfs)
+{
+        struct nitrox_device *ndev = pci_get_drvdata(pdev);
+        int err;
+        if (!num_vfs_valid(num_vfs)) {
+                dev_err(DEV(ndev), "Invalid num_vfs %d\n", num_vfs);
+                return -EINVAL;
+        }
+        if (pci_num_vf(pdev) == num_vfs)
+                return num_vfs;
+        err = pci_enable_sriov(pdev, num_vfs);
+        if (err) {
+                dev_err(DEV(ndev), "failed to enable PCI sriov %d\n", err);
+                return err;
+        }
+        dev_info(DEV(ndev), "Enabled VF(s) %d\n", num_vfs);
+        ndev->num_vfs = num_vfs;
+        ndev->mode = num_vfs_to_mode(num_vfs);
+        /* set bit in flags */
+        set_bit(__NDEV_SRIOV_BIT, &ndev->flags);
+        /* cleanup PF resources */
+        pf_sriov_cleanup(ndev);
+        config_nps_core_vfcfg_mode(ndev, ndev->mode);
+        return num_vfs;
+}
+static int nitrox_sriov_disable(struct pci_dev *pdev)
+{
+        struct nitrox_device *ndev = pci_get_drvdata(pdev);
+        if (!test_bit(__NDEV_SRIOV_BIT, &ndev->flags))
+                return 0;
+        if (pci_vfs_assigned(pdev)) {
+                dev_warn(DEV(ndev), "VFs are attached to VM. Can't disable SR-IOV\n");
+                return -EPERM;
+        }
+        pci_disable_sriov(pdev);
+        /* clear bit in flags */
+        clear_bit(__NDEV_SRIOV_BIT, &ndev->flags);
+        ndev->num_vfs = 0;
+        ndev->mode = __NDEV_MODE_PF;
+        config_nps_core_vfcfg_mode(ndev, ndev->mode);
+        return pf_sriov_init(ndev);
+}
+int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+        if (!num_vfs)
+                return nitrox_sriov_disable(pdev);
+        return nitrox_sriov_enable(pdev, num_vfs);
+}
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
index 94b5bcf5b628..ca4630b8395f 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
@@ -102,7 +102,7 @@ static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
        ctx->u.aes.key_len = key_len / 2;
        sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
-        return crypto_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len);
+        return crypto_sync_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len);
 }
 static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
@@ -151,12 +151,13 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
            (ctx->u.aes.key_len != AES_KEYSIZE_256))
                fallback = 1;
        if (fallback) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq,
+                                               ctx->u.aes.tfm_skcipher);
                /* Use the fallback to process the request for any
                 * unsupported unit sizes or key sizes
                 */
-                skcipher_request_set_tfm(subreq, ctx->u.aes.tfm_skcipher);
+                skcipher_request_set_sync_tfm(subreq, ctx->u.aes.tfm_skcipher);
                skcipher_request_set_callback(subreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -203,12 +204,12 @@ static int ccp_aes_xts_decrypt(struct ablkcipher_request *req)
 static int ccp_aes_xts_cra_init(struct crypto_tfm *tfm)
 {
        struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct crypto_skcipher *fallback_tfm;
+        struct crypto_sync_skcipher *fallback_tfm;
        ctx->complete = ccp_aes_xts_complete;
        ctx->u.aes.key_len = 0;
-        fallback_tfm = crypto_alloc_skcipher("xts(aes)", 0,
+        fallback_tfm = crypto_alloc_sync_skcipher("xts(aes)", 0,
                                             CRYPTO_ALG_ASYNC |
                                             CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(fallback_tfm)) {
@@ -226,7 +227,7 @@ static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm)
 {
        struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(ctx->u.aes.tfm_skcipher);
+        crypto_free_sync_skcipher(ctx->u.aes.tfm_skcipher);
 }
 static int ccp_register_aes_xts_alg(struct list_head *head,
diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h
index b9fd090c46c2..28819e11db96 100644
--- a/drivers/crypto/ccp/ccp-crypto.h
+++ b/drivers/crypto/ccp/ccp-crypto.h
@@ -88,7 +88,7 @@ static inline struct ccp_crypto_ahash_alg *
 /***** AES related defines *****/
 struct ccp_aes_ctx {
        /* Fallback cipher for XTS with unsupported unit sizes */
-        struct crypto_skcipher *tfm_skcipher;
+        struct crypto_sync_skcipher *tfm_skcipher;
        /* Cipher used to generate CMAC K1/K2 keys */
        struct crypto_cipher *tfm_cipher;
diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c
index 72790d88236d..d64a78ccc03e 100644
--- a/drivers/crypto/ccp/psp-dev.c
+++ b/drivers/crypto/ccp/psp-dev.c
@@ -31,8 +31,9 @@
                ((psp_master->api_major) >= _maj &&     \
                 (psp_master->api_minor) >= _min)
-#define DEVICE_NAME     "sev"
+#define DEVICE_NAME             "sev"
-#define SEV_FW_FILE     "amd/sev.fw"
+#define SEV_FW_FILE             "amd/sev.fw"
+#define SEV_FW_NAME_SIZE        64
 static DEFINE_MUTEX(sev_cmd_mutex);
 static struct sev_misc_dev *misc_dev;
@@ -423,7 +424,7 @@ EXPORT_SYMBOL_GPL(psp_copy_user_blob);
 static int sev_get_api_version(void)
 {
        struct sev_user_data_status *status;
-        int error, ret;
+        int error = 0, ret;
        status = &psp_master->status_cmd_buf;
        ret = sev_platform_status(status, &error);
@@ -440,6 +441,41 @@ static int sev_get_api_version(void)
        return 0;
 }
+static int sev_get_firmware(struct device *dev,
+                            const struct firmware **firmware)
+{
+        char fw_name_specific[SEV_FW_NAME_SIZE];
+        char fw_name_subset[SEV_FW_NAME_SIZE];
+        snprintf(fw_name_specific, sizeof(fw_name_specific),
+                 "amd/amd_sev_fam%.2xh_model%.2xh.sbin",
+                 boot_cpu_data.x86, boot_cpu_data.x86_model);
+        snprintf(fw_name_subset, sizeof(fw_name_subset),
+                 "amd/amd_sev_fam%.2xh_model%.1xxh.sbin",
+                 boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4);
+        /* Check for SEV FW for a particular model.
+         * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h
+         *
+         * or
+         *
+         * Check for SEV FW common to a subset of models.
+         * Ex. amd_sev_fam17h_model0xh.sbin for
+         *     Family 17h Model 00h -- Family 17h Model 0Fh
+         *
+         * or
+         *
+         * Fall-back to using generic name: sev.fw
+         */
+        if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) ||
+            (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) ||
+            (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0))
+                return 0;
+        return -ENOENT;
+}
 /* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
 static int sev_update_firmware(struct device *dev)
 {
@@ -449,9 +485,10 @@ static int sev_update_firmware(struct device *dev)
        struct page *p;
        u64 data_size;
-        ret = request_firmware(&firmware, SEV_FW_FILE, dev);
+        if (sev_get_firmware(dev, &firmware) == -ENOENT) {
-        if (ret < 0)
+                dev_dbg(dev, "No SEV firmware file present\n");
                return -1;
+        }
        /*
         * SEV FW expects the physical address given to it to be 32
diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c
index 71734f254fd1..b75dc7db2d4a 100644
--- a/drivers/crypto/ccp/sp-platform.c
+++ b/drivers/crypto/ccp/sp-platform.c
@@ -33,8 +33,31 @@ struct sp_platform {
        unsigned int irq_count;
 };
-static const struct acpi_device_id sp_acpi_match[];
+static const struct sp_dev_vdata dev_vdata[] = {
-static const struct of_device_id sp_of_match[];
+        {
+                .bar = 0,
+#ifdef CONFIG_CRYPTO_DEV_SP_CCP
+                .ccp_vdata = &ccpv3_platform,
+#endif
+        },
+};
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id sp_acpi_match[] = {
+        { "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] },
+        { },
+};
+MODULE_DEVICE_TABLE(acpi, sp_acpi_match);
+#endif
+#ifdef CONFIG_OF
+static const struct of_device_id sp_of_match[] = {
+        { .compatible = "amd,ccp-seattle-v1a",
+          .data = (const void *)&dev_vdata[0] },
+        { },
+};
+MODULE_DEVICE_TABLE(of, sp_of_match);
+#endif
 static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev)
 {
@@ -201,32 +224,6 @@ static int sp_platform_resume(struct platform_device *pdev)
 }
 #endif
-static const struct sp_dev_vdata dev_vdata[] = {
-        {
-                .bar = 0,
-#ifdef CONFIG_CRYPTO_DEV_SP_CCP
-                .ccp_vdata = &ccpv3_platform,
-#endif
-        },
-};
-#ifdef CONFIG_ACPI
-static const struct acpi_device_id sp_acpi_match[] = {
-        { "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] },
-        { },
-};
-MODULE_DEVICE_TABLE(acpi, sp_acpi_match);
-#endif
-#ifdef CONFIG_OF
-static const struct of_device_id sp_of_match[] = {
-        { .compatible = "amd,ccp-seattle-v1a",
-          .data = (const void *)&dev_vdata[0] },
-        { },
-};
-MODULE_DEVICE_TABLE(of, sp_of_match);
-#endif
 static struct platform_driver sp_platform_driver = {
        .driver = {
                .name = "ccp",
diff --git a/drivers/crypto/ccree/cc_hw_queue_defs.h b/drivers/crypto/ccree/cc_hw_queue_defs.h
index a091ae57f902..45985b955d2c 100644
--- a/drivers/crypto/ccree/cc_hw_queue_defs.h
+++ b/drivers/crypto/ccree/cc_hw_queue_defs.h
@@ -449,8 +449,7 @@ static inline void set_flow_mode(struct cc_hw_desc *pdesc,
 * @pdesc: pointer HW descriptor struct
 * @mode:  Any one of the modes defined in [CC7x-DESC]
 */
-static inline void set_cipher_mode(struct cc_hw_desc *pdesc,
+static inline void set_cipher_mode(struct cc_hw_desc *pdesc, int mode)
-                                   enum drv_cipher_mode mode)
 {
        pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_MODE, mode);
 }
@@ -461,8 +460,7 @@ static inline void set_cipher_mode(struct cc_hw_desc *pdesc,
 * @pdesc: pointer HW descriptor struct
 * @mode: Any one of the modes defined in [CC7x-DESC]
 */
-static inline void set_cipher_config0(struct cc_hw_desc *pdesc,
+static inline void set_cipher_config0(struct cc_hw_desc *pdesc, int mode)
-                                      enum drv_crypto_direction mode)
 {
        pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_CONF0, mode);
 }
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 010bbf607797..db203f8be429 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -673,7 +673,7 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src,
        return min(srclen, dstlen);
 }
-static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
+static int chcr_cipher_fallback(struct crypto_sync_skcipher *cipher,
                                u32 flags,
                                struct scatterlist *src,
                                struct scatterlist *dst,
@@ -683,9 +683,9 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
 {
        int err;
-        SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
-        skcipher_request_set_tfm(subreq, cipher);
+        skcipher_request_set_sync_tfm(subreq, cipher);
        skcipher_request_set_callback(subreq, flags, NULL, NULL);
        skcipher_request_set_crypt(subreq, src, dst,
                                   nbytes, iv);
@@ -856,13 +856,14 @@ static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
        struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
        int err = 0;
-        crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(ablkctx->sw_cipher,
-        crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
+                                CRYPTO_TFM_REQ_MASK);
-                                  CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_set_flags(ablkctx->sw_cipher,
-        err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
+                                cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
+        err = crypto_sync_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
        tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
        tfm->crt_flags |=
-                crypto_skcipher_get_flags(ablkctx->sw_cipher) &
+                crypto_sync_skcipher_get_flags(ablkctx->sw_cipher) &
                CRYPTO_TFM_RES_MASK;
        return err;
 }
@@ -1337,8 +1338,7 @@ static int chcr_device_init(struct chcr_context *ctx)
                }
                ctx->dev = u_ctx->dev;
                adap = padap(ctx->dev);
-                ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
+                ntxq = u_ctx->lldi.ntxq;
-                                    adap->vres.ncrypto_fc);
                rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
                txq_perchan = ntxq / u_ctx->lldi.nchan;
                spin_lock(&ctx->dev->lock_chcr_dev);
@@ -1369,8 +1369,8 @@ static int chcr_cra_init(struct crypto_tfm *tfm)
        struct chcr_context *ctx = crypto_tfm_ctx(tfm);
        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
-        ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
+        ablkctx->sw_cipher = crypto_alloc_sync_skcipher(alg->cra_name, 0,
-                                CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+                                CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(ablkctx->sw_cipher)) {
                pr_err("failed to allocate fallback for %s\n", alg->cra_name);
                return PTR_ERR(ablkctx->sw_cipher);
@@ -1399,8 +1399,8 @@ static int chcr_rfc3686_init(struct crypto_tfm *tfm)
        /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
         * cannot be used as fallback in chcr_handle_cipher_response
         */
-        ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
+        ablkctx->sw_cipher = crypto_alloc_sync_skcipher("ctr(aes)", 0,
-                                CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+                                CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(ablkctx->sw_cipher)) {
                pr_err("failed to allocate fallback for %s\n", alg->cra_name);
                return PTR_ERR(ablkctx->sw_cipher);
@@ -1415,7 +1415,7 @@ static void chcr_cra_exit(struct crypto_tfm *tfm)
        struct chcr_context *ctx = crypto_tfm_ctx(tfm);
        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
-        crypto_free_skcipher(ablkctx->sw_cipher);
+        crypto_free_sync_skcipher(ablkctx->sw_cipher);
        if (ablkctx->aes_generic)
                crypto_free_cipher(ablkctx->aes_generic);
 }
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index 62249d4ed373..2c472e3c6aeb 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -43,7 +43,7 @@ static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
 static struct cxgb4_uld_info chcr_uld_info = {
        .name = DRV_MODULE_NAME,
        .nrxq = MAX_ULD_QSETS,
-        .ntxq = MAX_ULD_QSETS,
+        /* Max ntxq will be derived from fw config file*/
        .rxq_size = 1024,
        .add = chcr_uld_add,
        .state_change = chcr_uld_state_change,
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 0d2c70c344f3..d37ef41f9ebe 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -170,7 +170,7 @@ static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm)
 }
 struct ablk_ctx {
-        struct crypto_skcipher *sw_cipher;
+        struct crypto_sync_skcipher *sw_cipher;
        struct crypto_cipher *aes_generic;
        __be32 key_ctx_hdr;
        unsigned int enckey_len;
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c
index 0997e166ea57..20209e29f814 100644
--- a/drivers/crypto/chelsio/chtls/chtls_cm.c
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -234,8 +234,7 @@ static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
        return;
 out:
-        if (skb)
+        kfree_skb(skb);
-                kfree_skb(skb);
 }
 static void release_tcp_port(struct sock *sk)
@@ -406,12 +405,10 @@ static int wait_for_states(struct sock *sk, unsigned int states)
 int chtls_disconnect(struct sock *sk, int flags)
 {
-        struct chtls_sock *csk;
        struct tcp_sock *tp;
        int err;
        tp = tcp_sk(sk);
-        csk = rcu_dereference_sk_user_data(sk);
        chtls_purge_recv_queue(sk);
        chtls_purge_receive_queue(sk);
        chtls_purge_write_queue(sk);
@@ -1014,7 +1011,6 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
                                    const struct cpl_pass_accept_req *req,
                                    struct chtls_dev *cdev)
 {
-        const struct tcphdr *tcph;
        struct inet_sock *newinet;
        const struct iphdr *iph;
        struct net_device *ndev;
@@ -1036,7 +1032,6 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
        if (!dst)
                goto free_sk;
-        tcph = (struct tcphdr *)(iph + 1);
        n = dst_neigh_lookup(dst, &iph->saddr);
        if (!n)
                goto free_sk;
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
index f59b044ebd25..f472c51abe56 100644
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -272,8 +272,7 @@ static void chtls_free_uld(struct chtls_dev *cdev)
        for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
                kfree_skb(cdev->rspq_skb_cache[i]);
        kfree(cdev->lldi);
-        if (cdev->askb)
+        kfree_skb(cdev->askb);
-                kfree_skb(cdev->askb);
        kfree(cdev);
 }
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index 56bd28174f52..4e6ff32f8a7e 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -28,9 +28,24 @@
 #define DCP_MAX_CHANS   4
 #define DCP_BUF_SZ      PAGE_SIZE
+#define DCP_SHA_PAY_SZ  64
 #define DCP_ALIGNMENT   64
+/*
+ * Null hashes to align with hw behavior on imx6sl and ull
+ * these are flipped for consistency with hw output
+ */
+static const uint8_t sha1_null_hash[] =
+        "\x09\x07\xd8\xaf\x90\x18\x60\x95\xef\xbf"
+        "\x55\x32\x0d\x4b\x6b\x5e\xee\xa3\x39\xda";
+static const uint8_t sha256_null_hash[] =
+        "\x55\xb8\x52\x78\x1b\x99\x95\xa4"
+        "\x4c\x93\x9b\x64\xe4\x41\xae\x27"
+        "\x24\xb9\x6f\x99\xc8\xf4\xfb\x9a"
+        "\x14\x1c\xfc\x98\x42\xc4\xb0\xe3";
 /* DCP DMA descriptor. */
 struct dcp_dma_desc {
        uint32_t        next_cmd_addr;
@@ -48,6 +63,7 @@ struct dcp_coherent_block {
        uint8_t                 aes_in_buf[DCP_BUF_SZ];
        uint8_t                 aes_out_buf[DCP_BUF_SZ];
        uint8_t                 sha_in_buf[DCP_BUF_SZ];
+        uint8_t                 sha_out_buf[DCP_SHA_PAY_SZ];
        uint8_t                 aes_key[2 * AES_KEYSIZE_128];
@@ -84,7 +100,7 @@ struct dcp_async_ctx {
        unsigned int                    hot:1;
        /* Crypto-specific context */
-        struct crypto_skcipher          *fallback;
+        struct crypto_sync_skcipher     *fallback;
        unsigned int                    key_len;
        uint8_t                         key[AES_KEYSIZE_128];
 };
@@ -99,6 +115,11 @@ struct dcp_sha_req_ctx {
        unsigned int    fini:1;
 };
+struct dcp_export_state {
+        struct dcp_sha_req_ctx req_ctx;
+        struct dcp_async_ctx async_ctx;
+};
 /*
 * There can even be only one instance of the MXS DCP due to the
 * design of Linux Crypto API.
@@ -209,6 +230,12 @@ static int mxs_dcp_run_aes(struct dcp_async_ctx *actx,
        dma_addr_t dst_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_out_buf,
                                             DCP_BUF_SZ, DMA_FROM_DEVICE);
+        if (actx->fill % AES_BLOCK_SIZE) {
+                dev_err(sdcp->dev, "Invalid block size!\n");
+                ret = -EINVAL;
+                goto aes_done_run;
+        }
        /* Fill in the DMA descriptor. */
        desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE |
                    MXS_DCP_CONTROL0_INTERRUPT |
@@ -238,6 +265,7 @@ static int mxs_dcp_run_aes(struct dcp_async_ctx *actx,
        ret = mxs_dcp_start_dma(actx);
+aes_done_run:
        dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128,
                         DMA_TO_DEVICE);
        dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE);
@@ -264,13 +292,15 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
        uint8_t *out_tmp, *src_buf, *dst_buf = NULL;
        uint32_t dst_off = 0;
+        uint32_t last_out_len = 0;
        uint8_t *key = sdcp->coh->aes_key;
        int ret = 0;
        int split = 0;
-        unsigned int i, len, clen, rem = 0;
+        unsigned int i, len, clen, rem = 0, tlen = 0;
        int init = 0;
+        bool limit_hit = false;
        actx->fill = 0;
@@ -289,6 +319,11 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
        for_each_sg(req->src, src, nents, i) {
                src_buf = sg_virt(src);
                len = sg_dma_len(src);
+                tlen += len;
+                limit_hit = tlen > req->nbytes;
+                if (limit_hit)
+                        len = req->nbytes - (tlen - len);
                do {
                        if (actx->fill + len > out_off)
@@ -305,13 +340,15 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
                         * If we filled the buffer or this is the last SG,
                         * submit the buffer.
                         */
-                        if (actx->fill == out_off || sg_is_last(src)) {
+                        if (actx->fill == out_off || sg_is_last(src) ||
+                                limit_hit) {
                                ret = mxs_dcp_run_aes(actx, req, init);
                                if (ret)
                                        return ret;
                                init = 0;
                                out_tmp = out_buf;
+                                last_out_len = actx->fill;
                                while (dst && actx->fill) {
                                        if (!split) {
                                                dst_buf = sg_virt(dst);
@@ -334,6 +371,19 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
                                }
                        }
                } while (len);
+                if (limit_hit)
+                        break;
+        }
+        /* Copy the IV for CBC for chaining */
+        if (!rctx->ecb) {
+                if (rctx->enc)
+                        memcpy(req->info, out_buf+(last_out_len-AES_BLOCK_SIZE),
+                                AES_BLOCK_SIZE);
+                else
+                        memcpy(req->info, in_buf+(last_out_len-AES_BLOCK_SIZE),
+                                AES_BLOCK_SIZE);
        }
        return ret;
@@ -380,10 +430,10 @@ static int mxs_dcp_block_fallback(struct ablkcipher_request *req, int enc)
 {
        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
        struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-        SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
        int ret;
-        skcipher_request_set_tfm(subreq, ctx->fallback);
+        skcipher_request_set_sync_tfm(subreq, ctx->fallback);
        skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL);
        skcipher_request_set_crypt(subreq, req->src, req->dst,
                                   req->nbytes, req->info);
@@ -464,16 +514,16 @@ static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
         * but is supported by in-kernel software implementation, we use
         * software fallback.
         */
-        crypto_skcipher_clear_flags(actx->fallback, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(actx->fallback, CRYPTO_TFM_REQ_MASK);
-        crypto_skcipher_set_flags(actx->fallback,
+        crypto_sync_skcipher_set_flags(actx->fallback,
                                  tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
-        ret = crypto_skcipher_setkey(actx->fallback, key, len);
+        ret = crypto_sync_skcipher_setkey(actx->fallback, key, len);
        if (!ret)
                return 0;
        tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK;
-        tfm->base.crt_flags |= crypto_skcipher_get_flags(actx->fallback) &
+        tfm->base.crt_flags |= crypto_sync_skcipher_get_flags(actx->fallback) &
                               CRYPTO_TFM_RES_MASK;
        return ret;
@@ -482,11 +532,10 @@ static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 static int mxs_dcp_aes_fallback_init(struct crypto_tfm *tfm)
 {
        const char *name = crypto_tfm_alg_name(tfm);
-        const uint32_t flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
        struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm);
-        struct crypto_skcipher *blk;
+        struct crypto_sync_skcipher *blk;
-        blk = crypto_alloc_skcipher(name, 0, flags);
+        blk = crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(blk))
                return PTR_ERR(blk);
@@ -499,7 +548,7 @@ static void mxs_dcp_aes_fallback_exit(struct crypto_tfm *tfm)
 {
        struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(actx->fallback);
+        crypto_free_sync_skcipher(actx->fallback);
 }
 /*
@@ -513,8 +562,6 @@ static int mxs_dcp_run_sha(struct ahash_request *req)
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm);
        struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req);
-        struct hash_alg_common *halg = crypto_hash_alg_common(tfm);
        struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan];
        dma_addr_t digest_phys = 0;
@@ -536,10 +583,23 @@ static int mxs_dcp_run_sha(struct ahash_request *req)
        desc->payload = 0;
        desc->status = 0;
+        /*
+         * Align driver with hw behavior when generating null hashes
+         */
+        if (rctx->init && rctx->fini && desc->size == 0) {
+                struct hash_alg_common *halg = crypto_hash_alg_common(tfm);
+                const uint8_t *sha_buf =
+                        (actx->alg == MXS_DCP_CONTROL1_HASH_SELECT_SHA1) ?
+                        sha1_null_hash : sha256_null_hash;
+                memcpy(sdcp->coh->sha_out_buf, sha_buf, halg->digestsize);
+                ret = 0;
+                goto done_run;
+        }
        /* Set HASH_TERM bit for last transfer block. */
        if (rctx->fini) {
-                digest_phys = dma_map_single(sdcp->dev, req->result,
+                digest_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_out_buf,
-                                             halg->digestsize, DMA_FROM_DEVICE);
+                                             DCP_SHA_PAY_SZ, DMA_FROM_DEVICE);
                desc->control0 |= MXS_DCP_CONTROL0_HASH_TERM;
                desc->payload = digest_phys;
        }
@@ -547,9 +607,10 @@ static int mxs_dcp_run_sha(struct ahash_request *req)
        ret = mxs_dcp_start_dma(actx);
        if (rctx->fini)
-                dma_unmap_single(sdcp->dev, digest_phys, halg->digestsize,
+                dma_unmap_single(sdcp->dev, digest_phys, DCP_SHA_PAY_SZ,
                                 DMA_FROM_DEVICE);
+done_run:
        dma_unmap_single(sdcp->dev, buf_phys, DCP_BUF_SZ, DMA_TO_DEVICE);
        return ret;
@@ -567,6 +628,7 @@ static int dcp_sha_req_to_buf(struct crypto_async_request *arq)
        const int nents = sg_nents(req->src);
        uint8_t *in_buf = sdcp->coh->sha_in_buf;
+        uint8_t *out_buf = sdcp->coh->sha_out_buf;
        uint8_t *src_buf;
@@ -621,11 +683,9 @@ static int dcp_sha_req_to_buf(struct crypto_async_request *arq)
                actx->fill = 0;
-                /* For some reason, the result is flipped. */
+                /* For some reason the result is flipped */
-                for (i = 0; i < halg->digestsize / 2; i++) {
+                for (i = 0; i < halg->digestsize; i++)
-                        swap(req->result[i],
+                        req->result[i] = out_buf[halg->digestsize - i - 1];
-                             req->result[halg->digestsize - i - 1]);
-                }
        }
        return 0;
@@ -766,14 +826,32 @@ static int dcp_sha_digest(struct ahash_request *req)
        return dcp_sha_finup(req);
 }
-static int dcp_sha_noimport(struct ahash_request *req, const void *in)
+static int dcp_sha_import(struct ahash_request *req, const void *in)
 {
-        return -ENOSYS;
+        struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req);
+        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+        struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm);
+        const struct dcp_export_state *export = in;
+        memset(rctx, 0, sizeof(struct dcp_sha_req_ctx));
+        memset(actx, 0, sizeof(struct dcp_async_ctx));
+        memcpy(rctx, &export->req_ctx, sizeof(struct dcp_sha_req_ctx));
+        memcpy(actx, &export->async_ctx, sizeof(struct dcp_async_ctx));
+        return 0;
 }
-static int dcp_sha_noexport(struct ahash_request *req, void *out)
+static int dcp_sha_export(struct ahash_request *req, void *out)
 {
-        return -ENOSYS;
+        struct dcp_sha_req_ctx *rctx_state = ahash_request_ctx(req);
+        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+        struct dcp_async_ctx *actx_state = crypto_ahash_ctx(tfm);
+        struct dcp_export_state *export = out;
+        memcpy(&export->req_ctx, rctx_state, sizeof(struct dcp_sha_req_ctx));
+        memcpy(&export->async_ctx, actx_state, sizeof(struct dcp_async_ctx));
+        return 0;
 }
 static int dcp_sha_cra_init(struct crypto_tfm *tfm)
@@ -846,10 +924,11 @@ static struct ahash_alg dcp_sha1_alg = {
        .final  = dcp_sha_final,
        .finup  = dcp_sha_finup,
        .digest = dcp_sha_digest,
-        .import = dcp_sha_noimport,
+        .import = dcp_sha_import,
-        .export = dcp_sha_noexport,
+        .export = dcp_sha_export,
        .halg   = {
                .digestsize     = SHA1_DIGEST_SIZE,
+                .statesize      = sizeof(struct dcp_export_state),
                .base           = {
                        .cra_name               = "sha1",
                        .cra_driver_name        = "sha1-dcp",
@@ -872,10 +951,11 @@ static struct ahash_alg dcp_sha256_alg = {
        .final  = dcp_sha_final,
        .finup  = dcp_sha_finup,
        .digest = dcp_sha_digest,
-        .import = dcp_sha_noimport,
+        .import = dcp_sha_import,
-        .export = dcp_sha_noexport,
+        .export = dcp_sha_export,
        .halg   = {
                .digestsize     = SHA256_DIGEST_SIZE,
+                .statesize      = sizeof(struct dcp_export_state),
                .base           = {
                        .cra_name               = "sha256",
                        .cra_driver_name        = "sha256-dcp",
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 9019f6b67986..a553ffddb11b 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -522,9 +522,9 @@ static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
                  !!(mode & FLAGS_CBC));
        if (req->nbytes < aes_fallback_sz) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                skcipher_request_set_tfm(subreq, ctx->fallback);
+                skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                skcipher_request_set_callback(subreq, req->base.flags, NULL,
                                              NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -564,11 +564,11 @@ static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
        memcpy(ctx->key, key, keylen);
        ctx->keylen = keylen;
-        crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
-        crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
+        crypto_sync_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
                                                 CRYPTO_TFM_REQ_MASK);
-        ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+        ret = crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
        if (!ret)
                return 0;
@@ -613,11 +613,10 @@ static int omap_aes_crypt_req(struct crypto_engine *engine,
 static int omap_aes_cra_init(struct crypto_tfm *tfm)
 {
        const char *name = crypto_tfm_alg_name(tfm);
-        const u32 flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
        struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct crypto_skcipher *blk;
+        struct crypto_sync_skcipher *blk;
-        blk = crypto_alloc_skcipher(name, 0, flags);
+        blk = crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(blk))
                return PTR_ERR(blk);
@@ -667,7 +666,7 @@ static void omap_aes_cra_exit(struct crypto_tfm *tfm)
        struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
        if (ctx->fallback)
-                crypto_free_skcipher(ctx->fallback);
+                crypto_free_sync_skcipher(ctx->fallback);
        ctx->fallback = NULL;
 }
diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h
index fc3b46a85809..7e02920ef6f8 100644
--- a/drivers/crypto/omap-aes.h
+++ b/drivers/crypto/omap-aes.h
@@ -101,7 +101,7 @@ struct omap_aes_ctx {
        int             keylen;
        u32             key[AES_KEYSIZE_256 / sizeof(u32)];
        u8              nonce[4];
-        struct crypto_skcipher  *fallback;
+        struct crypto_sync_skcipher     *fallback;
        struct crypto_skcipher  *ctr;
 };
diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c
index 321d5e2ac833..a28f1d18fe01 100644
--- a/drivers/crypto/picoxcell_crypto.c
+++ b/drivers/crypto/picoxcell_crypto.c
@@ -171,7 +171,7 @@ struct spacc_ablk_ctx {
         * The fallback cipher. If the operation can't be done in hardware,
         * fallback to a software version.
         */
-        struct crypto_skcipher          *sw_cipher;
+        struct crypto_sync_skcipher     *sw_cipher;
 };
 /* AEAD cipher context. */
@@ -799,17 +799,17 @@ static int spacc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
                 * Set the fallback transform to use the same request flags as
                 * the hardware transform.
                 */
-                crypto_skcipher_clear_flags(ctx->sw_cipher,
+                crypto_sync_skcipher_clear_flags(ctx->sw_cipher,
                                            CRYPTO_TFM_REQ_MASK);
-                crypto_skcipher_set_flags(ctx->sw_cipher,
+                crypto_sync_skcipher_set_flags(ctx->sw_cipher,
                                          cipher->base.crt_flags &
                                          CRYPTO_TFM_REQ_MASK);
-                err = crypto_skcipher_setkey(ctx->sw_cipher, key, len);
+                err = crypto_sync_skcipher_setkey(ctx->sw_cipher, key, len);
                tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
                tfm->crt_flags |=
-                        crypto_skcipher_get_flags(ctx->sw_cipher) &
+                        crypto_sync_skcipher_get_flags(ctx->sw_cipher) &
                        CRYPTO_TFM_RES_MASK;
                if (err)
@@ -914,7 +914,7 @@ static int spacc_ablk_do_fallback(struct ablkcipher_request *req,
        struct crypto_tfm *old_tfm =
            crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
        struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm);
-        SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher);
        int err;
        /*
@@ -922,7 +922,7 @@ static int spacc_ablk_do_fallback(struct ablkcipher_request *req,
         * the ciphering has completed, put the old transform back into the
         * request.
         */
-        skcipher_request_set_tfm(subreq, ctx->sw_cipher);
+        skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher);
        skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL);
        skcipher_request_set_crypt(subreq, req->src, req->dst,
                                   req->nbytes, req->info);
@@ -1020,9 +1020,8 @@ static int spacc_ablk_cra_init(struct crypto_tfm *tfm)
        ctx->generic.flags = spacc_alg->type;
        ctx->generic.engine = engine;
        if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
-                ctx->sw_cipher = crypto_alloc_skcipher(
+                ctx->sw_cipher = crypto_alloc_sync_skcipher(
-                        alg->cra_name, 0, CRYPTO_ALG_ASYNC |
+                        alg->cra_name, 0, CRYPTO_ALG_NEED_FALLBACK);
-                                          CRYPTO_ALG_NEED_FALLBACK);
                if (IS_ERR(ctx->sw_cipher)) {
                        dev_warn(engine->dev, "failed to allocate fallback for %s\n",
                                 alg->cra_name);
@@ -1041,7 +1040,7 @@ static void spacc_ablk_cra_exit(struct crypto_tfm *tfm)
 {
        struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(ctx->sw_cipher);
+        crypto_free_sync_skcipher(ctx->sw_cipher);
 }
 static int spacc_ablk_encrypt(struct ablkcipher_request *req)
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 1138e41d6805..d2698299896f 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -113,6 +113,13 @@ struct qat_alg_aead_ctx {
        struct crypto_shash *hash_tfm;
        enum icp_qat_hw_auth_algo qat_hash_alg;
        struct qat_crypto_instance *inst;
+        union {
+                struct sha1_state sha1;
+                struct sha256_state sha256;
+                struct sha512_state sha512;
+        };
+        char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */
+        char opad[SHA512_BLOCK_SIZE];
 };
 struct qat_alg_ablkcipher_ctx {
@@ -148,37 +155,32 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
                                  unsigned int auth_keylen)
 {
        SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
-        struct sha1_state sha1;
-        struct sha256_state sha256;
-        struct sha512_state sha512;
        int block_size = crypto_shash_blocksize(ctx->hash_tfm);
        int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
-        char ipad[block_size];
-        char opad[block_size];
        __be32 *hash_state_out;
        __be64 *hash512_state_out;
        int i, offset;
-        memset(ipad, 0, block_size);
+        memset(ctx->ipad, 0, block_size);
-        memset(opad, 0, block_size);
+        memset(ctx->opad, 0, block_size);
        shash->tfm = ctx->hash_tfm;
        shash->flags = 0x0;
        if (auth_keylen > block_size) {
                int ret = crypto_shash_digest(shash, auth_key,
-                                              auth_keylen, ipad);
+                                              auth_keylen, ctx->ipad);
                if (ret)
                        return ret;
-                memcpy(opad, ipad, digest_size);
+                memcpy(ctx->opad, ctx->ipad, digest_size);
        } else {
-                memcpy(ipad, auth_key, auth_keylen);
+                memcpy(ctx->ipad, auth_key, auth_keylen);
-                memcpy(opad, auth_key, auth_keylen);
+                memcpy(ctx->opad, auth_key, auth_keylen);
        }
        for (i = 0; i < block_size; i++) {
-                char *ipad_ptr = ipad + i;
+                char *ipad_ptr = ctx->ipad + i;
-                char *opad_ptr = opad + i;
+                char *opad_ptr = ctx->opad + i;
                *ipad_ptr ^= HMAC_IPAD_VALUE;
                *opad_ptr ^= HMAC_OPAD_VALUE;
        }
@@ -186,7 +188,7 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
        if (crypto_shash_init(shash))
                return -EFAULT;
-        if (crypto_shash_update(shash, ipad, block_size))
+        if (crypto_shash_update(shash, ctx->ipad, block_size))
                return -EFAULT;
        hash_state_out = (__be32 *)hash->sha.state1;
@@ -194,22 +196,22 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
        switch (ctx->qat_hash_alg) {
        case ICP_QAT_HW_AUTH_ALGO_SHA1:
-                if (crypto_shash_export(shash, &sha1))
+                if (crypto_shash_export(shash, &ctx->sha1))
                        return -EFAULT;
                for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
-                        *hash_state_out = cpu_to_be32(*(sha1.state + i));
+                        *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA256:
-                if (crypto_shash_export(shash, &sha256))
+                if (crypto_shash_export(shash, &ctx->sha256))
                        return -EFAULT;
                for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
-                        *hash_state_out = cpu_to_be32(*(sha256.state + i));
+                        *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA512:
-                if (crypto_shash_export(shash, &sha512))
+                if (crypto_shash_export(shash, &ctx->sha512))
                        return -EFAULT;
                for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
-                        *hash512_state_out = cpu_to_be64(*(sha512.state + i));
+                        *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
                break;
        default:
                return -EFAULT;
@@ -218,7 +220,7 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
        if (crypto_shash_init(shash))
                return -EFAULT;
-        if (crypto_shash_update(shash, opad, block_size))
+        if (crypto_shash_update(shash, ctx->opad, block_size))
                return -EFAULT;
        offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
@@ -227,28 +229,28 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
        switch (ctx->qat_hash_alg) {
        case ICP_QAT_HW_AUTH_ALGO_SHA1:
-                if (crypto_shash_export(shash, &sha1))
+                if (crypto_shash_export(shash, &ctx->sha1))
                        return -EFAULT;
                for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
-                        *hash_state_out = cpu_to_be32(*(sha1.state + i));
+                        *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA256:
-                if (crypto_shash_export(shash, &sha256))
+                if (crypto_shash_export(shash, &ctx->sha256))
                        return -EFAULT;
                for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
-                        *hash_state_out = cpu_to_be32(*(sha256.state + i));
+                        *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA512:
-                if (crypto_shash_export(shash, &sha512))
+                if (crypto_shash_export(shash, &ctx->sha512))
                        return -EFAULT;
                for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
-                        *hash512_state_out = cpu_to_be64(*(sha512.state + i));
+                        *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
                break;
        default:
                return -EFAULT;
        }
-        memzero_explicit(ipad, block_size);
+        memzero_explicit(ctx->ipad, block_size);
-        memzero_explicit(opad, block_size);
+        memzero_explicit(ctx->opad, block_size);
        return 0;
 }
diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c
index ea4d96bf47e8..585e1cab9ae3 100644
--- a/drivers/crypto/qce/ablkcipher.c
+++ b/drivers/crypto/qce/ablkcipher.c
@@ -189,7 +189,7 @@ static int qce_ablkcipher_setkey(struct crypto_ablkcipher *ablk, const u8 *key,
        memcpy(ctx->enc_key, key, keylen);
        return 0;
 fallback:
-        ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+        ret = crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
        if (!ret)
                ctx->enc_keylen = keylen;
        return ret;
@@ -212,9 +212,9 @@ static int qce_ablkcipher_crypt(struct ablkcipher_request *req, int encrypt)
        if (IS_AES(rctx->flags) && ctx->enc_keylen != AES_KEYSIZE_128 &&
            ctx->enc_keylen != AES_KEYSIZE_256) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                skcipher_request_set_tfm(subreq, ctx->fallback);
+                skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                skcipher_request_set_callback(subreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -245,9 +245,8 @@ static int qce_ablkcipher_init(struct crypto_tfm *tfm)
        memset(ctx, 0, sizeof(*ctx));
        tfm->crt_ablkcipher.reqsize = sizeof(struct qce_cipher_reqctx);
-        ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(tfm), 0,
+        ctx->fallback = crypto_alloc_sync_skcipher(crypto_tfm_alg_name(tfm),
-                                              CRYPTO_ALG_ASYNC |
+                                                   0, CRYPTO_ALG_NEED_FALLBACK);
-                                              CRYPTO_ALG_NEED_FALLBACK);
        return PTR_ERR_OR_ZERO(ctx->fallback);
 }
@@ -255,7 +254,7 @@ static void qce_ablkcipher_exit(struct crypto_tfm *tfm)
 {
        struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(ctx->fallback);
+        crypto_free_sync_skcipher(ctx->fallback);
 }
 struct qce_ablkcipher_def {
diff --git a/drivers/crypto/qce/cipher.h b/drivers/crypto/qce/cipher.h
index 2b0278bb6e92..ee055bfe98a0 100644
--- a/drivers/crypto/qce/cipher.h
+++ b/drivers/crypto/qce/cipher.h
@@ -22,7 +22,7 @@
 struct qce_cipher_ctx {
        u8 enc_key[QCE_MAX_KEY_SIZE];
        unsigned int enc_keylen;
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
 };
 /**
diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
index faa282074e5a..0064be0e3941 100644
--- a/drivers/crypto/s5p-sss.c
+++ b/drivers/crypto/s5p-sss.c
@@ -249,8 +249,8 @@ struct s5p_aes_reqctx {
 struct s5p_aes_ctx {
        struct s5p_aes_dev              *dev;
-        uint8_t                         aes_key[AES_MAX_KEY_SIZE];
+        u8                              aes_key[AES_MAX_KEY_SIZE];
-        uint8_t                         nonce[CTR_RFC3686_NONCE_SIZE];
+        u8                              nonce[CTR_RFC3686_NONCE_SIZE];
        int                             keylen;
 };
@@ -475,9 +475,9 @@ static void s5p_sg_done(struct s5p_aes_dev *dev)
 }
 /* Calls the completion. Cannot be called with dev->lock hold. */
-static void s5p_aes_complete(struct s5p_aes_dev *dev, int err)
+static void s5p_aes_complete(struct ablkcipher_request *req, int err)
 {
-        dev->req->base.complete(&dev->req->base, err);
+        req->base.complete(&req->base, err);
 }
 static void s5p_unset_outdata(struct s5p_aes_dev *dev)
@@ -491,7 +491,7 @@ static void s5p_unset_indata(struct s5p_aes_dev *dev)
 }
 static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,
-                            struct scatterlist **dst)
+                           struct scatterlist **dst)
 {
        void *pages;
        int len;
@@ -518,46 +518,28 @@ static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,
 static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
 {
-        int err;
+        if (!sg->length)
+                return -EINVAL;
-        if (!sg->length) {
-                err = -EINVAL;
-                goto exit;
-        }
-        err = dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE);
+        if (!dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE))
-        if (!err) {
+                return -ENOMEM;
-                err = -ENOMEM;
-                goto exit;
-        }
        dev->sg_dst = sg;
-        err = 0;
-exit:
+        return 0;
-        return err;
 }
 static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
 {
-        int err;
+        if (!sg->length)
+                return -EINVAL;
-        if (!sg->length) {
-                err = -EINVAL;
-                goto exit;
-        }
-        err = dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE);
+        if (!dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE))
-        if (!err) {
+                return -ENOMEM;
-                err = -ENOMEM;
-                goto exit;
-        }
        dev->sg_src = sg;
-        err = 0;
-exit:
+        return 0;
-        return err;
 }
 /*
@@ -655,14 +637,14 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
 {
        struct platform_device *pdev = dev_id;
        struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
+        struct ablkcipher_request *req;
        int err_dma_tx = 0;
        int err_dma_rx = 0;
        int err_dma_hx = 0;
        bool tx_end = false;
        bool hx_end = false;
        unsigned long flags;
-        uint32_t status;
+        u32 status, st_bits;
-        u32 st_bits;
        int err;
        spin_lock_irqsave(&dev->lock, flags);
@@ -727,7 +709,7 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
                spin_unlock_irqrestore(&dev->lock, flags);
-                s5p_aes_complete(dev, 0);
+                s5p_aes_complete(dev->req, 0);
                /* Device is still busy */
                tasklet_schedule(&dev->tasklet);
        } else {
@@ -752,11 +734,12 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
 error:
        s5p_sg_done(dev);
        dev->busy = false;
+        req = dev->req;
        if (err_dma_hx == 1)
                s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
        spin_unlock_irqrestore(&dev->lock, flags);
-        s5p_aes_complete(dev, err);
+        s5p_aes_complete(req, err);
 hash_irq_end:
        /*
@@ -1830,7 +1813,7 @@ static struct ahash_alg algs_sha1_md5_sha256[] = {
 };
 static void s5p_set_aes(struct s5p_aes_dev *dev,
-                        const uint8_t *key, const uint8_t *iv,
+                        const u8 *key, const u8 *iv, const u8 *ctr,
                        unsigned int keylen)
 {
        void __iomem *keystart;
@@ -1838,6 +1821,9 @@ static void s5p_set_aes(struct s5p_aes_dev *dev,
        if (iv)
                memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10);
+        if (ctr)
+                memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), ctr, 0x10);
        if (keylen == AES_KEYSIZE_256)
                keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0);
        else if (keylen == AES_KEYSIZE_192)
@@ -1887,7 +1873,7 @@ static int s5p_set_indata_start(struct s5p_aes_dev *dev,
 }
 static int s5p_set_outdata_start(struct s5p_aes_dev *dev,
-                                struct ablkcipher_request *req)
+                                 struct ablkcipher_request *req)
 {
        struct scatterlist *sg;
        int err;
@@ -1916,11 +1902,12 @@ static int s5p_set_outdata_start(struct s5p_aes_dev *dev,
 static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
 {
        struct ablkcipher_request *req = dev->req;
-        uint32_t aes_control;
+        u32 aes_control;
        unsigned long flags;
        int err;
-        u8 *iv;
+        u8 *iv, *ctr;
+        /* This sets bit [13:12] to 00, which selects 128-bit counter */
        aes_control = SSS_AES_KEY_CHANGE_MODE;
        if (mode & FLAGS_AES_DECRYPT)
                aes_control |= SSS_AES_MODE_DECRYPT;
@@ -1928,11 +1915,14 @@ static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
        if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {
                aes_control |= SSS_AES_CHAIN_MODE_CBC;
                iv = req->info;
+                ctr = NULL;
        } else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {
                aes_control |= SSS_AES_CHAIN_MODE_CTR;
-                iv = req->info;
+                iv = NULL;
+                ctr = req->info;
        } else {
                iv = NULL; /* AES_ECB */
+                ctr = NULL;
        }
        if (dev->ctx->keylen == AES_KEYSIZE_192)
@@ -1964,7 +1954,7 @@ static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
                goto outdata_error;
        SSS_AES_WRITE(dev, AES_CONTROL, aes_control);
-        s5p_set_aes(dev, dev->ctx->aes_key, iv, dev->ctx->keylen);
+        s5p_set_aes(dev, dev->ctx->aes_key, iv, ctr, dev->ctx->keylen);
        s5p_set_dma_indata(dev,  dev->sg_src);
        s5p_set_dma_outdata(dev, dev->sg_dst);
@@ -1983,7 +1973,7 @@ indata_error:
        s5p_sg_done(dev);
        dev->busy = false;
        spin_unlock_irqrestore(&dev->lock, flags);
-        s5p_aes_complete(dev, err);
+        s5p_aes_complete(req, err);
 }
 static void s5p_tasklet_cb(unsigned long data)
@@ -2024,7 +2014,7 @@ static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
        err = ablkcipher_enqueue_request(&dev->queue, req);
        if (dev->busy) {
                spin_unlock_irqrestore(&dev->lock, flags);
-                goto exit;
+                return err;
        }
        dev->busy = true;
@@ -2032,7 +2022,6 @@ static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
        tasklet_schedule(&dev->tasklet);
-exit:
        return err;
 }
@@ -2043,7 +2032,8 @@ static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
        struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
        struct s5p_aes_dev *dev = ctx->dev;
-        if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
+        if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE) &&
+                        ((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) {
                dev_err(dev->dev, "request size is not exact amount of AES blocks\n");
                return -EINVAL;
        }
@@ -2054,7 +2044,7 @@ static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 }
 static int s5p_aes_setkey(struct crypto_ablkcipher *cipher,
-                          const uint8_t *key, unsigned int keylen)
+                          const u8 *key, unsigned int keylen)
 {
        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
        struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -2090,6 +2080,11 @@ static int s5p_aes_cbc_decrypt(struct ablkcipher_request *req)
        return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);
 }
+static int s5p_aes_ctr_crypt(struct ablkcipher_request *req)
+{
+        return s5p_aes_crypt(req, FLAGS_AES_CTR);
+}
 static int s5p_aes_cra_init(struct crypto_tfm *tfm)
 {
        struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -2144,6 +2139,28 @@ static struct crypto_alg algs[] = {
                        .decrypt        = s5p_aes_cbc_decrypt,
                }
        },
+        {
+                .cra_name               = "ctr(aes)",
+                .cra_driver_name        = "ctr-aes-s5p",
+                .cra_priority           = 100,
+                .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
+                                          CRYPTO_ALG_ASYNC |
+                                          CRYPTO_ALG_KERN_DRIVER_ONLY,
+                .cra_blocksize          = AES_BLOCK_SIZE,
+                .cra_ctxsize            = sizeof(struct s5p_aes_ctx),
+                .cra_alignmask          = 0x0f,
+                .cra_type               = &crypto_ablkcipher_type,
+                .cra_module             = THIS_MODULE,
+                .cra_init               = s5p_aes_cra_init,
+                .cra_u.ablkcipher = {
+                        .min_keysize    = AES_MIN_KEY_SIZE,
+                        .max_keysize    = AES_MAX_KEY_SIZE,
+                        .ivsize         = AES_BLOCK_SIZE,
+                        .setkey         = s5p_aes_setkey,
+                        .encrypt        = s5p_aes_ctr_crypt,
+                        .decrypt        = s5p_aes_ctr_crypt,
+                }
+        },
 };
 static int s5p_aes_probe(struct platform_device *pdev)
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index e7540a5b8197..bbf166a97ad3 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -149,7 +149,7 @@ struct sahara_ctx {
        /* AES-specific context */
        int keylen;
        u8 key[AES_KEYSIZE_128];
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
 };
 struct sahara_aes_reqctx {
@@ -621,14 +621,14 @@ static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
        /*
         * The requested key size is not supported by HW, do a fallback.
         */
-        crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
+        crypto_sync_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
-        crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
+        crypto_sync_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
                                                 CRYPTO_TFM_REQ_MASK);
-        ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+        ret = crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
        tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK;
-        tfm->base.crt_flags |= crypto_skcipher_get_flags(ctx->fallback) &
+        tfm->base.crt_flags |= crypto_sync_skcipher_get_flags(ctx->fallback) &
                               CRYPTO_TFM_RES_MASK;
        return ret;
 }
@@ -666,9 +666,9 @@ static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
        int err;
        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                skcipher_request_set_tfm(subreq, ctx->fallback);
+                skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                skcipher_request_set_callback(subreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -688,9 +688,9 @@ static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
        int err;
        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                skcipher_request_set_tfm(subreq, ctx->fallback);
+                skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                skcipher_request_set_callback(subreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -710,9 +710,9 @@ static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
        int err;
        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                skcipher_request_set_tfm(subreq, ctx->fallback);
+                skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                skcipher_request_set_callback(subreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -732,9 +732,9 @@ static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
        int err;
        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
-                SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
-                skcipher_request_set_tfm(subreq, ctx->fallback);
+                skcipher_request_set_sync_tfm(subreq, ctx->fallback);
                skcipher_request_set_callback(subreq, req->base.flags,
                                              NULL, NULL);
                skcipher_request_set_crypt(subreq, req->src, req->dst,
@@ -752,8 +752,7 @@ static int sahara_aes_cra_init(struct crypto_tfm *tfm)
        const char *name = crypto_tfm_alg_name(tfm);
        struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
-        ctx->fallback = crypto_alloc_skcipher(name, 0,
+        ctx->fallback = crypto_alloc_sync_skcipher(name, 0,
-                                              CRYPTO_ALG_ASYNC |
                                              CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(ctx->fallback)) {
                pr_err("Error allocating fallback algo %s\n", name);
@@ -769,7 +768,7 @@ static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
 {
        struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
-        crypto_free_skcipher(ctx->fallback);
+        crypto_free_sync_skcipher(ctx->fallback);
 }
 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index b71895871be3..c5c5ff82b52e 100644
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -32,7 +32,7 @@
 #include "aesp8-ppc.h"
 struct p8_aes_cbc_ctx {
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
        struct aes_key enc_key;
        struct aes_key dec_key;
 };
@@ -40,11 +40,11 @@ struct p8_aes_cbc_ctx {
 static int p8_aes_cbc_init(struct crypto_tfm *tfm)
 {
        const char *alg = crypto_tfm_alg_name(tfm);
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
        struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
-        fallback = crypto_alloc_skcipher(alg, 0,
+        fallback = crypto_alloc_sync_skcipher(alg, 0,
-                        CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+                                              CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(fallback)) {
                printk(KERN_ERR
@@ -53,7 +53,7 @@ static int p8_aes_cbc_init(struct crypto_tfm *tfm)
                return PTR_ERR(fallback);
        }
-        crypto_skcipher_set_flags(
+        crypto_sync_skcipher_set_flags(
                fallback,
                crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
        ctx->fallback = fallback;
@@ -66,7 +66,7 @@ static void p8_aes_cbc_exit(struct crypto_tfm *tfm)
        struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
        if (ctx->fallback) {
-                crypto_free_skcipher(ctx->fallback);
+                crypto_free_sync_skcipher(ctx->fallback);
                ctx->fallback = NULL;
        }
 }
@@ -86,7 +86,7 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
        pagefault_enable();
        preempt_enable();
-        ret += crypto_skcipher_setkey(ctx->fallback, key, keylen);
+        ret += crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
        return ret;
 }
@@ -100,8 +100,8 @@ static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
        if (in_interrupt()) {
-                SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_tfm(req, ctx->fallback);
+                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
                ret = crypto_skcipher_encrypt(req);
@@ -139,8 +139,8 @@ static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
        if (in_interrupt()) {
-                SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_tfm(req, ctx->fallback);
+                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
                ret = crypto_skcipher_decrypt(req);
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index cd777c75291d..8a2fe092cb8e 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -32,18 +32,18 @@
 #include "aesp8-ppc.h"
 struct p8_aes_ctr_ctx {
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
        struct aes_key enc_key;
 };
 static int p8_aes_ctr_init(struct crypto_tfm *tfm)
 {
        const char *alg = crypto_tfm_alg_name(tfm);
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
        struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
-        fallback = crypto_alloc_skcipher(alg, 0,
+        fallback = crypto_alloc_sync_skcipher(alg, 0,
-                        CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+                                              CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(fallback)) {
                printk(KERN_ERR
                       "Failed to allocate transformation for '%s': %ld\n",
@@ -51,7 +51,7 @@ static int p8_aes_ctr_init(struct crypto_tfm *tfm)
                return PTR_ERR(fallback);
        }
-        crypto_skcipher_set_flags(
+        crypto_sync_skcipher_set_flags(
                fallback,
                crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
        ctx->fallback = fallback;
@@ -64,7 +64,7 @@ static void p8_aes_ctr_exit(struct crypto_tfm *tfm)
        struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
        if (ctx->fallback) {
-                crypto_free_skcipher(ctx->fallback);
+                crypto_free_sync_skcipher(ctx->fallback);
                ctx->fallback = NULL;
        }
 }
@@ -83,7 +83,7 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
        pagefault_enable();
        preempt_enable();
-        ret += crypto_skcipher_setkey(ctx->fallback, key, keylen);
+        ret += crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
        return ret;
 }
@@ -119,8 +119,8 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
        if (in_interrupt()) {
-                SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_tfm(req, ctx->fallback);
+                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
                ret = crypto_skcipher_encrypt(req);
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index e9954a7d4694..ecd64e5cc5bb 100644
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -33,7 +33,7 @@
 #include "aesp8-ppc.h"
 struct p8_aes_xts_ctx {
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
        struct aes_key enc_key;
        struct aes_key dec_key;
        struct aes_key tweak_key;
@@ -42,11 +42,11 @@ struct p8_aes_xts_ctx {
 static int p8_aes_xts_init(struct crypto_tfm *tfm)
 {
        const char *alg = crypto_tfm_alg_name(tfm);
-        struct crypto_skcipher *fallback;
+        struct crypto_sync_skcipher *fallback;
        struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
-        fallback = crypto_alloc_skcipher(alg, 0,
+        fallback = crypto_alloc_sync_skcipher(alg, 0,
-                        CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+                                              CRYPTO_ALG_NEED_FALLBACK);
        if (IS_ERR(fallback)) {
                printk(KERN_ERR
                        "Failed to allocate transformation for '%s': %ld\n",
@@ -54,7 +54,7 @@ static int p8_aes_xts_init(struct crypto_tfm *tfm)
                return PTR_ERR(fallback);
        }
-        crypto_skcipher_set_flags(
+        crypto_sync_skcipher_set_flags(
                fallback,
                crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
        ctx->fallback = fallback;
@@ -67,7 +67,7 @@ static void p8_aes_xts_exit(struct crypto_tfm *tfm)
        struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
        if (ctx->fallback) {
-                crypto_free_skcipher(ctx->fallback);
+                crypto_free_sync_skcipher(ctx->fallback);
                ctx->fallback = NULL;
        }
 }
@@ -92,7 +92,7 @@ static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
        pagefault_enable();
        preempt_enable();
-        ret += crypto_skcipher_setkey(ctx->fallback, key, keylen);
+        ret += crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
        return ret;
 }
@@ -109,8 +109,8 @@ static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
        if (in_interrupt()) {
-                SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_tfm(req, ctx->fallback);
+                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
                ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index e1fa6baf4e8e..bb3096bf2cc6 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -559,7 +559,12 @@ static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result
                }
                memset(result + size, 0, JOURNAL_MAC_SIZE - size);
        } else {
-                __u8 digest[size];
+                __u8 digest[HASH_MAX_DIGESTSIZE];
+                if (WARN_ON(size > sizeof(digest))) {
+                        dm_integrity_io_error(ic, "digest_size", -EINVAL);
+                        goto err;
+                }
                r = crypto_shash_final(desc, digest);
                if (unlikely(r)) {
                        dm_integrity_io_error(ic, "crypto_shash_final", r);
@@ -1324,7 +1329,7 @@ static void integrity_metadata(struct work_struct *w)
                struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
                char *checksums;
                unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
-                char checksums_onstack[ic->tag_size + extra_space];
+                char checksums_onstack[HASH_MAX_DIGESTSIZE];
                unsigned sectors_to_process = dio->range.n_sectors;
                sector_t sector = dio->range.logical_sector;
@@ -1333,8 +1338,14 @@ static void integrity_metadata(struct work_struct *w)
                checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
                                    GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
-                if (!checksums)
+                if (!checksums) {
                        checksums = checksums_onstack;
+                        if (WARN_ON(extra_space &&
+                                    digest_size > sizeof(checksums_onstack))) {
+                                r = -EINVAL;
+                                goto error;
+                        }
+                }
                __bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) {
                        unsigned pos;
@@ -1546,7 +1557,7 @@ retry_kmap:
                                } while (++s < ic->sectors_per_block);
 #ifdef INTERNAL_VERIFY
                                if (ic->internal_hash) {
-                                        char checksums_onstack[max(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)];
+                                        char checksums_onstack[max(HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
                                        integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
                                        if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
@@ -1596,7 +1607,7 @@ retry_kmap:
                                if (ic->internal_hash) {
                                        unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
                                        if (unlikely(digest_size > ic->tag_size)) {
-                                                char checksums_onstack[digest_size];
+                                                char checksums_onstack[HASH_MAX_DIGESTSIZE];
                                                integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
                                                memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
                                        } else
@@ -2023,7 +2034,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
                                    unlikely(from_replay) &&
 #endif
                                    ic->internal_hash) {
-                                        char test_tag[max(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)];
+                                        char test_tag[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
                                        integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
                                                                  (char *)access_journal_data(ic, i, l), test_tag);
diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c
index 684af08d0747..0ce04e5b4afb 100644
--- a/drivers/md/dm-verity-fec.c
+++ b/drivers/md/dm-verity-fec.c
@@ -212,12 +212,15 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
        struct dm_verity_fec_io *fio = fec_io(io);
        u64 block, ileaved;
        u8 *bbuf, *rs_block;
-        u8 want_digest[v->digest_size];
+        u8 want_digest[HASH_MAX_DIGESTSIZE];
        unsigned n, k;
        if (neras)
                *neras = 0;
+        if (WARN_ON(v->digest_size > sizeof(want_digest)))
+                return -EINVAL;
        /*
         * read each of the rsn data blocks that are part of the RS block, and
         * interleave contents to available bufs
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index 267322693ed5..9a6065a3fa46 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -520,10 +520,20 @@ setup_sge_txq_uld(struct adapter *adap, unsigned int uld_type,
        txq_info = kzalloc(sizeof(*txq_info), GFP_KERNEL);
        if (!txq_info)
                return -ENOMEM;
+        if (uld_type == CXGB4_ULD_CRYPTO) {
+                i = min_t(int, adap->vres.ncrypto_fc,
+                          num_online_cpus());
+                txq_info->ntxq = rounddown(i, adap->params.nports);
+                if (txq_info->ntxq <= 0) {
+                        dev_warn(adap->pdev_dev, "Crypto Tx Queues can't be zero\n");
+                        kfree(txq_info);
+                        return -EINVAL;
+                }
-        i = min_t(int, uld_info->ntxq, num_online_cpus());
+        } else {
-        txq_info->ntxq = roundup(i, adap->params.nports);
+                i = min_t(int, uld_info->ntxq, num_online_cpus());
+                txq_info->ntxq = roundup(i, adap->params.nports);
+        }
        txq_info->uldtxq = kcalloc(txq_info->ntxq, sizeof(struct sge_uld_txq),
                                   GFP_KERNEL);
        if (!txq_info->uldtxq) {
@@ -546,11 +556,14 @@ static void uld_queue_init(struct adapter *adap, unsigned int uld_type,
                           struct cxgb4_lld_info *lli)
 {
        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+        int tx_uld_type = TX_ULD(uld_type);
+        struct sge_uld_txq_info *txq_info = adap->sge.uld_txq_info[tx_uld_type];
        lli->rxq_ids = rxq_info->rspq_id;
        lli->nrxq = rxq_info->nrxq;
        lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq;
        lli->nciq = rxq_info->nciq;
+        lli->ntxq = txq_info->ntxq;
 }
 int t4_uld_mem_alloc(struct adapter *adap)
@@ -634,7 +647,6 @@ static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
        lld->ports = adap->port;
        lld->vr = &adap->vres;
        lld->mtus = adap->params.mtus;
-        lld->ntxq = adap->sge.ofldqsets;
        lld->nchan = adap->params.nports;
        lld->nports = adap->params.nports;
        lld->wr_cred = adap->params.ofldq_wr_cred;
diff --git a/drivers/net/ppp/ppp_mppe.c b/drivers/net/ppp/ppp_mppe.c
index a205750b431b..7ccdc62c6052 100644
--- a/drivers/net/ppp/ppp_mppe.c
+++ b/drivers/net/ppp/ppp_mppe.c
@@ -95,7 +95,7 @@ static inline void sha_pad_init(struct sha_pad *shapad)
 * State for an MPPE (de)compressor.
 */
 struct ppp_mppe_state {
-        struct crypto_skcipher *arc4;
+        struct crypto_sync_skcipher *arc4;
        struct shash_desc *sha1;
        unsigned char *sha1_digest;
        unsigned char master_key[MPPE_MAX_KEY_LEN];
@@ -155,15 +155,15 @@ static void get_new_key_from_sha(struct ppp_mppe_state * state)
 static void mppe_rekey(struct ppp_mppe_state * state, int initial_key)
 {
        struct scatterlist sg_in[1], sg_out[1];
-        SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
-        skcipher_request_set_tfm(req, state->arc4);
+        skcipher_request_set_sync_tfm(req, state->arc4);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        get_new_key_from_sha(state);
        if (!initial_key) {
-                crypto_skcipher_setkey(state->arc4, state->sha1_digest,
+                crypto_sync_skcipher_setkey(state->arc4, state->sha1_digest,
-                                       state->keylen);
+                                            state->keylen);
                sg_init_table(sg_in, 1);
                sg_init_table(sg_out, 1);
                setup_sg(sg_in, state->sha1_digest, state->keylen);
@@ -181,7 +181,8 @@ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key)
                state->session_key[1] = 0x26;
                state->session_key[2] = 0x9e;
        }
-        crypto_skcipher_setkey(state->arc4, state->session_key, state->keylen);
+        crypto_sync_skcipher_setkey(state->arc4, state->session_key,
+                                    state->keylen);
        skcipher_request_zero(req);
 }
@@ -203,7 +204,7 @@ static void *mppe_alloc(unsigned char *options, int optlen)
                goto out;
-        state->arc4 = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
+        state->arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
        if (IS_ERR(state->arc4)) {
                state->arc4 = NULL;
                goto out_free;
@@ -250,7 +251,7 @@ out_free:
                crypto_free_shash(state->sha1->tfm);
                kzfree(state->sha1);
        }
-        crypto_free_skcipher(state->arc4);
+        crypto_free_sync_skcipher(state->arc4);
        kfree(state);
 out:
        return NULL;
@@ -266,7 +267,7 @@ static void mppe_free(void *arg)
                kfree(state->sha1_digest);
                crypto_free_shash(state->sha1->tfm);
                kzfree(state->sha1);
-                crypto_free_skcipher(state->arc4);
+                crypto_free_sync_skcipher(state->arc4);
                kfree(state);
        }
 }
@@ -366,7 +367,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf,
              int isize, int osize)
 {
        struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg;
-        SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
        int proto;
        int err;
        struct scatterlist sg_in[1], sg_out[1];
@@ -426,7 +427,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf,
        setup_sg(sg_in, ibuf, isize);
        setup_sg(sg_out, obuf, osize);
-        skcipher_request_set_tfm(req, state->arc4);
+        skcipher_request_set_sync_tfm(req, state->arc4);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg_in, sg_out, isize, NULL);
        err = crypto_skcipher_encrypt(req);
@@ -480,7 +481,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
                int osize)
 {
        struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg;
-        SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
        unsigned ccount;
        int flushed = MPPE_BITS(ibuf) & MPPE_BIT_FLUSHED;
        struct scatterlist sg_in[1], sg_out[1];
@@ -615,7 +616,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
        setup_sg(sg_in, ibuf, 1);
        setup_sg(sg_out, obuf, 1);
-        skcipher_request_set_tfm(req, state->arc4);
+        skcipher_request_set_sync_tfm(req, state->arc4);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg_in, sg_out, 1, NULL);
        if (crypto_skcipher_decrypt(req)) {
diff --git a/drivers/soc/fsl/dpio/dpio-service.c b/drivers/soc/fsl/dpio/dpio-service.c
index 9b17f72349ed..321a92613a7e 100644
--- a/drivers/soc/fsl/dpio/dpio-service.c
+++ b/drivers/soc/fsl/dpio/dpio-service.c
@@ -310,6 +310,37 @@ int dpaa2_io_service_rearm(struct dpaa2_io *d,
 EXPORT_SYMBOL_GPL(dpaa2_io_service_rearm);
 /**
+ * dpaa2_io_service_pull_fq() - pull dequeue functions from a fq.
+ * @d: the given DPIO service.
+ * @fqid: the given frame queue id.
+ * @s: the dpaa2_io_store object for the result.
+ *
+ * Return 0 for success, or error code for failure.
+ */
+int dpaa2_io_service_pull_fq(struct dpaa2_io *d, u32 fqid,
+                             struct dpaa2_io_store *s)
+{
+        struct qbman_pull_desc pd;
+        int err;
+        qbman_pull_desc_clear(&pd);
+        qbman_pull_desc_set_storage(&pd, s->vaddr, s->paddr, 1);
+        qbman_pull_desc_set_numframes(&pd, (u8)s->max);
+        qbman_pull_desc_set_fq(&pd, fqid);
+        d = service_select(d);
+        if (!d)
+                return -ENODEV;
+        s->swp = d->swp;
+        err = qbman_swp_pull(d->swp, &pd);
+        if (err)
+                s->swp = NULL;
+        return err;
+}
+EXPORT_SYMBOL(dpaa2_io_service_pull_fq);
+/**
 * dpaa2_io_service_pull_channel() - pull dequeue functions from a channel.
 * @d: the given DPIO service.
 * @channelid: the given channel id.
@@ -342,6 +373,33 @@ int dpaa2_io_service_pull_channel(struct dpaa2_io *d, u32 channelid,
 EXPORT_SYMBOL_GPL(dpaa2_io_service_pull_channel);
 /**
+ * dpaa2_io_service_enqueue_fq() - Enqueue a frame to a frame queue.
+ * @d: the given DPIO service.
+ * @fqid: the given frame queue id.
+ * @fd: the frame descriptor which is enqueued.
+ *
+ * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,
+ * or -ENODEV if there is no dpio service.
+ */
+int dpaa2_io_service_enqueue_fq(struct dpaa2_io *d,
+                                u32 fqid,
+                                const struct dpaa2_fd *fd)
+{
+        struct qbman_eq_desc ed;
+        d = service_select(d);
+        if (!d)
+                return -ENODEV;
+        qbman_eq_desc_clear(&ed);
+        qbman_eq_desc_set_no_orp(&ed, 0);
+        qbman_eq_desc_set_fq(&ed, fqid);
+        return qbman_swp_enqueue(d->swp, &ed, fd);
+}
+EXPORT_SYMBOL(dpaa2_io_service_enqueue_fq);
+/**
 * dpaa2_io_service_enqueue_qd() - Enqueue a frame to a QD.
 * @d: the given DPIO service.
 * @qdid: the given queuing destination id.
diff --git a/drivers/staging/rtl8192e/rtllib_crypt_tkip.c b/drivers/staging/rtl8192e/rtllib_crypt_tkip.c
index 9f18be14dda6..f38f1f74fcd6 100644
--- a/drivers/staging/rtl8192e/rtllib_crypt_tkip.c
+++ b/drivers/staging/rtl8192e/rtllib_crypt_tkip.c
@@ -49,9 +49,9 @@ struct rtllib_tkip_data {
        u32 dot11RSNAStatsTKIPLocalMICFailures;
        int key_idx;
-        struct crypto_skcipher *rx_tfm_arc4;
+        struct crypto_sync_skcipher *rx_tfm_arc4;
        struct crypto_shash *rx_tfm_michael;
-        struct crypto_skcipher *tx_tfm_arc4;
+        struct crypto_sync_skcipher *tx_tfm_arc4;
        struct crypto_shash *tx_tfm_michael;
        /* scratch buffers for virt_to_page() (crypto API) */
        u8 rx_hdr[16];
@@ -66,8 +66,7 @@ static void *rtllib_tkip_init(int key_idx)
        if (priv == NULL)
                goto fail;
        priv->key_idx = key_idx;
-        priv->tx_tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0,
+        priv->tx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
-                                                  CRYPTO_ALG_ASYNC);
        if (IS_ERR(priv->tx_tfm_arc4)) {
                pr_debug("Could not allocate crypto API arc4\n");
                priv->tx_tfm_arc4 = NULL;
@@ -81,8 +80,7 @@ static void *rtllib_tkip_init(int key_idx)
                goto fail;
        }
-        priv->rx_tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0,
+        priv->rx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
-                                                  CRYPTO_ALG_ASYNC);
        if (IS_ERR(priv->rx_tfm_arc4)) {
                pr_debug("Could not allocate crypto API arc4\n");
                priv->rx_tfm_arc4 = NULL;
@@ -100,9 +98,9 @@ static void *rtllib_tkip_init(int key_idx)
 fail:
        if (priv) {
                crypto_free_shash(priv->tx_tfm_michael);
-                crypto_free_skcipher(priv->tx_tfm_arc4);
+                crypto_free_sync_skcipher(priv->tx_tfm_arc4);
                crypto_free_shash(priv->rx_tfm_michael);
-                crypto_free_skcipher(priv->rx_tfm_arc4);
+                crypto_free_sync_skcipher(priv->rx_tfm_arc4);
                kfree(priv);
        }
@@ -116,9 +114,9 @@ static void rtllib_tkip_deinit(void *priv)
        if (_priv) {
                crypto_free_shash(_priv->tx_tfm_michael);
-                crypto_free_skcipher(_priv->tx_tfm_arc4);
+                crypto_free_sync_skcipher(_priv->tx_tfm_arc4);
                crypto_free_shash(_priv->rx_tfm_michael);
-                crypto_free_skcipher(_priv->rx_tfm_arc4);
+                crypto_free_sync_skcipher(_priv->rx_tfm_arc4);
        }
        kfree(priv);
 }
@@ -337,7 +335,7 @@ static int rtllib_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
        *pos++ = (tkey->tx_iv32 >> 24) & 0xff;
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4);
                icv = skb_put(skb, 4);
                crc = ~crc32_le(~0, pos, len);
@@ -349,8 +347,8 @@ static int rtllib_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
                sg_init_one(&sg, pos, len+4);
-                crypto_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
+                crypto_sync_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
-                skcipher_request_set_tfm(req, tkey->tx_tfm_arc4);
+                skcipher_request_set_sync_tfm(req, tkey->tx_tfm_arc4);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
                ret = crypto_skcipher_encrypt(req);
@@ -420,7 +418,7 @@ static int rtllib_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
        pos += 8;
        if (!tcb_desc->bHwSec || (skb->cb[0] == 1)) {
-                SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4);
                if ((iv32 < tkey->rx_iv32 ||
                    (iv32 == tkey->rx_iv32 && iv16 <= tkey->rx_iv16)) &&
@@ -447,8 +445,8 @@ static int rtllib_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
                sg_init_one(&sg, pos, plen+4);
-                crypto_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
+                crypto_sync_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
-                skcipher_request_set_tfm(req, tkey->rx_tfm_arc4);
+                skcipher_request_set_sync_tfm(req, tkey->rx_tfm_arc4);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
                err = crypto_skcipher_decrypt(req);
@@ -664,9 +662,9 @@ static int rtllib_tkip_set_key(void *key, int len, u8 *seq, void *priv)
        struct rtllib_tkip_data *tkey = priv;
        int keyidx;
        struct crypto_shash *tfm = tkey->tx_tfm_michael;
-        struct crypto_skcipher *tfm2 = tkey->tx_tfm_arc4;
+        struct crypto_sync_skcipher *tfm2 = tkey->tx_tfm_arc4;
        struct crypto_shash *tfm3 = tkey->rx_tfm_michael;
-        struct crypto_skcipher *tfm4 = tkey->rx_tfm_arc4;
+        struct crypto_sync_skcipher *tfm4 = tkey->rx_tfm_arc4;
        keyidx = tkey->key_idx;
        memset(tkey, 0, sizeof(*tkey));
diff --git a/drivers/staging/rtl8192e/rtllib_crypt_wep.c b/drivers/staging/rtl8192e/rtllib_crypt_wep.c
index b3343a5d0fd6..d11ec39171d5 100644
--- a/drivers/staging/rtl8192e/rtllib_crypt_wep.c
+++ b/drivers/staging/rtl8192e/rtllib_crypt_wep.c
@@ -27,8 +27,8 @@ struct prism2_wep_data {
        u8 key[WEP_KEY_LEN + 1];
        u8 key_len;
        u8 key_idx;
-        struct crypto_skcipher *tx_tfm;
+        struct crypto_sync_skcipher *tx_tfm;
-        struct crypto_skcipher *rx_tfm;
+        struct crypto_sync_skcipher *rx_tfm;
 };
@@ -41,13 +41,13 @@ static void *prism2_wep_init(int keyidx)
                goto fail;
        priv->key_idx = keyidx;
-        priv->tx_tfm = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
+        priv->tx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
        if (IS_ERR(priv->tx_tfm)) {
                pr_debug("rtllib_crypt_wep: could not allocate crypto API arc4\n");
                priv->tx_tfm = NULL;
                goto fail;
        }
-        priv->rx_tfm = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
+        priv->rx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
        if (IS_ERR(priv->rx_tfm)) {
                pr_debug("rtllib_crypt_wep: could not allocate crypto API arc4\n");
                priv->rx_tfm = NULL;
@@ -61,8 +61,8 @@ static void *prism2_wep_init(int keyidx)
 fail:
        if (priv) {
-                crypto_free_skcipher(priv->tx_tfm);
+                crypto_free_sync_skcipher(priv->tx_tfm);
-                crypto_free_skcipher(priv->rx_tfm);
+                crypto_free_sync_skcipher(priv->rx_tfm);
                kfree(priv);
        }
        return NULL;
@@ -74,8 +74,8 @@ static void prism2_wep_deinit(void *priv)
        struct prism2_wep_data *_priv = priv;
        if (_priv) {
-                crypto_free_skcipher(_priv->tx_tfm);
+                crypto_free_sync_skcipher(_priv->tx_tfm);
-                crypto_free_skcipher(_priv->rx_tfm);
+                crypto_free_sync_skcipher(_priv->rx_tfm);
        }
        kfree(priv);
 }
@@ -135,7 +135,7 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
        memcpy(key + 3, wep->key, wep->key_len);
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm);
                /* Append little-endian CRC32 and encrypt it to produce ICV */
                crc = ~crc32_le(~0, pos, len);
@@ -146,8 +146,8 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
                icv[3] = crc >> 24;
                sg_init_one(&sg, pos, len+4);
-                crypto_skcipher_setkey(wep->tx_tfm, key, klen);
+                crypto_sync_skcipher_setkey(wep->tx_tfm, key, klen);
-                skcipher_request_set_tfm(req, wep->tx_tfm);
+                skcipher_request_set_sync_tfm(req, wep->tx_tfm);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
                err = crypto_skcipher_encrypt(req);
@@ -199,11 +199,11 @@ static int prism2_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
        plen = skb->len - hdr_len - 8;
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm);
                sg_init_one(&sg, pos, plen+4);
-                crypto_skcipher_setkey(wep->rx_tfm, key, klen);
+                crypto_sync_skcipher_setkey(wep->rx_tfm, key, klen);
-                skcipher_request_set_tfm(req, wep->rx_tfm);
+                skcipher_request_set_sync_tfm(req, wep->rx_tfm);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
                err = crypto_skcipher_decrypt(req);
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
index 1088fa0aee0e..829fa4bd253c 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
@@ -53,9 +53,9 @@ struct ieee80211_tkip_data {
        int key_idx;
-        struct crypto_skcipher *rx_tfm_arc4;
+        struct crypto_sync_skcipher *rx_tfm_arc4;
        struct crypto_shash *rx_tfm_michael;
-        struct crypto_skcipher *tx_tfm_arc4;
+        struct crypto_sync_skcipher *tx_tfm_arc4;
        struct crypto_shash *tx_tfm_michael;
        /* scratch buffers for virt_to_page() (crypto API) */
@@ -71,8 +71,7 @@ static void *ieee80211_tkip_init(int key_idx)
                goto fail;
        priv->key_idx = key_idx;
-        priv->tx_tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0,
+        priv->tx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
-                        CRYPTO_ALG_ASYNC);
        if (IS_ERR(priv->tx_tfm_arc4)) {
                printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate "
                                "crypto API arc4\n");
@@ -88,8 +87,7 @@ static void *ieee80211_tkip_init(int key_idx)
                goto fail;
        }
-        priv->rx_tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0,
+        priv->rx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
-                        CRYPTO_ALG_ASYNC);
        if (IS_ERR(priv->rx_tfm_arc4)) {
                printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate "
                                "crypto API arc4\n");
@@ -110,9 +108,9 @@ static void *ieee80211_tkip_init(int key_idx)
 fail:
        if (priv) {
                crypto_free_shash(priv->tx_tfm_michael);
-                crypto_free_skcipher(priv->tx_tfm_arc4);
+                crypto_free_sync_skcipher(priv->tx_tfm_arc4);
                crypto_free_shash(priv->rx_tfm_michael);
-                crypto_free_skcipher(priv->rx_tfm_arc4);
+                crypto_free_sync_skcipher(priv->rx_tfm_arc4);
                kfree(priv);
        }
@@ -126,9 +124,9 @@ static void ieee80211_tkip_deinit(void *priv)
        if (_priv) {
                crypto_free_shash(_priv->tx_tfm_michael);
-                crypto_free_skcipher(_priv->tx_tfm_arc4);
+                crypto_free_sync_skcipher(_priv->tx_tfm_arc4);
                crypto_free_shash(_priv->rx_tfm_michael);
-                crypto_free_skcipher(_priv->rx_tfm_arc4);
+                crypto_free_sync_skcipher(_priv->rx_tfm_arc4);
        }
        kfree(priv);
 }
@@ -340,7 +338,7 @@ static int ieee80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
        *pos++ = (tkey->tx_iv32 >> 24) & 0xff;
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4);
                icv = skb_put(skb, 4);
                crc = ~crc32_le(~0, pos, len);
@@ -348,9 +346,9 @@ static int ieee80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
                icv[1] = crc >> 8;
                icv[2] = crc >> 16;
                icv[3] = crc >> 24;
-                crypto_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
+                crypto_sync_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
                sg_init_one(&sg, pos, len+4);
-                skcipher_request_set_tfm(req, tkey->tx_tfm_arc4);
+                skcipher_request_set_sync_tfm(req, tkey->tx_tfm_arc4);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
                ret = crypto_skcipher_encrypt(req);
@@ -418,7 +416,7 @@ static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
        pos += 8;
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4);
                if (iv32 < tkey->rx_iv32 ||
                (iv32 == tkey->rx_iv32 && iv16 <= tkey->rx_iv16)) {
@@ -440,10 +438,10 @@ static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
                plen = skb->len - hdr_len - 12;
-                crypto_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
+                crypto_sync_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
                sg_init_one(&sg, pos, plen+4);
-                skcipher_request_set_tfm(req, tkey->rx_tfm_arc4);
+                skcipher_request_set_sync_tfm(req, tkey->rx_tfm_arc4);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
@@ -663,9 +661,9 @@ static int ieee80211_tkip_set_key(void *key, int len, u8 *seq, void *priv)
        struct ieee80211_tkip_data *tkey = priv;
        int keyidx;
        struct crypto_shash *tfm = tkey->tx_tfm_michael;
-        struct crypto_skcipher *tfm2 = tkey->tx_tfm_arc4;
+        struct crypto_sync_skcipher *tfm2 = tkey->tx_tfm_arc4;
        struct crypto_shash *tfm3 = tkey->rx_tfm_michael;
-        struct crypto_skcipher *tfm4 = tkey->rx_tfm_arc4;
+        struct crypto_sync_skcipher *tfm4 = tkey->rx_tfm_arc4;
        keyidx = tkey->key_idx;
        memset(tkey, 0, sizeof(*tkey));
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c
index b9f86be9e52b..d4a1bf0caa7a 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c
@@ -32,8 +32,8 @@ struct prism2_wep_data {
        u8 key[WEP_KEY_LEN + 1];
        u8 key_len;
        u8 key_idx;
-        struct crypto_skcipher *tx_tfm;
+        struct crypto_sync_skcipher *tx_tfm;
-        struct crypto_skcipher *rx_tfm;
+        struct crypto_sync_skcipher *rx_tfm;
 };
@@ -46,10 +46,10 @@ static void *prism2_wep_init(int keyidx)
                return NULL;
        priv->key_idx = keyidx;
-        priv->tx_tfm = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
+        priv->tx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
        if (IS_ERR(priv->tx_tfm))
                goto free_priv;
-        priv->rx_tfm = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
+        priv->rx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
        if (IS_ERR(priv->rx_tfm))
                goto free_tx;
@@ -58,7 +58,7 @@ static void *prism2_wep_init(int keyidx)
        return priv;
 free_tx:
-        crypto_free_skcipher(priv->tx_tfm);
+        crypto_free_sync_skcipher(priv->tx_tfm);
 free_priv:
        kfree(priv);
        return NULL;
@@ -70,8 +70,8 @@ static void prism2_wep_deinit(void *priv)
        struct prism2_wep_data *_priv = priv;
        if (_priv) {
-                crypto_free_skcipher(_priv->tx_tfm);
+                crypto_free_sync_skcipher(_priv->tx_tfm);
-                crypto_free_skcipher(_priv->rx_tfm);
+                crypto_free_sync_skcipher(_priv->rx_tfm);
        }
        kfree(priv);
 }
@@ -128,7 +128,7 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
        memcpy(key + 3, wep->key, wep->key_len);
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm);
                /* Append little-endian CRC32 and encrypt it to produce ICV */
                crc = ~crc32_le(~0, pos, len);
@@ -138,10 +138,10 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
                icv[2] = crc >> 16;
                icv[3] = crc >> 24;
-                crypto_skcipher_setkey(wep->tx_tfm, key, klen);
+                crypto_sync_skcipher_setkey(wep->tx_tfm, key, klen);
                sg_init_one(&sg, pos, len+4);
-                skcipher_request_set_tfm(req, wep->tx_tfm);
+                skcipher_request_set_sync_tfm(req, wep->tx_tfm);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
@@ -193,12 +193,12 @@ static int prism2_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
        plen = skb->len - hdr_len - 8;
        if (!tcb_desc->bHwSec) {
-                SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm);
-                crypto_skcipher_setkey(wep->rx_tfm, key, klen);
+                crypto_sync_skcipher_setkey(wep->rx_tfm, key, klen);
                sg_init_one(&sg, pos, plen+4);
-                skcipher_request_set_tfm(req, wep->rx_tfm);
+                skcipher_request_set_sync_tfm(req, wep->rx_tfm);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
diff --git a/drivers/usb/wusbcore/crypto.c b/drivers/usb/wusbcore/crypto.c
index aff50eb09ca9..68ddee86a886 100644
--- a/drivers/usb/wusbcore/crypto.c
+++ b/drivers/usb/wusbcore/crypto.c
@@ -189,7 +189,7 @@ struct wusb_mac_scratch {
 * NOTE: blen is not aligned to a block size, we'll pad zeros, that's
 *       what sg[4] is for. Maybe there is a smarter way to do this.
 */
-static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc,
+static int wusb_ccm_mac(struct crypto_sync_skcipher *tfm_cbc,
                        struct crypto_cipher *tfm_aes,
                        struct wusb_mac_scratch *scratch,
                        void *mic,
@@ -198,7 +198,7 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc,
                        size_t blen)
 {
        int result = 0;
-        SKCIPHER_REQUEST_ON_STACK(req, tfm_cbc);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm_cbc);
        struct scatterlist sg[4], sg_dst;
        void *dst_buf;
        size_t dst_size;
@@ -224,7 +224,7 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc,
        if (!dst_buf)
                goto error_dst_buf;
-        iv = kzalloc(crypto_skcipher_ivsize(tfm_cbc), GFP_KERNEL);
+        iv = kzalloc(crypto_sync_skcipher_ivsize(tfm_cbc), GFP_KERNEL);
        if (!iv)
                goto error_iv;
@@ -251,7 +251,7 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc,
        sg_set_page(&sg[3], ZERO_PAGE(0), zero_padding, 0);
        sg_init_one(&sg_dst, dst_buf, dst_size);
-        skcipher_request_set_tfm(req, tfm_cbc);
+        skcipher_request_set_sync_tfm(req, tfm_cbc);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, &sg_dst, dst_size, iv);
        result = crypto_skcipher_encrypt(req);
@@ -298,19 +298,19 @@ ssize_t wusb_prf(void *out, size_t out_size,
 {
        ssize_t result, bytes = 0, bitr;
        struct aes_ccm_nonce n = *_n;
-        struct crypto_skcipher *tfm_cbc;
+        struct crypto_sync_skcipher *tfm_cbc;
        struct crypto_cipher *tfm_aes;
        struct wusb_mac_scratch *scratch;
        u64 sfn = 0;
        __le64 sfn_le;
-        tfm_cbc = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC);
+        tfm_cbc = crypto_alloc_sync_skcipher("cbc(aes)", 0, 0);
        if (IS_ERR(tfm_cbc)) {
                result = PTR_ERR(tfm_cbc);
                printk(KERN_ERR "E: can't load CBC(AES): %d\n", (int)result);
                goto error_alloc_cbc;
        }
-        result = crypto_skcipher_setkey(tfm_cbc, key, 16);
+        result = crypto_sync_skcipher_setkey(tfm_cbc, key, 16);
        if (result < 0) {
                printk(KERN_ERR "E: can't set CBC key: %d\n", (int)result);
                goto error_setkey_cbc;
@@ -351,7 +351,7 @@ error_setkey_aes:
        crypto_free_cipher(tfm_aes);
 error_alloc_aes:
 error_setkey_cbc:
-        crypto_free_skcipher(tfm_cbc);
+        crypto_free_sync_skcipher(tfm_cbc);
 error_alloc_cbc:
        return result;
 }
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index 39c20ef26db4..79debfc9cef9 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -83,10 +83,6 @@ static inline bool fscrypt_valid_enc_modes(u32 contents_mode,
            filenames_mode == FS_ENCRYPTION_MODE_AES_256_CTS)
                return true;
-        if (contents_mode == FS_ENCRYPTION_MODE_SPECK128_256_XTS &&
-            filenames_mode == FS_ENCRYPTION_MODE_SPECK128_256_CTS)
-                return true;
        return false;
 }
diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c
index e997ca51192f..7874c9bb2fc5 100644
--- a/fs/crypto/keyinfo.c
+++ b/fs/crypto/keyinfo.c
@@ -174,16 +174,6 @@ static struct fscrypt_mode {
                .cipher_str = "cts(cbc(aes))",
                .keysize = 16,
        },
-        [FS_ENCRYPTION_MODE_SPECK128_256_XTS] = {
-                .friendly_name = "Speck128/256-XTS",
-                .cipher_str = "xts(speck128)",
-                .keysize = 64,
-        },
-        [FS_ENCRYPTION_MODE_SPECK128_256_CTS] = {
-                .friendly_name = "Speck128/256-CTS-CBC",
-                .cipher_str = "cts(cbc(speck128))",
-                .keysize = 32,
-        },
 };
 static struct fscrypt_mode *
diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h
index e328b52425a8..22e6f412c595 100644
--- a/include/crypto/acompress.h
+++ b/include/crypto/acompress.h
@@ -234,6 +234,34 @@ static inline void acomp_request_set_params(struct acomp_req *req,
                req->flags |= CRYPTO_ACOMP_ALLOC_OUTPUT;
 }
+static inline void crypto_stat_compress(struct acomp_req *req, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->compress_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->compress_cnt);
+                atomic64_add(req->slen, &tfm->base.__crt_alg->compress_tlen);
+        }
+#endif
+}
+static inline void crypto_stat_decompress(struct acomp_req *req, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->compress_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->decompress_cnt);
+                atomic64_add(req->slen, &tfm->base.__crt_alg->decompress_tlen);
+        }
+#endif
+}
 /**
 * crypto_acomp_compress() -- Invoke asynchronous compress operation
 *
@@ -246,8 +274,11 @@ static inline void acomp_request_set_params(struct acomp_req *req,
 static inline int crypto_acomp_compress(struct acomp_req *req)
 {
        struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+        int ret;
-        return tfm->compress(req);
+        ret = tfm->compress(req);
+        crypto_stat_compress(req, ret);
+        return ret;
 }
 /**
@@ -262,8 +293,11 @@ static inline int crypto_acomp_compress(struct acomp_req *req)
 static inline int crypto_acomp_decompress(struct acomp_req *req)
 {
        struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+        int ret;
-        return tfm->decompress(req);
+        ret = tfm->decompress(req);
+        crypto_stat_decompress(req, ret);
+        return ret;
 }
 #endif
diff --git a/include/crypto/aead.h b/include/crypto/aead.h
index 1e26f790b03f..0d765d7bfb82 100644
--- a/include/crypto/aead.h
+++ b/include/crypto/aead.h
@@ -306,6 +306,34 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req)
        return __crypto_aead_cast(req->base.tfm);
 }
+static inline void crypto_stat_aead_encrypt(struct aead_request *req, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->aead_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->encrypt_cnt);
+                atomic64_add(req->cryptlen, &tfm->base.__crt_alg->encrypt_tlen);
+        }
+#endif
+}
+static inline void crypto_stat_aead_decrypt(struct aead_request *req, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->aead_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->decrypt_cnt);
+                atomic64_add(req->cryptlen, &tfm->base.__crt_alg->decrypt_tlen);
+        }
+#endif
+}
 /**
 * crypto_aead_encrypt() - encrypt plaintext
 * @req: reference to the aead_request handle that holds all information
@@ -328,11 +356,14 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req)
 static inline int crypto_aead_encrypt(struct aead_request *req)
 {
        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        int ret;
        if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
-                return -ENOKEY;
+                ret = -ENOKEY;
+        else
-        return crypto_aead_alg(aead)->encrypt(req);
+                ret = crypto_aead_alg(aead)->encrypt(req);
+        crypto_stat_aead_encrypt(req, ret);
+        return ret;
 }
 /**
@@ -360,14 +391,16 @@ static inline int crypto_aead_encrypt(struct aead_request *req)
 static inline int crypto_aead_decrypt(struct aead_request *req)
 {
        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        int ret;
        if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
-                return -ENOKEY;
+                ret = -ENOKEY;
+        else if (req->cryptlen < crypto_aead_authsize(aead))
-        if (req->cryptlen < crypto_aead_authsize(aead))
+                ret = -EINVAL;
-                return -EINVAL;
+        else
+                ret = crypto_aead_alg(aead)->decrypt(req);
-        return crypto_aead_alg(aead)->decrypt(req);
+        crypto_stat_aead_decrypt(req, ret);
+        return ret;
 }
 /**
diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h
index b5e11de4d497..afac71119396 100644
--- a/include/crypto/akcipher.h
+++ b/include/crypto/akcipher.h
@@ -271,6 +271,62 @@ static inline unsigned int crypto_akcipher_maxsize(struct crypto_akcipher *tfm)
        return alg->max_size(tfm);
 }
+static inline void crypto_stat_akcipher_encrypt(struct akcipher_request *req,
+                                                int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->akcipher_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->encrypt_cnt);
+                atomic64_add(req->src_len, &tfm->base.__crt_alg->encrypt_tlen);
+        }
+#endif
+}
+static inline void crypto_stat_akcipher_decrypt(struct akcipher_request *req,
+                                                int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->akcipher_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->decrypt_cnt);
+                atomic64_add(req->src_len, &tfm->base.__crt_alg->decrypt_tlen);
+        }
+#endif
+}
+static inline void crypto_stat_akcipher_sign(struct akcipher_request *req,
+                                             int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY)
+                atomic_inc(&tfm->base.__crt_alg->akcipher_err_cnt);
+        else
+                atomic_inc(&tfm->base.__crt_alg->sign_cnt);
+#endif
+}
+static inline void crypto_stat_akcipher_verify(struct akcipher_request *req,
+                                               int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY)
+                atomic_inc(&tfm->base.__crt_alg->akcipher_err_cnt);
+        else
+                atomic_inc(&tfm->base.__crt_alg->verify_cnt);
+#endif
+}
 /**
 * crypto_akcipher_encrypt() - Invoke public key encrypt operation
 *
@@ -285,8 +341,11 @@ static inline int crypto_akcipher_encrypt(struct akcipher_request *req)
 {
        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
        struct akcipher_alg *alg = crypto_akcipher_alg(tfm);
+        int ret;
-        return alg->encrypt(req);
+        ret = alg->encrypt(req);
+        crypto_stat_akcipher_encrypt(req, ret);
+        return ret;
 }
 /**
@@ -303,8 +362,11 @@ static inline int crypto_akcipher_decrypt(struct akcipher_request *req)
 {
        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
        struct akcipher_alg *alg = crypto_akcipher_alg(tfm);
+        int ret;
-        return alg->decrypt(req);
+        ret = alg->decrypt(req);
+        crypto_stat_akcipher_decrypt(req, ret);
+        return ret;
 }
 /**
@@ -321,8 +383,11 @@ static inline int crypto_akcipher_sign(struct akcipher_request *req)
 {
        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
        struct akcipher_alg *alg = crypto_akcipher_alg(tfm);
+        int ret;
-        return alg->sign(req);
+        ret = alg->sign(req);
+        crypto_stat_akcipher_sign(req, ret);
+        return ret;
 }
 /**
@@ -339,8 +404,11 @@ static inline int crypto_akcipher_verify(struct akcipher_request *req)
 {
        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
        struct akcipher_alg *alg = crypto_akcipher_alg(tfm);
+        int ret;
-        return alg->verify(req);
+        ret = alg->verify(req);
+        crypto_stat_akcipher_verify(req, ret);
+        return ret;
 }
 /**
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index bd5e8ccf1687..4a5ad10e75f0 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -20,8 +20,10 @@
 /*
 * Maximum values for blocksize and alignmask, used to allocate
 * static buffers that are big enough for any combination of
- * ciphers and architectures.
+ * algs and architectures. Ciphers have a lower maximum size.
 */
+#define MAX_ALGAPI_BLOCKSIZE            160
+#define MAX_ALGAPI_ALIGNMASK            63
 #define MAX_CIPHER_BLOCKSIZE            16
 #define MAX_CIPHER_ALIGNMASK            15
@@ -425,4 +427,14 @@ static inline void crypto_yield(u32 flags)
 #endif
 }
+int crypto_register_notifier(struct notifier_block *nb);
+int crypto_unregister_notifier(struct notifier_block *nb);
+/* Crypto notification events. */
+enum {
+        CRYPTO_MSG_ALG_REQUEST,
+        CRYPTO_MSG_ALG_REGISTER,
+        CRYPTO_MSG_ALG_LOADED,
+};
 #endif  /* _CRYPTO_ALGAPI_H */
diff --git a/include/crypto/cbc.h b/include/crypto/cbc.h
index f5b8bfc22e6d..3bf28beefa33 100644
--- a/include/crypto/cbc.h
+++ b/include/crypto/cbc.h
@@ -113,7 +113,7 @@ static inline int crypto_cbc_decrypt_inplace(
        unsigned int bsize = crypto_skcipher_blocksize(tfm);
        unsigned int nbytes = walk->nbytes;
        u8 *src = walk->src.virt.addr;
-        u8 last_iv[bsize];
+        u8 last_iv[MAX_CIPHER_BLOCKSIZE];
        /* Start of the last block. */
        src += nbytes - (nbytes & (bsize - 1)) - bsize;
diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h
index b83d66073db0..f76302d99e2b 100644
--- a/include/crypto/chacha20.h
+++ b/include/crypto/chacha20.h
@@ -13,13 +13,12 @@
 #define CHACHA20_IV_SIZE        16
 #define CHACHA20_KEY_SIZE       32
 #define CHACHA20_BLOCK_SIZE     64
-#define CHACHA20_BLOCK_WORDS    (CHACHA20_BLOCK_SIZE / sizeof(u32))
 struct chacha20_ctx {
        u32 key[8];
 };
-void chacha20_block(u32 *state, u32 *stream);
+void chacha20_block(u32 *state, u8 *stream);
 void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv);
 int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
                           unsigned int keysize);
diff --git a/include/crypto/hash.h b/include/crypto/hash.h
index 76e432cab75d..bc7796600338 100644
--- a/include/crypto/hash.h
+++ b/include/crypto/hash.h
@@ -151,9 +151,13 @@ struct shash_desc {
        void *__ctx[] CRYPTO_MINALIGN_ATTR;
 };
+#define HASH_MAX_DIGESTSIZE      64
+#define HASH_MAX_DESCSIZE       360
+#define HASH_MAX_STATESIZE      512
 #define SHASH_DESC_ON_STACK(shash, ctx)                           \
        char __##shash##_desc[sizeof(struct shash_desc) +         \
-                crypto_shash_descsize(ctx)] CRYPTO_MINALIGN_ATTR; \
+                HASH_MAX_DESCSIZE] CRYPTO_MINALIGN_ATTR; \
        struct shash_desc *shash = (struct shash_desc *)__##shash##_desc
 /**
@@ -408,6 +412,32 @@ static inline void *ahash_request_ctx(struct ahash_request *req)
 int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
                        unsigned int keylen);
+static inline void crypto_stat_ahash_update(struct ahash_request *req, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY)
+                atomic_inc(&tfm->base.__crt_alg->hash_err_cnt);
+        else
+                atomic64_add(req->nbytes, &tfm->base.__crt_alg->hash_tlen);
+#endif
+}
+static inline void crypto_stat_ahash_final(struct ahash_request *req, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->hash_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->hash_cnt);
+                atomic64_add(req->nbytes, &tfm->base.__crt_alg->hash_tlen);
+        }
+#endif
+}
 /**
 * crypto_ahash_finup() - update and finalize message digest
 * @req: reference to the ahash_request handle that holds all information
@@ -522,7 +552,11 @@ static inline int crypto_ahash_init(struct ahash_request *req)
 */
 static inline int crypto_ahash_update(struct ahash_request *req)
 {
-        return crypto_ahash_reqtfm(req)->update(req);
+        int ret;
+        ret = crypto_ahash_reqtfm(req)->update(req);
+        crypto_stat_ahash_update(req, ret);
+        return ret;
 }
 /**
diff --git a/include/crypto/internal/cryptouser.h b/include/crypto/internal/cryptouser.h
new file mode 100644
index 000000000000..8db299c25566
--- /dev/null
+++ b/include/crypto/internal/cryptouser.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <net/netlink.h>
+struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact);
+int crypto_dump_reportstat(struct sk_buff *skb, struct netlink_callback *cb);
+int crypto_reportstat(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, struct nlattr **attrs);
+int crypto_dump_reportstat_done(struct netlink_callback *cb);
diff --git a/include/crypto/internal/geniv.h b/include/crypto/internal/geniv.h
index 2bcfb931bc5b..71be24cd59bd 100644
--- a/include/crypto/internal/geniv.h
+++ b/include/crypto/internal/geniv.h
@@ -20,7 +20,7 @@
 struct aead_geniv_ctx {
        spinlock_t lock;
        struct crypto_aead *child;
-        struct crypto_skcipher *sknull;
+        struct crypto_sync_skcipher *sknull;
        u8 salt[] __attribute__ ((aligned(__alignof__(u32))));
 };
diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h
index 1bde0a6514fa..f517ba6d3a27 100644
--- a/include/crypto/kpp.h
+++ b/include/crypto/kpp.h
@@ -268,6 +268,42 @@ struct kpp_secret {
        unsigned short len;
 };
+static inline void crypto_stat_kpp_set_secret(struct crypto_kpp *tfm, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        if (ret)
+                atomic_inc(&tfm->base.__crt_alg->kpp_err_cnt);
+        else
+                atomic_inc(&tfm->base.__crt_alg->setsecret_cnt);
+#endif
+}
+static inline void crypto_stat_kpp_generate_public_key(struct kpp_request *req,
+                                                       int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+        if (ret)
+                atomic_inc(&tfm->base.__crt_alg->kpp_err_cnt);
+        else
+                atomic_inc(&tfm->base.__crt_alg->generate_public_key_cnt);
+#endif
+}
+static inline void crypto_stat_kpp_compute_shared_secret(struct kpp_request *req,
+                                                         int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+        if (ret)
+                atomic_inc(&tfm->base.__crt_alg->kpp_err_cnt);
+        else
+                atomic_inc(&tfm->base.__crt_alg->compute_shared_secret_cnt);
+#endif
+}
 /**
 * crypto_kpp_set_secret() - Invoke kpp operation
 *
@@ -287,8 +323,11 @@ static inline int crypto_kpp_set_secret(struct crypto_kpp *tfm,
                                        const void *buffer, unsigned int len)
 {
        struct kpp_alg *alg = crypto_kpp_alg(tfm);
+        int ret;
-        return alg->set_secret(tfm, buffer, len);
+        ret = alg->set_secret(tfm, buffer, len);
+        crypto_stat_kpp_set_secret(tfm, ret);
+        return ret;
 }
 /**
@@ -308,8 +347,11 @@ static inline int crypto_kpp_generate_public_key(struct kpp_request *req)
 {
        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
        struct kpp_alg *alg = crypto_kpp_alg(tfm);
+        int ret;
-        return alg->generate_public_key(req);
+        ret = alg->generate_public_key(req);
+        crypto_stat_kpp_generate_public_key(req, ret);
+        return ret;
 }
 /**
@@ -326,8 +368,11 @@ static inline int crypto_kpp_compute_shared_secret(struct kpp_request *req)
 {
        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
        struct kpp_alg *alg = crypto_kpp_alg(tfm);
+        int ret;
-        return alg->compute_shared_secret(req);
+        ret = alg->compute_shared_secret(req);
+        crypto_stat_kpp_compute_shared_secret(req, ret);
+        return ret;
 }
 /**
diff --git a/include/crypto/mcryptd.h b/include/crypto/mcryptd.h
deleted file mode 100644
index b67404fc4b34..000000000000
--- a/include/crypto/mcryptd.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Software async multibuffer crypto daemon headers
- *
- *    Author:
- *             Tim Chen <tim.c.chen@linux.intel.com>
- *
- *    Copyright (c) 2014, Intel Corporation.
- */
-#ifndef _CRYPTO_MCRYPT_H
-#define _CRYPTO_MCRYPT_H
-#include <linux/crypto.h>
-#include <linux/kernel.h>
-#include <crypto/hash.h>
-struct mcryptd_ahash {
-        struct crypto_ahash base;
-};
-static inline struct mcryptd_ahash *__mcryptd_ahash_cast(
-        struct crypto_ahash *tfm)
-{
-        return (struct mcryptd_ahash *)tfm;
-}
-struct mcryptd_cpu_queue {
-        struct crypto_queue queue;
-        spinlock_t q_lock;
-        struct work_struct work;
-};
-struct mcryptd_queue {
-        struct mcryptd_cpu_queue __percpu *cpu_queue;
-};
-struct mcryptd_instance_ctx {
-        struct crypto_spawn spawn;
-        struct mcryptd_queue *queue;
-};
-struct mcryptd_hash_ctx {
-        struct crypto_ahash *child;
-        struct mcryptd_alg_state *alg_state;
-};
-struct mcryptd_tag {
-        /* seq number of request */
-        unsigned seq_num;
-        /* arrival time of request */
-        unsigned long arrival;
-        unsigned long expire;
-        int     cpu;
-};
-struct mcryptd_hash_request_ctx {
-        struct list_head waiter;
-        crypto_completion_t complete;
-        struct mcryptd_tag tag;
-        struct crypto_hash_walk walk;
-        u8 *out;
-        int flag;
-        struct ahash_request areq;
-};
-struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
-                                        u32 type, u32 mask);
-struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm);
-struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req);
-void mcryptd_free_ahash(struct mcryptd_ahash *tfm);
-void mcryptd_flusher(struct work_struct *work);
-enum mcryptd_req_type {
-        MCRYPTD_NONE,
-        MCRYPTD_UPDATE,
-        MCRYPTD_FINUP,
-        MCRYPTD_DIGEST,
-        MCRYPTD_FINAL
-};
-struct mcryptd_alg_cstate {
-        unsigned long next_flush;
-        unsigned next_seq_num;
-        bool    flusher_engaged;
-        struct  delayed_work flush;
-        int     cpu;
-        struct  mcryptd_alg_state *alg_state;
-        void    *mgr;
-        spinlock_t work_lock;
-        struct list_head work_list;
-        struct list_head flush_list;
-};
-struct mcryptd_alg_state {
-        struct mcryptd_alg_cstate __percpu *alg_cstate;
-        unsigned long (*flusher)(struct mcryptd_alg_cstate *cstate);
-};
-/* return delay in jiffies from current time */
-static inline unsigned long get_delay(unsigned long t)
-{
-        long delay;
-        delay = (long) t - (long) jiffies;
-        if (delay <= 0)
-                return 0;
-        else
-                return (unsigned long) delay;
-}
-void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay);
-#endif
diff --git a/include/crypto/morus1280_glue.h b/include/crypto/morus1280_glue.h
index b26dd70efd9a..ba782e10065e 100644
--- a/include/crypto/morus1280_glue.h
+++ b/include/crypto/morus1280_glue.h
@@ -82,7 +82,7 @@ void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead);
        { \
        } \
        \
-        struct aead_alg crypto_morus1280_##id##_algs[] = {\
+        static struct aead_alg crypto_morus1280_##id##_algs[] = {\
                { \
                        .setkey = crypto_morus1280_glue_setkey, \
                        .setauthsize = crypto_morus1280_glue_setauthsize, \
diff --git a/include/crypto/morus640_glue.h b/include/crypto/morus640_glue.h
index 90c8db07e740..27fa790a2362 100644
--- a/include/crypto/morus640_glue.h
+++ b/include/crypto/morus640_glue.h
@@ -82,7 +82,7 @@ void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead);
        { \
        } \
        \
-        struct aead_alg crypto_morus640_##id##_algs[] = {\
+        static struct aead_alg crypto_morus640_##id##_algs[] = {\
                { \
                        .setkey = crypto_morus640_glue_setkey, \
                        .setauthsize = crypto_morus640_glue_setauthsize, \
diff --git a/include/crypto/null.h b/include/crypto/null.h
index 15aeef6e30ef..0ef577cc00e3 100644
--- a/include/crypto/null.h
+++ b/include/crypto/null.h
@@ -9,7 +9,7 @@
 #define NULL_DIGEST_SIZE        0
 #define NULL_IV_SIZE            0
-struct crypto_skcipher *crypto_get_default_null_skcipher(void);
+struct crypto_sync_skcipher *crypto_get_default_null_skcipher(void);
 void crypto_put_default_null_skcipher(void);
 #endif
diff --git a/include/crypto/rng.h b/include/crypto/rng.h
index b95ede354a66..6d258f5b68f1 100644
--- a/include/crypto/rng.h
+++ b/include/crypto/rng.h
@@ -122,6 +122,29 @@ static inline void crypto_free_rng(struct crypto_rng *tfm)
        crypto_destroy_tfm(tfm, crypto_rng_tfm(tfm));
 }
+static inline void crypto_stat_rng_seed(struct crypto_rng *tfm, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY)
+                atomic_inc(&tfm->base.__crt_alg->rng_err_cnt);
+        else
+                atomic_inc(&tfm->base.__crt_alg->seed_cnt);
+#endif
+}
+static inline void crypto_stat_rng_generate(struct crypto_rng *tfm,
+                                            unsigned int dlen, int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&tfm->base.__crt_alg->rng_err_cnt);
+        } else {
+                atomic_inc(&tfm->base.__crt_alg->generate_cnt);
+                atomic64_add(dlen, &tfm->base.__crt_alg->generate_tlen);
+        }
+#endif
+}
 /**
 * crypto_rng_generate() - get random number
 * @tfm: cipher handle
@@ -140,7 +163,11 @@ static inline int crypto_rng_generate(struct crypto_rng *tfm,
                                      const u8 *src, unsigned int slen,
                                      u8 *dst, unsigned int dlen)
 {
-        return crypto_rng_alg(tfm)->generate(tfm, src, slen, dst, dlen);
+        int ret;
+        ret = crypto_rng_alg(tfm)->generate(tfm, src, slen, dst, dlen);
+        crypto_stat_rng_generate(tfm, dlen, ret);
+        return ret;
 }
 /**
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index 2f327f090c3e..925f547cdcfa 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -65,6 +65,10 @@ struct crypto_skcipher {
        struct crypto_tfm base;
 };
+struct crypto_sync_skcipher {
+        struct crypto_skcipher base;
+};
 /**
 * struct skcipher_alg - symmetric key cipher definition
 * @min_keysize: Minimum key size supported by the transformation. This is the
@@ -139,9 +143,17 @@ struct skcipher_alg {
        struct crypto_alg base;
 };
-#define SKCIPHER_REQUEST_ON_STACK(name, tfm) \
+#define MAX_SYNC_SKCIPHER_REQSIZE      384
+/*
+ * This performs a type-check against the "tfm" argument to make sure
+ * all users have the correct skcipher tfm for doing on-stack requests.
+ */
+#define SYNC_SKCIPHER_REQUEST_ON_STACK(name, tfm) \
        char __##name##_desc[sizeof(struct skcipher_request) + \
-                crypto_skcipher_reqsize(tfm)] CRYPTO_MINALIGN_ATTR; \
+                             MAX_SYNC_SKCIPHER_REQSIZE + \
+                             (!(sizeof((struct crypto_sync_skcipher *)1 == \
+                                       (typeof(tfm))1))) \
+                            ] CRYPTO_MINALIGN_ATTR; \
        struct skcipher_request *name = (void *)__##name##_desc
 /**
@@ -197,6 +209,9 @@ static inline struct crypto_skcipher *__crypto_skcipher_cast(
 struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name,
                                              u32 type, u32 mask);
+struct crypto_sync_skcipher *crypto_alloc_sync_skcipher(const char *alg_name,
+                                              u32 type, u32 mask);
 static inline struct crypto_tfm *crypto_skcipher_tfm(
        struct crypto_skcipher *tfm)
 {
@@ -212,6 +227,11 @@ static inline void crypto_free_skcipher(struct crypto_skcipher *tfm)
        crypto_destroy_tfm(tfm, crypto_skcipher_tfm(tfm));
 }
+static inline void crypto_free_sync_skcipher(struct crypto_sync_skcipher *tfm)
+{
+        crypto_free_skcipher(&tfm->base);
+}
 /**
 * crypto_has_skcipher() - Search for the availability of an skcipher.
 * @alg_name: is the cra_name / name or cra_driver_name / driver name of the
@@ -280,6 +300,12 @@ static inline unsigned int crypto_skcipher_ivsize(struct crypto_skcipher *tfm)
        return tfm->ivsize;
 }
+static inline unsigned int crypto_sync_skcipher_ivsize(
+        struct crypto_sync_skcipher *tfm)
+{
+        return crypto_skcipher_ivsize(&tfm->base);
+}
 static inline unsigned int crypto_skcipher_alg_chunksize(
        struct skcipher_alg *alg)
 {
@@ -356,6 +382,12 @@ static inline unsigned int crypto_skcipher_blocksize(
        return crypto_tfm_alg_blocksize(crypto_skcipher_tfm(tfm));
 }
+static inline unsigned int crypto_sync_skcipher_blocksize(
+        struct crypto_sync_skcipher *tfm)
+{
+        return crypto_skcipher_blocksize(&tfm->base);
+}
 static inline unsigned int crypto_skcipher_alignmask(
        struct crypto_skcipher *tfm)
 {
@@ -379,6 +411,24 @@ static inline void crypto_skcipher_clear_flags(struct crypto_skcipher *tfm,
        crypto_tfm_clear_flags(crypto_skcipher_tfm(tfm), flags);
 }
+static inline u32 crypto_sync_skcipher_get_flags(
+        struct crypto_sync_skcipher *tfm)
+{
+        return crypto_skcipher_get_flags(&tfm->base);
+}
+static inline void crypto_sync_skcipher_set_flags(
+        struct crypto_sync_skcipher *tfm, u32 flags)
+{
+        crypto_skcipher_set_flags(&tfm->base, flags);
+}
+static inline void crypto_sync_skcipher_clear_flags(
+        struct crypto_sync_skcipher *tfm, u32 flags)
+{
+        crypto_skcipher_clear_flags(&tfm->base, flags);
+}
 /**
 * crypto_skcipher_setkey() - set key for cipher
 * @tfm: cipher handle
@@ -401,6 +451,12 @@ static inline int crypto_skcipher_setkey(struct crypto_skcipher *tfm,
        return tfm->setkey(tfm, key, keylen);
 }
+static inline int crypto_sync_skcipher_setkey(struct crypto_sync_skcipher *tfm,
+                                         const u8 *key, unsigned int keylen)
+{
+        return crypto_skcipher_setkey(&tfm->base, key, keylen);
+}
 static inline unsigned int crypto_skcipher_default_keysize(
        struct crypto_skcipher *tfm)
 {
@@ -422,6 +478,40 @@ static inline struct crypto_skcipher *crypto_skcipher_reqtfm(
        return __crypto_skcipher_cast(req->base.tfm);
 }
+static inline struct crypto_sync_skcipher *crypto_sync_skcipher_reqtfm(
+        struct skcipher_request *req)
+{
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        return container_of(tfm, struct crypto_sync_skcipher, base);
+}
+static inline void crypto_stat_skcipher_encrypt(struct skcipher_request *req,
+                                                int ret, struct crypto_alg *alg)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&alg->cipher_err_cnt);
+        } else {
+                atomic_inc(&alg->encrypt_cnt);
+                atomic64_add(req->cryptlen, &alg->encrypt_tlen);
+        }
+#endif
+}
+static inline void crypto_stat_skcipher_decrypt(struct skcipher_request *req,
+                                                int ret, struct crypto_alg *alg)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&alg->cipher_err_cnt);
+        } else {
+                atomic_inc(&alg->decrypt_cnt);
+                atomic64_add(req->cryptlen, &alg->decrypt_tlen);
+        }
+#endif
+}
 /**
 * crypto_skcipher_encrypt() - encrypt plaintext
 * @req: reference to the skcipher_request handle that holds all information
@@ -436,11 +526,14 @@ static inline struct crypto_skcipher *crypto_skcipher_reqtfm(
 static inline int crypto_skcipher_encrypt(struct skcipher_request *req)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        int ret;
        if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
-                return -ENOKEY;
+                ret = -ENOKEY;
+        else
-        return tfm->encrypt(req);
+                ret = tfm->encrypt(req);
+        crypto_stat_skcipher_encrypt(req, ret, tfm->base.__crt_alg);
+        return ret;
 }
 /**
@@ -457,11 +550,14 @@ static inline int crypto_skcipher_encrypt(struct skcipher_request *req)
 static inline int crypto_skcipher_decrypt(struct skcipher_request *req)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        int ret;
        if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
-                return -ENOKEY;
+                ret = -ENOKEY;
+        else
-        return tfm->decrypt(req);
+                ret = tfm->decrypt(req);
+        crypto_stat_skcipher_decrypt(req, ret, tfm->base.__crt_alg);
+        return ret;
 }
 /**
@@ -500,6 +596,12 @@ static inline void skcipher_request_set_tfm(struct skcipher_request *req,
        req->base.tfm = crypto_skcipher_tfm(tfm);
 }
+static inline void skcipher_request_set_sync_tfm(struct skcipher_request *req,
+                                            struct crypto_sync_skcipher *tfm)
+{
+        skcipher_request_set_tfm(req, &tfm->base);
+}
 static inline struct skcipher_request *skcipher_request_cast(
        struct crypto_async_request *req)
 {
diff --git a/include/crypto/speck.h b/include/crypto/speck.h
deleted file mode 100644
index 73cfc952d405..000000000000
--- a/include/crypto/speck.h
+++ /dev/null
@@ -1,62 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Common values for the Speck algorithm
- */
-#ifndef _CRYPTO_SPECK_H
-#define _CRYPTO_SPECK_H
-#include <linux/types.h>
-/* Speck128 */
-#define SPECK128_BLOCK_SIZE     16
-#define SPECK128_128_KEY_SIZE   16
-#define SPECK128_128_NROUNDS    32
-#define SPECK128_192_KEY_SIZE   24
-#define SPECK128_192_NROUNDS    33
-#define SPECK128_256_KEY_SIZE   32
-#define SPECK128_256_NROUNDS    34
-struct speck128_tfm_ctx {
-        u64 round_keys[SPECK128_256_NROUNDS];
-        int nrounds;
-};
-void crypto_speck128_encrypt(const struct speck128_tfm_ctx *ctx,
-                             u8 *out, const u8 *in);
-void crypto_speck128_decrypt(const struct speck128_tfm_ctx *ctx,
-                             u8 *out, const u8 *in);
-int crypto_speck128_setkey(struct speck128_tfm_ctx *ctx, const u8 *key,
-                           unsigned int keysize);
-/* Speck64 */
-#define SPECK64_BLOCK_SIZE      8
-#define SPECK64_96_KEY_SIZE     12
-#define SPECK64_96_NROUNDS      26
-#define SPECK64_128_KEY_SIZE    16
-#define SPECK64_128_NROUNDS     27
-struct speck64_tfm_ctx {
-        u32 round_keys[SPECK64_128_NROUNDS];
-        int nrounds;
-};
-void crypto_speck64_encrypt(const struct speck64_tfm_ctx *ctx,
-                            u8 *out, const u8 *in);
-void crypto_speck64_decrypt(const struct speck64_tfm_ctx *ctx,
-                            u8 *out, const u8 *in);
-int crypto_speck64_setkey(struct speck64_tfm_ctx *ctx, const u8 *key,
-                          unsigned int keysize);
-#endif /* _CRYPTO_SPECK_H */
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index db192becfec4..97cfe29b3f0a 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -198,7 +198,6 @@ struct ftrace_likely_data {
 */
 #define __pure                  __attribute__((pure))
 #define __aligned(x)            __attribute__((aligned(x)))
-#define __aligned_largest       __attribute__((aligned))
 #define __printf(a, b)          __attribute__((format(printf, a, b)))
 #define __scanf(a, b)           __attribute__((format(scanf, a, b)))
 #define __maybe_unused          __attribute__((unused))
diff --git a/include/linux/cpufeature.h b/include/linux/cpufeature.h
index 986c06c88d81..84d3c81b5978 100644
--- a/include/linux/cpufeature.h
+++ b/include/linux/cpufeature.h
@@ -45,7 +45,7 @@
 * 'asm/cpufeature.h' of your favorite architecture.
 */
 #define module_cpu_feature_match(x, __initfunc)                 \
-static struct cpu_feature const cpu_feature_match_ ## x[] =     \
+static struct cpu_feature const __maybe_unused cpu_feature_match_ ## x[] = \
        { { .feature = cpu_feature(x) }, { } };                 \
 MODULE_DEVICE_TABLE(cpu, cpu_feature_match_ ## x);              \
                                                                \
diff --git a/include/linux/crc-t10dif.h b/include/linux/crc-t10dif.h
index 1fe0cfcdea30..6bb0c0bf357b 100644
--- a/include/linux/crc-t10dif.h
+++ b/include/linux/crc-t10dif.h
@@ -6,6 +6,7 @@
 #define CRC_T10DIF_DIGEST_SIZE 2
 #define CRC_T10DIF_BLOCK_SIZE 1
+#define CRC_T10DIF_STRING "crct10dif"
 extern __u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer,
                                size_t len);
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index e8839d3a7559..3634ad6fe202 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -454,6 +454,33 @@ struct compress_alg {
 * @cra_refcnt: internally used
 * @cra_destroy: internally used
 *
+ * All following statistics are for this crypto_alg
+ * @encrypt_cnt:        number of encrypt requests
+ * @decrypt_cnt:        number of decrypt requests
+ * @compress_cnt:       number of compress requests
+ * @decompress_cnt:     number of decompress requests
+ * @generate_cnt:       number of RNG generate requests
+ * @seed_cnt:           number of times the rng was seeded
+ * @hash_cnt:           number of hash requests
+ * @sign_cnt:           number of sign requests
+ * @setsecret_cnt:      number of setsecrey operation
+ * @generate_public_key_cnt:    number of generate_public_key operation
+ * @verify_cnt:                 number of verify operation
+ * @compute_shared_secret_cnt:  number of compute_shared_secret operation
+ * @encrypt_tlen:       total data size handled by encrypt requests
+ * @decrypt_tlen:       total data size handled by decrypt requests
+ * @compress_tlen:      total data size handled by compress requests
+ * @decompress_tlen:    total data size handled by decompress requests
+ * @generate_tlen:      total data size of generated data by the RNG
+ * @hash_tlen:          total data size hashed
+ * @akcipher_err_cnt:   number of error for akcipher requests
+ * @cipher_err_cnt:     number of error for akcipher requests
+ * @compress_err_cnt:   number of error for akcipher requests
+ * @aead_err_cnt:       number of error for akcipher requests
+ * @hash_err_cnt:       number of error for akcipher requests
+ * @rng_err_cnt:        number of error for akcipher requests
+ * @kpp_err_cnt:        number of error for akcipher requests
+ *
 * The struct crypto_alg describes a generic Crypto API algorithm and is common
 * for all of the transformations. Any variable not documented here shall not
 * be used by a cipher implementation as it is internal to the Crypto API.
@@ -487,6 +514,45 @@ struct crypto_alg {
        void (*cra_destroy)(struct crypto_alg *alg);
        
        struct module *cra_module;
+        union {
+                atomic_t encrypt_cnt;
+                atomic_t compress_cnt;
+                atomic_t generate_cnt;
+                atomic_t hash_cnt;
+                atomic_t setsecret_cnt;
+        };
+        union {
+                atomic64_t encrypt_tlen;
+                atomic64_t compress_tlen;
+                atomic64_t generate_tlen;
+                atomic64_t hash_tlen;
+        };
+        union {
+                atomic_t akcipher_err_cnt;
+                atomic_t cipher_err_cnt;
+                atomic_t compress_err_cnt;
+                atomic_t aead_err_cnt;
+                atomic_t hash_err_cnt;
+                atomic_t rng_err_cnt;
+                atomic_t kpp_err_cnt;
+        };
+        union {
+                atomic_t decrypt_cnt;
+                atomic_t decompress_cnt;
+                atomic_t seed_cnt;
+                atomic_t generate_public_key_cnt;
+        };
+        union {
+                atomic64_t decrypt_tlen;
+                atomic64_t decompress_tlen;
+        };
+        union {
+                atomic_t verify_cnt;
+                atomic_t compute_shared_secret_cnt;
+        };
+        atomic_t sign_cnt;
 } CRYPTO_MINALIGN_ATTR;
 /*
@@ -907,6 +973,38 @@ static inline struct crypto_ablkcipher *crypto_ablkcipher_reqtfm(
        return __crypto_ablkcipher_cast(req->base.tfm);
 }
+static inline void crypto_stat_ablkcipher_encrypt(struct ablkcipher_request *req,
+                                                  int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct ablkcipher_tfm *crt =
+                crypto_ablkcipher_crt(crypto_ablkcipher_reqtfm(req));
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&crt->base->base.__crt_alg->cipher_err_cnt);
+        } else {
+                atomic_inc(&crt->base->base.__crt_alg->encrypt_cnt);
+                atomic64_add(req->nbytes, &crt->base->base.__crt_alg->encrypt_tlen);
+        }
+#endif
+}
+static inline void crypto_stat_ablkcipher_decrypt(struct ablkcipher_request *req,
+                                                  int ret)
+{
+#ifdef CONFIG_CRYPTO_STATS
+        struct ablkcipher_tfm *crt =
+                crypto_ablkcipher_crt(crypto_ablkcipher_reqtfm(req));
+        if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
+                atomic_inc(&crt->base->base.__crt_alg->cipher_err_cnt);
+        } else {
+                atomic_inc(&crt->base->base.__crt_alg->decrypt_cnt);
+                atomic64_add(req->nbytes, &crt->base->base.__crt_alg->decrypt_tlen);
+        }
+#endif
+}
 /**
 * crypto_ablkcipher_encrypt() - encrypt plaintext
 * @req: reference to the ablkcipher_request handle that holds all information
@@ -922,7 +1020,11 @@ static inline int crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
 {
        struct ablkcipher_tfm *crt =
                crypto_ablkcipher_crt(crypto_ablkcipher_reqtfm(req));
-        return crt->encrypt(req);
+        int ret;
+        ret = crt->encrypt(req);
+        crypto_stat_ablkcipher_encrypt(req, ret);
+        return ret;
 }
 /**
@@ -940,7 +1042,11 @@ static inline int crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
 {
        struct ablkcipher_tfm *crt =
                crypto_ablkcipher_crt(crypto_ablkcipher_reqtfm(req));
-        return crt->decrypt(req);
+        int ret;
+        ret = crt->decrypt(req);
+        crypto_stat_ablkcipher_decrypt(req, ret);
+        return ret;
 }
 /**
diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h
index f27cb14088a4..5160f06ffbac 100644
--- a/include/linux/fsl/mc.h
+++ b/include/linux/fsl/mc.h
@@ -405,6 +405,7 @@ extern struct device_type fsl_mc_bus_dpcon_type;
 extern struct device_type fsl_mc_bus_dpmcp_type;
 extern struct device_type fsl_mc_bus_dpmac_type;
 extern struct device_type fsl_mc_bus_dprtc_type;
+extern struct device_type fsl_mc_bus_dpseci_type;
 static inline bool is_fsl_mc_bus_dprc(const struct fsl_mc_device *mc_dev)
 {
@@ -451,6 +452,11 @@ static inline bool is_fsl_mc_bus_dprtc(const struct fsl_mc_device *mc_dev)
        return mc_dev->dev.type == &fsl_mc_bus_dprtc_type;
 }
+static inline bool is_fsl_mc_bus_dpseci(const struct fsl_mc_device *mc_dev)
+{
+        return mc_dev->dev.type == &fsl_mc_bus_dpseci_type;
+}
 /*
 * Data Path Buffer Pool (DPBP) API
 * Contains initialization APIs and runtime control APIs for DPBP
diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
index bee0827766a3..c0b93e0ff0c0 100644
--- a/include/linux/hw_random.h
+++ b/include/linux/hw_random.h
@@ -33,7 +33,8 @@
 *                      and max is a multiple of 4 and >= 32 bytes.
 * @priv:               Private data, for use by the RNG driver.
 * @quality:            Estimation of true entropy in RNG's bitstream
- *                      (per mill).
+ *                      (in bits of entropy per 1024 bits of input;
+ *                      valid values: 1 to 1024, or 0 for unknown).
 */
 struct hwrng {
        const char *name;
diff --git a/include/linux/sunrpc/gss_krb5.h b/include/linux/sunrpc/gss_krb5.h
index 7df625d41e35..f6e8ceafafd8 100644
--- a/include/linux/sunrpc/gss_krb5.h
+++ b/include/linux/sunrpc/gss_krb5.h
@@ -71,10 +71,10 @@ struct gss_krb5_enctype {
        const u32               keyed_cksum;    /* is it a keyed cksum? */
        const u32               keybytes;       /* raw key len, in bytes */
        const u32               keylength;      /* final key len, in bytes */
-        u32 (*encrypt) (struct crypto_skcipher *tfm,
+        u32 (*encrypt) (struct crypto_sync_skcipher *tfm,
                        void *iv, void *in, void *out,
                        int length);            /* encryption function */
-        u32 (*decrypt) (struct crypto_skcipher *tfm,
+        u32 (*decrypt) (struct crypto_sync_skcipher *tfm,
                        void *iv, void *in, void *out,
                        int length);            /* decryption function */
        u32 (*mk_key) (const struct gss_krb5_enctype *gk5e,
@@ -98,12 +98,12 @@ struct krb5_ctx {
        u32                     enctype;
        u32                     flags;
        const struct gss_krb5_enctype *gk5e; /* enctype-specific info */
-        struct crypto_skcipher  *enc;
+        struct crypto_sync_skcipher *enc;
-        struct crypto_skcipher  *seq;
+        struct crypto_sync_skcipher *seq;
-        struct crypto_skcipher *acceptor_enc;
+        struct crypto_sync_skcipher *acceptor_enc;
-        struct crypto_skcipher *initiator_enc;
+        struct crypto_sync_skcipher *initiator_enc;
-        struct crypto_skcipher *acceptor_enc_aux;
+        struct crypto_sync_skcipher *acceptor_enc_aux;
-        struct crypto_skcipher *initiator_enc_aux;
+        struct crypto_sync_skcipher *initiator_enc_aux;
        u8                      Ksess[GSS_KRB5_MAX_KEYLEN]; /* session key */
        u8                      cksum[GSS_KRB5_MAX_KEYLEN];
        s32                     endtime;
@@ -262,24 +262,24 @@ gss_unwrap_kerberos(struct gss_ctx *ctx_id, int offset,
 u32
-krb5_encrypt(struct crypto_skcipher *key,
+krb5_encrypt(struct crypto_sync_skcipher *key,
             void *iv, void *in, void *out, int length);
 u32
-krb5_decrypt(struct crypto_skcipher *key,
+krb5_decrypt(struct crypto_sync_skcipher *key,
             void *iv, void *in, void *out, int length); 
 int
-gss_encrypt_xdr_buf(struct crypto_skcipher *tfm, struct xdr_buf *outbuf,
+gss_encrypt_xdr_buf(struct crypto_sync_skcipher *tfm, struct xdr_buf *outbuf,
                    int offset, struct page **pages);
 int
-gss_decrypt_xdr_buf(struct crypto_skcipher *tfm, struct xdr_buf *inbuf,
+gss_decrypt_xdr_buf(struct crypto_sync_skcipher *tfm, struct xdr_buf *inbuf,
                    int offset);
 s32
 krb5_make_seq_num(struct krb5_ctx *kctx,
-                struct crypto_skcipher *key,
+                struct crypto_sync_skcipher *key,
                int direction,
                u32 seqnum, unsigned char *cksum, unsigned char *buf);
@@ -320,12 +320,12 @@ gss_krb5_aes_decrypt(struct krb5_ctx *kctx, u32 offset,
 int
 krb5_rc4_setup_seq_key(struct krb5_ctx *kctx,
-                       struct crypto_skcipher *cipher,
+                       struct crypto_sync_skcipher *cipher,
                       unsigned char *cksum);
 int
 krb5_rc4_setup_enc_key(struct krb5_ctx *kctx,
-                       struct crypto_skcipher *cipher,
+                       struct crypto_sync_skcipher *cipher,
                       s32 seqnum);
 void
 gss_krb5_make_confounder(char *p, u32 conflen);
diff --git a/include/soc/fsl/dpaa2-fd.h b/include/soc/fsl/dpaa2-fd.h
index 2576abaa7779..90ae8d191f1a 100644
--- a/include/soc/fsl/dpaa2-fd.h
+++ b/include/soc/fsl/dpaa2-fd.h
@@ -66,6 +66,15 @@ struct dpaa2_fd {
 #define SG_BPID_MASK            0x3FFF
 #define SG_FINAL_FLAG_MASK      0x1
 #define SG_FINAL_FLAG_SHIFT     15
+#define FL_SHORT_LEN_FLAG_MASK  0x1
+#define FL_SHORT_LEN_FLAG_SHIFT 14
+#define FL_SHORT_LEN_MASK       0x3FFFF
+#define FL_OFFSET_MASK          0x0FFF
+#define FL_FORMAT_MASK          0x3
+#define FL_FORMAT_SHIFT         12
+#define FL_BPID_MASK            0x3FFF
+#define FL_FINAL_FLAG_MASK      0x1
+#define FL_FINAL_FLAG_SHIFT     15
 /* Error bits in FD CTRL */
 #define FD_CTRL_ERR_MASK        0x000000FF
@@ -435,4 +444,237 @@ static inline void dpaa2_sg_set_final(struct dpaa2_sg_entry *sg, bool final)
        sg->format_offset |= cpu_to_le16(final << SG_FINAL_FLAG_SHIFT);
 }
+/**
+ * struct dpaa2_fl_entry - structure for frame list entry.
+ * @addr:          address in the FLE
+ * @len:           length in the FLE
+ * @bpid:          buffer pool ID
+ * @format_offset: format, offset, and short-length fields
+ * @frc:           frame context
+ * @ctrl:          control bits...including pta, pvt1, pvt2, err, etc
+ * @flc:           flow context address
+ */
+struct dpaa2_fl_entry {
+        __le64 addr;
+        __le32 len;
+        __le16 bpid;
+        __le16 format_offset;
+        __le32 frc;
+        __le32 ctrl;
+        __le64 flc;
+};
+enum dpaa2_fl_format {
+        dpaa2_fl_single = 0,
+        dpaa2_fl_res,
+        dpaa2_fl_sg
+};
+/**
+ * dpaa2_fl_get_addr() - get the addr field of FLE
+ * @fle: the given frame list entry
+ *
+ * Return the address in the frame list entry.
+ */
+static inline dma_addr_t dpaa2_fl_get_addr(const struct dpaa2_fl_entry *fle)
+{
+        return (dma_addr_t)le64_to_cpu(fle->addr);
+}
+/**
+ * dpaa2_fl_set_addr() - Set the addr field of FLE
+ * @fle: the given frame list entry
+ * @addr: the address needs to be set in frame list entry
+ */
+static inline void dpaa2_fl_set_addr(struct dpaa2_fl_entry *fle,
+                                     dma_addr_t addr)
+{
+        fle->addr = cpu_to_le64(addr);
+}
+/**
+ * dpaa2_fl_get_frc() - Get the frame context in the FLE
+ * @fle: the given frame list entry
+ *
+ * Return the frame context field in the frame lsit entry.
+ */
+static inline u32 dpaa2_fl_get_frc(const struct dpaa2_fl_entry *fle)
+{
+        return le32_to_cpu(fle->frc);
+}
+/**
+ * dpaa2_fl_set_frc() - Set the frame context in the FLE
+ * @fle: the given frame list entry
+ * @frc: the frame context needs to be set in frame list entry
+ */
+static inline void dpaa2_fl_set_frc(struct dpaa2_fl_entry *fle, u32 frc)
+{
+        fle->frc = cpu_to_le32(frc);
+}
+/**
+ * dpaa2_fl_get_ctrl() - Get the control bits in the FLE
+ * @fle: the given frame list entry
+ *
+ * Return the control bits field in the frame list entry.
+ */
+static inline u32 dpaa2_fl_get_ctrl(const struct dpaa2_fl_entry *fle)
+{
+        return le32_to_cpu(fle->ctrl);
+}
+/**
+ * dpaa2_fl_set_ctrl() - Set the control bits in the FLE
+ * @fle: the given frame list entry
+ * @ctrl: the control bits to be set in the frame list entry
+ */
+static inline void dpaa2_fl_set_ctrl(struct dpaa2_fl_entry *fle, u32 ctrl)
+{
+        fle->ctrl = cpu_to_le32(ctrl);
+}
+/**
+ * dpaa2_fl_get_flc() - Get the flow context in the FLE
+ * @fle: the given frame list entry
+ *
+ * Return the flow context in the frame list entry.
+ */
+static inline dma_addr_t dpaa2_fl_get_flc(const struct dpaa2_fl_entry *fle)
+{
+        return (dma_addr_t)le64_to_cpu(fle->flc);
+}
+/**
+ * dpaa2_fl_set_flc() - Set the flow context field of FLE
+ * @fle: the given frame list entry
+ * @flc_addr: the flow context needs to be set in frame list entry
+ */
+static inline void dpaa2_fl_set_flc(struct dpaa2_fl_entry *fle,
+                                    dma_addr_t flc_addr)
+{
+        fle->flc = cpu_to_le64(flc_addr);
+}
+static inline bool dpaa2_fl_short_len(const struct dpaa2_fl_entry *fle)
+{
+        return !!((le16_to_cpu(fle->format_offset) >>
+                  FL_SHORT_LEN_FLAG_SHIFT) & FL_SHORT_LEN_FLAG_MASK);
+}
+/**
+ * dpaa2_fl_get_len() - Get the length in the FLE
+ * @fle: the given frame list entry
+ *
+ * Return the length field in the frame list entry.
+ */
+static inline u32 dpaa2_fl_get_len(const struct dpaa2_fl_entry *fle)
+{
+        if (dpaa2_fl_short_len(fle))
+                return le32_to_cpu(fle->len) & FL_SHORT_LEN_MASK;
+        return le32_to_cpu(fle->len);
+}
+/**
+ * dpaa2_fl_set_len() - Set the length field of FLE
+ * @fle: the given frame list entry
+ * @len: the length needs to be set in frame list entry
+ */
+static inline void dpaa2_fl_set_len(struct dpaa2_fl_entry *fle, u32 len)
+{
+        fle->len = cpu_to_le32(len);
+}
+/**
+ * dpaa2_fl_get_offset() - Get the offset field in the frame list entry
+ * @fle: the given frame list entry
+ *
+ * Return the offset.
+ */
+static inline u16 dpaa2_fl_get_offset(const struct dpaa2_fl_entry *fle)
+{
+        return le16_to_cpu(fle->format_offset) & FL_OFFSET_MASK;
+}
+/**
+ * dpaa2_fl_set_offset() - Set the offset field of FLE
+ * @fle: the given frame list entry
+ * @offset: the offset needs to be set in frame list entry
+ */
+static inline void dpaa2_fl_set_offset(struct dpaa2_fl_entry *fle, u16 offset)
+{
+        fle->format_offset &= cpu_to_le16(~FL_OFFSET_MASK);
+        fle->format_offset |= cpu_to_le16(offset);
+}
+/**
+ * dpaa2_fl_get_format() - Get the format field in the FLE
+ * @fle: the given frame list entry
+ *
+ * Return the format.
+ */
+static inline enum dpaa2_fl_format dpaa2_fl_get_format(const struct dpaa2_fl_entry *fle)
+{
+        return (enum dpaa2_fl_format)((le16_to_cpu(fle->format_offset) >>
+                                       FL_FORMAT_SHIFT) & FL_FORMAT_MASK);
+}
+/**
+ * dpaa2_fl_set_format() - Set the format field of FLE
+ * @fle: the given frame list entry
+ * @format: the format needs to be set in frame list entry
+ */
+static inline void dpaa2_fl_set_format(struct dpaa2_fl_entry *fle,
+                                       enum dpaa2_fl_format format)
+{
+        fle->format_offset &= cpu_to_le16(~(FL_FORMAT_MASK << FL_FORMAT_SHIFT));
+        fle->format_offset |= cpu_to_le16(format << FL_FORMAT_SHIFT);
+}
+/**
+ * dpaa2_fl_get_bpid() - Get the bpid field in the FLE
+ * @fle: the given frame list entry
+ *
+ * Return the buffer pool id.
+ */
+static inline u16 dpaa2_fl_get_bpid(const struct dpaa2_fl_entry *fle)
+{
+        return le16_to_cpu(fle->bpid) & FL_BPID_MASK;
+}
+/**
+ * dpaa2_fl_set_bpid() - Set the bpid field of FLE
+ * @fle: the given frame list entry
+ * @bpid: buffer pool id to be set
+ */
+static inline void dpaa2_fl_set_bpid(struct dpaa2_fl_entry *fle, u16 bpid)
+{
+        fle->bpid &= cpu_to_le16(~(FL_BPID_MASK));
+        fle->bpid |= cpu_to_le16(bpid);
+}
+/**
+ * dpaa2_fl_is_final() - Check final bit in FLE
+ * @fle: the given frame list entry
+ *
+ * Return bool.
+ */
+static inline bool dpaa2_fl_is_final(const struct dpaa2_fl_entry *fle)
+{
+        return !!(le16_to_cpu(fle->format_offset) >> FL_FINAL_FLAG_SHIFT);
+}
+/**
+ * dpaa2_fl_set_final() - Set the final bit in FLE
+ * @fle: the given frame list entry
+ * @final: the final boolean to be set
+ */
+static inline void dpaa2_fl_set_final(struct dpaa2_fl_entry *fle, bool final)
+{
+        fle->format_offset &= cpu_to_le16((~(FL_FINAL_FLAG_MASK <<
+                                             FL_FINAL_FLAG_SHIFT)) & 0xFFFF);
+        fle->format_offset |= cpu_to_le16(final << FL_FINAL_FLAG_SHIFT);
+}
 #endif /* __FSL_DPAA2_FD_H */
diff --git a/include/soc/fsl/dpaa2-global.h b/include/soc/fsl/dpaa2-global.h
index 9bc0713346a8..2bfc379d3dc9 100644
--- a/include/soc/fsl/dpaa2-global.h
+++ b/include/soc/fsl/dpaa2-global.h
@@ -174,4 +174,19 @@ static inline const struct dpaa2_fd *dpaa2_dq_fd(const struct dpaa2_dq *dq)
        return (const struct dpaa2_fd *)&dq->dq.fd[0];
 }
+#define DPAA2_CSCN_SIZE         sizeof(struct dpaa2_dq)
+#define DPAA2_CSCN_ALIGN        16
+#define DPAA2_CSCN_STATE_CG     BIT(0)
+/**
+ * dpaa2_cscn_state_congested() - Check congestion state
+ * @cscn: congestion SCN (delivered to WQ or memory)
+ *
+i * Return true is congested.
+ */
+static inline bool dpaa2_cscn_state_congested(struct dpaa2_dq *cscn)
+{
+        return !!(cscn->scn.state & DPAA2_CSCN_STATE_CG);
+}
 #endif /* __FSL_DPAA2_GLOBAL_H */
diff --git a/include/soc/fsl/dpaa2-io.h b/include/soc/fsl/dpaa2-io.h
index ab51e40d11db..70997ab2146c 100644
--- a/include/soc/fsl/dpaa2-io.h
+++ b/include/soc/fsl/dpaa2-io.h
@@ -97,9 +97,13 @@ void dpaa2_io_service_deregister(struct dpaa2_io *service,
 int dpaa2_io_service_rearm(struct dpaa2_io *service,
                           struct dpaa2_io_notification_ctx *ctx);
+int dpaa2_io_service_pull_fq(struct dpaa2_io *d, u32 fqid,
+                             struct dpaa2_io_store *s);
 int dpaa2_io_service_pull_channel(struct dpaa2_io *d, u32 channelid,
                                  struct dpaa2_io_store *s);
+int dpaa2_io_service_enqueue_fq(struct dpaa2_io *d, u32 fqid,
+                                const struct dpaa2_fd *fd);
 int dpaa2_io_service_enqueue_qd(struct dpaa2_io *d, u32 qdid, u8 prio,
                                u16 qdbin, const struct dpaa2_fd *fd);
 int dpaa2_io_service_release(struct dpaa2_io *d, u32 bpid,
diff --git a/include/uapi/linux/cryptouser.h b/include/uapi/linux/cryptouser.h
index 19bf0ca6d635..6dafbc3e4414 100644
--- a/include/uapi/linux/cryptouser.h
+++ b/include/uapi/linux/cryptouser.h
@@ -29,6 +29,7 @@ enum {
        CRYPTO_MSG_UPDATEALG,
        CRYPTO_MSG_GETALG,
        CRYPTO_MSG_DELRNG,
+        CRYPTO_MSG_GETSTAT,
        __CRYPTO_MSG_MAX
 };
 #define CRYPTO_MSG_MAX (__CRYPTO_MSG_MAX - 1)
@@ -50,6 +51,16 @@ enum crypto_attr_type_t {
        CRYPTOCFGA_REPORT_AKCIPHER,     /* struct crypto_report_akcipher */
        CRYPTOCFGA_REPORT_KPP,          /* struct crypto_report_kpp */
        CRYPTOCFGA_REPORT_ACOMP,        /* struct crypto_report_acomp */
+        CRYPTOCFGA_STAT_LARVAL,         /* struct crypto_stat */
+        CRYPTOCFGA_STAT_HASH,           /* struct crypto_stat */
+        CRYPTOCFGA_STAT_BLKCIPHER,      /* struct crypto_stat */
+        CRYPTOCFGA_STAT_AEAD,           /* struct crypto_stat */
+        CRYPTOCFGA_STAT_COMPRESS,       /* struct crypto_stat */
+        CRYPTOCFGA_STAT_RNG,            /* struct crypto_stat */
+        CRYPTOCFGA_STAT_CIPHER,         /* struct crypto_stat */
+        CRYPTOCFGA_STAT_AKCIPHER,       /* struct crypto_stat */
+        CRYPTOCFGA_STAT_KPP,            /* struct crypto_stat */
+        CRYPTOCFGA_STAT_ACOMP,          /* struct crypto_stat */
        __CRYPTOCFGA_MAX
 #define CRYPTOCFGA_MAX (__CRYPTOCFGA_MAX - 1)
@@ -65,6 +76,47 @@ struct crypto_user_alg {
        __u32 cru_flags;
 };
+struct crypto_stat {
+        char type[CRYPTO_MAX_NAME];
+        union {
+                __u32 stat_encrypt_cnt;
+                __u32 stat_compress_cnt;
+                __u32 stat_generate_cnt;
+                __u32 stat_hash_cnt;
+                __u32 stat_setsecret_cnt;
+        };
+        union {
+                __u64 stat_encrypt_tlen;
+                __u64 stat_compress_tlen;
+                __u64 stat_generate_tlen;
+                __u64 stat_hash_tlen;
+        };
+        union {
+                __u32 stat_akcipher_err_cnt;
+                __u32 stat_cipher_err_cnt;
+                __u32 stat_compress_err_cnt;
+                __u32 stat_aead_err_cnt;
+                __u32 stat_hash_err_cnt;
+                __u32 stat_rng_err_cnt;
+                __u32 stat_kpp_err_cnt;
+        };
+        union {
+                __u32 stat_decrypt_cnt;
+                __u32 stat_decompress_cnt;
+                __u32 stat_seed_cnt;
+                __u32 stat_generate_public_key_cnt;
+        };
+        union {
+                __u64 stat_decrypt_tlen;
+                __u64 stat_decompress_tlen;
+        };
+        union {
+                __u32 stat_verify_cnt;
+                __u32 stat_compute_shared_secret_cnt;
+        };
+        __u32 stat_sign_cnt;
+};
 struct crypto_report_larval {
        char type[CRYPTO_MAX_NAME];
 };
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index 73e01918f996..a441ea1bfe6d 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -279,8 +279,8 @@ struct fsxattr {
 #define FS_ENCRYPTION_MODE_AES_256_CTS          4
 #define FS_ENCRYPTION_MODE_AES_128_CBC          5
 #define FS_ENCRYPTION_MODE_AES_128_CTS          6
-#define FS_ENCRYPTION_MODE_SPECK128_256_XTS     7
+#define FS_ENCRYPTION_MODE_SPECK128_256_XTS     7 /* Removed, do not use. */
-#define FS_ENCRYPTION_MODE_SPECK128_256_CTS     8
+#define FS_ENCRYPTION_MODE_SPECK128_256_CTS     8 /* Removed, do not use. */
 struct fscrypt_policy {
        __u8 version;
diff --git a/lib/chacha20.c b/lib/chacha20.c
index c1cc50fb68c9..d907fec6a9ed 100644
--- a/lib/chacha20.c
+++ b/lib/chacha20.c
@@ -16,9 +16,9 @@
 #include <asm/unaligned.h>
 #include <crypto/chacha20.h>
-void chacha20_block(u32 *state, u32 *stream)
+void chacha20_block(u32 *state, u8 *stream)
 {
-        u32 x[16], *out = stream;
+        u32 x[16];
        int i;
        for (i = 0; i < ARRAY_SIZE(x); i++)
@@ -67,7 +67,7 @@ void chacha20_block(u32 *state, u32 *stream)
        }
        for (i = 0; i < ARRAY_SIZE(x); i++)
-                out[i] = cpu_to_le32(x[i] + state[i]);
+                put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]);
        state[12]++;
 }
diff --git a/lib/crc-t10dif.c b/lib/crc-t10dif.c
index 1ad33e555805..4d0d47c1ffbd 100644
--- a/lib/crc-t10dif.c
+++ b/lib/crc-t10dif.c
@@ -14,10 +14,47 @@
 #include <linux/err.h>
 #include <linux/init.h>
 #include <crypto/hash.h>
+#include <crypto/algapi.h>
 #include <linux/static_key.h>
+#include <linux/notifier.h>
-static struct crypto_shash *crct10dif_tfm;
+static struct crypto_shash __rcu *crct10dif_tfm;
 static struct static_key crct10dif_fallback __read_mostly;
+static DEFINE_MUTEX(crc_t10dif_mutex);
+static int crc_t10dif_rehash(struct notifier_block *self, unsigned long val, void *data)
+{
+        struct crypto_alg *alg = data;
+        struct crypto_shash *new, *old;
+        if (val != CRYPTO_MSG_ALG_LOADED ||
+            static_key_false(&crct10dif_fallback) ||
+            strncmp(alg->cra_name, CRC_T10DIF_STRING, strlen(CRC_T10DIF_STRING)))
+                return 0;
+        mutex_lock(&crc_t10dif_mutex);
+        old = rcu_dereference_protected(crct10dif_tfm,
+                                        lockdep_is_held(&crc_t10dif_mutex));
+        if (!old) {
+                mutex_unlock(&crc_t10dif_mutex);
+                return 0;
+        }
+        new = crypto_alloc_shash("crct10dif", 0, 0);
+        if (IS_ERR(new)) {
+                mutex_unlock(&crc_t10dif_mutex);
+                return 0;
+        }
+        rcu_assign_pointer(crct10dif_tfm, new);
+        mutex_unlock(&crc_t10dif_mutex);
+        synchronize_rcu();
+        crypto_free_shash(old);
+        return 0;
+}
+static struct notifier_block crc_t10dif_nb = {
+        .notifier_call = crc_t10dif_rehash,
+};
 __u16 crc_t10dif_update(__u16 crc, const unsigned char *buffer, size_t len)
 {
@@ -30,11 +67,14 @@ __u16 crc_t10dif_update(__u16 crc, const unsigned char *buffer, size_t len)
        if (static_key_false(&crct10dif_fallback))
                return crc_t10dif_generic(crc, buffer, len);
-        desc.shash.tfm = crct10dif_tfm;
+        rcu_read_lock();
+        desc.shash.tfm = rcu_dereference(crct10dif_tfm);
        desc.shash.flags = 0;
        *(__u16 *)desc.ctx = crc;
        err = crypto_shash_update(&desc.shash, buffer, len);
+        rcu_read_unlock();
        BUG_ON(err);
        return *(__u16 *)desc.ctx;
@@ -49,6 +89,7 @@ EXPORT_SYMBOL(crc_t10dif);
 static int __init crc_t10dif_mod_init(void)
 {
+        crypto_register_notifier(&crc_t10dif_nb);
        crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0);
        if (IS_ERR(crct10dif_tfm)) {
                static_key_slow_inc(&crct10dif_fallback);
@@ -59,12 +100,24 @@ static int __init crc_t10dif_mod_init(void)
 static void __exit crc_t10dif_mod_fini(void)
 {
+        crypto_unregister_notifier(&crc_t10dif_nb);
        crypto_free_shash(crct10dif_tfm);
 }
 module_init(crc_t10dif_mod_init);
 module_exit(crc_t10dif_mod_fini);
+static int crc_t10dif_transform_show(char *buffer, const struct kernel_param *kp)
+{
+        if (static_key_false(&crct10dif_fallback))
+                return sprintf(buffer, "fallback\n");
+        return sprintf(buffer, "%s\n",
+                crypto_tfm_alg_driver_name(crypto_shash_tfm(crct10dif_tfm)));
+}
+module_param_call(transform, NULL, crc_t10dif_transform_show, NULL, 0644);
 MODULE_DESCRIPTION("T10 DIF CRC calculation");
 MODULE_LICENSE("GPL");
 MODULE_SOFTDEP("pre: crct10dif");
diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c
index 02172c408ff2..5d6724cee38f 100644
--- a/net/ceph/crypto.c
+++ b/net/ceph/crypto.c
@@ -46,9 +46,9 @@ static int set_secret(struct ceph_crypto_key *key, void *buf)
                goto fail;
        }
-        /* crypto_alloc_skcipher() allocates with GFP_KERNEL */
+        /* crypto_alloc_sync_skcipher() allocates with GFP_KERNEL */
        noio_flag = memalloc_noio_save();
-        key->tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC);
+        key->tfm = crypto_alloc_sync_skcipher("cbc(aes)", 0, 0);
        memalloc_noio_restore(noio_flag);
        if (IS_ERR(key->tfm)) {
                ret = PTR_ERR(key->tfm);
@@ -56,7 +56,7 @@ static int set_secret(struct ceph_crypto_key *key, void *buf)
                goto fail;
        }
-        ret = crypto_skcipher_setkey(key->tfm, key->key, key->len);
+        ret = crypto_sync_skcipher_setkey(key->tfm, key->key, key->len);
        if (ret)
                goto fail;
@@ -136,7 +136,7 @@ void ceph_crypto_key_destroy(struct ceph_crypto_key *key)
        if (key) {
                kfree(key->key);
                key->key = NULL;
-                crypto_free_skcipher(key->tfm);
+                crypto_free_sync_skcipher(key->tfm);
                key->tfm = NULL;
        }
 }
@@ -216,7 +216,7 @@ static void teardown_sgtable(struct sg_table *sgt)
 static int ceph_aes_crypt(const struct ceph_crypto_key *key, bool encrypt,
                          void *buf, int buf_len, int in_len, int *pout_len)
 {
-        SKCIPHER_REQUEST_ON_STACK(req, key->tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, key->tfm);
        struct sg_table sgt;
        struct scatterlist prealloc_sg;
        char iv[AES_BLOCK_SIZE] __aligned(8);
@@ -232,7 +232,7 @@ static int ceph_aes_crypt(const struct ceph_crypto_key *key, bool encrypt,
                return ret;
        memcpy(iv, aes_iv, AES_BLOCK_SIZE);
-        skcipher_request_set_tfm(req, key->tfm);
+        skcipher_request_set_sync_tfm(req, key->tfm);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sgt.sgl, sgt.sgl, crypt_len, iv);
diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h
index bb45c7d43739..96ef4d860bc9 100644
--- a/net/ceph/crypto.h
+++ b/net/ceph/crypto.h
@@ -13,7 +13,7 @@ struct ceph_crypto_key {
        struct ceph_timespec created;
        int len;
        void *key;
-        struct crypto_skcipher *tfm;
+        struct crypto_sync_skcipher *tfm;
 };
 int ceph_crypto_key_clone(struct ceph_crypto_key *dst,
diff --git a/net/mac802154/llsec.c b/net/mac802154/llsec.c
index 2fb703d70803..7e29f88dbf6a 100644
--- a/net/mac802154/llsec.c
+++ b/net/mac802154/llsec.c
@@ -146,18 +146,18 @@ llsec_key_alloc(const struct ieee802154_llsec_key *template)
                        goto err_tfm;
        }
-        key->tfm0 = crypto_alloc_skcipher("ctr(aes)", 0, CRYPTO_ALG_ASYNC);
+        key->tfm0 = crypto_alloc_sync_skcipher("ctr(aes)", 0, 0);
        if (IS_ERR(key->tfm0))
                goto err_tfm;
-        if (crypto_skcipher_setkey(key->tfm0, template->key,
+        if (crypto_sync_skcipher_setkey(key->tfm0, template->key,
                                   IEEE802154_LLSEC_KEY_SIZE))
                goto err_tfm0;
        return key;
 err_tfm0:
-        crypto_free_skcipher(key->tfm0);
+        crypto_free_sync_skcipher(key->tfm0);
 err_tfm:
        for (i = 0; i < ARRAY_SIZE(key->tfm); i++)
                if (key->tfm[i])
@@ -177,7 +177,7 @@ static void llsec_key_release(struct kref *ref)
        for (i = 0; i < ARRAY_SIZE(key->tfm); i++)
                crypto_free_aead(key->tfm[i]);
-        crypto_free_skcipher(key->tfm0);
+        crypto_free_sync_skcipher(key->tfm0);
        kzfree(key);
 }
@@ -622,7 +622,7 @@ llsec_do_encrypt_unauth(struct sk_buff *skb, const struct mac802154_llsec *sec,
 {
        u8 iv[16];
        struct scatterlist src;
-        SKCIPHER_REQUEST_ON_STACK(req, key->tfm0);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, key->tfm0);
        int err, datalen;
        unsigned char *data;
@@ -632,7 +632,7 @@ llsec_do_encrypt_unauth(struct sk_buff *skb, const struct mac802154_llsec *sec,
        datalen = skb_tail_pointer(skb) - data;
        sg_init_one(&src, data, datalen);
-        skcipher_request_set_tfm(req, key->tfm0);
+        skcipher_request_set_sync_tfm(req, key->tfm0);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &src, &src, datalen, iv);
        err = crypto_skcipher_encrypt(req);
@@ -840,7 +840,7 @@ llsec_do_decrypt_unauth(struct sk_buff *skb, const struct mac802154_llsec *sec,
        unsigned char *data;
        int datalen;
        struct scatterlist src;
-        SKCIPHER_REQUEST_ON_STACK(req, key->tfm0);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, key->tfm0);
        int err;
        llsec_geniv(iv, dev_addr, &hdr->sec);
@@ -849,7 +849,7 @@ llsec_do_decrypt_unauth(struct sk_buff *skb, const struct mac802154_llsec *sec,
        sg_init_one(&src, data, datalen);
-        skcipher_request_set_tfm(req, key->tfm0);
+        skcipher_request_set_sync_tfm(req, key->tfm0);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &src, &src, datalen, iv);
diff --git a/net/mac802154/llsec.h b/net/mac802154/llsec.h
index 6f3b658e3279..8be46d74dc39 100644
--- a/net/mac802154/llsec.h
+++ b/net/mac802154/llsec.h
@@ -29,7 +29,7 @@ struct mac802154_llsec_key {
        /* one tfm for each authsize (4/8/16) */
        struct crypto_aead *tfm[3];
-        struct crypto_skcipher *tfm0;
+        struct crypto_sync_skcipher *tfm0;
        struct kref ref;
 };
diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index 0a7c49e8e053..382196e57a26 100644
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -435,7 +435,7 @@ struct rxrpc_connection {
        struct sk_buff_head     rx_queue;       /* received conn-level packets */
        const struct rxrpc_security *security;  /* applied security module */
        struct key              *server_key;    /* security for this service */
-        struct crypto_skcipher  *cipher;        /* encryption handle */
+        struct crypto_sync_skcipher *cipher;    /* encryption handle */
        struct rxrpc_crypt      csum_iv;        /* packet checksum base */
        unsigned long           flags;
        unsigned long           events;
diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c
index cea16838d588..cbef9ea43dec 100644
--- a/net/rxrpc/rxkad.c
+++ b/net/rxrpc/rxkad.c
@@ -46,7 +46,7 @@ struct rxkad_level2_hdr {
 * alloc routine, but since we have it to hand, we use it to decrypt RESPONSE
 * packets
 */
-static struct crypto_skcipher *rxkad_ci;
+static struct crypto_sync_skcipher *rxkad_ci;
 static DEFINE_MUTEX(rxkad_ci_mutex);
 /*
@@ -54,7 +54,7 @@ static DEFINE_MUTEX(rxkad_ci_mutex);
 */
 static int rxkad_init_connection_security(struct rxrpc_connection *conn)
 {
-        struct crypto_skcipher *ci;
+        struct crypto_sync_skcipher *ci;
        struct rxrpc_key_token *token;
        int ret;
@@ -63,14 +63,14 @@ static int rxkad_init_connection_security(struct rxrpc_connection *conn)
        token = conn->params.key->payload.data[0];
        conn->security_ix = token->security_index;
-        ci = crypto_alloc_skcipher("pcbc(fcrypt)", 0, CRYPTO_ALG_ASYNC);
+        ci = crypto_alloc_sync_skcipher("pcbc(fcrypt)", 0, 0);
        if (IS_ERR(ci)) {
                _debug("no cipher");
                ret = PTR_ERR(ci);
                goto error;
        }
-        if (crypto_skcipher_setkey(ci, token->kad->session_key,
+        if (crypto_sync_skcipher_setkey(ci, token->kad->session_key,
                                   sizeof(token->kad->session_key)) < 0)
                BUG();
@@ -104,7 +104,7 @@ error:
 static int rxkad_prime_packet_security(struct rxrpc_connection *conn)
 {
        struct rxrpc_key_token *token;
-        SKCIPHER_REQUEST_ON_STACK(req, conn->cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, conn->cipher);
        struct scatterlist sg;
        struct rxrpc_crypt iv;
        __be32 *tmpbuf;
@@ -128,7 +128,7 @@ static int rxkad_prime_packet_security(struct rxrpc_connection *conn)
        tmpbuf[3] = htonl(conn->security_ix);
        sg_init_one(&sg, tmpbuf, tmpsize);
-        skcipher_request_set_tfm(req, conn->cipher);
+        skcipher_request_set_sync_tfm(req, conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &sg, &sg, tmpsize, iv.x);
        crypto_skcipher_encrypt(req);
@@ -167,7 +167,7 @@ static int rxkad_secure_packet_auth(const struct rxrpc_call *call,
        memset(&iv, 0, sizeof(iv));
        sg_init_one(&sg, sechdr, 8);
-        skcipher_request_set_tfm(req, call->conn->cipher);
+        skcipher_request_set_sync_tfm(req, call->conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x);
        crypto_skcipher_encrypt(req);
@@ -212,7 +212,7 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call,
        memcpy(&iv, token->kad->session_key, sizeof(iv));
        sg_init_one(&sg[0], sechdr, sizeof(rxkhdr));
-        skcipher_request_set_tfm(req, call->conn->cipher);
+        skcipher_request_set_sync_tfm(req, call->conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &sg[0], &sg[0], sizeof(rxkhdr), iv.x);
        crypto_skcipher_encrypt(req);
@@ -250,7 +250,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call,
                               void *sechdr)
 {
        struct rxrpc_skb_priv *sp;
-        SKCIPHER_REQUEST_ON_STACK(req, call->conn->cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, call->conn->cipher);
        struct rxrpc_crypt iv;
        struct scatterlist sg;
        u32 x, y;
@@ -279,7 +279,7 @@ static int rxkad_secure_packet(struct rxrpc_call *call,
        call->crypto_buf[1] = htonl(x);
        sg_init_one(&sg, call->crypto_buf, 8);
-        skcipher_request_set_tfm(req, call->conn->cipher);
+        skcipher_request_set_sync_tfm(req, call->conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x);
        crypto_skcipher_encrypt(req);
@@ -352,7 +352,7 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
        /* start the decryption afresh */
        memset(&iv, 0, sizeof(iv));
-        skcipher_request_set_tfm(req, call->conn->cipher);
+        skcipher_request_set_sync_tfm(req, call->conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, 8, iv.x);
        crypto_skcipher_decrypt(req);
@@ -450,7 +450,7 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
        token = call->conn->params.key->payload.data[0];
        memcpy(&iv, token->kad->session_key, sizeof(iv));
-        skcipher_request_set_tfm(req, call->conn->cipher);
+        skcipher_request_set_sync_tfm(req, call->conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, len, iv.x);
        crypto_skcipher_decrypt(req);
@@ -506,7 +506,7 @@ static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb,
                               unsigned int offset, unsigned int len,
                               rxrpc_seq_t seq, u16 expected_cksum)
 {
-        SKCIPHER_REQUEST_ON_STACK(req, call->conn->cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, call->conn->cipher);
        struct rxrpc_crypt iv;
        struct scatterlist sg;
        bool aborted;
@@ -529,7 +529,7 @@ static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb,
        call->crypto_buf[1] = htonl(x);
        sg_init_one(&sg, call->crypto_buf, 8);
-        skcipher_request_set_tfm(req, call->conn->cipher);
+        skcipher_request_set_sync_tfm(req, call->conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, &sg, &sg, 8, iv.x);
        crypto_skcipher_encrypt(req);
@@ -755,7 +755,7 @@ static void rxkad_encrypt_response(struct rxrpc_connection *conn,
                                   struct rxkad_response *resp,
                                   const struct rxkad_key *s2)
 {
-        SKCIPHER_REQUEST_ON_STACK(req, conn->cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, conn->cipher);
        struct rxrpc_crypt iv;
        struct scatterlist sg[1];
@@ -764,7 +764,7 @@ static void rxkad_encrypt_response(struct rxrpc_connection *conn,
        sg_init_table(sg, 1);
        sg_set_buf(sg, &resp->encrypted, sizeof(resp->encrypted));
-        skcipher_request_set_tfm(req, conn->cipher);
+        skcipher_request_set_sync_tfm(req, conn->cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, sizeof(resp->encrypted), iv.x);
        crypto_skcipher_encrypt(req);
@@ -1021,7 +1021,7 @@ static void rxkad_decrypt_response(struct rxrpc_connection *conn,
                                   struct rxkad_response *resp,
                                   const struct rxrpc_crypt *session_key)
 {
-        SKCIPHER_REQUEST_ON_STACK(req, rxkad_ci);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, rxkad_ci);
        struct scatterlist sg[1];
        struct rxrpc_crypt iv;
@@ -1031,7 +1031,7 @@ static void rxkad_decrypt_response(struct rxrpc_connection *conn,
        ASSERT(rxkad_ci != NULL);
        mutex_lock(&rxkad_ci_mutex);
-        if (crypto_skcipher_setkey(rxkad_ci, session_key->x,
+        if (crypto_sync_skcipher_setkey(rxkad_ci, session_key->x,
                                   sizeof(*session_key)) < 0)
                BUG();
@@ -1039,7 +1039,7 @@ static void rxkad_decrypt_response(struct rxrpc_connection *conn,
        sg_init_table(sg, 1);
        sg_set_buf(sg, &resp->encrypted, sizeof(resp->encrypted));
-        skcipher_request_set_tfm(req, rxkad_ci);
+        skcipher_request_set_sync_tfm(req, rxkad_ci);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, sizeof(resp->encrypted), iv.x);
        crypto_skcipher_decrypt(req);
@@ -1218,7 +1218,7 @@ static void rxkad_clear(struct rxrpc_connection *conn)
        _enter("");
        if (conn->cipher)
-                crypto_free_skcipher(conn->cipher);
+                crypto_free_sync_skcipher(conn->cipher);
 }
 /*
@@ -1228,7 +1228,7 @@ static int rxkad_init(void)
 {
        /* pin the cipher we need so that the crypto layer doesn't invoke
         * keventd to go get it */
-        rxkad_ci = crypto_alloc_skcipher("pcbc(fcrypt)", 0, CRYPTO_ALG_ASYNC);
+        rxkad_ci = crypto_alloc_sync_skcipher("pcbc(fcrypt)", 0, 0);
        return PTR_ERR_OR_ZERO(rxkad_ci);
 }
@@ -1238,7 +1238,7 @@ static int rxkad_init(void)
 static void rxkad_exit(void)
 {
        if (rxkad_ci)
-                crypto_free_skcipher(rxkad_ci);
+                crypto_free_sync_skcipher(rxkad_ci);
 }
 /*
diff --git a/net/sunrpc/auth_gss/gss_krb5_crypto.c b/net/sunrpc/auth_gss/gss_krb5_crypto.c
index 0220e1ca5280..4f43383971ba 100644
--- a/net/sunrpc/auth_gss/gss_krb5_crypto.c
+++ b/net/sunrpc/auth_gss/gss_krb5_crypto.c
@@ -53,7 +53,7 @@
 u32
 krb5_encrypt(
-        struct crypto_skcipher *tfm,
+        struct crypto_sync_skcipher *tfm,
        void * iv,
        void * in,
        void * out,
@@ -62,24 +62,24 @@ krb5_encrypt(
        u32 ret = -EINVAL;
        struct scatterlist sg[1];
        u8 local_iv[GSS_KRB5_MAX_BLOCKSIZE] = {0};
-        SKCIPHER_REQUEST_ON_STACK(req, tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
-        if (length % crypto_skcipher_blocksize(tfm) != 0)
+        if (length % crypto_sync_skcipher_blocksize(tfm) != 0)
                goto out;
-        if (crypto_skcipher_ivsize(tfm) > GSS_KRB5_MAX_BLOCKSIZE) {
+        if (crypto_sync_skcipher_ivsize(tfm) > GSS_KRB5_MAX_BLOCKSIZE) {
                dprintk("RPC:       gss_k5encrypt: tfm iv size too large %d\n",
-                        crypto_skcipher_ivsize(tfm));
+                        crypto_sync_skcipher_ivsize(tfm));
                goto out;
        }
        if (iv)
-                memcpy(local_iv, iv, crypto_skcipher_ivsize(tfm));
+                memcpy(local_iv, iv, crypto_sync_skcipher_ivsize(tfm));
        memcpy(out, in, length);
        sg_init_one(sg, out, length);
-        skcipher_request_set_tfm(req, tfm);
+        skcipher_request_set_sync_tfm(req, tfm);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, length, local_iv);
@@ -92,7 +92,7 @@ out:
 u32
 krb5_decrypt(
-     struct crypto_skcipher *tfm,
+     struct crypto_sync_skcipher *tfm,
     void * iv,
     void * in,
     void * out,
@@ -101,23 +101,23 @@ krb5_decrypt(
        u32 ret = -EINVAL;
        struct scatterlist sg[1];
        u8 local_iv[GSS_KRB5_MAX_BLOCKSIZE] = {0};
-        SKCIPHER_REQUEST_ON_STACK(req, tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
-        if (length % crypto_skcipher_blocksize(tfm) != 0)
+        if (length % crypto_sync_skcipher_blocksize(tfm) != 0)
                goto out;
-        if (crypto_skcipher_ivsize(tfm) > GSS_KRB5_MAX_BLOCKSIZE) {
+        if (crypto_sync_skcipher_ivsize(tfm) > GSS_KRB5_MAX_BLOCKSIZE) {
                dprintk("RPC:       gss_k5decrypt: tfm iv size too large %d\n",
-                        crypto_skcipher_ivsize(tfm));
+                        crypto_sync_skcipher_ivsize(tfm));
                goto out;
        }
        if (iv)
-                memcpy(local_iv,iv, crypto_skcipher_ivsize(tfm));
+                memcpy(local_iv, iv, crypto_sync_skcipher_ivsize(tfm));
        memcpy(out, in, length);
        sg_init_one(sg, out, length);
-        skcipher_request_set_tfm(req, tfm);
+        skcipher_request_set_sync_tfm(req, tfm);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, length, local_iv);
@@ -466,7 +466,8 @@ encryptor(struct scatterlist *sg, void *data)
 {
        struct encryptor_desc *desc = data;
        struct xdr_buf *outbuf = desc->outbuf;
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(desc->req);
+        struct crypto_sync_skcipher *tfm =
+                crypto_sync_skcipher_reqtfm(desc->req);
        struct page *in_page;
        int thislen = desc->fraglen + sg->length;
        int fraglen, ret;
@@ -492,7 +493,7 @@ encryptor(struct scatterlist *sg, void *data)
        desc->fraglen += sg->length;
        desc->pos += sg->length;
-        fraglen = thislen & (crypto_skcipher_blocksize(tfm) - 1);
+        fraglen = thislen & (crypto_sync_skcipher_blocksize(tfm) - 1);
        thislen -= fraglen;
        if (thislen == 0)
@@ -526,16 +527,16 @@ encryptor(struct scatterlist *sg, void *data)
 }
 int
-gss_encrypt_xdr_buf(struct crypto_skcipher *tfm, struct xdr_buf *buf,
+gss_encrypt_xdr_buf(struct crypto_sync_skcipher *tfm, struct xdr_buf *buf,
                    int offset, struct page **pages)
 {
        int ret;
        struct encryptor_desc desc;
-        SKCIPHER_REQUEST_ON_STACK(req, tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
-        BUG_ON((buf->len - offset) % crypto_skcipher_blocksize(tfm) != 0);
+        BUG_ON((buf->len - offset) % crypto_sync_skcipher_blocksize(tfm) != 0);
-        skcipher_request_set_tfm(req, tfm);
+        skcipher_request_set_sync_tfm(req, tfm);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        memset(desc.iv, 0, sizeof(desc.iv));
@@ -567,7 +568,8 @@ decryptor(struct scatterlist *sg, void *data)
 {
        struct decryptor_desc *desc = data;
        int thislen = desc->fraglen + sg->length;
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(desc->req);
+        struct crypto_sync_skcipher *tfm =
+                crypto_sync_skcipher_reqtfm(desc->req);
        int fraglen, ret;
        /* Worst case is 4 fragments: head, end of page 1, start
@@ -578,7 +580,7 @@ decryptor(struct scatterlist *sg, void *data)
        desc->fragno++;
        desc->fraglen += sg->length;
-        fraglen = thislen & (crypto_skcipher_blocksize(tfm) - 1);
+        fraglen = thislen & (crypto_sync_skcipher_blocksize(tfm) - 1);
        thislen -= fraglen;
        if (thislen == 0)
@@ -608,17 +610,17 @@ decryptor(struct scatterlist *sg, void *data)
 }
 int
-gss_decrypt_xdr_buf(struct crypto_skcipher *tfm, struct xdr_buf *buf,
+gss_decrypt_xdr_buf(struct crypto_sync_skcipher *tfm, struct xdr_buf *buf,
                    int offset)
 {
        int ret;
        struct decryptor_desc desc;
-        SKCIPHER_REQUEST_ON_STACK(req, tfm);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
        /* XXXJBF: */
-        BUG_ON((buf->len - offset) % crypto_skcipher_blocksize(tfm) != 0);
+        BUG_ON((buf->len - offset) % crypto_sync_skcipher_blocksize(tfm) != 0);
-        skcipher_request_set_tfm(req, tfm);
+        skcipher_request_set_sync_tfm(req, tfm);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        memset(desc.iv, 0, sizeof(desc.iv));
@@ -672,12 +674,12 @@ xdr_extend_head(struct xdr_buf *buf, unsigned int base, unsigned int shiftlen)
 }
 static u32
-gss_krb5_cts_crypt(struct crypto_skcipher *cipher, struct xdr_buf *buf,
+gss_krb5_cts_crypt(struct crypto_sync_skcipher *cipher, struct xdr_buf *buf,
                   u32 offset, u8 *iv, struct page **pages, int encrypt)
 {
        u32 ret;
        struct scatterlist sg[1];
-        SKCIPHER_REQUEST_ON_STACK(req, cipher);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(req, cipher);
        u8 *data;
        struct page **save_pages;
        u32 len = buf->len - offset;
@@ -706,7 +708,7 @@ gss_krb5_cts_crypt(struct crypto_skcipher *cipher, struct xdr_buf *buf,
        sg_init_one(sg, data, len);
-        skcipher_request_set_tfm(req, cipher);
+        skcipher_request_set_sync_tfm(req, cipher);
        skcipher_request_set_callback(req, 0, NULL, NULL);
        skcipher_request_set_crypt(req, sg, sg, len, iv);
@@ -735,7 +737,7 @@ gss_krb5_aes_encrypt(struct krb5_ctx *kctx, u32 offset,
        struct xdr_netobj hmac;
        u8 *cksumkey;
        u8 *ecptr;
-        struct crypto_skcipher *cipher, *aux_cipher;
+        struct crypto_sync_skcipher *cipher, *aux_cipher;
        int blocksize;
        struct page **save_pages;
        int nblocks, nbytes;
@@ -754,7 +756,7 @@ gss_krb5_aes_encrypt(struct krb5_ctx *kctx, u32 offset,
                cksumkey = kctx->acceptor_integ;
                usage = KG_USAGE_ACCEPTOR_SEAL;
        }
-        blocksize = crypto_skcipher_blocksize(cipher);
+        blocksize = crypto_sync_skcipher_blocksize(cipher);
        /* hide the gss token header and insert the confounder */
        offset += GSS_KRB5_TOK_HDR_LEN;
@@ -807,7 +809,7 @@ gss_krb5_aes_encrypt(struct krb5_ctx *kctx, u32 offset,
        memset(desc.iv, 0, sizeof(desc.iv));
        if (cbcbytes) {
-                SKCIPHER_REQUEST_ON_STACK(req, aux_cipher);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, aux_cipher);
                desc.pos = offset + GSS_KRB5_TOK_HDR_LEN;
                desc.fragno = 0;
@@ -816,7 +818,7 @@ gss_krb5_aes_encrypt(struct krb5_ctx *kctx, u32 offset,
                desc.outbuf = buf;
                desc.req = req;
-                skcipher_request_set_tfm(req, aux_cipher);
+                skcipher_request_set_sync_tfm(req, aux_cipher);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                sg_init_table(desc.infrags, 4);
@@ -855,7 +857,7 @@ gss_krb5_aes_decrypt(struct krb5_ctx *kctx, u32 offset, struct xdr_buf *buf,
        struct xdr_buf subbuf;
        u32 ret = 0;
        u8 *cksum_key;
-        struct crypto_skcipher *cipher, *aux_cipher;
+        struct crypto_sync_skcipher *cipher, *aux_cipher;
        struct xdr_netobj our_hmac_obj;
        u8 our_hmac[GSS_KRB5_MAX_CKSUM_LEN];
        u8 pkt_hmac[GSS_KRB5_MAX_CKSUM_LEN];
@@ -874,7 +876,7 @@ gss_krb5_aes_decrypt(struct krb5_ctx *kctx, u32 offset, struct xdr_buf *buf,
                cksum_key = kctx->initiator_integ;
                usage = KG_USAGE_INITIATOR_SEAL;
        }
-        blocksize = crypto_skcipher_blocksize(cipher);
+        blocksize = crypto_sync_skcipher_blocksize(cipher);
        /* create a segment skipping the header and leaving out the checksum */
@@ -891,13 +893,13 @@ gss_krb5_aes_decrypt(struct krb5_ctx *kctx, u32 offset, struct xdr_buf *buf,
        memset(desc.iv, 0, sizeof(desc.iv));
        if (cbcbytes) {
-                SKCIPHER_REQUEST_ON_STACK(req, aux_cipher);
+                SYNC_SKCIPHER_REQUEST_ON_STACK(req, aux_cipher);
                desc.fragno = 0;
                desc.fraglen = 0;
                desc.req = req;
-                skcipher_request_set_tfm(req, aux_cipher);
+                skcipher_request_set_sync_tfm(req, aux_cipher);
                skcipher_request_set_callback(req, 0, NULL, NULL);
                sg_init_table(desc.frags, 4);
@@ -946,7 +948,8 @@ out_err:
 * Set the key of the given cipher.
 */
 int
-krb5_rc4_setup_seq_key(struct krb5_ctx *kctx, struct crypto_skcipher *cipher,
+krb5_rc4_setup_seq_key(struct krb5_ctx *kctx,
+                       struct crypto_sync_skcipher *cipher,
                       unsigned char *cksum)
 {
        struct crypto_shash *hmac;
@@ -994,7 +997,7 @@ krb5_rc4_setup_seq_key(struct krb5_ctx *kctx, struct crypto_skcipher *cipher,
        if (err)
                goto out_err;
-        err = crypto_skcipher_setkey(cipher, Kseq, kctx->gk5e->keylength);
+        err = crypto_sync_skcipher_setkey(cipher, Kseq, kctx->gk5e->keylength);
        if (err)
                goto out_err;
@@ -1012,7 +1015,8 @@ out_err:
 * Set the key of cipher kctx->enc.
 */
 int
-krb5_rc4_setup_enc_key(struct krb5_ctx *kctx, struct crypto_skcipher *cipher,
+krb5_rc4_setup_enc_key(struct krb5_ctx *kctx,
+                       struct crypto_sync_skcipher *cipher,
                       s32 seqnum)
 {
        struct crypto_shash *hmac;
@@ -1069,7 +1073,8 @@ krb5_rc4_setup_enc_key(struct krb5_ctx *kctx, struct crypto_skcipher *cipher,
        if (err)
                goto out_err;
-        err = crypto_skcipher_setkey(cipher, Kcrypt, kctx->gk5e->keylength);
+        err = crypto_sync_skcipher_setkey(cipher, Kcrypt,
+                                          kctx->gk5e->keylength);
        if (err)
                goto out_err;
diff --git a/net/sunrpc/auth_gss/gss_krb5_keys.c b/net/sunrpc/auth_gss/gss_krb5_keys.c
index f7fe2d2b851f..550fdf18d3b3 100644
--- a/net/sunrpc/auth_gss/gss_krb5_keys.c
+++ b/net/sunrpc/auth_gss/gss_krb5_keys.c
@@ -147,7 +147,7 @@ u32 krb5_derive_key(const struct gss_krb5_enctype *gk5e,
        size_t blocksize, keybytes, keylength, n;
        unsigned char *inblockdata, *outblockdata, *rawkey;
        struct xdr_netobj inblock, outblock;
-        struct crypto_skcipher *cipher;
+        struct crypto_sync_skcipher *cipher;
        u32 ret = EINVAL;
        blocksize = gk5e->blocksize;
@@ -157,11 +157,10 @@ u32 krb5_derive_key(const struct gss_krb5_enctype *gk5e,
        if ((inkey->len != keylength) || (outkey->len != keylength))
                goto err_return;
-        cipher = crypto_alloc_skcipher(gk5e->encrypt_name, 0,
+        cipher = crypto_alloc_sync_skcipher(gk5e->encrypt_name, 0, 0);
-                                       CRYPTO_ALG_ASYNC);
        if (IS_ERR(cipher))
                goto err_return;
-        if (crypto_skcipher_setkey(cipher, inkey->data, inkey->len))
+        if (crypto_sync_skcipher_setkey(cipher, inkey->data, inkey->len))
                goto err_return;
        /* allocate and set up buffers */
@@ -238,7 +237,7 @@ err_free_in:
        memset(inblockdata, 0, blocksize);
        kfree(inblockdata);
 err_free_cipher:
-        crypto_free_skcipher(cipher);
+        crypto_free_sync_skcipher(cipher);
 err_return:
        return ret;
 }
diff --git a/net/sunrpc/auth_gss/gss_krb5_mech.c b/net/sunrpc/auth_gss/gss_krb5_mech.c
index 7bb2514aadd9..7f0424dfa8f6 100644
--- a/net/sunrpc/auth_gss/gss_krb5_mech.c
+++ b/net/sunrpc/auth_gss/gss_krb5_mech.c
@@ -218,7 +218,7 @@ simple_get_netobj(const void *p, const void *end, struct xdr_netobj *res)
 static inline const void *
 get_key(const void *p, const void *end,
-        struct krb5_ctx *ctx, struct crypto_skcipher **res)
+        struct krb5_ctx *ctx, struct crypto_sync_skcipher **res)
 {
        struct xdr_netobj       key;
        int                     alg;
@@ -246,15 +246,14 @@ get_key(const void *p, const void *end,
        if (IS_ERR(p))
                goto out_err;
-        *res = crypto_alloc_skcipher(ctx->gk5e->encrypt_name, 0,
+        *res = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0);
-                                                        CRYPTO_ALG_ASYNC);
        if (IS_ERR(*res)) {
                printk(KERN_WARNING "gss_kerberos_mech: unable to initialize "
                        "crypto algorithm %s\n", ctx->gk5e->encrypt_name);
                *res = NULL;
                goto out_err_free_key;
        }
-        if (crypto_skcipher_setkey(*res, key.data, key.len)) {
+        if (crypto_sync_skcipher_setkey(*res, key.data, key.len)) {
                printk(KERN_WARNING "gss_kerberos_mech: error setting key for "
                        "crypto algorithm %s\n", ctx->gk5e->encrypt_name);
                goto out_err_free_tfm;
@@ -264,7 +263,7 @@ get_key(const void *p, const void *end,
        return p;
 out_err_free_tfm:
-        crypto_free_skcipher(*res);
+        crypto_free_sync_skcipher(*res);
 out_err_free_key:
        kfree(key.data);
        p = ERR_PTR(-EINVAL);
@@ -336,30 +335,30 @@ gss_import_v1_context(const void *p, const void *end, struct krb5_ctx *ctx)
        return 0;
 out_err_free_key2:
-        crypto_free_skcipher(ctx->seq);
+        crypto_free_sync_skcipher(ctx->seq);
 out_err_free_key1:
-        crypto_free_skcipher(ctx->enc);
+        crypto_free_sync_skcipher(ctx->enc);
 out_err_free_mech:
        kfree(ctx->mech_used.data);
 out_err:
        return PTR_ERR(p);
 }
-static struct crypto_skcipher *
+static struct crypto_sync_skcipher *
 context_v2_alloc_cipher(struct krb5_ctx *ctx, const char *cname, u8 *key)
 {
-        struct crypto_skcipher *cp;
+        struct crypto_sync_skcipher *cp;
-        cp = crypto_alloc_skcipher(cname, 0, CRYPTO_ALG_ASYNC);
+        cp = crypto_alloc_sync_skcipher(cname, 0, 0);
        if (IS_ERR(cp)) {
                dprintk("gss_kerberos_mech: unable to initialize "
                        "crypto algorithm %s\n", cname);
                return NULL;
        }
-        if (crypto_skcipher_setkey(cp, key, ctx->gk5e->keylength)) {
+        if (crypto_sync_skcipher_setkey(cp, key, ctx->gk5e->keylength)) {
                dprintk("gss_kerberos_mech: error setting key for "
                        "crypto algorithm %s\n", cname);
-                crypto_free_skcipher(cp);
+                crypto_free_sync_skcipher(cp);
                return NULL;
        }
        return cp;
@@ -413,9 +412,9 @@ context_derive_keys_des3(struct krb5_ctx *ctx, gfp_t gfp_mask)
        return 0;
 out_free_enc:
-        crypto_free_skcipher(ctx->enc);
+        crypto_free_sync_skcipher(ctx->enc);
 out_free_seq:
-        crypto_free_skcipher(ctx->seq);
+        crypto_free_sync_skcipher(ctx->seq);
 out_err:
        return -EINVAL;
 }
@@ -469,17 +468,15 @@ context_derive_keys_rc4(struct krb5_ctx *ctx)
        /*
         * allocate hash, and skciphers for data and seqnum encryption
         */
-        ctx->enc = crypto_alloc_skcipher(ctx->gk5e->encrypt_name, 0,
+        ctx->enc = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0);
-                                         CRYPTO_ALG_ASYNC);
        if (IS_ERR(ctx->enc)) {
                err = PTR_ERR(ctx->enc);
                goto out_err_free_hmac;
        }
-        ctx->seq = crypto_alloc_skcipher(ctx->gk5e->encrypt_name, 0,
+        ctx->seq = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0);
-                                         CRYPTO_ALG_ASYNC);
        if (IS_ERR(ctx->seq)) {
-                crypto_free_skcipher(ctx->enc);
+                crypto_free_sync_skcipher(ctx->enc);
                err = PTR_ERR(ctx->seq);
                goto out_err_free_hmac;
        }
@@ -591,7 +588,7 @@ context_derive_keys_new(struct krb5_ctx *ctx, gfp_t gfp_mask)
                        context_v2_alloc_cipher(ctx, "cbc(aes)",
                                                ctx->acceptor_seal);
                if (ctx->acceptor_enc_aux == NULL) {
-                        crypto_free_skcipher(ctx->initiator_enc_aux);
+                        crypto_free_sync_skcipher(ctx->initiator_enc_aux);
                        goto out_free_acceptor_enc;
                }
        }
@@ -599,9 +596,9 @@ context_derive_keys_new(struct krb5_ctx *ctx, gfp_t gfp_mask)
        return 0;
 out_free_acceptor_enc:
-        crypto_free_skcipher(ctx->acceptor_enc);
+        crypto_free_sync_skcipher(ctx->acceptor_enc);
 out_free_initiator_enc:
-        crypto_free_skcipher(ctx->initiator_enc);
+        crypto_free_sync_skcipher(ctx->initiator_enc);
 out_err:
        return -EINVAL;
 }
@@ -713,12 +710,12 @@ static void
 gss_delete_sec_context_kerberos(void *internal_ctx) {
        struct krb5_ctx *kctx = internal_ctx;
-        crypto_free_skcipher(kctx->seq);
+        crypto_free_sync_skcipher(kctx->seq);
-        crypto_free_skcipher(kctx->enc);
+        crypto_free_sync_skcipher(kctx->enc);
-        crypto_free_skcipher(kctx->acceptor_enc);
+        crypto_free_sync_skcipher(kctx->acceptor_enc);
-        crypto_free_skcipher(kctx->initiator_enc);
+        crypto_free_sync_skcipher(kctx->initiator_enc);
-        crypto_free_skcipher(kctx->acceptor_enc_aux);
+        crypto_free_sync_skcipher(kctx->acceptor_enc_aux);
-        crypto_free_skcipher(kctx->initiator_enc_aux);
+        crypto_free_sync_skcipher(kctx->initiator_enc_aux);
        kfree(kctx->mech_used.data);
        kfree(kctx);
 }
diff --git a/net/sunrpc/auth_gss/gss_krb5_seqnum.c b/net/sunrpc/auth_gss/gss_krb5_seqnum.c
index c8b9082f4a9d..fb6656295204 100644
--- a/net/sunrpc/auth_gss/gss_krb5_seqnum.c
+++ b/net/sunrpc/auth_gss/gss_krb5_seqnum.c
@@ -43,13 +43,12 @@ static s32
 krb5_make_rc4_seq_num(struct krb5_ctx *kctx, int direction, s32 seqnum,
                      unsigned char *cksum, unsigned char *buf)
 {
-        struct crypto_skcipher *cipher;
+        struct crypto_sync_skcipher *cipher;
        unsigned char plain[8];
        s32 code;
        dprintk("RPC:       %s:\n", __func__);
-        cipher = crypto_alloc_skcipher(kctx->gk5e->encrypt_name, 0,
+        cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, 0, 0);
-                                       CRYPTO_ALG_ASYNC);
        if (IS_ERR(cipher))
                return PTR_ERR(cipher);
@@ -68,12 +67,12 @@ krb5_make_rc4_seq_num(struct krb5_ctx *kctx, int direction, s32 seqnum,
        code = krb5_encrypt(cipher, cksum, plain, buf, 8);
 out:
-        crypto_free_skcipher(cipher);
+        crypto_free_sync_skcipher(cipher);
        return code;
 }
 s32
 krb5_make_seq_num(struct krb5_ctx *kctx,
-                struct crypto_skcipher *key,
+                struct crypto_sync_skcipher *key,
                int direction,
                u32 seqnum,
                unsigned char *cksum, unsigned char *buf)
@@ -101,13 +100,12 @@ static s32
 krb5_get_rc4_seq_num(struct krb5_ctx *kctx, unsigned char *cksum,
                     unsigned char *buf, int *direction, s32 *seqnum)
 {
-        struct crypto_skcipher *cipher;
+        struct crypto_sync_skcipher *cipher;
        unsigned char plain[8];
        s32 code;
        dprintk("RPC:       %s:\n", __func__);
-        cipher = crypto_alloc_skcipher(kctx->gk5e->encrypt_name, 0,
+        cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, 0, 0);
-                                       CRYPTO_ALG_ASYNC);
        if (IS_ERR(cipher))
                return PTR_ERR(cipher);
@@ -130,7 +128,7 @@ krb5_get_rc4_seq_num(struct krb5_ctx *kctx, unsigned char *cksum,
        *seqnum = ((plain[0] << 24) | (plain[1] << 16) |
                                        (plain[2] << 8) | (plain[3]));
 out:
-        crypto_free_skcipher(cipher);
+        crypto_free_sync_skcipher(cipher);
        return code;
 }
@@ -142,7 +140,7 @@ krb5_get_seq_num(struct krb5_ctx *kctx,
 {
        s32 code;
        unsigned char plain[8];
-        struct crypto_skcipher *key = kctx->seq;
+        struct crypto_sync_skcipher *key = kctx->seq;
        dprintk("RPC:       krb5_get_seq_num:\n");
diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c
index 39a2e672900b..3d975a4013d2 100644
--- a/net/sunrpc/auth_gss/gss_krb5_wrap.c
+++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c
@@ -174,7 +174,7 @@ gss_wrap_kerberos_v1(struct krb5_ctx *kctx, int offset,
        now = get_seconds();
-        blocksize = crypto_skcipher_blocksize(kctx->enc);
+        blocksize = crypto_sync_skcipher_blocksize(kctx->enc);
        gss_krb5_add_padding(buf, offset, blocksize);
        BUG_ON((buf->len - offset) % blocksize);
        plainlen = conflen + buf->len - offset;
@@ -239,10 +239,10 @@ gss_wrap_kerberos_v1(struct krb5_ctx *kctx, int offset,
                return GSS_S_FAILURE;
        if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) {
-                struct crypto_skcipher *cipher;
+                struct crypto_sync_skcipher *cipher;
                int err;
-                cipher = crypto_alloc_skcipher(kctx->gk5e->encrypt_name, 0,
+                cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name,
-                                               CRYPTO_ALG_ASYNC);
+                                                    0, 0);
                if (IS_ERR(cipher))
                        return GSS_S_FAILURE;
@@ -250,7 +250,7 @@ gss_wrap_kerberos_v1(struct krb5_ctx *kctx, int offset,
                err = gss_encrypt_xdr_buf(cipher, buf,
                                          offset + headlen - conflen, pages);
-                crypto_free_skcipher(cipher);
+                crypto_free_sync_skcipher(cipher);
                if (err)
                        return GSS_S_FAILURE;
        } else {
@@ -327,18 +327,18 @@ gss_unwrap_kerberos_v1(struct krb5_ctx *kctx, int offset, struct xdr_buf *buf)
                return GSS_S_BAD_SIG;
        if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) {
-                struct crypto_skcipher *cipher;
+                struct crypto_sync_skcipher *cipher;
                int err;
-                cipher = crypto_alloc_skcipher(kctx->gk5e->encrypt_name, 0,
+                cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name,
-                                               CRYPTO_ALG_ASYNC);
+                                                    0, 0);
                if (IS_ERR(cipher))
                        return GSS_S_FAILURE;
                krb5_rc4_setup_enc_key(kctx, cipher, seqnum);
                err = gss_decrypt_xdr_buf(cipher, buf, crypt_offset);
-                crypto_free_skcipher(cipher);
+                crypto_free_sync_skcipher(cipher);
                if (err)
                        return GSS_S_DEFECTIVE_TOKEN;
        } else {
@@ -371,7 +371,7 @@ gss_unwrap_kerberos_v1(struct krb5_ctx *kctx, int offset, struct xdr_buf *buf)
        /* Copy the data back to the right position.  XXX: Would probably be
         * better to copy and encrypt at the same time. */
-        blocksize = crypto_skcipher_blocksize(kctx->enc);
+        blocksize = crypto_sync_skcipher_blocksize(kctx->enc);
        data_start = ptr + (GSS_KRB5_TOK_HDR_LEN + kctx->gk5e->cksumlength) +
                                        conflen;
        orig_start = buf->head[0].iov_base + offset;
diff --git a/tools/crypto/getstat.c b/tools/crypto/getstat.c
new file mode 100644
index 000000000000..24115173a483
--- /dev/null
+++ b/tools/crypto/getstat.c
@@ -0,0 +1,294 @@
+/* Heavily copied from libkcapi 2015 - 2017, Stephan Mueller <smueller@chronox.de> */
+#include <errno.h>
+#include <linux/cryptouser.h>
+#include <linux/netlink.h>
+#include <linux/rtnetlink.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#define CR_RTA(x)  ((struct rtattr *)(((char *)(x)) + NLMSG_ALIGN(sizeof(struct crypto_user_alg))))
+static int get_stat(const char *drivername)
+{
+        struct {
+                struct nlmsghdr n;
+                struct crypto_user_alg cru;
+        } req;
+        struct sockaddr_nl nl;
+        int sd = 0, ret;
+        socklen_t addr_len;
+        struct iovec iov;
+        struct msghdr msg;
+        char buf[4096];
+        struct nlmsghdr *res_n = (struct nlmsghdr *)buf;
+        struct crypto_user_alg *cru_res = NULL;
+        int res_len = 0;
+        struct rtattr *tb[CRYPTOCFGA_MAX + 1];
+        struct rtattr *rta;
+        struct nlmsgerr *errmsg;
+        memset(&req, 0, sizeof(req));
+        memset(&buf, 0, sizeof(buf));
+        memset(&msg, 0, sizeof(msg));
+        req.n.nlmsg_len = NLMSG_LENGTH(sizeof(req.cru));
+        req.n.nlmsg_flags = NLM_F_REQUEST;
+        req.n.nlmsg_type = CRYPTO_MSG_GETSTAT;
+        req.n.nlmsg_seq = time(NULL);
+        strncpy(req.cru.cru_driver_name, drivername, strlen(drivername));
+        sd =  socket(AF_NETLINK, SOCK_RAW, NETLINK_CRYPTO);
+        if (sd < 0) {
+                fprintf(stderr, "Netlink error: cannot open netlink socket");
+                return -errno;
+        }
+        memset(&nl, 0, sizeof(nl));
+        nl.nl_family = AF_NETLINK;
+        if (bind(sd, (struct sockaddr *)&nl, sizeof(nl)) < 0) {
+                ret = -errno;
+                fprintf(stderr, "Netlink error: cannot bind netlink socket");
+                goto out;
+        }
+        /* sanity check that netlink socket was successfully opened */
+        addr_len = sizeof(nl);
+        if (getsockname(sd, (struct sockaddr *)&nl, &addr_len) < 0) {
+                ret = -errno;
+                printf("Netlink error: cannot getsockname");
+                goto out;
+        }
+        if (addr_len != sizeof(nl)) {
+                ret = -errno;
+                printf("Netlink error: wrong address length %d", addr_len);
+                goto out;
+        }
+        if (nl.nl_family != AF_NETLINK) {
+                ret = -errno;
+                printf("Netlink error: wrong address family %d",
+                                nl.nl_family);
+                goto out;
+        }
+        memset(&nl, 0, sizeof(nl));
+        nl.nl_family = AF_NETLINK;
+        iov.iov_base = (void *)&req.n;
+        iov.iov_len = req.n.nlmsg_len;
+        msg.msg_name = &nl;
+        msg.msg_namelen = sizeof(nl);
+        msg.msg_iov = &iov;
+        msg.msg_iovlen = 1;
+        if (sendmsg(sd, &msg, 0) < 0) {
+                ret = -errno;
+                printf("Netlink error: sendmsg failed");
+                goto out;
+        }
+        memset(buf, 0, sizeof(buf));
+        iov.iov_base = buf;
+        while (1) {
+                iov.iov_len = sizeof(buf);
+                ret = recvmsg(sd, &msg, 0);
+                if (ret < 0) {
+                        if (errno == EINTR || errno == EAGAIN)
+                                continue;
+                        ret = -errno;
+                        printf("Netlink error: netlink receive error");
+                        goto out;
+                }
+                if (ret == 0) {
+                        ret = -errno;
+                        printf("Netlink error: no data");
+                        goto out;
+                }
+                if (ret > sizeof(buf)) {
+                        ret = -errno;
+                        printf("Netlink error: received too much data");
+                        goto out;
+                }
+                break;
+        }
+        ret = -EFAULT;
+        res_len = res_n->nlmsg_len;
+        if (res_n->nlmsg_type == NLMSG_ERROR) {
+                errmsg = NLMSG_DATA(res_n);
+                fprintf(stderr, "Fail with %d\n", errmsg->error);
+                ret = errmsg->error;
+                goto out;
+        }
+        if (res_n->nlmsg_type == CRYPTO_MSG_GETSTAT) {
+                cru_res = NLMSG_DATA(res_n);
+                res_len -= NLMSG_SPACE(sizeof(*cru_res));
+        }
+        if (res_len < 0) {
+                printf("Netlink error: nlmsg len %d\n", res_len);
+                goto out;
+        }
+        if (!cru_res) {
+                ret = -EFAULT;
+                printf("Netlink error: no cru_res\n");
+                goto out;
+        }
+        rta = CR_RTA(cru_res);
+        memset(tb, 0, sizeof(struct rtattr *) * (CRYPTOCFGA_MAX + 1));
+        while (RTA_OK(rta, res_len)) {
+                if ((rta->rta_type <= CRYPTOCFGA_MAX) && (!tb[rta->rta_type]))
+                        tb[rta->rta_type] = rta;
+                rta = RTA_NEXT(rta, res_len);
+        }
+        if (res_len) {
+                printf("Netlink error: unprocessed data %d",
+                                res_len);
+                goto out;
+        }
+        if (tb[CRYPTOCFGA_STAT_HASH]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_HASH];
+                struct crypto_stat *rhash =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tHash\n\tHash: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        rhash->stat_hash_cnt, rhash->stat_hash_tlen,
+                        rhash->stat_hash_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_COMPRESS]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_COMPRESS];
+                struct crypto_stat *rblk =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tCompress\n\tCompress: %u bytes: %llu\n\tDecompress: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        rblk->stat_compress_cnt, rblk->stat_compress_tlen,
+                        rblk->stat_decompress_cnt, rblk->stat_decompress_tlen,
+                        rblk->stat_compress_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_ACOMP]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_ACOMP];
+                struct crypto_stat *rcomp =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tACompress\n\tCompress: %u bytes: %llu\n\tDecompress: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        rcomp->stat_compress_cnt, rcomp->stat_compress_tlen,
+                        rcomp->stat_decompress_cnt, rcomp->stat_decompress_tlen,
+                        rcomp->stat_compress_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_AEAD]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_AEAD];
+                struct crypto_stat *raead =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tAEAD\n\tEncrypt: %u bytes: %llu\n\tDecrypt: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        raead->stat_encrypt_cnt, raead->stat_encrypt_tlen,
+                        raead->stat_decrypt_cnt, raead->stat_decrypt_tlen,
+                        raead->stat_aead_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_BLKCIPHER]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_BLKCIPHER];
+                struct crypto_stat *rblk =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tCipher\n\tEncrypt: %u bytes: %llu\n\tDecrypt: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        rblk->stat_encrypt_cnt, rblk->stat_encrypt_tlen,
+                        rblk->stat_decrypt_cnt, rblk->stat_decrypt_tlen,
+                        rblk->stat_cipher_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_AKCIPHER]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_AKCIPHER];
+                struct crypto_stat *rblk =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tAkcipher\n\tEncrypt: %u bytes: %llu\n\tDecrypt: %u bytes: %llu\n\tSign: %u\n\tVerify: %u\n\tErrors: %u\n",
+                        drivername,
+                        rblk->stat_encrypt_cnt, rblk->stat_encrypt_tlen,
+                        rblk->stat_decrypt_cnt, rblk->stat_decrypt_tlen,
+                        rblk->stat_sign_cnt, rblk->stat_verify_cnt,
+                        rblk->stat_akcipher_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_CIPHER]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_CIPHER];
+                struct crypto_stat *rblk =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tcipher\n\tEncrypt: %u bytes: %llu\n\tDecrypt: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        rblk->stat_encrypt_cnt, rblk->stat_encrypt_tlen,
+                        rblk->stat_decrypt_cnt, rblk->stat_decrypt_tlen,
+                        rblk->stat_cipher_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_RNG]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_RNG];
+                struct crypto_stat *rrng =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tRNG\n\tSeed: %u\n\tGenerate: %u bytes: %llu\n\tErrors: %u\n",
+                        drivername,
+                        rrng->stat_seed_cnt,
+                        rrng->stat_generate_cnt, rrng->stat_generate_tlen,
+                        rrng->stat_rng_err_cnt);
+        } else if (tb[CRYPTOCFGA_STAT_KPP]) {
+                struct rtattr *rta = tb[CRYPTOCFGA_STAT_KPP];
+                struct crypto_stat *rkpp =
+                        (struct crypto_stat *)RTA_DATA(rta);
+                printf("%s\tKPP\n\tSetsecret: %u\n\tGenerate public key: %u\n\tCompute_shared_secret: %u\n\tErrors: %u\n",
+                        drivername,
+                        rkpp->stat_setsecret_cnt,
+                        rkpp->stat_generate_public_key_cnt,
+                        rkpp->stat_compute_shared_secret_cnt,
+                        rkpp->stat_kpp_err_cnt);
+        } else {
+                fprintf(stderr, "%s is of an unknown algorithm\n", drivername);
+        }
+        ret = 0;
+out:
+        close(sd);
+        return ret;
+}
+int main(int argc, const char *argv[])
+{
+        char buf[4096];
+        FILE *procfd;
+        int i, lastspace;
+        int ret;
+        procfd = fopen("/proc/crypto", "r");
+        if (!procfd) {
+                ret = errno;
+                fprintf(stderr, "Cannot open /proc/crypto %s\n", strerror(errno));
+                return ret;
+        }
+        if (argc > 1) {
+                if (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")) {
+                        printf("Usage: %s [-h|--help] display this help\n", argv[0]);
+                        printf("Usage: %s display all crypto statistics\n", argv[0]);
+                        printf("Usage: %s drivername1 drivername2 ... = display crypto statistics about drivername1 ...\n", argv[0]);
+                        return 0;
+                }
+                for (i = 1; i < argc; i++) {
+                        ret = get_stat(argv[i]);
+                        if (ret) {
+                                fprintf(stderr, "Failed with %s\n", strerror(-ret));
+                                return ret;
+                        }
+                }
+                return 0;
+        }
+        while (fgets(buf, sizeof(buf), procfd)) {
+                if (!strncmp(buf, "driver", 6)) {
+                        lastspace = 0;
+                        i = 0;
+                        while (i < strlen(buf)) {
+                                i++;
+                                if (buf[i] == ' ')
+                                        lastspace = i;
+                        }
+                        buf[strlen(buf) - 1] = '\0';
+                        ret = get_stat(buf + lastspace + 1);
+                        if (ret) {
+                                fprintf(stderr, "Failed with %s\n", strerror(-ret));
+                                goto out;
+                        }
+                }
+        }
+out:
+        fclose(procfd);
+        return ret;
+}
author	Linus Torvalds <torvalds@linux-foundation.org>	2018-10-25 19:43:35 -0400
committer	Linus Torvalds <torvalds@linux-foundation.org>	2018-10-25 19:43:35 -0400
commit	62606c224d72a98c35d21a849f95cccf95b0a252 (patch)
tree	6f6f3466451edf9baa2ea8b5f9fc558aa555c69a
parent	24ed334f33666f2ae929ccc08f72e7e72e353c64 (diff)
parent	a1c6fd4308d37f072e939a2782f24214115fc7e8 (diff)