Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto update from Herbert Xu:
 "Here is the crypto update for 3.20:

   - Added 192/256-bit key support to aesni GCM.
   - Added MIPS OCTEON MD5 support.
   - Fixed hwrng starvation and race conditions.
   - Added note that memzero_explicit is not a subsitute for memset.
   - Added user-space interface for crypto_rng.
   - Misc fixes"

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (71 commits)
  crypto: tcrypt - do not allocate iv on stack for aead speed tests
  crypto: testmgr - limit IV copy length in aead tests
  crypto: tcrypt - fix buflen reminder calculation
  crypto: testmgr - mark rfc4106(gcm(aes)) as fips_allowed
  crypto: caam - fix resource clean-up on error path for caam_jr_init
  crypto: caam - pair irq map and dispose in the same function
  crypto: ccp - terminate ccp_support array with empty element
  crypto: caam - remove unused local variable
  crypto: caam - remove dead code
  crypto: caam - don't emit ICV check failures to dmesg
  hwrng: virtio - drop extra empty line
  crypto: replace scatterwalk_sg_next with sg_next
  crypto: atmel - Free memory in error path
  crypto: doc - remove colons in comments
  crypto: seqiv - Ensure that IV size is at least 8 bytes
  crypto: cts - Weed out non-CBC algorithms
  MAINTAINERS: add linux-crypto to hw random
  crypto: cts - Remove bogus use of seqiv
  crypto: qat - don't need qat_auth_state struct
  crypto: algif_rng - fix sparse non static symbol warning
  ...

70 files changed, 1785 insertions, 709 deletions

diff --git a/MAINTAINERS b/MAINTAINERS
index cd8383e26ac8..3ac697a64280 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -4434,6 +4434,7 @@ F:	include/linux/hwmon*.h
 HARDWARE RANDOM NUMBER GENERATOR CORE
 M:      Matt Mackall <mpm@selenic.com>
 M:      Herbert Xu <herbert@gondor.apana.org.au>
+L:      linux-crypto@vger.kernel.org
 S:      Odd fixes
 F:      Documentation/hw_random.txt
 F:      drivers/char/hw_random/
diff --git a/arch/mips/cavium-octeon/Makefile b/arch/mips/cavium-octeon/Makefile
index 42f5f1a4b40a..69a8a8dabc2b 100644
--- a/arch/mips/cavium-octeon/Makefile
+++ b/arch/mips/cavium-octeon/Makefile
@@ -16,6 +16,7 @@ obj-y := cpu.o setup.o octeon-platform.o octeon-irq.o csrc-octeon.o
 obj-y += dma-octeon.o
 obj-y += octeon-memcpy.o
 obj-y += executive/
+obj-y += crypto/
 
 obj-$(CONFIG_MTD)                     += flash_setup.o
 obj-$(CONFIG_SMP)                     += smp.o
diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile
new file mode 100644
index 000000000000..a74f76d85a2f
--- /dev/null
+++ b/arch/mips/cavium-octeon/crypto/Makefile
@@ -0,0 +1,7 @@
+#
+# OCTEON-specific crypto modules.
+#
+
+obj-y += octeon-crypto.o
+
+obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o
diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.c b/arch/mips/cavium-octeon/crypto/octeon-crypto.c
new file mode 100644
index 000000000000..7c82ff463b65
--- /dev/null
+++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.c
@@ -0,0 +1,66 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2004-2012 Cavium Networks
+ */
+
+#include <asm/cop2.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+
+#include "octeon-crypto.h"
+
+/**
+ * Enable access to Octeon's COP2 crypto hardware for kernel use. Wrap any
+ * crypto operations in calls to octeon_crypto_enable/disable in order to make
+ * sure the state of COP2 isn't corrupted if userspace is also performing
+ * hardware crypto operations. Allocate the state parameter on the stack.
+ * Preemption must be disabled to prevent context switches.
+ *
+ * @state: Pointer to state structure to store current COP2 state in.
+ *
+ * Returns: Flags to be passed to octeon_crypto_disable()
+ */
+unsigned long octeon_crypto_enable(struct octeon_cop2_state *state)
+{
+        int status;
+        unsigned long flags;
+
+        local_irq_save(flags);
+        status = read_c0_status();
+        write_c0_status(status | ST0_CU2);
+        if (KSTK_STATUS(current) & ST0_CU2) {
+                octeon_cop2_save(&(current->thread.cp2));
+                KSTK_STATUS(current) &= ~ST0_CU2;
+                status &= ~ST0_CU2;
+        } else if (status & ST0_CU2) {
+                octeon_cop2_save(state);
+        }
+        local_irq_restore(flags);
+        return status & ST0_CU2;
+}
+EXPORT_SYMBOL_GPL(octeon_crypto_enable);
+
+/**
+ * Disable access to Octeon's COP2 crypto hardware in the kernel. This must be
+ * called after an octeon_crypto_enable() before any context switch or return to
+ * userspace.
+ *
+ * @state:      Pointer to COP2 state to restore
+ * @flags:      Return value from octeon_crypto_enable()
+ */
+void octeon_crypto_disable(struct octeon_cop2_state *state,
+                           unsigned long crypto_flags)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        if (crypto_flags & ST0_CU2)
+                octeon_cop2_restore(state);
+        else
+                write_c0_status(read_c0_status() & ~ST0_CU2);
+        local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(octeon_crypto_disable);
diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h
new file mode 100644
index 000000000000..e2a4aece9c24
--- /dev/null
+++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h
@@ -0,0 +1,75 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2012-2013 Cavium Inc., All Rights Reserved.
+ *
+ * MD5 instruction definitions added by Aaro Koskinen <aaro.koskinen@iki.fi>.
+ *
+ */
+#ifndef __LINUX_OCTEON_CRYPTO_H
+#define __LINUX_OCTEON_CRYPTO_H
+
+#include <linux/sched.h>
+#include <asm/mipsregs.h>
+
+#define OCTEON_CR_OPCODE_PRIORITY 300
+
+extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state);
+extern void octeon_crypto_disable(struct octeon_cop2_state *state,
+                                  unsigned long flags);
+
+/*
+ * Macros needed to implement MD5:
+ */
+
+/*
+ * The index can be 0-1.
+ */
+#define write_octeon_64bit_hash_dword(value, index)     \
+do {                                                    \
+        __asm__ __volatile__ (                          \
+        "dmtc2 %[rt],0x0048+" STR(index)                \
+        :                                               \
+        : [rt] "d" (value));                            \
+} while (0)
+
+/*
+ * The index can be 0-1.
+ */
+#define read_octeon_64bit_hash_dword(index)             \
+({                                                      \
+        u64 __value;                                    \
+                                                        \
+        __asm__ __volatile__ (                          \
+        "dmfc2 %[rt],0x0048+" STR(index)                \
+        : [rt] "=d" (__value)                           \
+        : );                                            \
+                                                        \
+        __value;                                        \
+})
+
+/*
+ * The index can be 0-6.
+ */
+#define write_octeon_64bit_block_dword(value, index)    \
+do {                                                    \
+        __asm__ __volatile__ (                          \
+        "dmtc2 %[rt],0x0040+" STR(index)                \
+        :                                               \
+        : [rt] "d" (value));                            \
+} while (0)
+
+/*
+ * The value is the final block dword (64-bit).
+ */
+#define octeon_md5_start(value)                         \
+do {                                                    \
+        __asm__ __volatile__ (                          \
+        "dmtc2 %[rt],0x4047"                            \
+        :                                               \
+        : [rt] "d" (value));                            \
+} while (0)
+
+#endif /* __LINUX_OCTEON_CRYPTO_H */
diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c
new file mode 100644
index 000000000000..b909881ba6c1
--- /dev/null
+++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c
@@ -0,0 +1,216 @@
+/*
+ * Cryptographic API.
+ *
+ * MD5 Message Digest Algorithm (RFC1321).
+ *
+ * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>.
+ *
+ * Based on crypto/md5.c, which is:
+ *
+ * Derived from cryptoapi implementation, originally based on the
+ * public domain implementation written by Colin Plumb in 1993.
+ *
+ * Copyright (c) Cryptoapi developers.
+ * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/md5.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+#include <linux/cryptohash.h>
+#include <asm/octeon/octeon.h>
+#include <crypto/internal/hash.h>
+
+#include "octeon-crypto.h"
+
+/*
+ * We pass everything as 64-bit. OCTEON can handle misaligned data.
+ */
+
+static void octeon_md5_store_hash(struct md5_state *ctx)
+{
+        u64 *hash = (u64 *)ctx->hash;
+
+        write_octeon_64bit_hash_dword(hash[0], 0);
+        write_octeon_64bit_hash_dword(hash[1], 1);
+}
+
+static void octeon_md5_read_hash(struct md5_state *ctx)
+{
+        u64 *hash = (u64 *)ctx->hash;
+
+        hash[0] = read_octeon_64bit_hash_dword(0);
+        hash[1] = read_octeon_64bit_hash_dword(1);
+}
+
+static void octeon_md5_transform(const void *_block)
+{
+        const u64 *block = _block;
+
+        write_octeon_64bit_block_dword(block[0], 0);
+        write_octeon_64bit_block_dword(block[1], 1);
+        write_octeon_64bit_block_dword(block[2], 2);
+        write_octeon_64bit_block_dword(block[3], 3);
+        write_octeon_64bit_block_dword(block[4], 4);
+        write_octeon_64bit_block_dword(block[5], 5);
+        write_octeon_64bit_block_dword(block[6], 6);
+        octeon_md5_start(block[7]);
+}
+
+static int octeon_md5_init(struct shash_desc *desc)
+{
+        struct md5_state *mctx = shash_desc_ctx(desc);
+
+        mctx->hash[0] = cpu_to_le32(0x67452301);
+        mctx->hash[1] = cpu_to_le32(0xefcdab89);
+        mctx->hash[2] = cpu_to_le32(0x98badcfe);
+        mctx->hash[3] = cpu_to_le32(0x10325476);
+        mctx->byte_count = 0;
+
+        return 0;
+}
+
+static int octeon_md5_update(struct shash_desc *desc, const u8 *data,
+                             unsigned int len)
+{
+        struct md5_state *mctx = shash_desc_ctx(desc);
+        const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f);
+        struct octeon_cop2_state state;
+        unsigned long flags;
+
+        mctx->byte_count += len;
+
+        if (avail > len) {
+                memcpy((char *)mctx->block + (sizeof(mctx->block) - avail),
+                       data, len);
+                return 0;
+        }
+
+        memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data,
+               avail);
+
+        local_bh_disable();
+        preempt_disable();
+        flags = octeon_crypto_enable(&state);
+        octeon_md5_store_hash(mctx);
+
+        octeon_md5_transform(mctx->block);
+        data += avail;
+        len -= avail;
+
+        while (len >= sizeof(mctx->block)) {
+                octeon_md5_transform(data);
+                data += sizeof(mctx->block);
+                len -= sizeof(mctx->block);
+        }
+
+        octeon_md5_read_hash(mctx);
+        octeon_crypto_disable(&state, flags);
+        preempt_enable();
+        local_bh_enable();
+
+        memcpy(mctx->block, data, len);
+
+        return 0;
+}
+
+static int octeon_md5_final(struct shash_desc *desc, u8 *out)
+{
+        struct md5_state *mctx = shash_desc_ctx(desc);
+        const unsigned int offset = mctx->byte_count & 0x3f;
+        char *p = (char *)mctx->block + offset;
+        int padding = 56 - (offset + 1);
+        struct octeon_cop2_state state;
+        unsigned long flags;
+
+        *p++ = 0x80;
+
+        local_bh_disable();
+        preempt_disable();
+        flags = octeon_crypto_enable(&state);
+        octeon_md5_store_hash(mctx);
+
+        if (padding < 0) {
+                memset(p, 0x00, padding + sizeof(u64));
+                octeon_md5_transform(mctx->block);
+                p = (char *)mctx->block;
+                padding = 56;
+        }
+
+        memset(p, 0, padding);
+        mctx->block[14] = cpu_to_le32(mctx->byte_count << 3);
+        mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29);
+        octeon_md5_transform(mctx->block);
+
+        octeon_md5_read_hash(mctx);
+        octeon_crypto_disable(&state, flags);
+        preempt_enable();
+        local_bh_enable();
+
+        memcpy(out, mctx->hash, sizeof(mctx->hash));
+        memset(mctx, 0, sizeof(*mctx));
+
+        return 0;
+}
+
+static int octeon_md5_export(struct shash_desc *desc, void *out)
+{
+        struct md5_state *ctx = shash_desc_ctx(desc);
+
+        memcpy(out, ctx, sizeof(*ctx));
+        return 0;
+}
+
+static int octeon_md5_import(struct shash_desc *desc, const void *in)
+{
+        struct md5_state *ctx = shash_desc_ctx(desc);
+
+        memcpy(ctx, in, sizeof(*ctx));
+        return 0;
+}
+
+static struct shash_alg alg = {
+        .digestsize     =       MD5_DIGEST_SIZE,
+        .init           =       octeon_md5_init,
+        .update         =       octeon_md5_update,
+        .final          =       octeon_md5_final,
+        .export         =       octeon_md5_export,
+        .import         =       octeon_md5_import,
+        .descsize       =       sizeof(struct md5_state),
+        .statesize      =       sizeof(struct md5_state),
+        .base           =       {
+                .cra_name       =       "md5",
+                .cra_driver_name=       "octeon-md5",
+                .cra_priority   =       OCTEON_CR_OPCODE_PRIORITY,
+                .cra_flags      =       CRYPTO_ALG_TYPE_SHASH,
+                .cra_blocksize  =       MD5_HMAC_BLOCK_SIZE,
+                .cra_module     =       THIS_MODULE,
+        }
+};
+
+static int __init md5_mod_init(void)
+{
+        if (!octeon_has_crypto())
+                return -ENOTSUPP;
+        return crypto_register_shash(&alg);
+}
+
+static void __exit md5_mod_fini(void)
+{
+        crypto_unregister_shash(&alg);
+}
+
+module_init(md5_mod_init);
+module_exit(md5_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("MD5 Message Digest Algorithm (OCTEON)");
+MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>");
diff --git a/arch/mips/cavium-octeon/executive/octeon-model.c b/arch/mips/cavium-octeon/executive/octeon-model.c
index e15b049b3bd7..b2104bd9ab3b 100644
--- a/arch/mips/cavium-octeon/executive/octeon-model.c
+++ b/arch/mips/cavium-octeon/executive/octeon-model.c
@@ -27,6 +27,9 @@
 
 #include <asm/octeon/octeon.h>
 
+enum octeon_feature_bits __octeon_feature_bits __read_mostly;
+EXPORT_SYMBOL_GPL(__octeon_feature_bits);
+
 /**
  * Read a byte of fuse data
  * @byte_addr:   address to read
@@ -103,6 +106,9 @@ static const char *__init octeon_model_get_string_buffer(uint32_t chip_id,
         else
                 suffix = "NSP";
 
+        if (!fus_dat2.s.nocrypto)
+                __octeon_feature_bits |= OCTEON_HAS_CRYPTO;
+
         /*
          * Assume pass number is encoded using <5:3><2:0>. Exceptions
          * will be fixed later.
diff --git a/arch/mips/include/asm/octeon/octeon-feature.h b/arch/mips/include/asm/octeon/octeon-feature.h
index c4fe81f47f53..8ebd3f579b84 100644
--- a/arch/mips/include/asm/octeon/octeon-feature.h
+++ b/arch/mips/include/asm/octeon/octeon-feature.h
@@ -46,8 +46,6 @@ enum octeon_feature {
         OCTEON_FEATURE_SAAD,
         /* Does this Octeon support the ZIP offload engine? */
         OCTEON_FEATURE_ZIP,
-        /* Does this Octeon support crypto acceleration using COP2? */
-        OCTEON_FEATURE_CRYPTO,
         OCTEON_FEATURE_DORM_CRYPTO,
         /* Does this Octeon support PCI express? */
         OCTEON_FEATURE_PCIE,
@@ -86,6 +84,21 @@ enum octeon_feature {
         OCTEON_MAX_FEATURE
 };
 
+enum octeon_feature_bits {
+        OCTEON_HAS_CRYPTO = 0x0001,     /* Crypto acceleration using COP2 */
+};
+extern enum octeon_feature_bits __octeon_feature_bits;
+
+/**
+ * octeon_has_crypto() - Check if this OCTEON has crypto acceleration support.
+ *
+ * Returns: Non-zero if the feature exists. Zero if the feature does not exist.
+ */
+static inline int octeon_has_crypto(void)
+{
+        return __octeon_feature_bits & OCTEON_HAS_CRYPTO;
+}
+
 /**
  * Determine if the current Octeon supports a specific feature. These
  * checks have been optimized to be fairly quick, but they should still
diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h
index d781f9e66884..6dfefd2d5cdf 100644
--- a/arch/mips/include/asm/octeon/octeon.h
+++ b/arch/mips/include/asm/octeon/octeon.h
@@ -44,11 +44,6 @@ extern int octeon_get_boot_num_arguments(void);
 extern const char *octeon_get_boot_argument(int arg);
 extern void octeon_hal_setup_reserved32(void);
 extern void octeon_user_io_init(void);
-struct octeon_cop2_state;
-extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state);
-extern void octeon_crypto_disable(struct octeon_cop2_state *state,
-                                  unsigned long flags);
-extern asmlinkage void octeon_cop2_restore(struct octeon_cop2_state *task);
 
 extern void octeon_init_cvmcount(void);
 extern void octeon_setup_delays(void);
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index 705408766ab0..2e48eb8813ff 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -497,7 +497,7 @@ module_init(aes_sparc64_mod_init);
 module_exit(aes_sparc64_mod_fini);
 
 MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated");
+MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
 
 MODULE_ALIAS_CRYPTO("aes");
 
diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c
index 641f55cb61c3..6bf2479a12fb 100644
--- a/arch/sparc/crypto/camellia_glue.c
+++ b/arch/sparc/crypto/camellia_glue.c
@@ -322,6 +322,6 @@ module_exit(camellia_sparc64_mod_fini);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated");
 
-MODULE_ALIAS_CRYPTO("aes");
+MODULE_ALIAS_CRYPTO("camellia");
 
 #include "crop_devid.c"
diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c
index d11500972994..dd6a34fa6e19 100644
--- a/arch/sparc/crypto/des_glue.c
+++ b/arch/sparc/crypto/des_glue.c
@@ -533,5 +533,6 @@ MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
 
 MODULE_ALIAS_CRYPTO("des");
+MODULE_ALIAS_CRYPTO("des3_ede");
 
 #include "crop_devid.c"
diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c
index 64c7ff5f72a9..b688731d7ede 100644
--- a/arch/sparc/crypto/md5_glue.c
+++ b/arch/sparc/crypto/md5_glue.c
@@ -183,7 +183,7 @@ module_init(md5_sparc64_mod_init);
 module_exit(md5_sparc64_mod_fini);
 
 MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated");
+MODULE_DESCRIPTION("MD5 Message Digest Algorithm, sparc64 md5 opcode accelerated");
 
 MODULE_ALIAS_CRYPTO("md5");
 
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 477e9d75149b..6bd2c6c95373 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,12 +32,23 @@
 #include <linux/linkage.h>
 #include <asm/inst.h>
 
+/*
+ * The following macros are used to move an (un)aligned 16 byte value to/from
+ * an XMM register.  This can done for either FP or integer values, for FP use
+ * movaps (move aligned packed single) or integer use movdqa (move double quad
+ * aligned).  It doesn't make a performance difference which instruction is used
+ * since Nehalem (original Core i7) was released.  However, the movaps is a byte
+ * shorter, so that is the one we'll use for now. (same for unaligned).
+ */
+#define MOVADQ  movaps
+#define MOVUDQ  movups
+
 #ifdef __x86_64__
+
 .data
 .align 16
 .Lgf128mul_x_ble_mask:
         .octa 0x00000000000000010000000000000087
-
 POLY:   .octa 0xC2000000000000000000000000000001
 TWOONE: .octa 0x00000001000000000000000000000001
 
@@ -89,6 +100,7 @@ enc:        .octa 0x2
 #define arg8 STACK_OFFSET+16(%r14)
 #define arg9 STACK_OFFSET+24(%r14)
 #define arg10 STACK_OFFSET+32(%r14)
+#define keysize 2*15*16(%arg1)
 #endif
 
 
@@ -213,10 +225,12 @@ enc:        .octa 0x2
 
 .macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
 XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+        MOVADQ     SHUF_MASK(%rip), %xmm14
         mov        arg7, %r10           # %r10 = AAD
         mov        arg8, %r12           # %r12 = aadLen
         mov        %r12, %r11
         pxor       %xmm\i, %xmm\i
+
 _get_AAD_loop\num_initial_blocks\operation:
         movd       (%r10), \TMP1
         pslldq     $12, \TMP1
@@ -225,16 +239,18 @@ _get_AAD_loop\num_initial_blocks\operation:
         add        $4, %r10
         sub        $4, %r12
         jne        _get_AAD_loop\num_initial_blocks\operation
+
         cmp        $16, %r11
         je         _get_AAD_loop2_done\num_initial_blocks\operation
+
         mov        $16, %r12
 _get_AAD_loop2\num_initial_blocks\operation:
         psrldq     $4, %xmm\i
         sub        $4, %r12
         cmp        %r11, %r12
         jne        _get_AAD_loop2\num_initial_blocks\operation
+
 _get_AAD_loop2_done\num_initial_blocks\operation:
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 
         xor        %r11, %r11 # initialise the data pointer offset as zero
@@ -243,59 +259,34 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
 
         mov        %arg5, %rax                      # %rax = *Y0
         movdqu     (%rax), \XMM0                    # XMM0 = Y0
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM   %xmm14, \XMM0
 
 .if (\i == 5) || (\i == 6) || (\i == 7)
+        MOVADQ          ONE(%RIP),\TMP1
+        MOVADQ          (%arg1),\TMP2
 .irpc index, \i_seq
-        paddd      ONE(%rip), \XMM0                 # INCR Y0
+        paddd      \TMP1, \XMM0                 # INCR Y0
         movdqa     \XMM0, %xmm\index
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
-
-.endr
-.irpc index, \i_seq
-        pxor       16*0(%arg1), %xmm\index
-.endr
-.irpc index, \i_seq
-        movaps 0x10(%rdi), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 1
-.endr
-.irpc index, \i_seq
-        movaps 0x20(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x30(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x40(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x50(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x60(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
+        pxor       \TMP2, %xmm\index
 .endr
-.irpc index, \i_seq
-        movaps 0x70(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x80(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x90(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
+        lea     0x10(%arg1),%r10
+        mov     keysize,%eax
+        shr     $2,%eax                         # 128->4, 192->6, 256->8
+        add     $5,%eax                       # 128->9, 192->11, 256->13
+
+aes_loop_initial_dec\num_initial_blocks:
+        MOVADQ  (%r10),\TMP1
+.irpc   index, \i_seq
+        AESENC  \TMP1, %xmm\index
 .endr
+        add     $16,%r10
+        sub     $1,%eax
+        jnz     aes_loop_initial_dec\num_initial_blocks
+
+        MOVADQ  (%r10), \TMP1
 .irpc index, \i_seq
-        movaps 0xa0(%arg1), \TMP1
-        AESENCLAST \TMP1, %xmm\index         # Round 10
+        AESENCLAST \TMP1, %xmm\index         # Last Round
 .endr
 .irpc index, \i_seq
         movdqu     (%arg3 , %r11, 1), \TMP1
@@ -305,10 +296,8 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
         add        $16, %r11
 
         movdqa     \TMP1, %xmm\index
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM         %xmm14, %xmm\index
-
-                # prepare plaintext/ciphertext for GHASH computation
+                # prepare plaintext/ciphertext for GHASH computation
 .endr
 .endif
         GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
@@ -338,30 +327,28 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
 * Precomputations for HashKey parallel with encryption of first 4 blocks.
 * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
 */
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM1
-        movdqa     SHUF_MASK(%rip), %xmm14
+        MOVADQ     ONE(%rip), \TMP1
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM1
         PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
 
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM2
-        movdqa     SHUF_MASK(%rip), %xmm14
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM2
         PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
 
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM3
-        movdqa     SHUF_MASK(%rip), %xmm14
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM3
         PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
 
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM4
-        movdqa     SHUF_MASK(%rip), %xmm14
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM4
         PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
 
-        pxor       16*0(%arg1), \XMM1
-        pxor       16*0(%arg1), \XMM2
-        pxor       16*0(%arg1), \XMM3
-        pxor       16*0(%arg1), \XMM4
+        MOVADQ     0(%arg1),\TMP1
+        pxor       \TMP1, \XMM1
+        pxor       \TMP1, \XMM2
+        pxor       \TMP1, \XMM3
+        pxor       \TMP1, \XMM4
         movdqa     \TMP3, \TMP5
         pshufd     $78, \TMP3, \TMP1
         pxor       \TMP3, \TMP1
@@ -399,7 +386,23 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
         pshufd     $78, \TMP5, \TMP1
         pxor       \TMP5, \TMP1
         movdqa     \TMP1, HashKey_4_k(%rsp)
-        movaps 0xa0(%arg1), \TMP2
+        lea        0xa0(%arg1),%r10
+        mov        keysize,%eax
+        shr        $2,%eax                      # 128->4, 192->6, 256->8
+        sub        $4,%eax                      # 128->0, 192->2, 256->4
+        jz         aes_loop_pre_dec_done\num_initial_blocks
+
+aes_loop_pre_dec\num_initial_blocks:
+        MOVADQ     (%r10),\TMP2
+.irpc   index, 1234
+        AESENC     \TMP2, %xmm\index
+.endr
+        add        $16,%r10
+        sub        $1,%eax
+        jnz        aes_loop_pre_dec\num_initial_blocks
+
+aes_loop_pre_dec_done\num_initial_blocks:
+        MOVADQ     (%r10), \TMP2
         AESENCLAST \TMP2, \XMM1
         AESENCLAST \TMP2, \XMM2
         AESENCLAST \TMP2, \XMM3
@@ -421,15 +424,11 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
         movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
         movdqa     \TMP1, \XMM4
         add        $64, %r11
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
         pxor       \XMMDst, \XMM1
 # combine GHASHed value with the corresponding ciphertext
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
 
 _initial_blocks_done\num_initial_blocks\operation:
@@ -451,6 +450,7 @@ _initial_blocks_done\num_initial_blocks\operation:
 
 .macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
 XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+        MOVADQ     SHUF_MASK(%rip), %xmm14
         mov        arg7, %r10           # %r10 = AAD
         mov        arg8, %r12           # %r12 = aadLen
         mov        %r12, %r11
@@ -472,7 +472,6 @@ _get_AAD_loop2\num_initial_blocks\operation:
         cmp        %r11, %r12
         jne        _get_AAD_loop2\num_initial_blocks\operation
 _get_AAD_loop2_done\num_initial_blocks\operation:
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 
         xor        %r11, %r11 # initialise the data pointer offset as zero
@@ -481,59 +480,35 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
 
         mov        %arg5, %rax                      # %rax = *Y0
         movdqu     (%rax), \XMM0                    # XMM0 = Y0
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM   %xmm14, \XMM0
 
 .if (\i == 5) || (\i == 6) || (\i == 7)
-.irpc index, \i_seq
-        paddd      ONE(%rip), \XMM0                 # INCR Y0
-        movdqa     \XMM0, %xmm\index
-        movdqa     SHUF_MASK(%rip), %xmm14
-        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
 
-.endr
-.irpc index, \i_seq
-        pxor       16*0(%arg1), %xmm\index
-.endr
-.irpc index, \i_seq
-        movaps 0x10(%rdi), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 1
-.endr
-.irpc index, \i_seq
-        movaps 0x20(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
+        MOVADQ          ONE(%RIP),\TMP1
+        MOVADQ          0(%arg1),\TMP2
 .irpc index, \i_seq
-        movaps 0x30(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
+        paddd           \TMP1, \XMM0                 # INCR Y0
+        MOVADQ          \XMM0, %xmm\index
+        PSHUFB_XMM      %xmm14, %xmm\index      # perform a 16 byte swap
+        pxor            \TMP2, %xmm\index
 .endr
-.irpc index, \i_seq
-        movaps 0x40(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x50(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x60(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x70(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x80(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
-.endr
-.irpc index, \i_seq
-        movaps 0x90(%arg1), \TMP1
-        AESENC     \TMP1, %xmm\index          # Round 2
+        lea     0x10(%arg1),%r10
+        mov     keysize,%eax
+        shr     $2,%eax                         # 128->4, 192->6, 256->8
+        add     $5,%eax                       # 128->9, 192->11, 256->13
+
+aes_loop_initial_enc\num_initial_blocks:
+        MOVADQ  (%r10),\TMP1
+.irpc   index, \i_seq
+        AESENC  \TMP1, %xmm\index
 .endr
+        add     $16,%r10
+        sub     $1,%eax
+        jnz     aes_loop_initial_enc\num_initial_blocks
+
+        MOVADQ  (%r10), \TMP1
 .irpc index, \i_seq
-        movaps 0xa0(%arg1), \TMP1
-        AESENCLAST \TMP1, %xmm\index         # Round 10
+        AESENCLAST \TMP1, %xmm\index         # Last Round
 .endr
 .irpc index, \i_seq
         movdqu     (%arg3 , %r11, 1), \TMP1
@@ -541,8 +516,6 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
         movdqu     %xmm\index, (%arg2 , %r11, 1)
         # write back plaintext/ciphertext for num_initial_blocks
         add        $16, %r11
-
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM         %xmm14, %xmm\index
 
                 # prepare plaintext/ciphertext for GHASH computation
@@ -575,30 +548,28 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
 * Precomputations for HashKey parallel with encryption of first 4 blocks.
 * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
 */
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM1
-        movdqa     SHUF_MASK(%rip), %xmm14
+        MOVADQ     ONE(%RIP),\TMP1
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM1
         PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
 
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM2
-        movdqa     SHUF_MASK(%rip), %xmm14
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM2
         PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
 
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM3
-        movdqa     SHUF_MASK(%rip), %xmm14
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM3
         PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
 
-        paddd      ONE(%rip), \XMM0              # INCR Y0
-        movdqa     \XMM0, \XMM4
-        movdqa     SHUF_MASK(%rip), %xmm14
+        paddd      \TMP1, \XMM0              # INCR Y0
+        MOVADQ     \XMM0, \XMM4
         PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
 
-        pxor       16*0(%arg1), \XMM1
-        pxor       16*0(%arg1), \XMM2
-        pxor       16*0(%arg1), \XMM3
-        pxor       16*0(%arg1), \XMM4
+        MOVADQ     0(%arg1),\TMP1
+        pxor       \TMP1, \XMM1
+        pxor       \TMP1, \XMM2
+        pxor       \TMP1, \XMM3
+        pxor       \TMP1, \XMM4
         movdqa     \TMP3, \TMP5
         pshufd     $78, \TMP3, \TMP1
         pxor       \TMP3, \TMP1
@@ -636,7 +607,23 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
         pshufd     $78, \TMP5, \TMP1
         pxor       \TMP5, \TMP1
         movdqa     \TMP1, HashKey_4_k(%rsp)
-        movaps 0xa0(%arg1), \TMP2
+        lea        0xa0(%arg1),%r10
+        mov        keysize,%eax
+        shr        $2,%eax                      # 128->4, 192->6, 256->8
+        sub        $4,%eax                      # 128->0, 192->2, 256->4
+        jz         aes_loop_pre_enc_done\num_initial_blocks
+
+aes_loop_pre_enc\num_initial_blocks:
+        MOVADQ     (%r10),\TMP2
+.irpc   index, 1234
+        AESENC     \TMP2, %xmm\index
+.endr
+        add        $16,%r10
+        sub        $1,%eax
+        jnz        aes_loop_pre_enc\num_initial_blocks
+
+aes_loop_pre_enc_done\num_initial_blocks:
+        MOVADQ     (%r10), \TMP2
         AESENCLAST \TMP2, \XMM1
         AESENCLAST \TMP2, \XMM2
         AESENCLAST \TMP2, \XMM3
@@ -655,15 +642,11 @@ _get_AAD_loop2_done\num_initial_blocks\operation:
         movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
 
         add        $64, %r11
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
         pxor       \XMMDst, \XMM1
 # combine GHASHed value with the corresponding ciphertext
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
-        movdqa     SHUF_MASK(%rip), %xmm14
         PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
 
 _initial_blocks_done\num_initial_blocks\operation:
@@ -794,7 +777,23 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
         AESENC    \TMP3, \XMM3
         AESENC    \TMP3, \XMM4
         PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
-        movaps 0xa0(%arg1), \TMP3
+        lea       0xa0(%arg1),%r10
+        mov       keysize,%eax
+        shr       $2,%eax                       # 128->4, 192->6, 256->8
+        sub       $4,%eax                       # 128->0, 192->2, 256->4
+        jz        aes_loop_par_enc_done
+
+aes_loop_par_enc:
+        MOVADQ    (%r10),\TMP3
+.irpc   index, 1234
+        AESENC    \TMP3, %xmm\index
+.endr
+        add       $16,%r10
+        sub       $1,%eax
+        jnz       aes_loop_par_enc
+
+aes_loop_par_enc_done:
+        MOVADQ    (%r10), \TMP3
         AESENCLAST \TMP3, \XMM1           # Round 10
         AESENCLAST \TMP3, \XMM2
         AESENCLAST \TMP3, \XMM3
@@ -986,8 +985,24 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
         AESENC    \TMP3, \XMM3
         AESENC    \TMP3, \XMM4
         PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
-        movaps 0xa0(%arg1), \TMP3
-        AESENCLAST \TMP3, \XMM1           # Round 10
+        lea       0xa0(%arg1),%r10
+        mov       keysize,%eax
+        shr       $2,%eax                       # 128->4, 192->6, 256->8
+        sub       $4,%eax                       # 128->0, 192->2, 256->4
+        jz        aes_loop_par_dec_done
+
+aes_loop_par_dec:
+        MOVADQ    (%r10),\TMP3
+.irpc   index, 1234
+        AESENC    \TMP3, %xmm\index
+.endr
+        add       $16,%r10
+        sub       $1,%eax
+        jnz       aes_loop_par_dec
+
+aes_loop_par_dec_done:
+        MOVADQ    (%r10), \TMP3
+        AESENCLAST \TMP3, \XMM1           # last round
         AESENCLAST \TMP3, \XMM2
         AESENCLAST \TMP3, \XMM3
         AESENCLAST \TMP3, \XMM4
@@ -1155,33 +1170,29 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
         pxor      \TMP6, \XMMDst            # reduced result is in XMMDst
 .endm
 
-/* Encryption of a single block done*/
-.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
 
-        pxor    (%arg1), \XMM0
-        movaps 16(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 32(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 48(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 64(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 80(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 96(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 112(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 128(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 144(%arg1), \TMP1
-        AESENC  \TMP1, \XMM0
-        movaps 160(%arg1), \TMP1
-        AESENCLAST      \TMP1, \XMM0
-.endm
+/* Encryption of a single block
+* uses eax & r10
+*/
 
+.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
 
+        pxor            (%arg1), \XMM0
+        mov             keysize,%eax
+        shr             $2,%eax                 # 128->4, 192->6, 256->8
+        add             $5,%eax                 # 128->9, 192->11, 256->13
+        lea             16(%arg1), %r10   # get first expanded key address
+
+_esb_loop_\@:
+        MOVADQ          (%r10),\TMP1
+        AESENC          \TMP1,\XMM0
+        add             $16,%r10
+        sub             $1,%eax
+        jnz             _esb_loop_\@
+
+        MOVADQ          (%r10),\TMP1
+        AESENCLAST      \TMP1,\XMM0
+.endm
 /*****************************************************************************
 * void aesni_gcm_dec(void *aes_ctx,    // AES Key schedule. Starts on a 16 byte boundary.
 *                   u8 *out,           // Plaintext output. Encrypt in-place is allowed.
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index ae855f4f64b7..947c6bf52c33 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -43,6 +43,7 @@
 #include <asm/crypto/glue_helper.h>
 #endif
 
+
 /* This data is stored at the end of the crypto_tfm struct.
  * It's a type of per "session" data storage location.
  * This needs to be 16 byte aligned.
@@ -182,7 +183,8 @@ static void aesni_gcm_enc_avx(void *ctx, u8 *out,
                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                         u8 *auth_tag, unsigned long auth_tag_len)
 {
-        if (plaintext_len < AVX_GEN2_OPTSIZE) {
+        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+        if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
                                 aad_len, auth_tag, auth_tag_len);
         } else {
@@ -197,7 +199,8 @@ static void aesni_gcm_dec_avx(void *ctx, u8 *out,
                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                         u8 *auth_tag, unsigned long auth_tag_len)
 {
-        if (ciphertext_len < AVX_GEN2_OPTSIZE) {
+        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+        if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
                                 aad_len, auth_tag, auth_tag_len);
         } else {
@@ -231,7 +234,8 @@ static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                         u8 *auth_tag, unsigned long auth_tag_len)
 {
-        if (plaintext_len < AVX_GEN2_OPTSIZE) {
+       struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+        if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
                                 aad_len, auth_tag, auth_tag_len);
         } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
@@ -250,7 +254,8 @@ static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                         u8 *auth_tag, unsigned long auth_tag_len)
 {
-        if (ciphertext_len < AVX_GEN2_OPTSIZE) {
+       struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+        if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
                                 aad, aad_len, auth_tag, auth_tag_len);
         } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
@@ -511,7 +516,7 @@ static int ctr_crypt(struct blkcipher_desc *desc,
         kernel_fpu_begin();
         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
-                                  nbytes & AES_BLOCK_MASK, walk.iv);
+                                      nbytes & AES_BLOCK_MASK, walk.iv);
                 nbytes &= AES_BLOCK_SIZE - 1;
                 err = blkcipher_walk_done(desc, &walk, nbytes);
         }
@@ -902,7 +907,8 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
         }
         /*Account for 4 byte nonce at the end.*/
         key_len -= 4;
-        if (key_len != AES_KEYSIZE_128) {
+        if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
+            key_len != AES_KEYSIZE_256) {
                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
                 return -EINVAL;
         }
@@ -1013,6 +1019,7 @@ static int __driver_rfc4106_encrypt(struct aead_request *req)
         __be32 counter = cpu_to_be32(1);
         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        u32 key_len = ctx->aes_key_expanded.key_length;
         void *aes_ctx = &(ctx->aes_key_expanded);
         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
         u8 iv_tab[16+AESNI_ALIGN];
@@ -1027,6 +1034,13 @@ static int __driver_rfc4106_encrypt(struct aead_request *req)
         /* to 8 or 12 bytes */
         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
                 return -EINVAL;
+        if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
+                return -EINVAL;
+        if (unlikely(key_len != AES_KEYSIZE_128 &&
+                     key_len != AES_KEYSIZE_192 &&
+                     key_len != AES_KEYSIZE_256))
+                return -EINVAL;
+
         /* IV below built */
         for (i = 0; i < 4; i++)
                 *(iv+i) = ctx->nonce[i];
@@ -1091,6 +1105,7 @@ static int __driver_rfc4106_decrypt(struct aead_request *req)
         int retval = 0;
         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        u32 key_len = ctx->aes_key_expanded.key_length;
         void *aes_ctx = &(ctx->aes_key_expanded);
         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
         u8 iv_and_authTag[32+AESNI_ALIGN];
@@ -1104,6 +1119,13 @@ static int __driver_rfc4106_decrypt(struct aead_request *req)
         if (unlikely((req->cryptlen < auth_tag_len) ||
                 (req->assoclen != 8 && req->assoclen != 12)))
                 return -EINVAL;
+        if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
+                return -EINVAL;
+        if (unlikely(key_len != AES_KEYSIZE_128 &&
+                     key_len != AES_KEYSIZE_192 &&
+                     key_len != AES_KEYSIZE_256))
+                return -EINVAL;
+
         /* Assuming we are supporting rfc4106 64-bit extended */
         /* sequence numbers We need to have the AAD length */
         /* equal to 8 or 12 bytes */
diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c
index 38a14f818ef1..d6fc59aaaadf 100644
--- a/arch/x86/crypto/des3_ede_glue.c
+++ b/arch/x86/crypto/des3_ede_glue.c
@@ -504,6 +504,4 @@ MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized");
 MODULE_ALIAS_CRYPTO("des3_ede");
 MODULE_ALIAS_CRYPTO("des3_ede-asm");
-MODULE_ALIAS_CRYPTO("des");
-MODULE_ALIAS_CRYPTO("des-asm");
 MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 87bbc9c1e681..50f4da44a304 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -427,6 +427,15 @@ config CRYPTO_MD5
         help
           MD5 message digest algorithm (RFC1321).
 
+config CRYPTO_MD5_OCTEON
+        tristate "MD5 digest algorithm (OCTEON)"
+        depends on CPU_CAVIUM_OCTEON
+        select CRYPTO_MD5
+        select CRYPTO_HASH
+        help
+          MD5 message digest algorithm (RFC1321) implemented
+          using OCTEON crypto instructions, when available.
+
 config CRYPTO_MD5_SPARC64
         tristate "MD5 digest algorithm (SPARC64)"
         depends on SPARC64
@@ -1505,6 +1514,15 @@ config CRYPTO_USER_API_SKCIPHER
           This option enables the user-spaces interface for symmetric
           key cipher algorithms.
 
+config CRYPTO_USER_API_RNG
+        tristate "User-space interface for random number generator algorithms"
+        depends on NET
+        select CRYPTO_RNG
+        select CRYPTO_USER_API
+        help
+          This option enables the user-spaces interface for random
+          number generator algorithms.
+
 config CRYPTO_HASH_INFO
         bool
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 1445b9100c05..ba19465f9ad3 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o
 obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o
 obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
 obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
+obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
 
 #
 # generic algorithms and the async_tx api
diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index 40886c489903..db201bca1581 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -69,6 +69,7 @@ static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk,
 static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len)
 {
         u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK);
+
         return max(start, end_page);
 }
 
@@ -86,7 +87,7 @@ static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk,
                 if (n == len_this_page)
                         break;
                 n -= len_this_page;
-                scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg));
+                scatterwalk_start(&walk->out, sg_next(walk->out.sg));
         }
 
         return bsize;
@@ -284,6 +285,7 @@ static int ablkcipher_walk_first(struct ablkcipher_request *req,
         walk->iv = req->info;
         if (unlikely(((unsigned long)walk->iv & alignmask))) {
                 int err = ablkcipher_copy_iv(walk, tfm, alignmask);
+
                 if (err)
                         return err;
         }
@@ -589,7 +591,8 @@ static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask)
         if (IS_ERR(inst))
                 goto put_tmpl;
 
-        if ((err = crypto_register_instance(tmpl, inst))) {
+        err = crypto_register_instance(tmpl, inst);
+        if (err) {
                 tmpl->free(inst);
                 goto put_tmpl;
         }
diff --git a/crypto/aead.c b/crypto/aead.c
index 547491e35c63..222271070b49 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -448,7 +448,8 @@ static int crypto_nivaead_default(struct crypto_alg *alg, u32 type, u32 mask)
         if (IS_ERR(inst))
                 goto put_tmpl;
 
-        if ((err = crypto_register_instance(tmpl, inst))) {
+        err = crypto_register_instance(tmpl, inst);
+        if (err) {
                 tmpl->free(inst);
                 goto put_tmpl;
         }
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 3e80d8b8be45..7f8b7edcadca 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -188,7 +188,7 @@ static int alg_setkey(struct sock *sk, char __user *ukey,
         err = type->setkey(ask->private, key, keylen);
 
 out:
-        sock_kfree_s(sk, key, keylen);
+        sock_kzfree_s(sk, key, keylen);
 
         return err;
 }
@@ -215,6 +215,13 @@ static int alg_setsockopt(struct socket *sock, int level, int optname,
                         goto unlock;
 
                 err = alg_setkey(sk, optval, optlen);
+                break;
+        case ALG_SET_AEAD_AUTHSIZE:
+                if (sock->state == SS_CONNECTED)
+                        goto unlock;
+                if (!type->setauthsize)
+                        goto unlock;
+                err = type->setauthsize(ask->private, optlen);
         }
 
 unlock:
@@ -387,7 +394,7 @@ int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con)
                 if (cmsg->cmsg_level != SOL_ALG)
                         continue;
 
-                switch(cmsg->cmsg_type) {
+                switch (cmsg->cmsg_type) {
                 case ALG_SET_IV:
                         if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv)))
                                 return -EINVAL;
diff --git a/crypto/ahash.c b/crypto/ahash.c
index f6a36a52d738..8acb886032ae 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -55,6 +55,7 @@ static int hash_walk_next(struct crypto_hash_walk *walk)
 
         if (offset & alignmask) {
                 unsigned int unaligned = alignmask + 1 - (offset & alignmask);
+
                 if (nbytes > unaligned)
                         nbytes = unaligned;
         }
@@ -120,7 +121,7 @@ int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
         if (!walk->total)
                 return 0;
 
-        walk->sg = scatterwalk_sg_next(walk->sg);
+        walk->sg = sg_next(walk->sg);
 
         return hash_walk_new_entry(walk);
 }
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 71a8143e23b1..83b04e0884b1 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -473,6 +473,7 @@ void crypto_unregister_template(struct crypto_template *tmpl)
         list = &tmpl->instances;
         hlist_for_each_entry(inst, list, list) {
                 int err = crypto_remove_alg(&inst->alg, &users);
+
                 BUG_ON(err);
         }
 
diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c
new file mode 100644
index 000000000000..67f612cfed97
--- /dev/null
+++ b/crypto/algif_rng.c
@@ -0,0 +1,192 @@
+/*
+ * algif_rng: User-space interface for random number generators
+ *
+ * This file provides the user-space API for random number generators.
+ *
+ * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, and the entire permission notice in its entirety,
+ *    including the disclaimer of warranties.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote
+ *    products derived from this software without specific prior
+ *    written permission.
+ *
+ * ALTERNATIVELY, this product may be distributed under the terms of
+ * the GNU General Public License, in which case the provisions of the GPL2
+ * are required INSTEAD OF the above restrictions.  (This clause is
+ * necessary due to a potential bad interaction between the GPL and
+ * the restrictions contained in a BSD-style copyright.)
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
+ * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <linux/module.h>
+#include <crypto/rng.h>
+#include <linux/random.h>
+#include <crypto/if_alg.h>
+#include <linux/net.h>
+#include <net/sock.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
+MODULE_DESCRIPTION("User-space interface for random number generators");
+
+struct rng_ctx {
+#define MAXSIZE 128
+        unsigned int len;
+        struct crypto_rng *drng;
+};
+
+static int rng_recvmsg(struct kiocb *unused, struct socket *sock,
+                       struct msghdr *msg, size_t len, int flags)
+{
+        struct sock *sk = sock->sk;
+        struct alg_sock *ask = alg_sk(sk);
+        struct rng_ctx *ctx = ask->private;
+        int err = -EFAULT;
+        int genlen = 0;
+        u8 result[MAXSIZE];
+
+        if (len == 0)
+                return 0;
+        if (len > MAXSIZE)
+                len = MAXSIZE;
+
+        /*
+         * although not strictly needed, this is a precaution against coding
+         * errors
+         */
+        memset(result, 0, len);
+
+        /*
+         * The enforcement of a proper seeding of an RNG is done within an
+         * RNG implementation. Some RNGs (DRBG, krng) do not need specific
+         * seeding as they automatically seed. The X9.31 DRNG will return
+         * an error if it was not seeded properly.
+         */
+        genlen = crypto_rng_get_bytes(ctx->drng, result, len);
+        if (genlen < 0)
+                return genlen;
+
+        err = memcpy_to_msg(msg, result, len);
+        memzero_explicit(result, genlen);
+
+        return err ? err : len;
+}
+
+static struct proto_ops algif_rng_ops = {
+        .family         =       PF_ALG,
+
+        .connect        =       sock_no_connect,
+        .socketpair     =       sock_no_socketpair,
+        .getname        =       sock_no_getname,
+        .ioctl          =       sock_no_ioctl,
+        .listen         =       sock_no_listen,
+        .shutdown       =       sock_no_shutdown,
+        .getsockopt     =       sock_no_getsockopt,
+        .mmap           =       sock_no_mmap,
+        .bind           =       sock_no_bind,
+        .accept         =       sock_no_accept,
+        .setsockopt     =       sock_no_setsockopt,
+        .poll           =       sock_no_poll,
+        .sendmsg        =       sock_no_sendmsg,
+        .sendpage       =       sock_no_sendpage,
+
+        .release        =       af_alg_release,
+        .recvmsg        =       rng_recvmsg,
+};
+
+static void *rng_bind(const char *name, u32 type, u32 mask)
+{
+        return crypto_alloc_rng(name, type, mask);
+}
+
+static void rng_release(void *private)
+{
+        crypto_free_rng(private);
+}
+
+static void rng_sock_destruct(struct sock *sk)
+{
+        struct alg_sock *ask = alg_sk(sk);
+        struct rng_ctx *ctx = ask->private;
+
+        sock_kfree_s(sk, ctx, ctx->len);
+        af_alg_release_parent(sk);
+}
+
+static int rng_accept_parent(void *private, struct sock *sk)
+{
+        struct rng_ctx *ctx;
+        struct alg_sock *ask = alg_sk(sk);
+        unsigned int len = sizeof(*ctx);
+
+        ctx = sock_kmalloc(sk, len, GFP_KERNEL);
+        if (!ctx)
+                return -ENOMEM;
+
+        ctx->len = len;
+
+        /*
+         * No seeding done at that point -- if multiple accepts are
+         * done on one RNG instance, each resulting FD points to the same
+         * state of the RNG.
+         */
+
+        ctx->drng = private;
+        ask->private = ctx;
+        sk->sk_destruct = rng_sock_destruct;
+
+        return 0;
+}
+
+static int rng_setkey(void *private, const u8 *seed, unsigned int seedlen)
+{
+        /*
+         * Check whether seedlen is of sufficient size is done in RNG
+         * implementations.
+         */
+        return crypto_rng_reset(private, (u8 *)seed, seedlen);
+}
+
+static const struct af_alg_type algif_type_rng = {
+        .bind           =       rng_bind,
+        .release        =       rng_release,
+        .accept         =       rng_accept_parent,
+        .setkey         =       rng_setkey,
+        .ops            =       &algif_rng_ops,
+        .name           =       "rng",
+        .owner          =       THIS_MODULE
+};
+
+static int __init rng_init(void)
+{
+        return af_alg_register_type(&algif_type_rng);
+}
+
+static void __exit rng_exit(void)
+{
+        int err = af_alg_unregister_type(&algif_type_rng);
+        BUG_ON(err);
+}
+
+module_init(rng_init);
+module_exit(rng_exit);
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index 6fc12c3fc4b9..0c8a1e5ccadf 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -330,6 +330,7 @@ static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock,
 
                 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
                 sg = sgl->sg;
+                sg_unmark_end(sg + sgl->cur);
                 do {
                         i = sgl->cur;
                         plen = min_t(int, len, PAGE_SIZE);
@@ -355,6 +356,9 @@ static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock,
                         sgl->cur++;
                 } while (len && sgl->cur < MAX_SGL_ENTS);
 
+                if (!size)
+                        sg_mark_end(sg + sgl->cur - 1);
+
                 ctx->merge = plen & (PAGE_SIZE - 1);
         }
 
@@ -401,6 +405,10 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
         ctx->merge = 0;
         sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
 
+        if (sgl->cur)
+                sg_unmark_end(sgl->sg + sgl->cur - 1);
+
+        sg_mark_end(sgl->sg + sgl->cur);
         get_page(page);
         sg_set_page(sgl->sg + sgl->cur, page, size, offset);
         sgl->cur++;
diff --git a/crypto/cts.c b/crypto/cts.c
index bd9405820e8a..e467ec0acf9f 100644
--- a/crypto/cts.c
+++ b/crypto/cts.c
@@ -290,6 +290,9 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb)
         if (!is_power_of_2(alg->cra_blocksize))
                 goto out_put_alg;
 
+        if (strncmp(alg->cra_name, "cbc(", 4))
+                goto out_put_alg;
+
         inst = crypto_alloc_instance("cts", alg);
         if (IS_ERR(inst))
                 goto out_put_alg;
@@ -307,8 +310,6 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb)
         inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
         inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
 
-        inst->alg.cra_blkcipher.geniv = "seqiv";
-
         inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx);
 
         inst->alg.cra_init = crypto_cts_init_tfm;
diff --git a/crypto/drbg.c b/crypto/drbg.c
index d748a1d0ca24..d8ff16e5c322 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -98,7 +98,6 @@
  */
 
 #include <crypto/drbg.h>
-#include <linux/string.h>
 
 /***************************************************************
  * Backend cipher definitions available to DRBG
@@ -223,15 +222,6 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags)
  * function. Thus, the function implicitly knows the size of the
  * buffer.
  *
- * The FIPS test can be called in an endless loop until it returns
- * true. Although the code looks like a potential for a deadlock, it
- * is not the case, because returning a false cannot mathematically
- * occur (except once when a reseed took place and the updated state
- * would is now set up such that the generation of new value returns
- * an identical one -- this is most unlikely and would happen only once).
- * Thus, if this function repeatedly returns false and thus would cause
- * a deadlock, the integrity of the entire kernel is lost.
- *
  * @drbg DRBG handle
  * @buf output buffer of random data to be checked
  *
@@ -258,6 +248,8 @@ static bool drbg_fips_continuous_test(struct drbg_state *drbg,
                 return false;
         }
         ret = memcmp(drbg->prev, buf, drbg_blocklen(drbg));
+        if (!ret)
+                panic("DRBG continuous self test failed\n");
         memcpy(drbg->prev, buf, drbg_blocklen(drbg));
         /* the test shall pass when the two compared values are not equal */
         return ret != 0;
@@ -498,9 +490,9 @@ static int drbg_ctr_df(struct drbg_state *drbg,
         ret = 0;
 
 out:
-        memzero_explicit(iv, drbg_blocklen(drbg));
-        memzero_explicit(temp, drbg_statelen(drbg));
-        memzero_explicit(pad, drbg_blocklen(drbg));
+        memset(iv, 0, drbg_blocklen(drbg));
+        memset(temp, 0, drbg_statelen(drbg));
+        memset(pad, 0, drbg_blocklen(drbg));
         return ret;
 }
 
@@ -574,9 +566,9 @@ static int drbg_ctr_update(struct drbg_state *drbg, struct list_head *seed,
         ret = 0;
 
 out:
-        memzero_explicit(temp, drbg_statelen(drbg) + drbg_blocklen(drbg));
+        memset(temp, 0, drbg_statelen(drbg) + drbg_blocklen(drbg));
         if (2 != reseed)
-                memzero_explicit(df_data, drbg_statelen(drbg));
+                memset(df_data, 0, drbg_statelen(drbg));
         return ret;
 }
 
@@ -634,7 +626,7 @@ static int drbg_ctr_generate(struct drbg_state *drbg,
                 len = ret;
 
 out:
-        memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg));
+        memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
         return len;
 }
 
@@ -872,7 +864,7 @@ static int drbg_hash_df(struct drbg_state *drbg,
         }
 
 out:
-        memzero_explicit(tmp, drbg_blocklen(drbg));
+        memset(tmp, 0, drbg_blocklen(drbg));
         return ret;
 }
 
@@ -916,7 +908,7 @@ static int drbg_hash_update(struct drbg_state *drbg, struct list_head *seed,
         ret = drbg_hash_df(drbg, drbg->C, drbg_statelen(drbg), &datalist2);
 
 out:
-        memzero_explicit(drbg->scratchpad, drbg_statelen(drbg));
+        memset(drbg->scratchpad, 0, drbg_statelen(drbg));
         return ret;
 }
 
@@ -951,7 +943,7 @@ static int drbg_hash_process_addtl(struct drbg_state *drbg,
                      drbg->scratchpad, drbg_blocklen(drbg));
 
 out:
-        memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg));
+        memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
         return ret;
 }
 
@@ -998,7 +990,7 @@ static int drbg_hash_hashgen(struct drbg_state *drbg,
         }
 
 out:
-        memzero_explicit(drbg->scratchpad,
+        memset(drbg->scratchpad, 0,
                (drbg_statelen(drbg) + drbg_blocklen(drbg)));
         return len;
 }
@@ -1047,7 +1039,7 @@ static int drbg_hash_generate(struct drbg_state *drbg,
         drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8);
 
 out:
-        memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg));
+        memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
         return len;
 }
 
diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c
index 79ca2278c2a3..3bd749c7bb70 100644
--- a/crypto/scatterwalk.c
+++ b/crypto/scatterwalk.c
@@ -62,7 +62,7 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out,
                 walk->offset += PAGE_SIZE - 1;
                 walk->offset &= PAGE_MASK;
                 if (walk->offset >= walk->sg->offset + walk->sg->length)
-                        scatterwalk_start(walk, scatterwalk_sg_next(walk->sg));
+                        scatterwalk_start(walk, sg_next(walk->sg));
         }
 }
 
@@ -116,7 +116,7 @@ void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
                         break;
 
                 offset += sg->length;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         scatterwalk_advance(&walk, start - offset);
@@ -136,7 +136,7 @@ int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes)
         do {
                 offset += sg->length;
                 n++;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
 
                 /* num_bytes is too large */
                 if (unlikely(!sg && (num_bytes < offset)))
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index 9daa854cc485..b7bb9a2f4a31 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -267,6 +267,12 @@ static struct crypto_instance *seqiv_ablkcipher_alloc(struct rtattr **tb)
         if (IS_ERR(inst))
                 goto out;
 
+        if (inst->alg.cra_ablkcipher.ivsize < sizeof(u64)) {
+                skcipher_geniv_free(inst);
+                inst = ERR_PTR(-EINVAL);
+                goto out;
+        }
+
         inst->alg.cra_ablkcipher.givencrypt = seqiv_givencrypt_first;
 
         inst->alg.cra_init = seqiv_init;
@@ -287,6 +293,12 @@ static struct crypto_instance *seqiv_aead_alloc(struct rtattr **tb)
         if (IS_ERR(inst))
                 goto out;
 
+        if (inst->alg.cra_aead.ivsize < sizeof(u64)) {
+                aead_geniv_free(inst);
+                inst = ERR_PTR(-EINVAL);
+                goto out;
+        }
+
         inst->alg.cra_aead.givencrypt = seqiv_aead_givencrypt_first;
 
         inst->alg.cra_init = seqiv_aead_init;
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 1d864e988ea9..4b9e23fa4204 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -250,19 +250,19 @@ static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
         int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
         int k, rem;
 
-        np = (np > XBUFSIZE) ? XBUFSIZE : np;
-        rem = buflen % PAGE_SIZE;
         if (np > XBUFSIZE) {
                 rem = PAGE_SIZE;
                 np = XBUFSIZE;
+        } else {
+                rem = buflen % PAGE_SIZE;
         }
+
         sg_init_table(sg, np);
-        for (k = 0; k < np; ++k) {
-                if (k == (np-1))
-                        sg_set_buf(&sg[k], xbuf[k], rem);
-                else
-                        sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE);
-        }
+        np--;
+        for (k = 0; k < np; k++)
+                sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE);
+
+        sg_set_buf(&sg[k], xbuf[k], rem);
 }
 
 static void test_aead_speed(const char *algo, int enc, unsigned int secs,
@@ -280,16 +280,20 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs,
         struct scatterlist *sgout;
         const char *e;
         void *assoc;
-        char iv[MAX_IVLEN];
+        char *iv;
         char *xbuf[XBUFSIZE];
         char *xoutbuf[XBUFSIZE];
         char *axbuf[XBUFSIZE];
         unsigned int *b_size;
         unsigned int iv_len;
 
+        iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
+        if (!iv)
+                return;
+
         if (aad_size >= PAGE_SIZE) {
                 pr_err("associate data length (%u) too big\n", aad_size);
-                return;
+                goto out_noxbuf;
         }
 
         if (enc == ENCRYPT)
@@ -355,7 +359,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs,
 
                         iv_len = crypto_aead_ivsize(tfm);
                         if (iv_len)
-                                memset(&iv, 0xff, iv_len);
+                                memset(iv, 0xff, iv_len);
 
                         crypto_aead_clear_flags(tfm, ~0);
                         printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
@@ -408,6 +412,7 @@ out_nooutbuf:
 out_noaxbuf:
         testmgr_free_buf(xbuf);
 out_noxbuf:
+        kfree(iv);
         return;
 }
 
@@ -764,10 +769,9 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret)
         if (ret == -EINPROGRESS || ret == -EBUSY) {
                 struct tcrypt_result *tr = req->base.data;
 
-                ret = wait_for_completion_interruptible(&tr->completion);
-                if (!ret)
-                        ret = tr->err;
+                wait_for_completion(&tr->completion);
                 reinit_completion(&tr->completion);
+                ret = tr->err;
         }
         return ret;
 }
@@ -993,10 +997,9 @@ static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret)
         if (ret == -EINPROGRESS || ret == -EBUSY) {
                 struct tcrypt_result *tr = req->base.data;
 
-                ret = wait_for_completion_interruptible(&tr->completion);
-                if (!ret)
-                        ret = tr->err;
+                wait_for_completion(&tr->completion);
                 reinit_completion(&tr->completion);
+                ret = tr->err;
         }
 
         return ret;
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 037368d34586..f4ed6d4205e7 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -181,10 +181,9 @@ static void testmgr_free_buf(char *buf[XBUFSIZE])
 static int wait_async_op(struct tcrypt_result *tr, int ret)
 {
         if (ret == -EINPROGRESS || ret == -EBUSY) {
-                ret = wait_for_completion_interruptible(&tr->completion);
-                if (!ret)
-                        ret = tr->err;
+                wait_for_completion(&tr->completion);
                 reinit_completion(&tr->completion);
+                ret = tr->err;
         }
         return ret;
 }
@@ -353,12 +352,11 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
                         break;
                 case -EINPROGRESS:
                 case -EBUSY:
-                        ret = wait_for_completion_interruptible(
-                                &tresult.completion);
-                        if (!ret && !(ret = tresult.err)) {
-                                reinit_completion(&tresult.completion);
+                        wait_for_completion(&tresult.completion);
+                        reinit_completion(&tresult.completion);
+                        ret = tresult.err;
+                        if (!ret)
                                 break;
-                        }
                         /* fall through */
                 default:
                         printk(KERN_ERR "alg: hash: digest failed "
@@ -431,7 +429,7 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
         struct scatterlist *sgout;
         const char *e, *d;
         struct tcrypt_result result;
-        unsigned int authsize;
+        unsigned int authsize, iv_len;
         void *input;
         void *output;
         void *assoc;
@@ -502,10 +500,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
 
                 memcpy(input, template[i].input, template[i].ilen);
                 memcpy(assoc, template[i].assoc, template[i].alen);
+                iv_len = crypto_aead_ivsize(tfm);
                 if (template[i].iv)
-                        memcpy(iv, template[i].iv, MAX_IVLEN);
+                        memcpy(iv, template[i].iv, iv_len);
                 else
-                        memset(iv, 0, MAX_IVLEN);
+                        memset(iv, 0, iv_len);
 
                 crypto_aead_clear_flags(tfm, ~0);
                 if (template[i].wk)
@@ -569,12 +568,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
                         break;
                 case -EINPROGRESS:
                 case -EBUSY:
-                        ret = wait_for_completion_interruptible(
-                                &result.completion);
-                        if (!ret && !(ret = result.err)) {
-                                reinit_completion(&result.completion);
+                        wait_for_completion(&result.completion);
+                        reinit_completion(&result.completion);
+                        ret = result.err;
+                        if (!ret)
                                 break;
-                        }
                 case -EBADMSG:
                         if (template[i].novrfy)
                                 /* verification failure was expected */
@@ -720,12 +718,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
                         break;
                 case -EINPROGRESS:
                 case -EBUSY:
-                        ret = wait_for_completion_interruptible(
-                                &result.completion);
-                        if (!ret && !(ret = result.err)) {
-                                reinit_completion(&result.completion);
+                        wait_for_completion(&result.completion);
+                        reinit_completion(&result.completion);
+                        ret = result.err;
+                        if (!ret)
                                 break;
-                        }
                 case -EBADMSG:
                         if (template[i].novrfy)
                                 /* verification failure was expected */
@@ -1002,12 +999,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc,
                         break;
                 case -EINPROGRESS:
                 case -EBUSY:
-                        ret = wait_for_completion_interruptible(
-                                &result.completion);
-                        if (!ret && !((ret = result.err))) {
-                                reinit_completion(&result.completion);
+                        wait_for_completion(&result.completion);
+                        reinit_completion(&result.completion);
+                        ret = result.err;
+                        if (!ret)
                                 break;
-                        }
                         /* fall through */
                 default:
                         pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n",
@@ -1097,12 +1093,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc,
                         break;
                 case -EINPROGRESS:
                 case -EBUSY:
-                        ret = wait_for_completion_interruptible(
-                                        &result.completion);
-                        if (!ret && !((ret = result.err))) {
-                                reinit_completion(&result.completion);
+                        wait_for_completion(&result.completion);
+                        reinit_completion(&result.completion);
+                        ret = result.err;
+                        if (!ret)
                                 break;
-                        }
                         /* fall through */
                 default:
                         pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n",
@@ -3299,6 +3294,7 @@ static const struct alg_test_desc alg_test_descs[] = {
         }, {
                 .alg = "rfc4106(gcm(aes))",
                 .test = alg_test_aead,
+                .fips_allowed = 1,
                 .suite = {
                         .aead = {
                                 .enc = {
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 1500cfd799a7..32a8a867f7f8 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -42,6 +42,7 @@
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/random.h>
+#include <linux/err.h>
 #include <asm/uaccess.h>
 
 
@@ -53,7 +54,10 @@
 static struct hwrng *current_rng;
 static struct task_struct *hwrng_fill;
 static LIST_HEAD(rng_list);
+/* Protects rng_list and current_rng */
 static DEFINE_MUTEX(rng_mutex);
+/* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */
+static DEFINE_MUTEX(reading_mutex);
 static int data_avail;
 static u8 *rng_buffer, *rng_fillbuf;
 static unsigned short current_quality;
@@ -66,6 +70,8 @@ module_param(default_quality, ushort, 0644);
 MODULE_PARM_DESC(default_quality,
                  "default entropy content of hwrng per mill");
 
+static void drop_current_rng(void);
+static int hwrng_init(struct hwrng *rng);
 static void start_khwrngd(void);
 
 static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size,
@@ -81,13 +87,83 @@ static void add_early_randomness(struct hwrng *rng)
         unsigned char bytes[16];
         int bytes_read;
 
+        mutex_lock(&reading_mutex);
         bytes_read = rng_get_data(rng, bytes, sizeof(bytes), 1);
+        mutex_unlock(&reading_mutex);
         if (bytes_read > 0)
                 add_device_randomness(bytes, bytes_read);
 }
 
-static inline int hwrng_init(struct hwrng *rng)
+static inline void cleanup_rng(struct kref *kref)
 {
+        struct hwrng *rng = container_of(kref, struct hwrng, ref);
+
+        if (rng->cleanup)
+                rng->cleanup(rng);
+
+        complete(&rng->cleanup_done);
+}
+
+static int set_current_rng(struct hwrng *rng)
+{
+        int err;
+
+        BUG_ON(!mutex_is_locked(&rng_mutex));
+
+        err = hwrng_init(rng);
+        if (err)
+                return err;
+
+        drop_current_rng();
+        current_rng = rng;
+
+        return 0;
+}
+
+static void drop_current_rng(void)
+{
+        BUG_ON(!mutex_is_locked(&rng_mutex));
+        if (!current_rng)
+                return;
+
+        /* decrease last reference for triggering the cleanup */
+        kref_put(&current_rng->ref, cleanup_rng);
+        current_rng = NULL;
+}
+
+/* Returns ERR_PTR(), NULL or refcounted hwrng */
+static struct hwrng *get_current_rng(void)
+{
+        struct hwrng *rng;
+
+        if (mutex_lock_interruptible(&rng_mutex))
+                return ERR_PTR(-ERESTARTSYS);
+
+        rng = current_rng;
+        if (rng)
+                kref_get(&rng->ref);
+
+        mutex_unlock(&rng_mutex);
+        return rng;
+}
+
+static void put_rng(struct hwrng *rng)
+{
+        /*
+         * Hold rng_mutex here so we serialize in case they set_current_rng
+         * on rng again immediately.
+         */
+        mutex_lock(&rng_mutex);
+        if (rng)
+                kref_put(&rng->ref, cleanup_rng);
+        mutex_unlock(&rng_mutex);
+}
+
+static int hwrng_init(struct hwrng *rng)
+{
+        if (kref_get_unless_zero(&rng->ref))
+                goto skip_init;
+
         if (rng->init) {
                 int ret;
 
@@ -95,6 +171,11 @@ static inline int hwrng_init(struct hwrng *rng)
                 if (ret)
                         return ret;
         }
+
+        kref_init(&rng->ref);
+        reinit_completion(&rng->cleanup_done);
+
+skip_init:
         add_early_randomness(rng);
 
         current_quality = rng->quality ? : default_quality;
@@ -108,12 +189,6 @@ static inline int hwrng_init(struct hwrng *rng)
         return 0;
 }
 
-static inline void hwrng_cleanup(struct hwrng *rng)
-{
-        if (rng && rng->cleanup)
-                rng->cleanup(rng);
-}
-
 static int rng_dev_open(struct inode *inode, struct file *filp)
 {
         /* enforce read-only access to this chrdev */
@@ -128,6 +203,7 @@ static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size,
                         int wait) {
         int present;
 
+        BUG_ON(!mutex_is_locked(&reading_mutex));
         if (rng->read)
                 return rng->read(rng, (void *)buffer, size, wait);
 
@@ -148,25 +224,27 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf,
         ssize_t ret = 0;
         int err = 0;
         int bytes_read, len;
+        struct hwrng *rng;
 
         while (size) {
-                if (mutex_lock_interruptible(&rng_mutex)) {
-                        err = -ERESTARTSYS;
+                rng = get_current_rng();
+                if (IS_ERR(rng)) {
+                        err = PTR_ERR(rng);
                         goto out;
                 }
-
-                if (!current_rng) {
+                if (!rng) {
                         err = -ENODEV;
-                        goto out_unlock;
+                        goto out;
                 }
 
+                mutex_lock(&reading_mutex);
                 if (!data_avail) {
-                        bytes_read = rng_get_data(current_rng, rng_buffer,
+                        bytes_read = rng_get_data(rng, rng_buffer,
                                 rng_buffer_size(),
                                 !(filp->f_flags & O_NONBLOCK));
                         if (bytes_read < 0) {
                                 err = bytes_read;
-                                goto out_unlock;
+                                goto out_unlock_reading;
                         }
                         data_avail = bytes_read;
                 }
@@ -174,7 +252,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf,
                 if (!data_avail) {
                         if (filp->f_flags & O_NONBLOCK) {
                                 err = -EAGAIN;
-                                goto out_unlock;
+                                goto out_unlock_reading;
                         }
                 } else {
                         len = data_avail;
@@ -186,14 +264,15 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf,
                         if (copy_to_user(buf + ret, rng_buffer + data_avail,
                                                                 len)) {
                                 err = -EFAULT;
-                                goto out_unlock;
+                                goto out_unlock_reading;
                         }
 
                         size -= len;
                         ret += len;
                 }
 
-                mutex_unlock(&rng_mutex);
+                mutex_unlock(&reading_mutex);
+                put_rng(rng);
 
                 if (need_resched())
                         schedule_timeout_interruptible(1);
@@ -205,8 +284,10 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf,
         }
 out:
         return ret ? : err;
-out_unlock:
-        mutex_unlock(&rng_mutex);
+
+out_unlock_reading:
+        mutex_unlock(&reading_mutex);
+        put_rng(rng);
         goto out;
 }
 
@@ -239,16 +320,9 @@ static ssize_t hwrng_attr_current_store(struct device *dev,
         err = -ENODEV;
         list_for_each_entry(rng, &rng_list, list) {
                 if (strcmp(rng->name, buf) == 0) {
-                        if (rng == current_rng) {
-                                err = 0;
-                                break;
-                        }
-                        err = hwrng_init(rng);
-                        if (err)
-                                break;
-                        hwrng_cleanup(current_rng);
-                        current_rng = rng;
                         err = 0;
+                        if (rng != current_rng)
+                                err = set_current_rng(rng);
                         break;
                 }
         }
@@ -261,17 +335,15 @@ static ssize_t hwrng_attr_current_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
 {
-        int err;
         ssize_t ret;
-        const char *name = "none";
+        struct hwrng *rng;
 
-        err = mutex_lock_interruptible(&rng_mutex);
-        if (err)
-                return -ERESTARTSYS;
-        if (current_rng)
-                name = current_rng->name;
-        ret = snprintf(buf, PAGE_SIZE, "%s\n", name);
-        mutex_unlock(&rng_mutex);
+        rng = get_current_rng();
+        if (IS_ERR(rng))
+                return PTR_ERR(rng);
+
+        ret = snprintf(buf, PAGE_SIZE, "%s\n", rng ? rng->name : "none");
+        put_rng(rng);
 
         return ret;
 }
@@ -305,14 +377,14 @@ static DEVICE_ATTR(rng_available, S_IRUGO,
                    NULL);
 
 
-static void unregister_miscdev(void)
+static void __exit unregister_miscdev(void)
 {
         device_remove_file(rng_miscdev.this_device, &dev_attr_rng_available);
         device_remove_file(rng_miscdev.this_device, &dev_attr_rng_current);
         misc_deregister(&rng_miscdev);
 }
 
-static int register_miscdev(void)
+static int __init register_miscdev(void)
 {
         int err;
 
@@ -342,15 +414,22 @@ static int hwrng_fillfn(void *unused)
         long rc;
 
         while (!kthread_should_stop()) {
-                if (!current_rng)
+                struct hwrng *rng;
+
+                rng = get_current_rng();
+                if (IS_ERR(rng) || !rng)
                         break;
-                rc = rng_get_data(current_rng, rng_fillbuf,
+                mutex_lock(&reading_mutex);
+                rc = rng_get_data(rng, rng_fillbuf,
                                   rng_buffer_size(), 1);
+                mutex_unlock(&reading_mutex);
+                put_rng(rng);
                 if (rc <= 0) {
                         pr_warn("hwrng: no data available\n");
                         msleep_interruptible(10000);
                         continue;
                 }
+                /* Outside lock, sure, but y'know: randomness. */
                 add_hwgenerator_randomness((void *)rng_fillbuf, rc,
                                            rc * current_quality * 8 >> 10);
         }
@@ -400,23 +479,16 @@ int hwrng_register(struct hwrng *rng)
                         goto out_unlock;
         }
 
+        init_completion(&rng->cleanup_done);
+        complete(&rng->cleanup_done);
+
         old_rng = current_rng;
-        if (!old_rng) {
-                err = hwrng_init(rng);
-                if (err)
-                        goto out_unlock;
-                current_rng = rng;
-        }
         err = 0;
         if (!old_rng) {
-                err = register_miscdev();
-                if (err) {
-                        hwrng_cleanup(rng);
-                        current_rng = NULL;
+                err = set_current_rng(rng);
+                if (err)
                         goto out_unlock;
-                }
         }
-        INIT_LIST_HEAD(&rng->list);
         list_add_tail(&rng->list, &rng_list);
 
         if (old_rng && !rng->init) {
@@ -439,42 +511,49 @@ EXPORT_SYMBOL_GPL(hwrng_register);
 
 void hwrng_unregister(struct hwrng *rng)
 {
-        int err;
-
         mutex_lock(&rng_mutex);
 
         list_del(&rng->list);
         if (current_rng == rng) {
-                hwrng_cleanup(rng);
-                if (list_empty(&rng_list)) {
-                        current_rng = NULL;
-                } else {
-                        current_rng = list_entry(rng_list.prev, struct hwrng, list);
-                        err = hwrng_init(current_rng);
-                        if (err)
-                                current_rng = NULL;
+                drop_current_rng();
+                if (!list_empty(&rng_list)) {
+                        struct hwrng *tail;
+
+                        tail = list_entry(rng_list.prev, struct hwrng, list);
+
+                        set_current_rng(tail);
                 }
         }
+
         if (list_empty(&rng_list)) {
-                unregister_miscdev();
+                mutex_unlock(&rng_mutex);
                 if (hwrng_fill)
                         kthread_stop(hwrng_fill);
-        }
+        } else
+                mutex_unlock(&rng_mutex);
 
-        mutex_unlock(&rng_mutex);
+        wait_for_completion(&rng->cleanup_done);
 }
 EXPORT_SYMBOL_GPL(hwrng_unregister);
 
-static void __exit hwrng_exit(void)
+static int __init hwrng_modinit(void)
+{
+        return register_miscdev();
+}
+
+static void __exit hwrng_modexit(void)
 {
         mutex_lock(&rng_mutex);
         BUG_ON(current_rng);
         kfree(rng_buffer);
         kfree(rng_fillbuf);
         mutex_unlock(&rng_mutex);
+
+        unregister_miscdev();
 }
 
-module_exit(hwrng_exit);
+module_init(hwrng_modinit);
+module_exit(hwrng_modexit);
 
 MODULE_DESCRIPTION("H/W Random Number Generator (RNG) driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
index 72295ea2fd1c..3fa2f8a009b3 100644
--- a/drivers/char/hw_random/virtio-rng.c
+++ b/drivers/char/hw_random/virtio-rng.c
@@ -39,7 +39,6 @@ struct virtrng_info {
         bool hwrng_removed;
 };
 
-
 static void random_recv_done(struct virtqueue *vq)
 {
         struct virtrng_info *vi = vq->vdev->priv;
diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c
index de8a7a48775a..69182e2cc3ea 100644
--- a/drivers/crypto/amcc/crypto4xx_sa.c
+++ b/drivers/crypto/amcc/crypto4xx_sa.c
@@ -34,29 +34,6 @@
 #include "crypto4xx_sa.h"
 #include "crypto4xx_core.h"
 
-u32 get_dynamic_sa_offset_iv_field(struct crypto4xx_ctx *ctx)
-{
-        u32 offset;
-        union dynamic_sa_contents cts;
-
-        if (ctx->direction == DIR_INBOUND)
-                cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_in))->sa_contents;
-        else
-                cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_out))->sa_contents;
-        offset = cts.bf.key_size
-                + cts.bf.inner_size
-                + cts.bf.outer_size
-                + cts.bf.spi
-                + cts.bf.seq_num0
-                + cts.bf.seq_num1
-                + cts.bf.seq_num_mask0
-                + cts.bf.seq_num_mask1
-                + cts.bf.seq_num_mask2
-                + cts.bf.seq_num_mask3;
-
-        return sizeof(struct dynamic_sa_ctl) + offset * 4;
-}
-
 u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx)
 {
         u32 offset;
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 53d1c330f8a8..6597aac9905d 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -673,9 +673,9 @@ err_map_out:
         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
                 DMA_TO_DEVICE);
 err_map_in:
+err_alloc:
         free_page((unsigned long)dd->buf_out);
         free_page((unsigned long)dd->buf_in);
-err_alloc:
         if (err)
                 pr_err("error: %d\n", err);
         return err;
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index d94f07c78e19..34db04addc18 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -102,10 +102,6 @@ struct atmel_sha_ctx {
         struct atmel_sha_dev    *dd;
 
         unsigned long           flags;
-
-        /* fallback stuff */
-        struct crypto_shash     *fallback;
-
 };
 
 #define ATMEL_SHA_QUEUE_LENGTH  50
@@ -974,19 +970,8 @@ static int atmel_sha_digest(struct ahash_request *req)
         return atmel_sha_init(req) ?: atmel_sha_finup(req);
 }
 
-static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
+static int atmel_sha_cra_init(struct crypto_tfm *tfm)
 {
-        struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm);
-        const char *alg_name = crypto_tfm_alg_name(tfm);
-
-        /* Allocate a fallback and abort if it failed. */
-        tctx->fallback = crypto_alloc_shash(alg_name, 0,
-                                            CRYPTO_ALG_NEED_FALLBACK);
-        if (IS_ERR(tctx->fallback)) {
-                pr_err("atmel-sha: fallback driver '%s' could not be loaded.\n",
-                                alg_name);
-                return PTR_ERR(tctx->fallback);
-        }
         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
                                  sizeof(struct atmel_sha_reqctx) +
                                  SHA_BUFFER_LEN + SHA512_BLOCK_SIZE);
@@ -994,19 +979,6 @@ static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
         return 0;
 }
 
-static int atmel_sha_cra_init(struct crypto_tfm *tfm)
-{
-        return atmel_sha_cra_init_alg(tfm, NULL);
-}
-
-static void atmel_sha_cra_exit(struct crypto_tfm *tfm)
-{
-        struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm);
-
-        crypto_free_shash(tctx->fallback);
-        tctx->fallback = NULL;
-}
-
 static struct ahash_alg sha_1_256_algs[] = {
 {
         .init           = atmel_sha_init,
@@ -1020,14 +992,12 @@ static struct ahash_alg sha_1_256_algs[] = {
                         .cra_name               = "sha1",
                         .cra_driver_name        = "atmel-sha1",
                         .cra_priority           = 100,
-                        .cra_flags              = CRYPTO_ALG_ASYNC |
-                                                CRYPTO_ALG_NEED_FALLBACK,
+                        .cra_flags              = CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = SHA1_BLOCK_SIZE,
                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
                         .cra_alignmask          = 0,
                         .cra_module             = THIS_MODULE,
                         .cra_init               = atmel_sha_cra_init,
-                        .cra_exit               = atmel_sha_cra_exit,
                 }
         }
 },
@@ -1043,14 +1013,12 @@ static struct ahash_alg sha_1_256_algs[] = {
                         .cra_name               = "sha256",
                         .cra_driver_name        = "atmel-sha256",
                         .cra_priority           = 100,
-                        .cra_flags              = CRYPTO_ALG_ASYNC |
-                                                CRYPTO_ALG_NEED_FALLBACK,
+                        .cra_flags              = CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = SHA256_BLOCK_SIZE,
                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
                         .cra_alignmask          = 0,
                         .cra_module             = THIS_MODULE,
                         .cra_init               = atmel_sha_cra_init,
-                        .cra_exit               = atmel_sha_cra_exit,
                 }
         }
 },
@@ -1068,14 +1036,12 @@ static struct ahash_alg sha_224_alg = {
                         .cra_name               = "sha224",
                         .cra_driver_name        = "atmel-sha224",
                         .cra_priority           = 100,
-                        .cra_flags              = CRYPTO_ALG_ASYNC |
-                                                CRYPTO_ALG_NEED_FALLBACK,
+                        .cra_flags              = CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = SHA224_BLOCK_SIZE,
                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
                         .cra_alignmask          = 0,
                         .cra_module             = THIS_MODULE,
                         .cra_init               = atmel_sha_cra_init,
-                        .cra_exit               = atmel_sha_cra_exit,
                 }
         }
 };
@@ -1093,14 +1059,12 @@ static struct ahash_alg sha_384_512_algs[] = {
                         .cra_name               = "sha384",
                         .cra_driver_name        = "atmel-sha384",
                         .cra_priority           = 100,
-                        .cra_flags              = CRYPTO_ALG_ASYNC |
-                                                CRYPTO_ALG_NEED_FALLBACK,
+                        .cra_flags              = CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = SHA384_BLOCK_SIZE,
                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
                         .cra_alignmask          = 0x3,
                         .cra_module             = THIS_MODULE,
                         .cra_init               = atmel_sha_cra_init,
-                        .cra_exit               = atmel_sha_cra_exit,
                 }
         }
 },
@@ -1116,14 +1080,12 @@ static struct ahash_alg sha_384_512_algs[] = {
                         .cra_name               = "sha512",
                         .cra_driver_name        = "atmel-sha512",
                         .cra_priority           = 100,
-                        .cra_flags              = CRYPTO_ALG_ASYNC |
-                                                CRYPTO_ALG_NEED_FALLBACK,
+                        .cra_flags              = CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = SHA512_BLOCK_SIZE,
                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
                         .cra_alignmask          = 0x3,
                         .cra_module             = THIS_MODULE,
                         .cra_init               = atmel_sha_cra_init,
-                        .cra_exit               = atmel_sha_cra_exit,
                 }
         }
 },
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index 5e7c896cde30..258772d9b22f 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -376,9 +376,9 @@ err_map_out:
         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
                 DMA_TO_DEVICE);
 err_map_in:
+err_alloc:
         free_page((unsigned long)dd->buf_out);
         free_page((unsigned long)dd->buf_in);
-err_alloc:
         if (err)
                 pr_err("error: %d\n", err);
         return err;
diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c
index 9ae149bddb6e..d9af9403ab6c 100644
--- a/drivers/crypto/bfin_crc.c
+++ b/drivers/crypto/bfin_crc.c
@@ -110,7 +110,7 @@ static int sg_count(struct scatterlist *sg_list)
 
         while (!sg_is_last(sg)) {
                 sg_nents++;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         return sg_nents;
@@ -744,7 +744,7 @@ static int __init bfin_crypto_crc_mod_init(void)
 
         ret = platform_driver_register(&bfin_crypto_crc_driver);
         if (ret) {
-                pr_info(KERN_ERR "unable to register driver\n");
+                pr_err("unable to register driver\n");
                 return ret;
         }
 
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 3187400daf31..29071a156cbe 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -2532,7 +2532,7 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
                 in_options = 0;
         } else {
                 src_dma = edesc->sec4_sg_dma;
-                sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents;
+                sec4_sg_index += edesc->src_nents + 1;
                 in_options = LDST_SGF;
         }
         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
@@ -2714,10 +2714,10 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         if (!all_contig) {
                 if (!is_gcm) {
                         sg_to_sec4_sg(req->assoc,
-                                      (assoc_nents ? : 1),
+                                      assoc_nents,
                                       edesc->sec4_sg +
                                       sec4_sg_index, 0);
-                        sec4_sg_index += assoc_nents ? : 1;
+                        sec4_sg_index += assoc_nents;
                 }
 
                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
@@ -2726,17 +2726,17 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 
                 if (is_gcm) {
                         sg_to_sec4_sg(req->assoc,
-                                      (assoc_nents ? : 1),
+                                      assoc_nents,
                                       edesc->sec4_sg +
                                       sec4_sg_index, 0);
-                        sec4_sg_index += assoc_nents ? : 1;
+                        sec4_sg_index += assoc_nents;
                 }
 
                 sg_to_sec4_sg_last(req->src,
-                                   (src_nents ? : 1),
+                                   src_nents,
                                    edesc->sec4_sg +
                                    sec4_sg_index, 0);
-                sec4_sg_index += src_nents ? : 1;
+                sec4_sg_index += src_nents;
         }
         if (dst_nents) {
                 sg_to_sec4_sg_last(req->dst, dst_nents,
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 70f1e6f37336..efba4ccd4fac 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -175,13 +175,10 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
 {
         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
         struct caam_ctrl __iomem *ctrl;
-        struct rng4tst __iomem *r4tst;
         u32 *desc, status, rdsta_val;
         int ret = 0, sh_idx;
 
         ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
-        r4tst = &ctrl->r4tst[0];
-
         desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL);
         if (!desc)
                 return -ENOMEM;
@@ -209,8 +206,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
                  * without any error (HW optimizations for later
                  * CAAM eras), then try again.
                  */
-                rdsta_val =
-                        rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
+                rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
                 if (status || !(rdsta_val & (1 << sh_idx)))
                         ret = -EAGAIN;
                 if (ret)
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 66d73bf54166..33e41ea83fcc 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -151,10 +151,15 @@ static void report_ccb_status(struct device *jrdev, const u32 status,
         else
                 snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
 
-        dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
-                status, error, idx_str, idx,
-                cha_str, cha_err_code,
-                err_str, err_err_code);
+        /*
+         * CCB ICV check failures are part of normal operation life;
+         * we leave the upper layers to do what they want with them.
+         */
+        if (err_id != JRSTA_CCBERR_ERRID_ICVCHK)
+                dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
+                        status, error, idx_str, idx,
+                        cha_str, cha_err_code,
+                        err_str, err_err_code);
 }
 
 static void report_jump_status(struct device *jrdev, const u32 status,
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 9b3ef1bc9bd7..b8b5d47acd7a 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -384,30 +384,28 @@ static int caam_jr_init(struct device *dev)
         if (error) {
                 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
                         jrp->ridx, jrp->irq);
-                irq_dispose_mapping(jrp->irq);
-                jrp->irq = 0;
-                return -EINVAL;
+                goto out_kill_deq;
         }
 
         error = caam_reset_hw_jr(dev);
         if (error)
-                return error;
+                goto out_free_irq;
 
+        error = -ENOMEM;
         jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
                                           &inpbusaddr, GFP_KERNEL);
+        if (!jrp->inpring)
+                goto out_free_irq;
 
         jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
                                           JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
+        if (!jrp->outring)
+                goto out_free_inpring;
 
         jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
                                GFP_KERNEL);
-
-        if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
-            (jrp->entinfo == NULL)) {
-                dev_err(dev, "can't allocate job rings for %d\n",
-                        jrp->ridx);
-                return -ENOMEM;
-        }
+        if (!jrp->entinfo)
+                goto out_free_outring;
 
         for (i = 0; i < JOBR_DEPTH; i++)
                 jrp->entinfo[i].desc_addr_dma = !0;
@@ -434,6 +432,19 @@ static int caam_jr_init(struct device *dev)
                   (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
 
         return 0;
+
+out_free_outring:
+        dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
+                          jrp->outring, outbusaddr);
+out_free_inpring:
+        dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
+                          jrp->inpring, inpbusaddr);
+        dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
+out_free_irq:
+        free_irq(jrp->irq, dev);
+out_kill_deq:
+        tasklet_kill(&jrp->irqtask);
+        return error;
 }
 
 
@@ -484,8 +495,10 @@ static int caam_jr_probe(struct platform_device *pdev)
 
         /* Now do the platform independent part */
         error = caam_jr_init(jrdev); /* now turn on hardware */
-        if (error)
+        if (error) {
+                irq_dispose_mapping(jrpriv->irq);
                 return error;
+        }
 
         jrpriv->dev = jrdev;
         spin_lock(&driver_data.jr_alloc_lock);
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index ce28a563effc..3b918218aa4c 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -37,7 +37,7 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
                 dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg),
                                    sg_dma_len(sg), offset);
                 sec4_sg_ptr++;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
                 sg_count--;
         }
         return sec4_sg_ptr - 1;
@@ -67,7 +67,7 @@ static inline int __sg_count(struct scatterlist *sg_list, int nbytes,
                 nbytes -= sg->length;
                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
                         *chained = true;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         return sg_nents;
@@ -93,7 +93,7 @@ static int dma_map_sg_chained(struct device *dev, struct scatterlist *sg,
                 int i;
                 for (i = 0; i < nents; i++) {
                         dma_map_sg(dev, sg, 1, dir);
-                        sg = scatterwalk_sg_next(sg);
+                        sg = sg_next(sg);
                 }
         } else {
                 dma_map_sg(dev, sg, nents, dir);
@@ -109,7 +109,7 @@ static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg,
                 int i;
                 for (i = 0; i < nents; i++) {
                         dma_unmap_sg(dev, sg, 1, dir);
-                        sg = scatterwalk_sg_next(sg);
+                        sg = sg_next(sg);
                 }
         } else {
                 dma_unmap_sg(dev, sg, nents, dir);
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index c6e6171eb6d3..ca29c120b85f 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -583,6 +583,7 @@ bool ccp_queues_suspended(struct ccp_device *ccp)
 #ifdef CONFIG_X86
 static const struct x86_cpu_id ccp_support[] = {
         { X86_VENDOR_AMD, 22, },
+        { },
 };
 #endif
 
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index f757a0f428bd..48f453555f1f 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -784,7 +784,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev,
                 struct buffer_desc *buf, gfp_t flags,
                 enum dma_data_direction dir)
 {
-        for (;nbytes > 0; sg = scatterwalk_sg_next(sg)) {
+        for (; nbytes > 0; sg = sg_next(sg)) {
                 unsigned len = min(nbytes, sg->length);
                 struct buffer_desc *next_buf;
                 u32 next_buf_phys;
@@ -982,7 +982,7 @@ static int hmac_inconsistent(struct scatterlist *sg, unsigned start,
                         break;
 
                 offset += sg->length;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
         return (start + nbytes > offset + sg->length);
 }
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index a392465d3e3f..1da6dc59d0dd 100644
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -177,7 +177,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg       *nx_dst,
                         break;
 
                 offset += sg_src->length;
-                sg_src = scatterwalk_sg_next(sg_src);
+                sg_src = sg_next(sg_src);
         }
 
         /* start - offset is the number of bytes to advance in the scatterlist
@@ -187,9 +187,9 @@ struct nx_sg *nx_walk_and_build(struct nx_sg       *nx_dst,
         while (len && (nx_sg - nx_dst) < sglen) {
                 n = scatterwalk_clamp(&walk, len);
                 if (!n) {
-                        /* In cases where we have scatterlist chain scatterwalk_sg_next
+                        /* In cases where we have scatterlist chain sg_next
                          * handles with it properly */
-                        scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
+                        scatterwalk_start(&walk, sg_next(walk.sg));
                         n = scatterwalk_clamp(&walk, len);
                 }
                 dst = scatterwalk_map(&walk);
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index f79dd410dede..42f95a4326b0 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -994,7 +994,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id)
 
                         scatterwalk_advance(&dd->in_walk, 4);
                         if (dd->in_sg->length == _calc_walked(in)) {
-                                dd->in_sg = scatterwalk_sg_next(dd->in_sg);
+                                dd->in_sg = sg_next(dd->in_sg);
                                 if (dd->in_sg) {
                                         scatterwalk_start(&dd->in_walk,
                                                           dd->in_sg);
@@ -1026,7 +1026,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id)
                         *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
                         scatterwalk_advance(&dd->out_walk, 4);
                         if (dd->out_sg->length == _calc_walked(out)) {
-                                dd->out_sg = scatterwalk_sg_next(dd->out_sg);
+                                dd->out_sg = sg_next(dd->out_sg);
                                 if (dd->out_sg) {
                                         scatterwalk_start(&dd->out_walk,
                                                           dd->out_sg);
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index e350f5be4d2e..46307098f8ba 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -921,7 +921,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id)
 
                         scatterwalk_advance(&dd->in_walk, 4);
                         if (dd->in_sg->length == _calc_walked(in)) {
-                                dd->in_sg = scatterwalk_sg_next(dd->in_sg);
+                                dd->in_sg = sg_next(dd->in_sg);
                                 if (dd->in_sg) {
                                         scatterwalk_start(&dd->in_walk,
                                                           dd->in_sg);
@@ -953,7 +953,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id)
                         *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i));
                         scatterwalk_advance(&dd->out_walk, 4);
                         if (dd->out_sg->length == _calc_walked(out)) {
-                                dd->out_sg = scatterwalk_sg_next(dd->out_sg);
+                                dd->out_sg = sg_next(dd->out_sg);
                                 if (dd->out_sg) {
                                         scatterwalk_start(&dd->out_walk,
                                                           dd->out_sg);
@@ -965,9 +965,9 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id)
                         }
                 }
 
-                dd->total -= DES_BLOCK_SIZE;
+                BUG_ON(dd->total < DES_BLOCK_SIZE);
 
-                BUG_ON(dd->total < 0);
+                dd->total -= DES_BLOCK_SIZE;
 
                 /* Clear IRQ status */
                 status &= ~DES_REG_IRQ_DATA_OUT;
diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h
index 2ed425664a16..19c0efa29ab3 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_devices.h
+++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h
@@ -47,7 +47,6 @@
 #ifndef ADF_ACCEL_DEVICES_H_
 #define ADF_ACCEL_DEVICES_H_
 #include <linux/module.h>
-#include <linux/atomic.h>
 #include <linux/list.h>
 #include <linux/proc_fs.h>
 #include <linux/io.h>
@@ -148,6 +147,11 @@ struct adf_hw_device_data {
         int (*alloc_irq)(struct adf_accel_dev *accel_dev);
         void (*free_irq)(struct adf_accel_dev *accel_dev);
         void (*enable_error_correction)(struct adf_accel_dev *accel_dev);
+        int (*init_admin_comms)(struct adf_accel_dev *accel_dev);
+        void (*exit_admin_comms)(struct adf_accel_dev *accel_dev);
+        int (*init_arb)(struct adf_accel_dev *accel_dev);
+        void (*exit_arb)(struct adf_accel_dev *accel_dev);
+        void (*enable_ints)(struct adf_accel_dev *accel_dev);
         const char *fw_name;
         uint32_t pci_dev_id;
         uint32_t fuses;
diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c
index 10ce4a2854ab..fa1fef824de2 100644
--- a/drivers/crypto/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/qat/qat_common/adf_aer.c
@@ -82,28 +82,15 @@ struct adf_reset_dev_data {
         struct work_struct reset_work;
 };
 
-#define PPDSTAT_OFFSET 0x7E
 static void adf_dev_restore(struct adf_accel_dev *accel_dev)
 {
         struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
         struct pci_dev *parent = pdev->bus->self;
-        uint16_t ppdstat = 0, bridge_ctl = 0;
-        int pending = 0;
+        uint16_t bridge_ctl = 0;
 
         pr_info("QAT: Resetting device qat_dev%d\n", accel_dev->accel_id);
-        pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat);
-        pending = ppdstat & PCI_EXP_DEVSTA_TRPND;
-        if (pending) {
-                int ctr = 0;
-
-                do {
-                        msleep(100);
-                        pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat);
-                        pending = ppdstat & PCI_EXP_DEVSTA_TRPND;
-                } while (pending && ctr++ < 10);
-        }
 
-        if (pending)
+        if (!pci_wait_for_pending_transaction(pdev))
                 pr_info("QAT: Transaction still in progress. Proceeding\n");
 
         pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl);
@@ -125,8 +112,9 @@ static void adf_device_reset_worker(struct work_struct *work)
 
         adf_dev_restarting_notify(accel_dev);
         adf_dev_stop(accel_dev);
+        adf_dev_shutdown(accel_dev);
         adf_dev_restore(accel_dev);
-        if (adf_dev_start(accel_dev)) {
+        if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) {
                 /* The device hanged and we can't restart it so stop here */
                 dev_err(&GET_DEV(accel_dev), "Restart device failed\n");
                 kfree(reset_data);
@@ -148,8 +136,8 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev,
 {
         struct adf_reset_dev_data *reset_data;
 
-        if (adf_dev_started(accel_dev) &&
-            !test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
+        if (!adf_dev_started(accel_dev) ||
+            test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
                 return 0;
 
         set_bit(ADF_STATUS_RESTARTING, &accel_dev->status);
diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c
index aba7f1d043fb..de16da9070a5 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg.c
+++ b/drivers/crypto/qat/qat_common/adf_cfg.c
@@ -50,6 +50,7 @@
 #include <linux/seq_file.h>
 #include "adf_accel_devices.h"
 #include "adf_cfg.h"
+#include "adf_common_drv.h"
 
 static DEFINE_MUTEX(qat_cfg_read_lock);
 
@@ -159,6 +160,7 @@ void adf_cfg_del_all(struct adf_accel_dev *accel_dev)
         down_write(&dev_cfg_data->lock);
         adf_cfg_section_del_all(&dev_cfg_data->sec_list);
         up_write(&dev_cfg_data->lock);
+        clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
 }
 
 /**
diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h
index 5e8f9d431e5d..a62e485c8786 100644
--- a/drivers/crypto/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/qat/qat_common/adf_common_drv.h
@@ -93,7 +93,7 @@ int adf_service_unregister(struct service_hndl *service);
 int adf_dev_init(struct adf_accel_dev *accel_dev);
 int adf_dev_start(struct adf_accel_dev *accel_dev);
 int adf_dev_stop(struct adf_accel_dev *accel_dev);
-int adf_dev_shutdown(struct adf_accel_dev *accel_dev);
+void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
 
 int adf_ctl_dev_register(void);
 void adf_ctl_dev_unregister(void);
diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c
index 7ee93f881db6..74207a6f0516 100644
--- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c
+++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c
@@ -282,6 +282,8 @@ static int adf_ctl_stop_devices(uint32_t id)
                         if (adf_dev_stop(accel_dev)) {
                                 pr_err("QAT: Failed to stop qat_dev%d\n", id);
                                 ret = -EFAULT;
+                        } else {
+                                adf_dev_shutdown(accel_dev);
                         }
                 }
         }
@@ -343,7 +345,9 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd,
         if (!adf_dev_started(accel_dev)) {
                 pr_info("QAT: Starting acceleration device qat_dev%d.\n",
                         ctl_data->device_id);
-                ret = adf_dev_start(accel_dev);
+                ret = adf_dev_init(accel_dev);
+                if (!ret)
+                        ret = adf_dev_start(accel_dev);
         } else {
                 pr_info("QAT: Acceleration device qat_dev%d already started.\n",
                         ctl_data->device_id);
@@ -351,6 +355,7 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd,
         if (ret) {
                 pr_err("QAT: Failed to start qat_dev%d\n", ctl_data->device_id);
                 adf_dev_stop(accel_dev);
+                adf_dev_shutdown(accel_dev);
         }
 out:
         kfree(ctl_data);
diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c
index 5c0e47a00a87..8f0ca498ab87 100644
--- a/drivers/crypto/qat/qat_common/adf_init.c
+++ b/drivers/crypto/qat/qat_common/adf_init.c
@@ -108,26 +108,47 @@ int adf_service_unregister(struct service_hndl *service)
 EXPORT_SYMBOL_GPL(adf_service_unregister);
 
 /**
- * adf_dev_start() - Start acceleration service for the given accel device
- * @accel_dev:    Pointer to acceleration device.
+ * adf_dev_init() - Init data structures and services for the given accel device
+ * @accel_dev: Pointer to acceleration device.
  *
- * Function notifies all the registered services that the acceleration device
- * is ready to be used.
- * To be used by QAT device specific drivers.
+ * Initialize the ring data structures and the admin comms and arbitration
+ * services.
  *
  * Return: 0 on success, error code othewise.
  */
-int adf_dev_start(struct adf_accel_dev *accel_dev)
+int adf_dev_init(struct adf_accel_dev *accel_dev)
 {
         struct service_hndl *service;
         struct list_head *list_itr;
         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 
+        if (!hw_data) {
+                dev_err(&GET_DEV(accel_dev),
+                        "QAT: Failed to init device - hw_data not set\n");
+                return -EFAULT;
+        }
+
         if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
                 pr_info("QAT: Device not configured\n");
                 return -EFAULT;
         }
-        set_bit(ADF_STATUS_STARTING, &accel_dev->status);
+
+        if (adf_init_etr_data(accel_dev)) {
+                dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
+                return -EFAULT;
+        }
+
+        if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
+                dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
+                return -EFAULT;
+        }
+
+        if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
+                dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
+                return -EFAULT;
+        }
+
+        hw_data->enable_ints(accel_dev);
 
         if (adf_ae_init(accel_dev)) {
                 pr_err("QAT: Failed to initialise Acceleration Engine\n");
@@ -178,6 +199,27 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
 
         hw_data->enable_error_correction(accel_dev);
 
+        return 0;
+}
+EXPORT_SYMBOL_GPL(adf_dev_init);
+
+/**
+ * adf_dev_start() - Start acceleration service for the given accel device
+ * @accel_dev:    Pointer to acceleration device.
+ *
+ * Function notifies all the registered services that the acceleration device
+ * is ready to be used.
+ * To be used by QAT device specific drivers.
+ *
+ * Return: 0 on success, error code othewise.
+ */
+int adf_dev_start(struct adf_accel_dev *accel_dev)
+{
+        struct service_hndl *service;
+        struct list_head *list_itr;
+
+        set_bit(ADF_STATUS_STARTING, &accel_dev->status);
+
         if (adf_ae_start(accel_dev)) {
                 pr_err("QAT: AE Start Failed\n");
                 return -EFAULT;
@@ -232,16 +274,15 @@ EXPORT_SYMBOL_GPL(adf_dev_start);
  */
 int adf_dev_stop(struct adf_accel_dev *accel_dev)
 {
-        struct adf_hw_device_data *hw_data = accel_dev->hw_device;
         struct service_hndl *service;
         struct list_head *list_itr;
-        int ret, wait = 0;
+        bool wait = false;
+        int ret;
 
         if (!adf_dev_started(accel_dev) &&
             !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) {
                 return 0;
         }
-        clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
         clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
         clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
 
@@ -258,7 +299,7 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev)
                 if (!ret) {
                         clear_bit(accel_dev->accel_id, &service->start_status);
                 } else if (ret == -EAGAIN) {
-                        wait = 1;
+                        wait = true;
                         clear_bit(accel_dev->accel_id, &service->start_status);
                 }
         }
@@ -278,13 +319,36 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev)
         if (wait)
                 msleep(100);
 
-        if (adf_dev_started(accel_dev)) {
+        if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
                 if (adf_ae_stop(accel_dev))
                         pr_err("QAT: failed to stop AE\n");
                 else
                         clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
         }
 
+        return 0;
+}
+EXPORT_SYMBOL_GPL(adf_dev_stop);
+
+/**
+ * adf_dev_shutdown() - shutdown acceleration services and data strucutures
+ * @accel_dev: Pointer to acceleration device
+ *
+ * Cleanup the ring data structures and the admin comms and arbitration
+ * services.
+ */
+void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
+{
+        struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+        struct service_hndl *service;
+        struct list_head *list_itr;
+
+        if (!hw_data) {
+                dev_err(&GET_DEV(accel_dev),
+                        "QAT: Failed to shutdown device - hw_data not set\n");
+                return;
+        }
+
         if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
                 if (adf_ae_fw_release(accel_dev))
                         pr_err("QAT: Failed to release the ucode\n");
@@ -335,9 +399,15 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev)
         if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
                 adf_cfg_del_all(accel_dev);
 
-        return 0;
+        if (hw_data->exit_arb)
+                hw_data->exit_arb(accel_dev);
+
+        if (hw_data->exit_admin_comms)
+                hw_data->exit_admin_comms(accel_dev);
+
+        adf_cleanup_etr_data(accel_dev);
 }
-EXPORT_SYMBOL_GPL(adf_dev_stop);
+EXPORT_SYMBOL_GPL(adf_dev_shutdown);
 
 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
 {
diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h
index c40546079981..a4869627fd57 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_internal.h
+++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h
@@ -48,7 +48,6 @@
 #define ADF_TRANSPORT_INTRN_H
 
 #include <linux/interrupt.h>
-#include <linux/atomic.h>
 #include <linux/spinlock_types.h>
 #include "adf_transport.h"
 
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h
index 5031f8c10d75..68f191b653b0 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hw.h
+++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h
@@ -301,5 +301,5 @@ struct icp_qat_hw_cipher_aes256_f8 {
 
 struct icp_qat_hw_cipher_algo_blk {
         struct icp_qat_hw_cipher_aes256_f8 aes;
-};
+} __aligned(64);
 #endif
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 19eea1c832ac..1dc5b0a17cf7 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -63,15 +63,15 @@
 #include "icp_qat_fw.h"
 #include "icp_qat_fw_la.h"
 
-#define QAT_AES_HW_CONFIG_ENC(alg) \
+#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
         ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
-                        ICP_QAT_HW_CIPHER_NO_CONVERT, \
-                        ICP_QAT_HW_CIPHER_ENCRYPT)
+                                       ICP_QAT_HW_CIPHER_NO_CONVERT, \
+                                       ICP_QAT_HW_CIPHER_ENCRYPT)
 
-#define QAT_AES_HW_CONFIG_DEC(alg) \
+#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
         ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
-                        ICP_QAT_HW_CIPHER_KEY_CONVERT, \
-                        ICP_QAT_HW_CIPHER_DECRYPT)
+                                       ICP_QAT_HW_CIPHER_KEY_CONVERT, \
+                                       ICP_QAT_HW_CIPHER_DECRYPT)
 
 static atomic_t active_dev;
 
@@ -102,25 +102,31 @@ struct qat_alg_cd {
         };
 } __aligned(64);
 
-#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk)
-
-struct qat_auth_state {
-        uint8_t data[MAX_AUTH_STATE_SIZE + 64];
-} __aligned(64);
-
-struct qat_alg_session_ctx {
+struct qat_alg_aead_ctx {
         struct qat_alg_cd *enc_cd;
-        dma_addr_t enc_cd_paddr;
         struct qat_alg_cd *dec_cd;
+        dma_addr_t enc_cd_paddr;
         dma_addr_t dec_cd_paddr;
-        struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl;
-        struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl;
-        struct qat_crypto_instance *inst;
-        struct crypto_tfm *tfm;
+        struct icp_qat_fw_la_bulk_req enc_fw_req;
+        struct icp_qat_fw_la_bulk_req dec_fw_req;
         struct crypto_shash *hash_tfm;
         enum icp_qat_hw_auth_algo qat_hash_alg;
+        struct qat_crypto_instance *inst;
+        struct crypto_tfm *tfm;
         uint8_t salt[AES_BLOCK_SIZE];
-        spinlock_t lock;        /* protects qat_alg_session_ctx struct */
+        spinlock_t lock;        /* protects qat_alg_aead_ctx struct */
+};
+
+struct qat_alg_ablkcipher_ctx {
+        struct icp_qat_hw_cipher_algo_blk *enc_cd;
+        struct icp_qat_hw_cipher_algo_blk *dec_cd;
+        dma_addr_t enc_cd_paddr;
+        dma_addr_t dec_cd_paddr;
+        struct icp_qat_fw_la_bulk_req enc_fw_req;
+        struct icp_qat_fw_la_bulk_req dec_fw_req;
+        struct qat_crypto_instance *inst;
+        struct crypto_tfm *tfm;
+        spinlock_t lock;        /* protects qat_alg_ablkcipher_ctx struct */
 };
 
 static int get_current_node(void)
@@ -144,43 +150,37 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 }
 
 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
-                                  struct qat_alg_session_ctx *ctx,
+                                  struct qat_alg_aead_ctx *ctx,
                                   const uint8_t *auth_key,
                                   unsigned int auth_keylen)
 {
-        struct qat_auth_state auth_state;
         SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
         struct sha1_state sha1;
         struct sha256_state sha256;
         struct sha512_state sha512;
         int block_size = crypto_shash_blocksize(ctx->hash_tfm);
         int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
-        uint8_t *ipad = auth_state.data;
-        uint8_t *opad = ipad + block_size;
+        char ipad[block_size];
+        char opad[block_size];
         __be32 *hash_state_out;
         __be64 *hash512_state_out;
         int i, offset;
 
-        memzero_explicit(auth_state.data, MAX_AUTH_STATE_SIZE + 64);
+        memset(ipad, 0, block_size);
+        memset(opad, 0, block_size);
         shash->tfm = ctx->hash_tfm;
         shash->flags = 0x0;
 
         if (auth_keylen > block_size) {
-                char buff[SHA512_BLOCK_SIZE];
                 int ret = crypto_shash_digest(shash, auth_key,
-                                              auth_keylen, buff);
+                                              auth_keylen, ipad);
                 if (ret)
                         return ret;
 
-                memcpy(ipad, buff, digest_size);
-                memcpy(opad, buff, digest_size);
-                memzero_explicit(ipad + digest_size, block_size - digest_size);
-                memzero_explicit(opad + digest_size, block_size - digest_size);
+                memcpy(opad, ipad, digest_size);
         } else {
                 memcpy(ipad, auth_key, auth_keylen);
                 memcpy(opad, auth_key, auth_keylen);
-                memzero_explicit(ipad + auth_keylen, block_size - auth_keylen);
-                memzero_explicit(opad + auth_keylen, block_size - auth_keylen);
         }
 
         for (i = 0; i < block_size; i++) {
@@ -267,8 +267,6 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
         header->comn_req_flags =
                 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
                                             QAT_COMN_PTR_TYPE_SGL);
-        ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
-                                           ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
                                   ICP_QAT_FW_LA_PARTIAL_NONE);
         ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
@@ -279,8 +277,9 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
                                        ICP_QAT_FW_LA_NO_UPDATE_STATE);
 }
 
-static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
-                                    int alg, struct crypto_authenc_keys *keys)
+static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx,
+                                         int alg,
+                                         struct crypto_authenc_keys *keys)
 {
         struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
         unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
@@ -289,7 +288,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
         struct icp_qat_hw_auth_algo_blk *hash =
                 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
                 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
-        struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req_tmpl;
+        struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
         void *ptr = &req_tmpl->cd_ctrl;
@@ -297,7 +296,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 
         /* CD setup */
-        cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg);
+        cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
         hash->sha.inner_setup.auth_config.config =
                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
@@ -311,6 +310,8 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
         /* Request setup */
         qat_alg_init_common_hdr(header);
         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+        ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+                                           ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
                                    ICP_QAT_FW_LA_RET_AUTH_RES);
         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
@@ -356,8 +357,9 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
         return 0;
 }
 
-static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
-                                    int alg, struct crypto_authenc_keys *keys)
+static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx,
+                                         int alg,
+                                         struct crypto_authenc_keys *keys)
 {
         struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
         unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
@@ -367,7 +369,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
                 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
                 sizeof(struct icp_qat_hw_auth_setup) +
                 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
-        struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req_tmpl;
+        struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
         void *ptr = &req_tmpl->cd_ctrl;
@@ -379,7 +381,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
                 sizeof(struct icp_qat_fw_la_cipher_req_params));
 
         /* CD setup */
-        cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg);
+        cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
         hash->sha.inner_setup.auth_config.config =
                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
@@ -394,6 +396,8 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
         /* Request setup */
         qat_alg_init_common_hdr(header);
         header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+        ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+                                           ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
                                    ICP_QAT_FW_LA_NO_RET_AUTH_RES);
         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
@@ -444,36 +448,91 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
         return 0;
 }
 
-static int qat_alg_init_sessions(struct qat_alg_session_ctx *ctx,
-                                 const uint8_t *key, unsigned int keylen)
+static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
+                                        struct icp_qat_fw_la_bulk_req *req,
+                                        struct icp_qat_hw_cipher_algo_blk *cd,
+                                        const uint8_t *key, unsigned int keylen)
 {
-        struct crypto_authenc_keys keys;
-        int alg;
+        struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+        struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+        struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
 
-        if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
-                return -EFAULT;
+        memcpy(cd->aes.key, key, keylen);
+        qat_alg_init_common_hdr(header);
+        header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
+        cd_pars->u.s.content_desc_params_sz =
+                                sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
+        /* Cipher CD config setup */
+        cd_ctrl->cipher_key_sz = keylen >> 3;
+        cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
+        cd_ctrl->cipher_cfg_offset = 0;
+        ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
+        ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
+}
 
-        if (crypto_authenc_extractkeys(&keys, key, keylen))
-                goto bad_key;
+static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
+                                        int alg, const uint8_t *key,
+                                        unsigned int keylen)
+{
+        struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
+        struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
+        struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+
+        qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
+        cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
+        enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
+}
 
-        switch (keys.enckeylen) {
+static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
+                                        int alg, const uint8_t *key,
+                                        unsigned int keylen)
+{
+        struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
+        struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
+        struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+
+        qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
+        cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
+        dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
+}
+
+static int qat_alg_validate_key(int key_len, int *alg)
+{
+        switch (key_len) {
         case AES_KEYSIZE_128:
-                alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
+                *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
                 break;
         case AES_KEYSIZE_192:
-                alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
+                *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
                 break;
         case AES_KEYSIZE_256:
-                alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
+                *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
                 break;
         default:
-                goto bad_key;
+                return -EINVAL;
         }
+        return 0;
+}
 
-        if (qat_alg_init_enc_session(ctx, alg, &keys))
+static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx,
+                                      const uint8_t *key, unsigned int keylen)
+{
+        struct crypto_authenc_keys keys;
+        int alg;
+
+        if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
+                return -EFAULT;
+
+        if (crypto_authenc_extractkeys(&keys, key, keylen))
+                goto bad_key;
+
+        if (qat_alg_validate_key(keys.enckeylen, &alg))
+                goto bad_key;
+
+        if (qat_alg_aead_init_enc_session(ctx, alg, &keys))
                 goto error;
 
-        if (qat_alg_init_dec_session(ctx, alg, &keys))
+        if (qat_alg_aead_init_dec_session(ctx, alg, &keys))
                 goto error;
 
         return 0;
@@ -484,22 +543,37 @@ error:
         return -EFAULT;
 }
 
-static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
-                          unsigned int keylen)
+static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
+                                            const uint8_t *key,
+                                            unsigned int keylen)
+{
+        int alg;
+
+        if (qat_alg_validate_key(keylen, &alg))
+                goto bad_key;
+
+        qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);
+        qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);
+        return 0;
+bad_key:
+        crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        return -EINVAL;
+}
+
+static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
+                               unsigned int keylen)
 {
-        struct qat_alg_session_ctx *ctx = crypto_aead_ctx(tfm);
+        struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
         struct device *dev;
 
         spin_lock(&ctx->lock);
         if (ctx->enc_cd) {
                 /* rekeying */
                 dev = &GET_DEV(ctx->inst->accel_dev);
-                memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
-                memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
-                memzero_explicit(&ctx->enc_fw_req_tmpl,
-                                 sizeof(struct icp_qat_fw_la_bulk_req));
-                memzero_explicit(&ctx->dec_fw_req_tmpl,
-                                 sizeof(struct icp_qat_fw_la_bulk_req));
+                memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+                memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+                memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+                memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
         } else {
                 /* new key */
                 int node = get_current_node();
@@ -512,16 +586,14 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
 
                 dev = &GET_DEV(inst->accel_dev);
                 ctx->inst = inst;
-                ctx->enc_cd = dma_zalloc_coherent(dev,
-                                                  sizeof(struct qat_alg_cd),
+                ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
                                                   &ctx->enc_cd_paddr,
                                                   GFP_ATOMIC);
                 if (!ctx->enc_cd) {
                         spin_unlock(&ctx->lock);
                         return -ENOMEM;
                 }
-                ctx->dec_cd = dma_zalloc_coherent(dev,
-                                                  sizeof(struct qat_alg_cd),
+                ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
                                                   &ctx->dec_cd_paddr,
                                                   GFP_ATOMIC);
                 if (!ctx->dec_cd) {
@@ -530,18 +602,18 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
                 }
         }
         spin_unlock(&ctx->lock);
-        if (qat_alg_init_sessions(ctx, key, keylen))
+        if (qat_alg_aead_init_sessions(ctx, key, keylen))
                 goto out_free_all;
 
         return 0;
 
 out_free_all:
-        memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
+        memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
                           ctx->dec_cd, ctx->dec_cd_paddr);
         ctx->dec_cd = NULL;
 out_free_enc:
-        memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
+        memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
                           ctx->enc_cd, ctx->enc_cd_paddr);
         ctx->enc_cd = NULL;
@@ -557,7 +629,8 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
         dma_addr_t blp = qat_req->buf.blp;
         dma_addr_t blpout = qat_req->buf.bloutp;
         size_t sz = qat_req->buf.sz;
-        int i, bufs = bl->num_bufs;
+        size_t sz_out = qat_req->buf.sz_out;
+        int i;
 
         for (i = 0; i < bl->num_bufs; i++)
                 dma_unmap_single(dev, bl->bufers[i].addr,
@@ -567,14 +640,14 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
         kfree(bl);
         if (blp != blpout) {
                 /* If out of place operation dma unmap only data */
-                int bufless = bufs - blout->num_mapped_bufs;
+                int bufless = blout->num_bufs - blout->num_mapped_bufs;
 
-                for (i = bufless; i < bufs; i++) {
+                for (i = bufless; i < blout->num_bufs; i++) {
                         dma_unmap_single(dev, blout->bufers[i].addr,
                                          blout->bufers[i].len,
                                          DMA_BIDIRECTIONAL);
                 }
-                dma_unmap_single(dev, blpout, sz, DMA_TO_DEVICE);
+                dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
                 kfree(blout);
         }
 }
@@ -587,19 +660,20 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
                                struct qat_crypto_request *qat_req)
 {
         struct device *dev = &GET_DEV(inst->accel_dev);
-        int i, bufs = 0, n = sg_nents(sgl), assoc_n = sg_nents(assoc);
+        int i, bufs = 0, sg_nctr = 0;
+        int n = sg_nents(sgl), assoc_n = sg_nents(assoc);
         struct qat_alg_buf_list *bufl;
         struct qat_alg_buf_list *buflout = NULL;
         dma_addr_t blp;
         dma_addr_t bloutp = 0;
         struct scatterlist *sg;
-        size_t sz = sizeof(struct qat_alg_buf_list) +
+        size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
                         ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
 
         if (unlikely(!n))
                 return -EINVAL;
 
-        bufl = kmalloc_node(sz, GFP_ATOMIC,
+        bufl = kzalloc_node(sz, GFP_ATOMIC,
                             dev_to_node(&GET_DEV(inst->accel_dev)));
         if (unlikely(!bufl))
                 return -ENOMEM;
@@ -620,15 +694,20 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
                         goto err;
                 bufs++;
         }
-        bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
-                                                 DMA_BIDIRECTIONAL);
-        bufl->bufers[bufs].len = ivlen;
-        if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
-                goto err;
-        bufs++;
+        if (ivlen) {
+                bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
+                                                         DMA_BIDIRECTIONAL);
+                bufl->bufers[bufs].len = ivlen;
+                if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
+                        goto err;
+                bufs++;
+        }
 
         for_each_sg(sgl, sg, n, i) {
-                int y = i + bufs;
+                int y = sg_nctr + bufs;
+
+                if (!sg->length)
+                        continue;
 
                 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
                                                       sg->length,
@@ -636,8 +715,9 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
                 bufl->bufers[y].len = sg->length;
                 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
                         goto err;
+                sg_nctr++;
         }
-        bufl->num_bufs = n + bufs;
+        bufl->num_bufs = sg_nctr + bufs;
         qat_req->buf.bl = bufl;
         qat_req->buf.blp = blp;
         qat_req->buf.sz = sz;
@@ -645,11 +725,15 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
         if (sgl != sglout) {
                 struct qat_alg_buf *bufers;
 
-                buflout = kmalloc_node(sz, GFP_ATOMIC,
+                n = sg_nents(sglout);
+                sz_out = sizeof(struct qat_alg_buf_list) +
+                        ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
+                sg_nctr = 0;
+                buflout = kzalloc_node(sz_out, GFP_ATOMIC,
                                        dev_to_node(&GET_DEV(inst->accel_dev)));
                 if (unlikely(!buflout))
                         goto err;
-                bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE);
+                bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
                 if (unlikely(dma_mapping_error(dev, bloutp)))
                         goto err;
                 bufers = buflout->bufers;
@@ -660,60 +744,62 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
                         bufers[i].addr = bufl->bufers[i].addr;
                 }
                 for_each_sg(sglout, sg, n, i) {
-                        int y = i + bufs;
+                        int y = sg_nctr + bufs;
+
+                        if (!sg->length)
+                                continue;
 
                         bufers[y].addr = dma_map_single(dev, sg_virt(sg),
                                                         sg->length,
                                                         DMA_BIDIRECTIONAL);
-                        buflout->bufers[y].len = sg->length;
                         if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
                                 goto err;
+                        bufers[y].len = sg->length;
+                        sg_nctr++;
                 }
-                buflout->num_bufs = n + bufs;
-                buflout->num_mapped_bufs = n;
+                buflout->num_bufs = sg_nctr + bufs;
+                buflout->num_mapped_bufs = sg_nctr;
                 qat_req->buf.blout = buflout;
                 qat_req->buf.bloutp = bloutp;
+                qat_req->buf.sz_out = sz_out;
         } else {
                 /* Otherwise set the src and dst to the same address */
                 qat_req->buf.bloutp = qat_req->buf.blp;
+                qat_req->buf.sz_out = 0;
         }
         return 0;
 err:
         dev_err(dev, "Failed to map buf for dma\n");
-        for_each_sg(sgl, sg, n + bufs, i) {
-                if (!dma_mapping_error(dev, bufl->bufers[i].addr)) {
+        sg_nctr = 0;
+        for (i = 0; i < n + bufs; i++)
+                if (!dma_mapping_error(dev, bufl->bufers[i].addr))
                         dma_unmap_single(dev, bufl->bufers[i].addr,
                                          bufl->bufers[i].len,
                                          DMA_BIDIRECTIONAL);
-                }
-        }
+
         if (!dma_mapping_error(dev, blp))
                 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
         kfree(bufl);
         if (sgl != sglout && buflout) {
-                for_each_sg(sglout, sg, n, i) {
-                        int y = i + bufs;
-
-                        if (!dma_mapping_error(dev, buflout->bufers[y].addr))
-                                dma_unmap_single(dev, buflout->bufers[y].addr,
-                                                 buflout->bufers[y].len,
+                n = sg_nents(sglout);
+                for (i = bufs; i < n + bufs; i++)
+                        if (!dma_mapping_error(dev, buflout->bufers[i].addr))
+                                dma_unmap_single(dev, buflout->bufers[i].addr,
+                                                 buflout->bufers[i].len,
                                                  DMA_BIDIRECTIONAL);
-                }
                 if (!dma_mapping_error(dev, bloutp))
-                        dma_unmap_single(dev, bloutp, sz, DMA_TO_DEVICE);
+                        dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
                 kfree(buflout);
         }
         return -ENOMEM;
 }
 
-void qat_alg_callback(void *resp)
+static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
+                                  struct qat_crypto_request *qat_req)
 {
-        struct icp_qat_fw_la_resp *qat_resp = resp;
-        struct qat_crypto_request *qat_req =
-                                (void *)(__force long)qat_resp->opaque_data;
-        struct qat_alg_session_ctx *ctx = qat_req->ctx;
+        struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
         struct qat_crypto_instance *inst = ctx->inst;
-        struct aead_request *areq = qat_req->areq;
+        struct aead_request *areq = qat_req->aead_req;
         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 
@@ -723,11 +809,35 @@ void qat_alg_callback(void *resp)
         areq->base.complete(&areq->base, res);
 }
 
-static int qat_alg_dec(struct aead_request *areq)
+static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
+                                        struct qat_crypto_request *qat_req)
+{
+        struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;
+        struct qat_crypto_instance *inst = ctx->inst;
+        struct ablkcipher_request *areq = qat_req->ablkcipher_req;
+        uint8_t stat_filed = qat_resp->comn_resp.comn_status;
+        int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
+
+        qat_alg_free_bufl(inst, qat_req);
+        if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
+                res = -EINVAL;
+        areq->base.complete(&areq->base, res);
+}
+
+void qat_alg_callback(void *resp)
+{
+        struct icp_qat_fw_la_resp *qat_resp = resp;
+        struct qat_crypto_request *qat_req =
+                                (void *)(__force long)qat_resp->opaque_data;
+
+        qat_req->cb(qat_resp, qat_req);
+}
+
+static int qat_alg_aead_dec(struct aead_request *areq)
 {
         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
-        struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
         struct icp_qat_fw_la_cipher_req_params *cipher_param;
         struct icp_qat_fw_la_auth_req_params *auth_param;
@@ -741,9 +851,10 @@ static int qat_alg_dec(struct aead_request *areq)
                 return ret;
 
         msg = &qat_req->req;
-        *msg = ctx->dec_fw_req_tmpl;
-        qat_req->ctx = ctx;
-        qat_req->areq = areq;
+        *msg = ctx->dec_fw_req;
+        qat_req->aead_ctx = ctx;
+        qat_req->aead_req = areq;
+        qat_req->cb = qat_aead_alg_callback;
         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
@@ -766,12 +877,12 @@ static int qat_alg_dec(struct aead_request *areq)
         return -EINPROGRESS;
 }
 
-static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv,
-                                int enc_iv)
+static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv,
+                                     int enc_iv)
 {
         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
-        struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
         struct icp_qat_fw_la_cipher_req_params *cipher_param;
         struct icp_qat_fw_la_auth_req_params *auth_param;
@@ -784,9 +895,10 @@ static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv,
                 return ret;
 
         msg = &qat_req->req;
-        *msg = ctx->enc_fw_req_tmpl;
-        qat_req->ctx = ctx;
-        qat_req->areq = areq;
+        *msg = ctx->enc_fw_req;
+        qat_req->aead_ctx = ctx;
+        qat_req->aead_req = areq;
+        qat_req->cb = qat_aead_alg_callback;
         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
@@ -815,31 +927,168 @@ static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv,
         return -EINPROGRESS;
 }
 
-static int qat_alg_enc(struct aead_request *areq)
+static int qat_alg_aead_enc(struct aead_request *areq)
 {
-        return qat_alg_enc_internal(areq, areq->iv, 0);
+        return qat_alg_aead_enc_internal(areq, areq->iv, 0);
 }
 
-static int qat_alg_genivenc(struct aead_givcrypt_request *req)
+static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req)
 {
         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq);
         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
-        struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
         __be64 seq;
 
         memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);
         seq = cpu_to_be64(req->seq);
         memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t),
                &seq, sizeof(uint64_t));
-        return qat_alg_enc_internal(&req->areq, req->giv, 1);
+        return qat_alg_aead_enc_internal(&req->areq, req->giv, 1);
 }
 
-static int qat_alg_init(struct crypto_tfm *tfm,
-                        enum icp_qat_hw_auth_algo hash, const char *hash_name)
+static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+                                     const uint8_t *key,
+                                     unsigned int keylen)
 {
-        struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+        struct device *dev;
+
+        spin_lock(&ctx->lock);
+        if (ctx->enc_cd) {
+                /* rekeying */
+                dev = &GET_DEV(ctx->inst->accel_dev);
+                memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+                memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+                memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+                memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
+        } else {
+                /* new key */
+                int node = get_current_node();
+                struct qat_crypto_instance *inst =
+                                qat_crypto_get_instance_node(node);
+                if (!inst) {
+                        spin_unlock(&ctx->lock);
+                        return -EINVAL;
+                }
+
+                dev = &GET_DEV(inst->accel_dev);
+                ctx->inst = inst;
+                ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
+                                                  &ctx->enc_cd_paddr,
+                                                  GFP_ATOMIC);
+                if (!ctx->enc_cd) {
+                        spin_unlock(&ctx->lock);
+                        return -ENOMEM;
+                }
+                ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
+                                                  &ctx->dec_cd_paddr,
+                                                  GFP_ATOMIC);
+                if (!ctx->dec_cd) {
+                        spin_unlock(&ctx->lock);
+                        goto out_free_enc;
+                }
+        }
+        spin_unlock(&ctx->lock);
+        if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))
+                goto out_free_all;
+
+        return 0;
+
+out_free_all:
+        memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));
+        dma_free_coherent(dev, sizeof(*ctx->enc_cd),
+                          ctx->dec_cd, ctx->dec_cd_paddr);
+        ctx->dec_cd = NULL;
+out_free_enc:
+        memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));
+        dma_free_coherent(dev, sizeof(*ctx->dec_cd),
+                          ctx->enc_cd, ctx->enc_cd_paddr);
+        ctx->enc_cd = NULL;
+        return -ENOMEM;
+}
+
+static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
+{
+        struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
+        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
+        struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
+        struct icp_qat_fw_la_cipher_req_params *cipher_param;
+        struct icp_qat_fw_la_bulk_req *msg;
+        int ret, ctr = 0;
+
+        ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,
+                                  NULL, 0, qat_req);
+        if (unlikely(ret))
+                return ret;
+
+        msg = &qat_req->req;
+        *msg = ctx->enc_fw_req;
+        qat_req->ablkcipher_ctx = ctx;
+        qat_req->ablkcipher_req = req;
+        qat_req->cb = qat_ablkcipher_alg_callback;
+        qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
+        qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
+        qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
+        cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
+        cipher_param->cipher_length = req->nbytes;
+        cipher_param->cipher_offset = 0;
+        memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+        do {
+                ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
+        } while (ret == -EAGAIN && ctr++ < 10);
+
+        if (ret == -EAGAIN) {
+                qat_alg_free_bufl(ctx->inst, qat_req);
+                return -EBUSY;
+        }
+        return -EINPROGRESS;
+}
+
+static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
+{
+        struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
+        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
+        struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
+        struct icp_qat_fw_la_cipher_req_params *cipher_param;
+        struct icp_qat_fw_la_bulk_req *msg;
+        int ret, ctr = 0;
+
+        ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,
+                                  NULL, 0, qat_req);
+        if (unlikely(ret))
+                return ret;
+
+        msg = &qat_req->req;
+        *msg = ctx->dec_fw_req;
+        qat_req->ablkcipher_ctx = ctx;
+        qat_req->ablkcipher_req = req;
+        qat_req->cb = qat_ablkcipher_alg_callback;
+        qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
+        qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
+        qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
+        cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
+        cipher_param->cipher_length = req->nbytes;
+        cipher_param->cipher_offset = 0;
+        memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+        do {
+                ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
+        } while (ret == -EAGAIN && ctr++ < 10);
+
+        if (ret == -EAGAIN) {
+                qat_alg_free_bufl(ctx->inst, qat_req);
+                return -EBUSY;
+        }
+        return -EINPROGRESS;
+}
+
+static int qat_alg_aead_init(struct crypto_tfm *tfm,
+                             enum icp_qat_hw_auth_algo hash,
+                             const char *hash_name)
+{
+        struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        memzero_explicit(ctx, sizeof(*ctx));
         ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
         if (IS_ERR(ctx->hash_tfm))
                 return -EFAULT;
@@ -851,24 +1100,24 @@ static int qat_alg_init(struct crypto_tfm *tfm,
         return 0;
 }
 
-static int qat_alg_sha1_init(struct crypto_tfm *tfm)
+static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm)
 {
-        return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
+        return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
 }
 
-static int qat_alg_sha256_init(struct crypto_tfm *tfm)
+static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm)
 {
-        return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
+        return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
 }
 
-static int qat_alg_sha512_init(struct crypto_tfm *tfm)
+static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm)
 {
-        return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
+        return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
 }
 
-static void qat_alg_exit(struct crypto_tfm *tfm)
+static void qat_alg_aead_exit(struct crypto_tfm *tfm)
 {
-        struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
         struct qat_crypto_instance *inst = ctx->inst;
         struct device *dev;
 
@@ -880,36 +1129,74 @@ static void qat_alg_exit(struct crypto_tfm *tfm)
 
         dev = &GET_DEV(inst->accel_dev);
         if (ctx->enc_cd) {
-                memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
+                memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
                                   ctx->enc_cd, ctx->enc_cd_paddr);
         }
         if (ctx->dec_cd) {
-                memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
+                memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
                                   ctx->dec_cd, ctx->dec_cd_paddr);
         }
         qat_crypto_put_instance(inst);
 }
 
+static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
+{
+        struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+        spin_lock_init(&ctx->lock);
+        tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
+                                        sizeof(struct qat_crypto_request);
+        ctx->tfm = tfm;
+        return 0;
+}
+
+static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm)
+{
+        struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct qat_crypto_instance *inst = ctx->inst;
+        struct device *dev;
+
+        if (!inst)
+                return;
+
+        dev = &GET_DEV(inst->accel_dev);
+        if (ctx->enc_cd) {
+                memset(ctx->enc_cd, 0,
+                       sizeof(struct icp_qat_hw_cipher_algo_blk));
+                dma_free_coherent(dev,
+                                  sizeof(struct icp_qat_hw_cipher_algo_blk),
+                                  ctx->enc_cd, ctx->enc_cd_paddr);
+        }
+        if (ctx->dec_cd) {
+                memset(ctx->dec_cd, 0,
+                       sizeof(struct icp_qat_hw_cipher_algo_blk));
+                dma_free_coherent(dev,
+                                  sizeof(struct icp_qat_hw_cipher_algo_blk),
+                                  ctx->dec_cd, ctx->dec_cd_paddr);
+        }
+        qat_crypto_put_instance(inst);
+}
+
 static struct crypto_alg qat_algs[] = { {
         .cra_name = "authenc(hmac(sha1),cbc(aes))",
         .cra_driver_name = "qat_aes_cbc_hmac_sha1",
         .cra_priority = 4001,
         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
         .cra_blocksize = AES_BLOCK_SIZE,
-        .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
+        .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
         .cra_alignmask = 0,
         .cra_type = &crypto_aead_type,
         .cra_module = THIS_MODULE,
-        .cra_init = qat_alg_sha1_init,
-        .cra_exit = qat_alg_exit,
+        .cra_init = qat_alg_aead_sha1_init,
+        .cra_exit = qat_alg_aead_exit,
         .cra_u = {
                 .aead = {
-                        .setkey = qat_alg_setkey,
-                        .decrypt = qat_alg_dec,
-                        .encrypt = qat_alg_enc,
-                        .givencrypt = qat_alg_genivenc,
+                        .setkey = qat_alg_aead_setkey,
+                        .decrypt = qat_alg_aead_dec,
+                        .encrypt = qat_alg_aead_enc,
+                        .givencrypt = qat_alg_aead_genivenc,
                         .ivsize = AES_BLOCK_SIZE,
                         .maxauthsize = SHA1_DIGEST_SIZE,
                 },
@@ -920,18 +1207,18 @@ static struct crypto_alg qat_algs[] = { {
         .cra_priority = 4001,
         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
         .cra_blocksize = AES_BLOCK_SIZE,
-        .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
+        .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
         .cra_alignmask = 0,
         .cra_type = &crypto_aead_type,
         .cra_module = THIS_MODULE,
-        .cra_init = qat_alg_sha256_init,
-        .cra_exit = qat_alg_exit,
+        .cra_init = qat_alg_aead_sha256_init,
+        .cra_exit = qat_alg_aead_exit,
         .cra_u = {
                 .aead = {
-                        .setkey = qat_alg_setkey,
-                        .decrypt = qat_alg_dec,
-                        .encrypt = qat_alg_enc,
-                        .givencrypt = qat_alg_genivenc,
+                        .setkey = qat_alg_aead_setkey,
+                        .decrypt = qat_alg_aead_dec,
+                        .encrypt = qat_alg_aead_enc,
+                        .givencrypt = qat_alg_aead_genivenc,
                         .ivsize = AES_BLOCK_SIZE,
                         .maxauthsize = SHA256_DIGEST_SIZE,
                 },
@@ -942,22 +1229,44 @@ static struct crypto_alg qat_algs[] = { {
         .cra_priority = 4001,
         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
         .cra_blocksize = AES_BLOCK_SIZE,
-        .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
+        .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
         .cra_alignmask = 0,
         .cra_type = &crypto_aead_type,
         .cra_module = THIS_MODULE,
-        .cra_init = qat_alg_sha512_init,
-        .cra_exit = qat_alg_exit,
+        .cra_init = qat_alg_aead_sha512_init,
+        .cra_exit = qat_alg_aead_exit,
         .cra_u = {
                 .aead = {
-                        .setkey = qat_alg_setkey,
-                        .decrypt = qat_alg_dec,
-                        .encrypt = qat_alg_enc,
-                        .givencrypt = qat_alg_genivenc,
+                        .setkey = qat_alg_aead_setkey,
+                        .decrypt = qat_alg_aead_dec,
+                        .encrypt = qat_alg_aead_enc,
+                        .givencrypt = qat_alg_aead_genivenc,
                         .ivsize = AES_BLOCK_SIZE,
                         .maxauthsize = SHA512_DIGEST_SIZE,
                 },
         },
+}, {
+        .cra_name = "cbc(aes)",
+        .cra_driver_name = "qat_aes_cbc",
+        .cra_priority = 4001,
+        .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize = AES_BLOCK_SIZE,
+        .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
+        .cra_alignmask = 0,
+        .cra_type = &crypto_ablkcipher_type,
+        .cra_module = THIS_MODULE,
+        .cra_init = qat_alg_ablkcipher_init,
+        .cra_exit = qat_alg_ablkcipher_exit,
+        .cra_u = {
+                .ablkcipher = {
+                        .setkey = qat_alg_ablkcipher_setkey,
+                        .decrypt = qat_alg_ablkcipher_decrypt,
+                        .encrypt = qat_alg_ablkcipher_encrypt,
+                        .min_keysize = AES_MIN_KEY_SIZE,
+                        .max_keysize = AES_MAX_KEY_SIZE,
+                        .ivsize = AES_BLOCK_SIZE,
+                },
+        },
 } };
 
 int qat_algs_register(void)
@@ -966,8 +1275,11 @@ int qat_algs_register(void)
                 int i;
 
                 for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
-                        qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_AEAD |
-                                                CRYPTO_ALG_ASYNC;
+                        qat_algs[i].cra_flags =
+                                (qat_algs[i].cra_type == &crypto_aead_type) ?
+                                CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC :
+                                CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
+
                 return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
         }
         return 0;
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index ab8468d11ddb..d503007b49e6 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -72,12 +72,24 @@ struct qat_crypto_request_buffs {
         struct qat_alg_buf_list *blout;
         dma_addr_t bloutp;
         size_t sz;
+        size_t sz_out;
 };
 
+struct qat_crypto_request;
+
 struct qat_crypto_request {
         struct icp_qat_fw_la_bulk_req req;
-        struct qat_alg_session_ctx *ctx;
-        struct aead_request *areq;
+        union {
+                struct qat_alg_aead_ctx *aead_ctx;
+                struct qat_alg_ablkcipher_ctx *ablkcipher_ctx;
+        };
+        union {
+                struct aead_request *aead_req;
+                struct ablkcipher_request *ablkcipher_req;
+        };
         struct qat_crypto_request_buffs buf;
+        void (*cb)(struct icp_qat_fw_la_resp *resp,
+                   struct qat_crypto_request *req);
 };
+
 #endif
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
index ef05825cc651..6a735d5c0e37 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
@@ -46,6 +46,7 @@
 */
 #include <adf_accel_devices.h>
 #include "adf_dh895xcc_hw_data.h"
+#include "adf_common_drv.h"
 #include "adf_drv.h"
 
 /* Worker thread to service arbiter mappings based on dev SKUs */
@@ -182,6 +183,19 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev)
         }
 }
 
+static void adf_enable_ints(struct adf_accel_dev *accel_dev)
+{
+        void __iomem *addr;
+
+        addr = (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr;
+
+        /* Enable bundle and misc interrupts */
+        ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET,
+                   ADF_DH895XCC_SMIA0_MASK);
+        ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET,
+                   ADF_DH895XCC_SMIA1_MASK);
+}
+
 void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
 {
         hw_data->dev_class = &dh895xcc_class;
@@ -206,6 +220,11 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
         hw_data->get_misc_bar_id = get_misc_bar_id;
         hw_data->get_sku = get_sku;
         hw_data->fw_name = ADF_DH895XCC_FW;
+        hw_data->init_admin_comms = adf_init_admin_comms;
+        hw_data->exit_admin_comms = adf_exit_admin_comms;
+        hw_data->init_arb = adf_init_arb;
+        hw_data->exit_arb = adf_exit_arb;
+        hw_data->enable_ints = adf_enable_ints;
 }
 
 void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
index 948f66be262b..8ffdb95c9804 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
@@ -90,9 +90,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
         struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
         int i;
 
-        adf_exit_admin_comms(accel_dev);
-        adf_exit_arb(accel_dev);
-        adf_cleanup_etr_data(accel_dev);
+        adf_dev_shutdown(accel_dev);
 
         for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
@@ -119,7 +117,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
         kfree(accel_dev);
 }
 
-static int qat_dev_start(struct adf_accel_dev *accel_dev)
+static int adf_dev_configure(struct adf_accel_dev *accel_dev)
 {
         int cpus = num_online_cpus();
         int banks = GET_MAX_BANKS(accel_dev);
@@ -206,7 +204,7 @@ static int qat_dev_start(struct adf_accel_dev *accel_dev)
                 goto err;
 
         set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
-        return adf_dev_start(accel_dev);
+        return 0;
 err:
         dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
         return -EINVAL;
@@ -217,7 +215,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
         struct adf_accel_dev *accel_dev;
         struct adf_accel_pci *accel_pci_dev;
         struct adf_hw_device_data *hw_data;
-        void __iomem *pmisc_bar_addr = NULL;
         char name[ADF_DEVICE_NAME_LENGTH];
         unsigned int i, bar_nr;
         int ret;
@@ -347,8 +344,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
                         ret = -EFAULT;
                         goto out_err;
                 }
-                if (i == ADF_DH895XCC_PMISC_BAR)
-                        pmisc_bar_addr = bar->virt_addr;
         }
         pci_set_master(pdev);
 
@@ -358,36 +353,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
                 goto out_err;
         }
 
-        if (adf_init_etr_data(accel_dev)) {
-                dev_err(&pdev->dev, "Failed initialize etr\n");
-                ret = -EFAULT;
-                goto out_err;
-        }
-
-        if (adf_init_admin_comms(accel_dev)) {
-                dev_err(&pdev->dev, "Failed initialize admin comms\n");
-                ret = -EFAULT;
-                goto out_err;
-        }
-
-        if (adf_init_arb(accel_dev)) {
-                dev_err(&pdev->dev, "Failed initialize hw arbiter\n");
-                ret = -EFAULT;
-                goto out_err;
-        }
         if (pci_save_state(pdev)) {
                 dev_err(&pdev->dev, "Failed to save pci state\n");
                 ret = -ENOMEM;
                 goto out_err;
         }
 
-        /* Enable bundle and misc interrupts */
-        ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET,
-                   ADF_DH895XCC_SMIA0_MASK);
-        ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET,
-                   ADF_DH895XCC_SMIA1_MASK);
+        ret = adf_dev_configure(accel_dev);
+        if (ret)
+                goto out_err;
+
+        ret = adf_dev_init(accel_dev);
+        if (ret)
+                goto out_err;
 
-        ret = qat_dev_start(accel_dev);
+        ret = adf_dev_start(accel_dev);
         if (ret) {
                 adf_dev_stop(accel_dev);
                 goto out_err;
diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c
index 0fb21e13f247..378cb768647f 100644
--- a/drivers/crypto/qce/dma.c
+++ b/drivers/crypto/qce/dma.c
@@ -64,7 +64,7 @@ int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents,
                         err = dma_map_sg(dev, sg, 1, dir);
                         if (!err)
                                 return -EFAULT;
-                        sg = scatterwalk_sg_next(sg);
+                        sg = sg_next(sg);
                 }
         } else {
                 err = dma_map_sg(dev, sg, nents, dir);
@@ -81,7 +81,7 @@ void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents,
         if (chained)
                 while (sg) {
                         dma_unmap_sg(dev, sg, 1, dir);
-                        sg = scatterwalk_sg_next(sg);
+                        sg = sg_next(sg);
                 }
         else
                 dma_unmap_sg(dev, sg, nents, dir);
@@ -100,7 +100,7 @@ int qce_countsg(struct scatterlist *sglist, int nbytes, bool *chained)
                 nbytes -= sg->length;
                 if (!sg_is_last(sg) && (sg + 1)->length == 0 && chained)
                         *chained = true;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         return nents;
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index f3385934eed2..5c5df1d17f90 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -285,7 +285,7 @@ static int qce_ahash_update(struct ahash_request *req)
                         break;
                 len += sg_dma_len(sg);
                 sg_last = sg;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         if (!sg_last)
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 220b92f7eabc..290a7f0a681f 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -940,7 +940,7 @@ static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
                         break;
                 }
                 nbytes -= sg->length;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         return nbytes;
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 067ec2193d71..ebbae8d3ce0d 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -743,7 +743,7 @@ static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
         if (unlikely(chained))
                 while (sg) {
                         dma_map_sg(dev, sg, 1, dir);
-                        sg = scatterwalk_sg_next(sg);
+                        sg = sg_next(sg);
                 }
         else
                 dma_map_sg(dev, sg, nents, dir);
@@ -755,7 +755,7 @@ static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
 {
         while (sg) {
                 dma_unmap_sg(dev, sg, 1, dir);
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 }
 
@@ -915,7 +915,7 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
                 link_tbl_ptr->j_extent = 0;
                 link_tbl_ptr++;
                 cryptlen -= sg_dma_len(sg);
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         /* adjust (decrease) last one (or two) entry's len to cryptlen */
@@ -1102,7 +1102,7 @@ static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained)
                 nbytes -= sg->length;
                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
                         *chained = true;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         return sg_nents;
diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c
index f831bb952b2f..d594ae962ed2 100644
--- a/drivers/crypto/ux500/cryp/cryp_core.c
+++ b/drivers/crypto/ux500/cryp/cryp_core.c
@@ -479,13 +479,13 @@ static void cryp_dma_setup_channel(struct cryp_device_data *device_data,
                 .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO,
                 .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
                 .dst_maxburst = 4,
-        };
+        };
         struct dma_slave_config cryp2mem = {
                 .direction = DMA_DEV_TO_MEM,
                 .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO,
                 .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
                 .src_maxburst = 4,
-        };
+        };
 
         dma_cap_zero(device_data->dma.mask);
         dma_cap_set(DMA_SLAVE, device_data->dma.mask);
@@ -814,7 +814,7 @@ static int get_nents(struct scatterlist *sg, int nbytes)
 
         while (nbytes > 0) {
                 nbytes -= sg->length;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
                 nents++;
         }
 
@@ -1774,8 +1774,8 @@ static int ux500_cryp_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume);
 
 static const struct of_device_id ux500_cryp_match[] = {
-        { .compatible = "stericsson,ux500-cryp" },
-        { },
+        { .compatible = "stericsson,ux500-cryp" },
+        { },
 };
 
 static struct platform_driver cryp_driver = {
diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h
index 88ea64e9a91c..178525e5f430 100644
--- a/include/crypto/if_alg.h
+++ b/include/crypto/if_alg.h
@@ -50,6 +50,7 @@ struct af_alg_type {
         void (*release)(void *private);
         int (*setkey)(void *private, const u8 *key, unsigned int keylen);
         int (*accept)(void *private, struct sock *sk);
+        int (*setauthsize)(void *private, unsigned int authsize);
 
         struct proto_ops *ops;
         struct module *owner;
diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h
index 7ef512f8631c..20e4226a2e14 100644
--- a/include/crypto/scatterwalk.h
+++ b/include/crypto/scatterwalk.h
@@ -33,21 +33,13 @@ static inline void scatterwalk_sg_chain(struct scatterlist *sg1, int num,
         sg1[num - 1].page_link |= 0x01;
 }
 
-static inline struct scatterlist *scatterwalk_sg_next(struct scatterlist *sg)
-{
-        if (sg_is_last(sg))
-                return NULL;
-
-        return (++sg)->length ? sg : sg_chain_ptr(sg);
-}
-
 static inline void scatterwalk_crypto_chain(struct scatterlist *head,
                                             struct scatterlist *sg,
                                             int chain, int num)
 {
         if (chain) {
                 head->length += sg->length;
-                sg = scatterwalk_sg_next(sg);
+                sg = sg_next(sg);
         }
 
         if (sg)
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index 9c8776d0ada8..fb5ef16d6a12 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -1147,7 +1147,7 @@ static inline void ablkcipher_request_free(struct ablkcipher_request *req)
  * cipher operation completes.
  *
  * The callback function is registered with the ablkcipher_request handle and
- * must comply with the following template:
+ * must comply with the following template
  *
  *      void callback_function(struct crypto_async_request *req, int error)
  */
@@ -1174,7 +1174,7 @@ static inline void ablkcipher_request_set_callback(
  *
  * For encryption, the source is treated as the plaintext and the
  * destination is the ciphertext. For a decryption operation, the use is
- * reversed: the source is the ciphertext and the destination is the plaintext.
+ * reversed - the source is the ciphertext and the destination is the plaintext.
  */
 static inline void ablkcipher_request_set_crypt(
         struct ablkcipher_request *req,
@@ -1412,6 +1412,9 @@ static inline int crypto_aead_encrypt(struct aead_request *req)
  */
 static inline int crypto_aead_decrypt(struct aead_request *req)
 {
+        if (req->cryptlen < crypto_aead_authsize(crypto_aead_reqtfm(req)))
+                return -EINVAL;
+
         return crypto_aead_crt(crypto_aead_reqtfm(req))->decrypt(req);
 }
 
@@ -1506,7 +1509,7 @@ static inline void aead_request_free(struct aead_request *req)
  * completes
  *
  * The callback function is registered with the aead_request handle and
- * must comply with the following template:
+ * must comply with the following template
  *
  *      void callback_function(struct crypto_async_request *req, int error)
  */
@@ -1533,7 +1536,7 @@ static inline void aead_request_set_callback(struct aead_request *req,
  *
  * For encryption, the source is treated as the plaintext and the
  * destination is the ciphertext. For a decryption operation, the use is
- * reversed: the source is the ciphertext and the destination is the plaintext.
+ * reversed - the source is the ciphertext and the destination is the plaintext.
  *
  * IMPORTANT NOTE AEAD requires an authentication tag (MAC). For decryption,
  *                the caller must concatenate the ciphertext followed by the
diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
index 914bb08cd738..eb7b414d232b 100644
--- a/include/linux/hw_random.h
+++ b/include/linux/hw_random.h
@@ -12,8 +12,10 @@
 #ifndef LINUX_HWRANDOM_H_
 #define LINUX_HWRANDOM_H_
 
+#include <linux/completion.h>
 #include <linux/types.h>
 #include <linux/list.h>
+#include <linux/kref.h>
 
 /**
  * struct hwrng - Hardware Random Number Generator driver
@@ -44,6 +46,8 @@ struct hwrng {
 
         /* internal. */
         struct list_head list;
+        struct kref ref;
+        struct completion cleanup_done;
 };
 
 /** Register a new Hardware Random Number Generator driver. */
diff --git a/lib/string.c b/lib/string.c
index cdd97f431ae2..ce81aaec3839 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -596,6 +596,11 @@ EXPORT_SYMBOL(memset);
  * @s: Pointer to the start of the area.
  * @count: The size of the area.
  *
+ * Note: usually using memset() is just fine (!), but in cases
+ * where clearing out _local_ data at the end of a scope is
+ * necessary, memzero_explicit() should be used instead in
+ * order to prevent the compiler from optimising away zeroing.
+ *
  * memzero_explicit() doesn't need an arch-specific version as
  * it just invokes the one of memset() implicitly.
  */